fix(status): stop false LOAD FAILED from a relayed adsb.lol 429 and the installations zoom prompt

Two independent triggers turned the global status chip red while every layer
was still populated (reproduced at JFK in Cockpit; timeline in the issue).

adsb.lol proxy: a non-OK upstream status was relayed verbatim even with a
12 s-old cache in hand, and the browser layer answers a 429 with a 45 s
cooldown in which every refresh reports "adsb.lol rate limited" without a
network call. The proxy now serves its cached body as STALE on a 429/5xx and
backs off from upstream for Retry-After (bounded), mirroring the OpenSky
proxy. A failure with nothing cached is still relayed.

Mapped installations: the "zoom in" guidance was stored in `error`, so the
manager reported a refresh failure and the chip a LOAD FAILED whenever the
view could not be bounded (Cockpit looks at the horizon). The status is now
set without an error string; the row reads the prompt from `statusMessage`,
and the chip ignores `error` for guidance statuses, matching the carve-out
layerFeedState already has.

server/providers/aircraft/adsb-lol.js

@@ -1,8 +1,16 @@
 /**
  * Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
  *
- * Proxies GET /api/adsblol/mil to https://api.adsb.lol/v2/mil. On upstream
- * failure, serves a stale cached response if one exists.
+ * Proxies GET /api/adsblol/mil to https://api.adsb.lol/v2/mil. When upstream
+ * fails — a thrown fetch OR a non-OK status such as 429 — the proxy serves its
+ * cached body as STALE instead of relaying the failure, and it backs off from
+ * upstream for the Retry-After period (bounded) so a rate limit is never
+ * hammered mid-cooldown. A failure with nothing cached is relayed as-is.
+ *
+ * Why the non-OK case matters: the browser layer treats a relayed 429 as a
+ * 45 s cooldown during which EVERY refresh (the 15 s poll and any tracking
+ * resolve) reports "adsb.lol rate limited", which the global status chip
+ * renders as LOAD FAILED while the map is still fully populated.
  *
  * @returns {import('vite').Plugin}
  */
@@ -11,19 +19,72 @@ export function adsbLolProxy() {
   let _cache = null;
   /** @type {number} Epoch-ms when the cache was populated. */
   let _cacheAt = 0;
+  /** @type {number} Epoch-ms until which upstream is not contacted. */
+  let _cooldownUntil = 0;
+  /** @type {number} Upstream status that started the current cooldown. */
+  let _cooldownStatus = 0;
   /** Response cache TTL (ms). */
   const CACHE_MS = 12000;
+  /** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
+  const RATE_LIMIT_COOLDOWN_MS = 30000;
+  /** Cooldown after a 5xx (ms). */
+  const SERVER_ERROR_COOLDOWN_MS = 15000;
+  /** Bounds for an upstream-supplied Retry-After (ms). */
+  const COOLDOWN_MIN_MS = 5000;
+  const COOLDOWN_MAX_MS = 120000;
+
+  const clampCooldown = (ms) =>
+    Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms));
+
+  /** Cooldown (ms) an upstream failure earns, honouring Retry-After when sane. */
+  function cooldownFor(upstream, now) {
+    const raw = upstream.headers?.get?.('retry-after');
+    if (raw) {
+      const seconds = Number(raw);
+      if (Number.isFinite(seconds) && seconds > 0)
+        return clampCooldown(seconds * 1000);
+      const at = Date.parse(raw);
+      if (Number.isFinite(at) && at > now) return clampCooldown(at - now);
+    }
+    return upstream.status === 429
+      ? RATE_LIMIT_COOLDOWN_MS
+      : SERVER_ERROR_COOLDOWN_MS;
+  }
+
+  function serve(res, status, body, cacheStatus, extra = {}) {
+    res.writeHead(status, {
+      'Content-Type': 'application/json',
+      'Cache-Control': 'no-store',
+      'X-ADS-B-Cache': cacheStatus,
+      ...extra,
+    });
+    res.end(body);
+  }
+
   const installMiddleware = (server) => {
     server.middlewares.use('/api/adsblol/mil', async (req, res) => {
       try {
         const now = Date.now();
         if (_cache && now - _cacheAt < CACHE_MS) {
-          res.writeHead(200, {
-            'Content-Type': 'application/json',
-            'Cache-Control': 'no-store',
-            'X-ADS-B-Cache': 'HIT',
-          });
-          res.end(_cache);
+          serve(res, 200, _cache, 'HIT');
+          return;
+        }
+        if (now < _cooldownUntil) {
+          const retryAfter = String(Math.ceil((_cooldownUntil - now) / 1000));
+          if (_cache) {
+            serve(res, 200, _cache, 'STALE', {
+              'X-ADS-B-Upstream-Status': String(_cooldownStatus),
+              'X-ADS-B-Retry-After-Seconds': retryAfter,
+            });
+            return;
+          }
+          serve(
+            res,
+            _cooldownStatus || 503,
+            JSON.stringify({ error: 'adsb.lol upstream cooling down' }),
+            'NONE',
+            { 'Retry-After': retryAfter },
+          );
           return;
         }
         const upstream = await fetch('https://api.adsb.lol/v2/mil', {
@@ -33,21 +94,32 @@ export function adsbLolProxy() {
         if (upstream.ok) {
           _cache = body;
           _cacheAt = now;
+          _cooldownUntil = 0;
+          _cooldownStatus = 0;
+          serve(res, 200, body, 'MISS');
+          return;
         }
-        res.writeHead(upstream.status, {
-          'Content-Type': 'application/json',
-          'Cache-Control': 'no-store',
-          'X-ADS-B-Cache': 'MISS',
-        });
-        res.end(body);
+        if (upstream.status === 429 || upstream.status >= 500) {
+          _cooldownUntil = now + cooldownFor(upstream, now);
+          _cooldownStatus = upstream.status;
+          console.warn(
+            `[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - now) / 1000)} s${_cache ? ', serving stale' : ''}`,
+          );
+          if (_cache) {
+            serve(res, 200, _cache, 'STALE', {
+              'X-ADS-B-Upstream-Status': String(upstream.status),
+              'X-ADS-B-Retry-After-Seconds': String(
+                Math.ceil((_cooldownUntil - now) / 1000),
+              ),
+            });
+            return;
+          }
+        }
+        serve(res, upstream.status, body, 'MISS');
       } catch (e) {
         console.error('[adsb.lol Proxy]', e.message);
         if (_cache) {
-          res.writeHead(200, {
-            'Content-Type': 'application/json',
-            'X-ADS-B-Cache': 'STALE',
-          });
-          res.end(_cache);
+          serve(res, 200, _cache, 'STALE');
           return;
         }
         res.writeHead(502, { 'Content-Type': 'application/json' });

src/data/manager.js

@@ -1,4 +1,5 @@
 import { governorRequestRender } from '../renderGovernor.js';
+import { GUIDANCE_STATUSES } from '../loadingFeedback.js';
 import { markDetectionSourcesChanged } from './detection.js';
 function cloneLayerParams(value) {
   if (Array.isArray(value)) return value.map(cloneLayerParams);
@@ -2233,6 +2234,12 @@ export class DataLayerManager {
       }
       return `${stateLabel} · ${source} · ${presentedError}`;
     }
+    // A guidance status carries its prompt in `statusMessage`, not `error`, so
+    // the row still tells the operator what to do without reporting a fault.
+    if (GUIDANCE_STATUSES.includes(String(stats.status || '').toLowerCase())
+        && typeof stats.statusMessage === 'string' && stats.statusMessage.trim()) {
+      return `${source} · ${stats.statusMessage.trim()}`;
+    }
     const ago = stats.lastUpdate ? this._timeAgo(stats.lastUpdate) : 'never';
     if (stats.loading) {
       const loadingLabel = typeof stats.loadingLabel === 'string' && stats.loadingLabel.trim()

src/data/manager.test.mjs

@@ -3256,3 +3256,14 @@ test('a layer that surrenders its row controls hides the block entirely', async
     else globalThis.document = originalDocument;
   }
 });
+
+test('layer metadata shows a guidance prompt without reporting it as a fault', () => {
+  const mgr = new DataLayerManager({});
+  const text = mgr._buildMetaText({
+    enabled: true,
+    source: 'OpenStreetMap',
+    stats: { status: 'zoom-in', error: null, count: 0, statusMessage: 'Zoom in to search mapped installations' },
+  });
+  assert.match(text, /Zoom in to search mapped installations$/);
+  assert.doesNotMatch(text, /UNAVAILABLE|DEGRADED/);
+});

src/data/militaryInstallations.js

@@ -425,12 +425,18 @@ function scheduleLoad() {
 async function loadInstallations() {
   if (!state.enabled || !state.viewer) return;
   const box = viewportBox(state.viewer);
+  // Guidance, not a fault: the layer chose not to query because the view is
+  // unbounded (a global view, or Cockpit looking at the horizon). Keep it out
+  // of `error` — the manager derives refresh failures and the global status
+  // chip derives LOAD FAILED from that field, and a "zoom in" prompt turned
+  // every Cockpit refresh into a reported failure. The row and toast read the
+  // prompt from `statusMessage` (installationFeedback) instead.
   if (!box) {
     state.abort?.abort();
     state.abort = null;
     state.loading = false;
     clearUnavailableRetry();
-    setInstallationStatus('zoom-in', 'Zoom in to load mapped installation context');
+    setInstallationStatus('zoom-in');
     return;
   }
   state.abort?.abort();

src/data/militaryInstallations.test.mjs

@@ -758,6 +758,8 @@ test('zoom-out aborts an active installation request and returns non-loading gui
     assert.equal(observedSignal.aborted, true);
     assert.equal(militaryInstallationsLayer.getStats().loading, false);
     assert.equal(militaryInstallationsLayer.getStats().status, 'zoom-in');
+    assert.equal(militaryInstallationsLayer.getStats().error, null, 'guidance is not a fault');
+    assert.match(militaryInstallationsLayer.getStats().statusMessage, /zoom in/i);
   } finally {
     militaryInstallationsLayer.destroy(viewer);
     globalThis.fetch = originalFetch;

src/loadingFeedback.js

@@ -5,6 +5,8 @@ export const LOADING_TERMINAL_DWELL_MS = 2200;
 export const LOADING_FAILURE_DWELL_MS = 5000;
 export const LOADING_LONG_THRESHOLD_MS = 30000;
 export const TRAFFIC_SYNC_CONFIRM_MS = 1500;
+/** Layer statuses that are user guidance, not feed faults (see manager.js layerFeedState). */
+export const GUIDANCE_STATUSES = Object.freeze(['zoom-in', 'empty', 'idle']);
 
 function finiteCount(value) {
   const number = Number(value);
@@ -20,7 +22,13 @@ export function normalizeLayerLoading(layer = {}) {
   const loading = lifecycleState === 'enabling' || disabling || stats.loading === true || stats.refreshing === true;
   const count = finiteCount(stats.count);
   const stoppingInstallations = layer.id === 'military-installations' && disabling;
-  const error = stoppingInstallations ? null : stats.error || stats.lastError || stats.managerRefreshError || null;
+  // Guidance statuses ask the user to act (zoom in, run a search). They are
+  // normal operation, never a batch failure — mirrors layerFeedState's carve-out
+  // so a prompt stored alongside the status cannot turn the chip red.
+  const guidance = GUIDANCE_STATUSES.includes(status);
+  const error = stoppingInstallations || guidance
+    ? null
+    : stats.error || stats.lastError || stats.managerRefreshError || null;
   const unavailable = !stoppingInstallations && (stats.unavailable === true
     || stats.available === false
     || ['unavailable', 'offline', 'down', 'error'].includes(status));

src/loadingFeedback.test.mjs

@@ -685,3 +685,17 @@ test('the loading ticker never runs hidden and stops after loading and notices s
     'the resample handler must be removed on teardown',
   );
 });
+
+test('a guidance status such as zoom-in never counts as a participant failure', () => {
+  const zoomIn = { id: 'military-installations', name: 'Mapped Installations', enabled: true,
+    stats: { status: 'zoom-in', error: 'Zoom in to load mapped installation context', loading: true, count: 0 } };
+  const loading = aggregateLayerLoading([zoomIn]);
+  assert.equal(loading.records[0].error, null);
+  assert.equal(loading.records[0].degraded, false);
+  let state = reduceLoadingFeedback(createLoadingFeedbackState(), loading, 1000);
+  state = reduceLoadingFeedback(state, loading, 1200);
+  const settled = aggregateLayerLoading([{ ...zoomIn, stats: { ...zoomIn.stats, loading: false } }]);
+  state = reduceLoadingFeedback(state, settled, 1500);
+  assert.equal(state.terminal, 'complete');
+  assert.equal(presentLoadingFeedback(state, settled, 1500).label, 'MAPPED SITES LOADED');
+});

src/tooling/liveProviders.test.mjs

@@ -230,3 +230,62 @@ test('AIS preview route ingests through the socket, returns tracks and disposes
   for (let i = 0; i < 100 && sockets.length < 2; i++) await delay(10);
   assert.equal(sockets.length, 2);
 });
+
+test('military aircraft route serves stale cache on an upstream 429 and cools down for Retry-After', async (t) => {
+  let now = Date.now();
+  let calls = 0;
+  t.mock.method(Date, 'now', () => now);
+  t.mock.method(console, 'warn', () => {});
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    if (calls === 1) return Response.json({ ac: [{ hex: 'abc123' }] });
+    return new Response(JSON.stringify({ error: 'rate limited' }), {
+      status: 429,
+      headers: { 'Retry-After': '20' },
+    });
+  });
+  const request = install(providers.adsbLolProxy());
+  const first = await request('/api/adsblol/mil');
+  assert.equal(first.statusCode, 200);
+  now += 13_000;
+  const limited = await request('/api/adsblol/mil');
+  assert.equal(calls, 2);
+  assert.equal(
+    limited.statusCode,
+    200,
+    'a 429 with a cached body is never relayed',
+  );
+  assert.equal(limited.body, first.body);
+  assert.equal(limited.headers['x-ads-b-cache'], 'STALE');
+  assert.equal(limited.headers['x-ads-b-upstream-status'], '429');
+  now += 5_000;
+  const cooling = await request('/api/adsblol/mil');
+  assert.equal(calls, 2, 'no upstream call inside the Retry-After window');
+  assert.equal(cooling.headers['x-ads-b-cache'], 'STALE');
+  now += 16_000;
+  await request('/api/adsblol/mil');
+  assert.equal(calls, 3, 'upstream is retried once Retry-After elapses');
+});
+
+test('military aircraft route relays an upstream 429 when nothing is cached', async (t) => {
+  t.mock.method(console, 'warn', () => {});
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    return new Response('{"error":"rate limited"}', { status: 429 });
+  });
+  const request = install(providers.adsbLolProxy());
+  const limited = await request('/api/adsblol/mil');
+  assert.equal(limited.statusCode, 429);
+  const again = await request('/api/adsblol/mil');
+  assert.equal(again.statusCode, 429);
+  assert.equal(
+    calls,
+    1,
+    'the cooldown still protects upstream with nothing cached',
+  );
+  assert.ok(
+    again.headers['retry-after'],
+    'a cooling-down miss carries Retry-After',
+  );
+});