fix: recover GNSS feed startup and honor retry cooldowns

src/layers/gnss/index.js

@@ -73,12 +73,15 @@ export function createGnssIntegrityLayer({
   let _stale = false;
   let _manager = null;
   let _cameraTimer = null;
-  let _cameraRefresh = null;
+  let _backgroundRefresh = null;
   let _lastQueryKey = null;
   let _lastRequestAt = -Infinity;
   let _removeCameraEnd = null;
   let _removeCameraChanged = null;
   let _coverage = null;
+  let _retryTimer = null;
+  let _retryAt = 0;
+  let _failureCount = 0;
   const _observations = new Map();
   const provenance = gnssProvenance({ windowMs });
 
@@ -89,6 +92,7 @@ export function createGnssIntegrityLayer({
 
   function scheduleCameraRefresh() {
     if (!_enabled || !_dataSource) return;
+    if (now() < _retryAt) return;
     const key = queryKey(viewAnchor(_viewer));
     if (!key || key === _lastQueryKey) return;
     clearTimeout(_cameraTimer);
@@ -97,34 +101,48 @@ export function createGnssIntegrityLayer({
     _cameraTimer = setTimeout(
       async () => {
         _cameraTimer = null;
-        if (!_enabled || _cameraRefresh) return;
+        if (!_enabled || _backgroundRefresh) return;
         const key = queryKey(viewAnchor(_viewer));
         if (!key || key === _lastQueryKey) return;
-        const controller = new AbortController();
-        _cameraRefresh = controller;
-        try {
-          if (_manager) {
-            // Use the normal refresh lane so counts, loading/errors and render
-            // demand settle together; wait out an older periodic request.
-            await _manager.refreshLayer(layer.id, {
-              signal: controller.signal,
-            });
-          } else {
-            await layer.update(undefined, { signal: controller.signal });
-          }
-        } finally {
-          if (_cameraRefresh === controller) {
-            _cameraRefresh = null;
-            // Movement during an in-flight request is folded into one update
-            // for the latest view, rather than lost until the minute tick.
-            scheduleCameraRefresh();
-          }
-        }
+        await refreshInBackground();
       },
       Math.max(600, 2500 - (now() - _lastRequestAt)),
     );
   }
 
+  async function refreshInBackground() {
+    if (!_enabled || _backgroundRefresh) return;
+    const controller = new AbortController();
+    _backgroundRefresh = controller;
+    try {
+      if (_manager)
+        await _manager.refreshLayer(layer.id, { signal: controller.signal });
+      else await layer.update(undefined, { signal: controller.signal });
+    } finally {
+      if (_backgroundRefresh === controller) {
+        _backgroundRefresh = null;
+        scheduleCameraRefresh();
+        scheduleRetry();
+      }
+    }
+  }
+
+  function scheduleRetry() {
+    clearTimeout(_retryTimer);
+    _retryTimer = null;
+    if (!_enabled || !_retryAt) return;
+    // Long provider cooldowns keep their full deadline. Browser timers cannot
+    // exceed ~24 days, so wake in bounded chunks without fetching early.
+    _retryTimer = setTimeout(
+      () => {
+        _retryTimer = null;
+        void refreshInBackground();
+      },
+      Math.min(2147483647, Math.max(0, _retryAt - now())),
+    );
+    _retryTimer?.unref?.();
+  }
+
   /** Age the store without new evidence, so expired cells leave the globe. */
   function expire() {
     pruneGnssObservations(_observations, now(), { windowMs });
@@ -205,8 +223,10 @@ export function createGnssIntegrityLayer({
     disable() {
       clearTimeout(_cameraTimer);
       _cameraTimer = null;
-      _cameraRefresh?.abort();
-      _cameraRefresh = null;
+      clearTimeout(_retryTimer);
+      _retryTimer = null;
+      _backgroundRefresh?.abort();
+      _backgroundRefresh = null;
       _request?.abort();
       _request = null;
       _enabled = false;
@@ -215,6 +235,14 @@ export function createGnssIntegrityLayer({
 
     async update(_viewerArg, { signal } = {}) {
       if (!_enabled || !_dataSource || signal?.aborted) return false;
+      if (now() < _retryAt) {
+        expire();
+        scheduleRetry();
+        return true;
+      }
+      _retryAt = 0;
+      clearTimeout(_retryTimer);
+      _retryTimer = null;
       const anchor = viewAnchor(_viewer);
       // No anchor yet (camera still settling) is not a failure: returning
       // false would make the manager reject the enable. The next interval
@@ -236,12 +264,14 @@ export function createGnssIntegrityLayer({
         longitude: Math.round(anchor.longitude),
         radiusNm: 250,
       };
+      let received = false;
       try {
         const snapshot = await source.getSnapshot(anchor, {
           signal: request.signal,
         });
         if (request.signal.aborted || _request !== request || !_enabled)
           return false;
+        received = true;
         // Date the rows by when the proxy observed them: a cached or stale
         // replay keeps its original age instead of restarting the window.
         // The proxy reports that age, so client and server clocks never mix.
@@ -255,19 +285,37 @@ export function createGnssIntegrityLayer({
         _stale = snapshot.stale === true;
         _lastUpdate = now();
         _lastError = null;
+        _retryAt = 0;
+        _failureCount = 0;
         return true;
       } catch (e) {
         if (request.signal.aborted || _request !== request || !_enabled)
           return false;
+        // A renderer/programming failure remains a real lifecycle failure.
+        // Only acquisition failures are kept enabled for recovery.
+        if (received) throw e;
         console.warn('[Data:GNSS] Fetch error:', e);
         _lastError = e?.message || 'GNSS integrity source unavailable';
         // Keep the last cells only while their evidence is inside the window.
         expire();
-        return false;
+        if (e?.retryable === false) return false;
+        _failureCount = Math.min(3, _failureCount + 1);
+        const delay =
+          Number.isFinite(e?.retryAfterMs) && e.retryAfterMs > 0
+            ? e.retryAfterMs
+            : Math.min(120000, 30000 * 2 ** (_failureCount - 1));
+        _retryAt = Math.max(
+          _retryAt,
+          Math.min(Number.MAX_VALUE, now() + delay),
+        );
+        // Enabling the display succeeded. Stats still report an unavailable
+        // or degraded feed; the manager keeps its normal refresh failure lane.
+        return true;
       } finally {
         signal?.removeEventListener('abort', onAbort);
         if (_request === request) _request = null;
         scheduleCameraRefresh();
+        scheduleRetry();
       }
     },
 
@@ -292,6 +340,8 @@ export function createGnssIntegrityLayer({
       _lastError = null;
       _stale = false;
       _coverage = null;
+      _retryAt = 0;
+      _failureCount = 0;
       _lastQueryKey = null;
       _manager = null;
     },
@@ -299,7 +349,15 @@ export function createGnssIntegrityLayer({
     getRowControls() {
       return {
         chips: [],
-        info: 'Regional sample · 250 nm radius · pan to sample another area',
+        info:
+          'Regional sample · 250 nm radius · pan to sample another area' +
+          (_lastError && _retryAt
+            ? ` · ${_lastError} · Retrying automatically${
+                _retryAt <= 8640000000000000
+                  ? ` at ${new Date(_retryAt).toISOString().slice(11, 19)} UTC`
+                  : ' after provider cooldown'
+              }`
+            : ''),
         infoTitle:
           'Updates automatically after pan or zoom and once a minute. The public feed covers a 250-nautical-mile circle around the view centre; zooming out does not make it a worldwide snapshot. Blank cells have insufficient observations.',
         legend: ['high', 'medium', 'low'].map((level, index) => ({
@@ -320,6 +378,9 @@ export function createGnssIntegrityLayer({
         count: _cells.length,
         lastUpdate: _lastUpdate,
         error: _lastError,
+        retryInSec: _retryAt
+          ? Math.max(0, Math.ceil((_retryAt - now()) / 1000))
+          : null,
         stale: _stale,
         provenance,
         coverage: _coverage,

src/layers/gnss/ownership.test.mjs

@@ -3,6 +3,7 @@ import test from 'node:test';
 import * as Cesium from 'cesium';
 import { createGnssIntegrityLayer, defaultGnssViewAnchor } from './index.js';
 import { LayerLifecycle } from '../../data/lifecycle.js';
+import { layerFeedState } from '../../data/feedState.js';
 
 const row = (hex, lat, degraded) => ({
   hex,
@@ -281,7 +282,7 @@ test('a failed refresh keeps the last cells and reports the error', async () =>
   await layer.update();
   assert.equal(layer.getStats().stale, true);
   fail = true;
-  assert.equal(await layer.update(), false);
+  assert.equal(await layer.update(), true);
   assert.equal(sources[0].entities.values.length, 1);
   assert.equal(layer.getStats().error, 'adsb.lol HTTP 502');
   assert.equal(layer.getStats().count, 1);
@@ -376,10 +377,10 @@ test('cells expire during prolonged fetch failures', async () => {
   await layer.update();
   fail = true;
   advance(20 * MIN);
-  assert.equal(await layer.update(), false);
+  assert.equal(await layer.update(), true);
   assert.equal(layer.getStats().count, 1, 'still inside the window');
   advance(11 * MIN);
-  assert.equal(await layer.update(), false);
+  assert.equal(await layer.update(), true);
   assert.equal(layer.getStats().count, 0);
   assert.equal(sources[0].entities.values.length, 0);
   assert.equal(layer.getStats().error, 'adsb.lol HTTP 502');
@@ -432,3 +433,125 @@ test('an identical snapshot leaves the drawn entities untouched', async () => {
   assert.equal(sources[0].entities.values.length, 1);
   assert.equal(sources[0].entities.values[0], first);
 });
+
+test('an initial feed failure stays enabled, reports unavailable, and recovers automatically', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout', 'setInterval'] });
+  let calls = 0;
+  const h = harness(
+    async () => {
+      calls++;
+      if (calls === 1) throw new Error('adsb.lol HTTP 502');
+      return { rows: threeHealthy() };
+    },
+    { initialize: false },
+  );
+  const manager = new LayerLifecycle(h.viewer);
+  manager.register(h.layer);
+  h.layer.attachDataManager(manager);
+  const changes = [];
+  manager.subscribe((c) => changes.push(c.type));
+  assert.equal(await manager.setEnabled(h.layer.id, true), true);
+  assert.equal(manager.isEnabled(h.layer.id), true);
+  assert.equal(layerFeedState(h.layer.getStats()), 'unavailable');
+  assert.equal(h.layer.getStats().retryInSec, 30);
+  assert.match(
+    h.layer.getRowControls().info,
+    /HTTP 502.*Retrying automatically at/,
+  );
+  assert.equal(changes.includes('visibility-failed'), false);
+  assert.equal(
+    await manager.refreshLayer(h.layer.id),
+    false,
+    'feed remains honestly unhealthy during cooldown',
+  );
+  assert.equal(calls, 1, 'a manual refresh cannot bypass cooldown');
+  h.advance(30000);
+  t.mock.timers.tick(30000);
+  await flush();
+  assert.equal(calls, 2);
+  assert.equal(manager.isEnabled(h.layer.id), true);
+  assert.equal(layerFeedState(h.layer.getStats()), 'nominal');
+  assert.equal(h.layer.getStats().count, 1);
+  assert.equal(h.layer.getStats().retryInSec, null);
+  assert.doesNotMatch(h.layer.getRowControls().info, /Retrying/);
+  await manager.destroyAll();
+});
+
+test('provider cooldown is preserved across movement and toggling; disable cancels retry work', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  let calls = 0;
+  const h = harness(async () => {
+    calls++;
+    if (calls === 1)
+      throw Object.assign(new Error('Rate limited (HTTP 429)'), {
+        retryAfterMs: 3600000,
+      });
+    return { rows: threeHealthy() };
+  });
+  h.layer.enable();
+  await h.layer.update();
+  h.setAnchor({ latitude: 10, longitude: 20 });
+  h.viewer.camera.moveEnd.raiseEvent();
+  h.advance(30000);
+  t.mock.timers.tick(30000);
+  await flush();
+  await h.layer.update();
+  assert.equal(calls, 1);
+  assert.equal(h.layer.getStats().retryInSec, 3570);
+  h.layer.disable();
+  h.advance(30000);
+  t.mock.timers.tick(30000);
+  await flush();
+  h.layer.enable();
+  await h.layer.update();
+  assert.equal(calls, 1, 'reenabling does not erase a provider deadline');
+  h.advance(3540000);
+  t.mock.timers.tick(3540000);
+  await flush();
+  assert.equal(calls, 2);
+  assert.deepEqual(h.anchors.at(-1), { latitude: 10, longitude: 20 });
+  h.layer.destroy();
+
+  const stopped = harness(async () => {
+    throw new Error('network unavailable');
+  });
+  stopped.layer.enable();
+  await stopped.layer.update();
+  stopped.layer.disable();
+  stopped.advance(30000);
+  t.mock.timers.tick(30000);
+  await flush();
+  assert.equal(stopped.anchors.length, 1, 'disabled layers do not retry');
+  stopped.layer.destroy();
+});
+
+test('repeated acquisition failures back off, while genuine non-retryable failures reject startup', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  let calls = 0;
+  const h = harness(async () => {
+    calls++;
+    throw new Error('network unavailable');
+  });
+  h.layer.enable();
+  await h.layer.update();
+  assert.equal(h.layer.getStats().retryInSec, 30);
+  h.advance(30000);
+  t.mock.timers.tick(30000);
+  await flush();
+  assert.equal(h.layer.getStats().retryInSec, 60);
+  h.advance(60000);
+  t.mock.timers.tick(60000);
+  await flush();
+  assert.equal(h.layer.getStats().retryInSec, 120);
+  assert.equal(calls, 3);
+  h.layer.destroy();
+  const fatal = harness(async () => {
+    throw Object.assign(new Error('invalid request (HTTP 400)'), {
+      retryable: false,
+    });
+  });
+  fatal.layer.enable();
+  assert.equal(await fatal.layer.update(), false);
+  assert.equal(fatal.layer.getStats().retryInSec, null);
+  fatal.layer.destroy();
+});

src/layers/gnss/source.js

@@ -1,8 +1,18 @@
 import { readResponseJsonCapped } from '../../sources/httpBody.js';
 
+function retryDelay(raw, nowMs) {
+  if (!raw) return null;
+  const seconds = Number(raw);
+  const ms = seconds * 1000;
+  if (Number.isFinite(ms) && seconds > 0) return Math.max(5000, ms);
+  const at = Date.parse(raw);
+  return Number.isFinite(at) && at > nowMs ? Math.max(5000, at - nowMs) : null;
+}
+
 /** Request integrity facts around a view anchor through the same-origin adsb.lol proxy. */
 export function createAdsbGnssSource({
   fetchImpl = (...args) => globalThis.fetch(...args),
+  now = () => Date.now(),
 } = {}) {
   return {
     async getSnapshot({ latitude, longitude } = {}, { signal } = {}) {
@@ -16,7 +26,25 @@ export function createAdsbGnssSource({
       const response = await fetchImpl(`/api/gnss-integrity?${params}`, {
         signal,
       });
-      if (!response.ok) throw new Error(`adsb.lol HTTP ${response.status}`);
+      if (!response.ok) {
+        const limited = response.status === 429;
+        const retryAfterMs = retryDelay(
+          response.headers.get('retry-after'),
+          now(),
+        );
+        throw Object.assign(
+          new Error(
+            limited
+              ? 'adsb.lol rate limited (HTTP 429)'
+              : `adsb.lol feed unavailable (HTTP ${response.status})`,
+          ),
+          {
+            status: response.status,
+            retryable: limited || response.status >= 500,
+            retryAfterMs: retryAfterMs ?? (limited ? 60000 : null),
+          },
+        );
+      }
       const payload = await readResponseJsonCapped(
         response,
         4 * 1024 * 1024,

src/layers/gnss/source.test.mjs

@@ -74,3 +74,38 @@ test('the source reports stale proxy data and rejects bad input or responses', a
     ),
   );
 });
+
+test('HTTP failures distinguish rate limiting, preserve Retry-After and classify retryability', async () => {
+  const at = Date.UTC(2026, 9, 13, 12);
+  for (const [status, header, delay, retryable] of [
+    [429, '3600', 3600000, true],
+    [429, new Date(at + 3600000).toUTCString(), 3600000, true],
+    [429, null, 60000, true],
+    [429, 'Infinity', 60000, true],
+    [502, '5', 5000, true],
+    [502, null, null, true],
+    [400, null, null, false],
+  ]) {
+    const source = createAdsbGnssSource({
+      now: () => at,
+      fetchImpl: async () =>
+        Response.json(
+          {},
+          { status, headers: header ? { 'Retry-After': header } : {} },
+        ),
+    });
+    await assert.rejects(
+      source.getSnapshot({ latitude: 50, longitude: 20 }),
+      (e) => {
+        assert.equal(e.status, status);
+        assert.equal(e.retryAfterMs, delay);
+        assert.equal(e.retryable, retryable);
+        assert.match(
+          e.message,
+          status === 429 ? /rate limited/ : /feed unavailable/,
+        );
+        return true;
+      },
+    );
+  }
+});