fix(cctv): space out failed background refreshes; test the stale paths

- A background catalog refresh that rejects now keeps the old catalog for
  another TTL, as an empty refresh already did, so a refresh that keeps
  failing is retried once per TTL instead of on every request.
- Trim notes are keyed by the source list they describe, and /sources asks
  for the list it served, so a request answered during a refresh never pairs
  the old cameras with the new list's notes.
- Tests: a failed background refresh (one warning, no unhandled rejection,
  no retry before the TTL, trim notes stay with their list) and an empty
  background refresh (old catalog kept, no upstream fetch before the TTL).
  The race assertion's message now names the failure; its timer is cleared.
- The Tarktee test no longer imports the new constant, so on main it fails
  on its own assertion instead of taking the whole file down.
- precompute-cctv-heights waits 60 s for a cold catalog, past Tarktee's 30 s.
- CURRENT-STATE notes the 30 s bound and the retry spacing.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

docs/CURRENT-STATE.md

@@ -4412,7 +4412,9 @@ silently demoting every later lookup for the session.
   ~1,100 cameras total, all RAW PRIOR poses, stills-first. The catalog lives for 15 minutes; once
   it expires, `/api/cctv` keeps answering from it while one background refresh reloads every
   pack, so only the very first load waits for the slowest pack (a live first load took about
-  22 s). Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s
+  22 s, and is bounded by Tarktee's 30 s). A background refresh that fails or comes back
+  empty keeps the old catalog and is retried after another 15 minutes, not on the next
+  request. Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s
   (`TARKTEE_FETCH_TIMEOUT_MS`) where other packs get 15 s.
 - **CCTV v3 UX — viewshed + calibration gizmo** (built 2026-07-05 and field
   validated 2026-07-21): the COVERAGE toggle is a

scripts/precompute-cctv-heights.mjs

@@ -326,8 +326,10 @@ async function main() {
       'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json',
   );
   let sidecar = await loadSidecar(output);
+  // A cold catalog waits for its slowest pack (Tarktee gets 30 s), so allow
+  // more than that.
   const response = await fetch(`${base}/api/cctv/sources`, {
-    signal: AbortSignal.timeout(30_000),
+    signal: AbortSignal.timeout(60_000),
   });
   if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`);
   const catalog = await response.json();

server/providers/cctv.js

@@ -143,7 +143,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
 
         if (url.pathname === '/sources') {
           const body = {
-            trimmedPacks: getCctvSources.trimmedPacks?.() ?? [],
+            trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [],
             sources: sources.map((source) => ({
               id: source.id,
               pack: source.pack,

server/providers/cctv/catalog.js

@@ -195,9 +195,11 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
   /** @type {Promise<Array<object>>|null} In-flight refresh, shared by concurrent
    * callers so a post-TTL burst launches ONE refetch, not one per request. */
   let _cctvSourceInflight = null;
-  /** @type {Array<{pack: string, available: number, served: number}>} Packs
-   * that serve fewer cameras than they offer. */
-  let _cctvTrimmedPacks = [];
+  /** @type {WeakMap<Array<object>, Array<{pack: string, available: number, served: number}>>}
+   * Packs that serve fewer cameras than they offer, keyed by the source list
+   * they describe, so a request served the old list during a background
+   * refresh never pairs it with the new list's trim notes. */
+  const _cctvTrimmedPacks = new WeakMap();
 
   /**
    * Assemble and cache the merged CCTV source list.
@@ -225,11 +227,16 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
         _cctvSourceInflight = null;
       });
       // A background refresh has no caller waiting on it: report its failure
-      // here, once per refresh rather than once per request it serves.
+      // here, once per refresh rather than once per request it serves. Keep
+      // the old catalog for another TTL, as an empty refresh does, so a refresh
+      // that keeps failing is retried once per TTL, not back to back.
       if (_cctvSourceCache.length) {
         _cctvSourceInflight.catch((error) => {
+          _cctvSourceCacheAt = Date.now();
           console.warn(
-            '[CCTV] background catalog refresh failed:',
+            '[CCTV] background catalog refresh failed; serving',
+            _cctvSourceCache.length,
+            'cameras from the last catalog:',
             error?.message || error,
           );
         });
@@ -329,7 +336,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
     }
     if (capped.length > 0 || _cctvSourceCache.length === 0) {
       _cctvSourceCache = capped;
-      _cctvTrimmedPacks = trimmedPacks;
+      _cctvTrimmedPacks.set(capped, trimmedPacks);
     } else {
       // Every source came back empty (all live packs timed out / upstream outage)
       // but a good catalog is already cached — serve it stale rather than blanking
@@ -343,7 +350,10 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
     return _cctvSourceCache;
   }
 
-  /** Packs serving fewer cameras than they offer, as of the last refresh. */
-  getCctvSources.trimmedPacks = () => _cctvTrimmedPacks;
+  /** Packs serving fewer cameras than they offer in `sources` (by default the
+   * current catalog). Pass the list a request was served to keep the two in
+   * step while a background refresh swaps the catalog. */
+  getCctvSources.trimmedPacks = (sources = _cctvSourceCache) =>
+    _cctvTrimmedPacks.get(sources) ?? [];
   return getCctvSources;
 }

src/data/cctvCatalogCap.test.mjs

@@ -162,18 +162,31 @@ test('a pack trimmed to its nearest cameras reports how many it offered', async
   assert.deepEqual(kept.region, { west: 0, south: 0, east: 0, north: 4 });
 });
 
+/** A CCTV_SOURCES_JSON value with `count` cameras named `<prefix>-<i>`. */
+const envSources = (prefix, count = 3) =>
+  JSON.stringify(
+    Array.from({ length: count }, (_, i) => ({
+      id: `${prefix}-${i}`,
+      name: `${prefix} ${i}`,
+      lat: 30 + i * 0.01,
+      lon: -97,
+      url: `https://example.test/${prefix}-${i}.jpg`,
+    })),
+  );
+
+/** Let a background refresh that needs no timers run to completion. */
+const settle = () => new Promise((resolve) => setTimeout(resolve, 10));
+
+/** Restore process.env to a snapshot taken with `{ ...process.env }`. */
+const restoreEnv = (saved) => {
+  for (const key of Object.keys(process.env)) {
+    if (!(key in saved)) delete process.env[key];
+  }
+  Object.assign(process.env, saved);
+};
+
 test('an expired catalog is served at once while a slow pack refreshes in the background', async (t) => {
   const saved = { ...process.env };
-  const envSources = (prefix) =>
-    JSON.stringify(
-      Array.from({ length: 3 }, (_, i) => ({
-        id: `${prefix}-${i}`,
-        name: `${prefix} ${i}`,
-        lat: 30 + i * 0.01,
-        lon: -97,
-        url: `https://example.test/${prefix}-${i}.jpg`,
-      })),
-    );
   try {
     // Live packs run alongside the env pack, so one slow upstream can hold
     // the refresh open.
@@ -203,14 +216,18 @@ test('an expired catalog is served at once while a slow pack refreshes in the ba
     now += CCTV_SOURCE_CACHE_MS + 1;
     hang = true;
     process.env.CCTV_SOURCES_JSON = envSources('new');
+    let timer;
     const served = await Promise.race([
       getSources(),
-      new Promise((resolve) => setTimeout(() => resolve('waited'), 200)),
+      new Promise((resolve) => {
+        timer = setTimeout(() => resolve('waited'), 200);
+      }),
     ]);
+    clearTimeout(timer);
     assert.notEqual(
       served,
       'waited',
-      'the request did not wait for the refresh',
+      'a stale request waited for the background refresh',
     );
     assert.deepEqual(
       served.map((s) => s.id),
@@ -240,9 +257,128 @@ test('an expired catalog is served at once while a slow pack refreshes in the ba
       ['new-0', 'new-1', 'new-2'],
     );
   } finally {
-    for (const key of Object.keys(process.env)) {
-      if (!(key in saved)) delete process.env[key];
+    restoreEnv(saved);
+  }
+});
+
+test('a failed background refresh keeps the catalog, warns once, and waits a TTL to retry', async (t) => {
+  const saved = { ...process.env };
+  const unhandled = [];
+  const onUnhandled = (reason) => unhandled.push(reason);
+  process.on('unhandledRejection', onUnhandled);
+  try {
+    process.env.CCTV_PREFER_AUSTIN = '0';
+    process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json';
+    process.env.CCTV_SOURCES_JSON = envSources('old', 9);
+    process.env.CCTV_MAX_SOURCES = '8';
+    let now = 1_000_000;
+    t.mock.method(Date, 'now', () => now);
+    // Nine cameras over a cap of eight make every refresh warn; throwing from
+    // that warning is how this test makes a refresh reject.
+    let failRefresh = false;
+    let failedRefreshes = 0;
+    const warnings = [];
+    t.mock.method(console, 'warn', (...args) => {
+      const text = args.join(' ');
+      if (failRefresh && text.includes('exceeds cap')) {
+        failedRefreshes += 1;
+        throw new Error('refresh broke');
+      }
+      warnings.push(text);
+    });
+    const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' });
+    const old = await getSources();
+    assert.equal(old.length, 8);
+
+    // Past the TTL the refresh throws: the old catalog is still served.
+    now += CCTV_SOURCE_CACHE_MS + 1;
+    failRefresh = true;
+    assert.equal(await getSources(), old);
+    await settle();
+    assert.equal(failedRefreshes, 1);
+    assert.equal(
+      warnings.filter((w) => w.includes('background catalog refresh failed'))
+        .length,
+      1,
+    );
+
+    // The failure is not retried on every request until the next TTL.
+    assert.equal(await getSources(), old);
+    assert.equal(await getSources(), old);
+    await settle();
+    assert.equal(failedRefreshes, 1, 'a failed refresh was retried at once');
+
+    // After it, a refresh that works replaces the catalog. With no live packs
+    // the refresh finishes before the request that started it returns.
+    now += CCTV_SOURCE_CACHE_MS + 1;
+    failRefresh = false;
+    process.env.CCTV_SOURCES_JSON = envSources('new');
+    await getSources();
+    await settle();
+    const fresh = await getSources();
+    assert.deepEqual(
+      fresh.map((s) => s.id),
+      ['new-0', 'new-1', 'new-2'],
+    );
+    // Trim notes stay with the list they describe, so a request served the
+    // old list during the refresh never reports the new list's notes.
+    assert.deepEqual(
+      getSources.trimmedPacks(old).map((p) => [p.pack, p.available, p.served]),
+      [['env', 9, 8]],
+    );
+    assert.deepEqual(getSources.trimmedPacks(fresh), []);
+    assert.deepEqual(getSources.trimmedPacks(), []);
+    assert.deepEqual(unhandled, []);
+  } finally {
+    process.off('unhandledRejection', onUnhandled);
+    restoreEnv(saved);
+  }
+});
+
+test('an empty background refresh keeps the catalog and waits a TTL before fetching again', async (t) => {
+  const saved = { ...process.env };
+  try {
+    process.env.CCTV_FORCE_AUSTIN = '1';
+    process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json';
+    process.env.CCTV_SOURCES_JSON = envSources('old');
+    let now = 1_000_000;
+    t.mock.method(Date, 'now', () => now);
+    t.mock.method(console, 'log', () => {});
+    const warnings = [];
+    t.mock.method(console, 'warn', (...args) => warnings.push(args.join(' ')));
+    let fetches = 0;
+    t.mock.method(globalThis, 'fetch', () => {
+      fetches += 1;
+      return Promise.reject(new Error('offline'));
+    });
+    const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' });
+    const old = await getSources();
+
+    // Past the TTL every pack comes back empty, as in an upstream outage.
+    now += CCTV_SOURCE_CACHE_MS + 1;
+    process.env.CCTV_SOURCES_JSON = '[]';
+    assert.equal(await getSources(), old);
+    for (let i = 0; i < 50; i += 1) {
+      if (warnings.some((w) => w.includes('source refresh returned empty')))
+        break;
+      await settle();
     }
-    Object.assign(process.env, saved);
+    assert.ok(
+      warnings.some((w) => w.includes('serving 3 stale cameras')),
+      'the empty refresh kept the old catalog',
+    );
+
+    // Until the next TTL, requests are served without touching any upstream.
+    const fetchesAfterRefresh = fetches;
+    assert.equal(await getSources(), old);
+    assert.equal(await getSources(), old);
+    await settle();
+    assert.equal(
+      fetches,
+      fetchesAfterRefresh,
+      'an empty refresh reran at once',
+    );
+  } finally {
+    restoreEnv(saved);
   }
 });

src/data/cctvEstonia.test.mjs

@@ -6,7 +6,6 @@ import {
   parseTarkteeDatexImages,
   parseTarkteeDatexLocations,
 } from '../../server/providers/cctv/sources.js';
-import { TARKTEE_FETCH_TIMEOUT_MS } from '../../server/providers/cctv/constants.js';
 
 const SAMPLE_LOCATIONS_XML = `<?xml version="1.0" encoding="UTF-8"?>
 <d2LogicalModel xmlns="http://datex2.eu/schema/2/2_0">
@@ -107,6 +106,5 @@ test('Tarktee DATEX requests wait 30 s, past the slow answers measured live', as
   );
   const cameras = await loadTarkteeSourcesFromDatex();
   assert.deepEqual(timeouts, [30_000, 30_000]);
-  assert.equal(TARKTEE_FETCH_TIMEOUT_MS, 30_000);
   assert.equal(cameras.length, 1);
 });