From f26586bb3e6e0b791b63c26df65777e7276ce36d Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:08:24 -0700 Subject: [PATCH] fix(cctv): serve the camera catalog while it refreshes An expired catalog made every /api/cctv request (camera list and frames) wait for the full refresh, which lasts as long as the slowest of the live packs: Tarktee's DATEX feeds took 2 to 16.6 s across ten requests. With Tarktee's timeout raised to 30 s, that wait could reach 30 s at each 15-minute refresh. The cached list is now returned while a single-flight refresh replaces it in the background; only a first load, with nothing to serve, waits. A background refresh that throws is logged. Co-Authored-By: Claude Opus 5.5 --- server/providers/cctv/catalog.js | 25 ++++++++--- src/data/cctvCatalogCap.test.mjs | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 75e58d2a6..396b62f8b 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -204,7 +204,8 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { * * Merges every source pack (live open-data packs, local file, env * variable), deduplicates by ID, shares the catalog cap fairly across - * packs, and caches for CCTV_SOURCE_CACHE_MS. + * packs, and caches for CCTV_SOURCE_CACHE_MS. Past that, the cached list is + * returned while a background refresh replaces it; only a first load waits. * * @returns {Promise>} Deduplicated, capped source list. */ @@ -219,10 +220,24 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { // Single-flight: a burst of requests arriving past the TTL shares ONE refresh // instead of each launching the full multi-provider refetch. The `.finally` // clears the ref so the next post-TTL cycle starts fresh. - if (_cctvSourceInflight) return _cctvSourceInflight; - _cctvSourceInflight = refreshCctvSources().finally(() => { - _cctvSourceInflight = null; - }); + if (!_cctvSourceInflight) { + _cctvSourceInflight = refreshCctvSources().finally(() => { + _cctvSourceInflight = null; + }); + } + // With a catalog in hand, answer with it while the refresh runs: the + // refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up + // to its 30 s timeout), and every /api/cctv request would otherwise wait + // for it. Only the first load, with nothing to serve yet, waits. + if (_cctvSourceCache.length) { + _cctvSourceInflight.catch((error) => { + console.warn( + '[CCTV] background catalog refresh failed:', + error?.message || error, + ); + }); + return _cctvSourceCache; + } return _cctvSourceInflight; } diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 9d93778db..46faa43de 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -6,6 +6,7 @@ import { } from '../../server/providers/cctv/cap.js'; import { CCTV_MAX_SOURCES_CEILING, + CCTV_SOURCE_CACHE_MS, DEFAULT_CCTV_MAX_SOURCES, } from '../../server/providers/cctv/constants.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; @@ -160,3 +161,77 @@ test('a pack trimmed to its nearest cameras reports how many it offered', async assert.equal(kept.available, 5); assert.deepEqual(kept.region, { west: 0, south: 0, east: 0, north: 4 }); }); + +test('an expired catalog is served at once while a slow pack refreshes in the background', async (t) => { + const saved = { ...process.env }; + const envSources = (prefix) => + JSON.stringify( + Array.from({ length: 3 }, (_, i) => ({ + id: `${prefix}-${i}`, + name: `${prefix} ${i}`, + lat: 30 + i * 0.01, + lon: -97, + url: `https://example.test/${prefix}-${i}.jpg`, + })), + ); + try { + // Live packs run alongside the env pack, so one slow upstream can hold + // the refresh open. + process.env.CCTV_FORCE_AUSTIN = '1'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old'); + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'warn', () => {}); + t.mock.method(console, 'log', () => {}); + const hung = []; + let hang = false; + t.mock.method(globalThis, 'fetch', () => + hang + ? new Promise((_, reject) => { + hung.push(() => reject(new Error('upstream gave up'))); + }) + : Promise.reject(new Error('offline')), + ); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + assert.deepEqual( + (await getSources()).map((s) => s.id), + ['old-0', 'old-1', 'old-2'], + ); + + // Past the TTL, with every live pack's upstream hanging. + now += CCTV_SOURCE_CACHE_MS + 1; + hang = true; + process.env.CCTV_SOURCES_JSON = envSources('new'); + const served = await Promise.race([ + getSources(), + new Promise((resolve) => setTimeout(() => resolve('waited'), 200)), + ]); + assert.notEqual( + served, + 'waited', + 'the request did not wait for the refresh', + ); + assert.deepEqual( + served.map((s) => s.id), + ['old-0', 'old-1', 'old-2'], + ); + + // When the slow pack gives up, the refreshed catalog replaces the old one. + for (const giveUp of hung) giveUp(); + for (let i = 0; i < 50; i += 1) { + const ids = (await getSources()).map((s) => s.id); + if (ids[0] === 'new-0') break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.deepEqual( + (await getSources()).map((s) => s.id), + ['new-0', 'new-1', 'new-2'], + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +});