diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index b9dc02918..269b1c7e9 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4412,7 +4412,9 @@ silently demoting every later lookup for the session. ~1,100 cameras total, all RAW PRIOR poses, stills-first. The catalog lives for 15 minutes; once it expires, `/api/cctv` keeps answering from it while one background refresh reloads every pack, so only the very first load waits for the slowest pack (a live first load took about - 22 s). Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s + 22 s, and is bounded by Tarktee's 30 s). A background refresh that fails or comes back + empty keeps the old catalog and is retried after another 15 minutes, not on the next + request. Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s (`TARKTEE_FETCH_TIMEOUT_MS`) where other packs get 15 s. - **CCTV v3 UX — viewshed + calibration gizmo** (built 2026-07-05 and field validated 2026-07-21): the COVERAGE toggle is a diff --git a/scripts/precompute-cctv-heights.mjs b/scripts/precompute-cctv-heights.mjs index 648768341..2d55f4168 100644 --- a/scripts/precompute-cctv-heights.mjs +++ b/scripts/precompute-cctv-heights.mjs @@ -326,8 +326,10 @@ async function main() { 'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json', ); let sidecar = await loadSidecar(output); + // A cold catalog waits for its slowest pack (Tarktee gets 30 s), so allow + // more than that. const response = await fetch(`${base}/api/cctv/sources`, { - signal: AbortSignal.timeout(30_000), + signal: AbortSignal.timeout(60_000), }); if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`); const catalog = await response.json(); diff --git a/server/providers/cctv.js b/server/providers/cctv.js index b24f38fa1..e549753e8 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -143,7 +143,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { if (url.pathname === '/sources') { const body = { - trimmedPacks: getCctvSources.trimmedPacks?.() ?? [], + trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [], sources: sources.map((source) => ({ id: source.id, pack: source.pack, diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 840ca9ce1..5e933bc02 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -195,9 +195,11 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { /** @type {Promise>|null} In-flight refresh, shared by concurrent * callers so a post-TTL burst launches ONE refetch, not one per request. */ let _cctvSourceInflight = null; - /** @type {Array<{pack: string, available: number, served: number}>} Packs - * that serve fewer cameras than they offer. */ - let _cctvTrimmedPacks = []; + /** @type {WeakMap, Array<{pack: string, available: number, served: number}>>} + * Packs that serve fewer cameras than they offer, keyed by the source list + * they describe, so a request served the old list during a background + * refresh never pairs it with the new list's trim notes. */ + const _cctvTrimmedPacks = new WeakMap(); /** * Assemble and cache the merged CCTV source list. @@ -225,11 +227,16 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { _cctvSourceInflight = null; }); // A background refresh has no caller waiting on it: report its failure - // here, once per refresh rather than once per request it serves. + // here, once per refresh rather than once per request it serves. Keep + // the old catalog for another TTL, as an empty refresh does, so a refresh + // that keeps failing is retried once per TTL, not back to back. if (_cctvSourceCache.length) { _cctvSourceInflight.catch((error) => { + _cctvSourceCacheAt = Date.now(); console.warn( - '[CCTV] background catalog refresh failed:', + '[CCTV] background catalog refresh failed; serving', + _cctvSourceCache.length, + 'cameras from the last catalog:', error?.message || error, ); }); @@ -329,7 +336,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { } if (capped.length > 0 || _cctvSourceCache.length === 0) { _cctvSourceCache = capped; - _cctvTrimmedPacks = trimmedPacks; + _cctvTrimmedPacks.set(capped, trimmedPacks); } else { // Every source came back empty (all live packs timed out / upstream outage) // but a good catalog is already cached — serve it stale rather than blanking @@ -343,7 +350,10 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { return _cctvSourceCache; } - /** Packs serving fewer cameras than they offer, as of the last refresh. */ - getCctvSources.trimmedPacks = () => _cctvTrimmedPacks; + /** Packs serving fewer cameras than they offer in `sources` (by default the + * current catalog). Pass the list a request was served to keep the two in + * step while a background refresh swaps the catalog. */ + getCctvSources.trimmedPacks = (sources = _cctvSourceCache) => + _cctvTrimmedPacks.get(sources) ?? []; return getCctvSources; } diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 26775cb48..7ef4b1f7b 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -162,18 +162,31 @@ test('a pack trimmed to its nearest cameras reports how many it offered', async assert.deepEqual(kept.region, { west: 0, south: 0, east: 0, north: 4 }); }); +/** A CCTV_SOURCES_JSON value with `count` cameras named `-`. */ +const envSources = (prefix, count = 3) => + JSON.stringify( + Array.from({ length: count }, (_, i) => ({ + id: `${prefix}-${i}`, + name: `${prefix} ${i}`, + lat: 30 + i * 0.01, + lon: -97, + url: `https://example.test/${prefix}-${i}.jpg`, + })), + ); + +/** Let a background refresh that needs no timers run to completion. */ +const settle = () => new Promise((resolve) => setTimeout(resolve, 10)); + +/** Restore process.env to a snapshot taken with `{ ...process.env }`. */ +const restoreEnv = (saved) => { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); +}; + test('an expired catalog is served at once while a slow pack refreshes in the background', async (t) => { const saved = { ...process.env }; - const envSources = (prefix) => - JSON.stringify( - Array.from({ length: 3 }, (_, i) => ({ - id: `${prefix}-${i}`, - name: `${prefix} ${i}`, - lat: 30 + i * 0.01, - lon: -97, - url: `https://example.test/${prefix}-${i}.jpg`, - })), - ); try { // Live packs run alongside the env pack, so one slow upstream can hold // the refresh open. @@ -203,14 +216,18 @@ test('an expired catalog is served at once while a slow pack refreshes in the ba now += CCTV_SOURCE_CACHE_MS + 1; hang = true; process.env.CCTV_SOURCES_JSON = envSources('new'); + let timer; const served = await Promise.race([ getSources(), - new Promise((resolve) => setTimeout(() => resolve('waited'), 200)), + new Promise((resolve) => { + timer = setTimeout(() => resolve('waited'), 200); + }), ]); + clearTimeout(timer); assert.notEqual( served, 'waited', - 'the request did not wait for the refresh', + 'a stale request waited for the background refresh', ); assert.deepEqual( served.map((s) => s.id), @@ -240,9 +257,128 @@ test('an expired catalog is served at once while a slow pack refreshes in the ba ['new-0', 'new-1', 'new-2'], ); } finally { - for (const key of Object.keys(process.env)) { - if (!(key in saved)) delete process.env[key]; + restoreEnv(saved); + } +}); + +test('a failed background refresh keeps the catalog, warns once, and waits a TTL to retry', async (t) => { + const saved = { ...process.env }; + const unhandled = []; + const onUnhandled = (reason) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + try { + process.env.CCTV_PREFER_AUSTIN = '0'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old', 9); + process.env.CCTV_MAX_SOURCES = '8'; + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + // Nine cameras over a cap of eight make every refresh warn; throwing from + // that warning is how this test makes a refresh reject. + let failRefresh = false; + let failedRefreshes = 0; + const warnings = []; + t.mock.method(console, 'warn', (...args) => { + const text = args.join(' '); + if (failRefresh && text.includes('exceeds cap')) { + failedRefreshes += 1; + throw new Error('refresh broke'); + } + warnings.push(text); + }); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + const old = await getSources(); + assert.equal(old.length, 8); + + // Past the TTL the refresh throws: the old catalog is still served. + now += CCTV_SOURCE_CACHE_MS + 1; + failRefresh = true; + assert.equal(await getSources(), old); + await settle(); + assert.equal(failedRefreshes, 1); + assert.equal( + warnings.filter((w) => w.includes('background catalog refresh failed')) + .length, + 1, + ); + + // The failure is not retried on every request until the next TTL. + assert.equal(await getSources(), old); + assert.equal(await getSources(), old); + await settle(); + assert.equal(failedRefreshes, 1, 'a failed refresh was retried at once'); + + // After it, a refresh that works replaces the catalog. With no live packs + // the refresh finishes before the request that started it returns. + now += CCTV_SOURCE_CACHE_MS + 1; + failRefresh = false; + process.env.CCTV_SOURCES_JSON = envSources('new'); + await getSources(); + await settle(); + const fresh = await getSources(); + assert.deepEqual( + fresh.map((s) => s.id), + ['new-0', 'new-1', 'new-2'], + ); + // Trim notes stay with the list they describe, so a request served the + // old list during the refresh never reports the new list's notes. + assert.deepEqual( + getSources.trimmedPacks(old).map((p) => [p.pack, p.available, p.served]), + [['env', 9, 8]], + ); + assert.deepEqual(getSources.trimmedPacks(fresh), []); + assert.deepEqual(getSources.trimmedPacks(), []); + assert.deepEqual(unhandled, []); + } finally { + process.off('unhandledRejection', onUnhandled); + restoreEnv(saved); + } +}); + +test('an empty background refresh keeps the catalog and waits a TTL before fetching again', async (t) => { + const saved = { ...process.env }; + try { + process.env.CCTV_FORCE_AUSTIN = '1'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old'); + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'log', () => {}); + const warnings = []; + t.mock.method(console, 'warn', (...args) => warnings.push(args.join(' '))); + let fetches = 0; + t.mock.method(globalThis, 'fetch', () => { + fetches += 1; + return Promise.reject(new Error('offline')); + }); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + const old = await getSources(); + + // Past the TTL every pack comes back empty, as in an upstream outage. + now += CCTV_SOURCE_CACHE_MS + 1; + process.env.CCTV_SOURCES_JSON = '[]'; + assert.equal(await getSources(), old); + for (let i = 0; i < 50; i += 1) { + if (warnings.some((w) => w.includes('source refresh returned empty'))) + break; + await settle(); } - Object.assign(process.env, saved); + assert.ok( + warnings.some((w) => w.includes('serving 3 stale cameras')), + 'the empty refresh kept the old catalog', + ); + + // Until the next TTL, requests are served without touching any upstream. + const fetchesAfterRefresh = fetches; + assert.equal(await getSources(), old); + assert.equal(await getSources(), old); + await settle(); + assert.equal( + fetches, + fetchesAfterRefresh, + 'an empty refresh reran at once', + ); + } finally { + restoreEnv(saved); } }); diff --git a/src/data/cctvEstonia.test.mjs b/src/data/cctvEstonia.test.mjs index 0d578a26c..b46f9eb06 100644 --- a/src/data/cctvEstonia.test.mjs +++ b/src/data/cctvEstonia.test.mjs @@ -6,7 +6,6 @@ import { parseTarkteeDatexImages, parseTarkteeDatexLocations, } from '../../server/providers/cctv/sources.js'; -import { TARKTEE_FETCH_TIMEOUT_MS } from '../../server/providers/cctv/constants.js'; const SAMPLE_LOCATIONS_XML = ` @@ -107,6 +106,5 @@ test('Tarktee DATEX requests wait 30 s, past the slow answers measured live', as ); const cameras = await loadTarkteeSourcesFromDatex(); assert.deepEqual(timeouts, [30_000, 30_000]); - assert.equal(TARKTEE_FETCH_TIMEOUT_MS, 30_000); assert.equal(cameras.length, 1); });