From 6b3f7a3ec11c8d3b5311bbc9635c9eef6a6d2e1a Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:37:59 -0700 Subject: [PATCH 1/5] fix(cctv): give Tarktee's slow DATEX feeds 30 s instead of the shared 15 s The Estonian Transpordiamet feeds answered in 2 to 16.6 s across ten requests (120 and 160 KB), and both must arrive for any camera to load, so the shared 15 s catalog timeout dropped the whole pack on about a third of loads ("Tarktee DATEX download error: The operation was aborted due to timeout"), for the 15 minutes until the next catalog refresh. TARKTEE_FETCH_TIMEOUT_MS gives the two requests 30 s; other packs keep 15 s. Live: 175 cameras in 3.3 s. Co-Authored-By: Claude Opus 5.5 --- server/providers/cctv/constants.js | 4 ++++ server/providers/cctv/sources.js | 5 +++-- src/data/cctvEstonia.test.mjs | 26 ++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 20b963770..b0e768c83 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -439,6 +439,10 @@ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; * pending forever — a hung fetch aborts, the loader returns [], and * serve-stale/other packs take over. */ export const CCTV_SOURCE_FETCH_TIMEOUT_MS = 15 * 1000; +/** Tarktee's two DATEX feeds (120 and 160 KB) answered in 2 to 16.6 s across + * ten requests on 2026-10-18, so the shared 15 s timeout dropped the Estonian + * pack on about a third of loads. Both must arrive for any camera to load. */ +export const TARKTEE_FETCH_TIMEOUT_MS = 30 * 1000; /** Individual CCTV image fetches must settle before the active 10-second * client refresh cadence. A bounded miss can fall through to Street View or * the synthetic frame instead of leaving the browser preview pending. */ diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index c3d11e752..f9711ea0c 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -70,6 +70,7 @@ import { TORONTO_GROUND_ELEVATION_M, TORONTO_MAX_CATALOG_BYTES, CCTV_SOURCE_FETCH_TIMEOUT_MS, + TARKTEE_FETCH_TIMEOUT_MS, DELDOT_CCTV_URL, DEFAULT_DELDOT_MAX_SOURCES, DELDOT_ANCHORS, @@ -1194,11 +1195,11 @@ export async function loadTarkteeSourcesFromDatex() { const [locResp, imgResp] = await Promise.all([ fetch(TARKTEE_LOCATIONS_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), fetch(TARKTEE_IMAGES_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), ]); if (!locResp.ok) { diff --git a/src/data/cctvEstonia.test.mjs b/src/data/cctvEstonia.test.mjs index 7c0ebaba1..0d578a26c 100644 --- a/src/data/cctvEstonia.test.mjs +++ b/src/data/cctvEstonia.test.mjs @@ -2,9 +2,11 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { loadTallinnSourcesFromCatalog, + loadTarkteeSourcesFromDatex, parseTarkteeDatexImages, parseTarkteeDatexLocations, } from '../../server/providers/cctv/sources.js'; +import { TARKTEE_FETCH_TIMEOUT_MS } from '../../server/providers/cctv/constants.js'; const SAMPLE_LOCATIONS_XML = ` @@ -84,3 +86,27 @@ test('Tarktee DATEX image parser rejects non-official hosts', () => { const images = parseTarkteeDatexImages(xml); assert.equal(images.size, 0); }); + +test('Tarktee DATEX requests wait 30 s, past the slow answers measured live', async (t) => { + const timeouts = []; + const timeout = AbortSignal.timeout; + t.mock.method(AbortSignal, 'timeout', (ms) => { + timeouts.push(ms); + return timeout.call(AbortSignal, ms); + }); + t.mock.method(console, 'log', () => {}); + t.mock.method( + globalThis, + 'fetch', + async (url) => + new Response( + String(url).includes('roadCameraLocations') + ? SAMPLE_LOCATIONS_XML + : SAMPLE_IMAGES_XML, + ), + ); + const cameras = await loadTarkteeSourcesFromDatex(); + assert.deepEqual(timeouts, [30_000, 30_000]); + assert.equal(TARKTEE_FETCH_TIMEOUT_MS, 30_000); + assert.equal(cameras.length, 1); +}); From f26586bb3e6e0b791b63c26df65777e7276ce36d Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:08:24 -0700 Subject: [PATCH 2/5] fix(cctv): serve the camera catalog while it refreshes An expired catalog made every /api/cctv request (camera list and frames) wait for the full refresh, which lasts as long as the slowest of the live packs: Tarktee's DATEX feeds took 2 to 16.6 s across ten requests. With Tarktee's timeout raised to 30 s, that wait could reach 30 s at each 15-minute refresh. The cached list is now returned while a single-flight refresh replaces it in the background; only a first load, with nothing to serve, waits. A background refresh that throws is logged. Co-Authored-By: Claude Opus 5.5 --- server/providers/cctv/catalog.js | 25 ++++++++--- src/data/cctvCatalogCap.test.mjs | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 75e58d2a6..396b62f8b 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -204,7 +204,8 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { * * Merges every source pack (live open-data packs, local file, env * variable), deduplicates by ID, shares the catalog cap fairly across - * packs, and caches for CCTV_SOURCE_CACHE_MS. + * packs, and caches for CCTV_SOURCE_CACHE_MS. Past that, the cached list is + * returned while a background refresh replaces it; only a first load waits. * * @returns {Promise>} Deduplicated, capped source list. */ @@ -219,10 +220,24 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { // Single-flight: a burst of requests arriving past the TTL shares ONE refresh // instead of each launching the full multi-provider refetch. The `.finally` // clears the ref so the next post-TTL cycle starts fresh. - if (_cctvSourceInflight) return _cctvSourceInflight; - _cctvSourceInflight = refreshCctvSources().finally(() => { - _cctvSourceInflight = null; - }); + if (!_cctvSourceInflight) { + _cctvSourceInflight = refreshCctvSources().finally(() => { + _cctvSourceInflight = null; + }); + } + // With a catalog in hand, answer with it while the refresh runs: the + // refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up + // to its 30 s timeout), and every /api/cctv request would otherwise wait + // for it. Only the first load, with nothing to serve yet, waits. + if (_cctvSourceCache.length) { + _cctvSourceInflight.catch((error) => { + console.warn( + '[CCTV] background catalog refresh failed:', + error?.message || error, + ); + }); + return _cctvSourceCache; + } return _cctvSourceInflight; } diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 9d93778db..46faa43de 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -6,6 +6,7 @@ import { } from '../../server/providers/cctv/cap.js'; import { CCTV_MAX_SOURCES_CEILING, + CCTV_SOURCE_CACHE_MS, DEFAULT_CCTV_MAX_SOURCES, } from '../../server/providers/cctv/constants.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; @@ -160,3 +161,77 @@ test('a pack trimmed to its nearest cameras reports how many it offered', async assert.equal(kept.available, 5); assert.deepEqual(kept.region, { west: 0, south: 0, east: 0, north: 4 }); }); + +test('an expired catalog is served at once while a slow pack refreshes in the background', async (t) => { + const saved = { ...process.env }; + const envSources = (prefix) => + JSON.stringify( + Array.from({ length: 3 }, (_, i) => ({ + id: `${prefix}-${i}`, + name: `${prefix} ${i}`, + lat: 30 + i * 0.01, + lon: -97, + url: `https://example.test/${prefix}-${i}.jpg`, + })), + ); + try { + // Live packs run alongside the env pack, so one slow upstream can hold + // the refresh open. + process.env.CCTV_FORCE_AUSTIN = '1'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old'); + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'warn', () => {}); + t.mock.method(console, 'log', () => {}); + const hung = []; + let hang = false; + t.mock.method(globalThis, 'fetch', () => + hang + ? new Promise((_, reject) => { + hung.push(() => reject(new Error('upstream gave up'))); + }) + : Promise.reject(new Error('offline')), + ); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + assert.deepEqual( + (await getSources()).map((s) => s.id), + ['old-0', 'old-1', 'old-2'], + ); + + // Past the TTL, with every live pack's upstream hanging. + now += CCTV_SOURCE_CACHE_MS + 1; + hang = true; + process.env.CCTV_SOURCES_JSON = envSources('new'); + const served = await Promise.race([ + getSources(), + new Promise((resolve) => setTimeout(() => resolve('waited'), 200)), + ]); + assert.notEqual( + served, + 'waited', + 'the request did not wait for the refresh', + ); + assert.deepEqual( + served.map((s) => s.id), + ['old-0', 'old-1', 'old-2'], + ); + + // When the slow pack gives up, the refreshed catalog replaces the old one. + for (const giveUp of hung) giveUp(); + for (let i = 0; i < 50; i += 1) { + const ids = (await getSources()).map((s) => s.id); + if (ids[0] === 'new-0') break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.deepEqual( + (await getSources()).map((s) => s.id), + ['new-0', 'new-1', 'new-2'], + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}); From 1b66d284c46c3a4fd97935a9ee2452f4c92f9846 Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:12:13 -0700 Subject: [PATCH 3/5] docs(cctv): CURRENT-STATE describes the background catalog refresh and Tarktee's 30 s Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9562dc9f5..9eb1dd751 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4397,7 +4397,11 @@ silently demoting every later lookup for the session. GATHERING stations carry 2,256 in-collection presets, prioritized to 300 against seven anchors on the main road spine. `CCTV_MAX_SOURCES` is a 4,000 catalog-wide ceiling shared round-robin across packs, so a lower global cap thins every region instead of starving the last pack. - ~1,100 cameras total, all RAW PRIOR poses, stills-first. + ~1,100 cameras total, all RAW PRIOR poses, stills-first. The catalog lives for 15 minutes; once + it expires, `/api/cctv` keeps answering from it while one background refresh reloads every + pack, so only the very first load waits for the slowest pack (a live first load took about + 22 s). Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s + (`TARKTEE_FETCH_TIMEOUT_MS`) where other packs get 15 s. - **CCTV v3 UX — viewshed + calibration gizmo** (built 2026-07-05 and field validated 2026-07-21): the COVERAGE toggle is a tri-state cycle `OFF → ON → VIEWSHED`; viewshed mode renders each visible camera's frustum From ca9dbea06000b0cfeeea2f77addfa334ae2f2cfc Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:13:36 -0700 Subject: [PATCH 4/5] docs(changelog): fragment for #1234 Co-Authored-By: Claude Opus 5.5 --- changelog.d/1234-cctv-catalog-background-refresh.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog.d/1234-cctv-catalog-background-refresh.md diff --git a/changelog.d/1234-cctv-catalog-background-refresh.md b/changelog.d/1234-cctv-catalog-background-refresh.md new file mode 100644 index 000000000..2d3706b14 --- /dev/null +++ b/changelog.d/1234-cctv-catalog-background-refresh.md @@ -0,0 +1,5 @@ +- Serve the camera catalog while it refreshes. An expired catalog made every + camera request wait for the slowest pack to reload; it is now answered at + once while one background refresh runs, so only the first load waits. Estonia's + slow Tarktee feeds get 30 s instead of 15 s, so the pack no longer drops out + on about a third of loads (quanticsoul4772, #1234). From 5b12f51e9897c7a85c1a799a01cbf1b6b54d5ef8 Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:28:56 -0700 Subject: [PATCH 5/5] fix(cctv): warn once per failed background refresh; test the single flight Attach the background-refresh warning where the refresh starts, so a failed refresh logs once instead of once per request served meanwhile. The test now checks that requests during a refresh share it, and turns the hang off before releasing the fetches so a retrying loader can't stall it. Co-Authored-By: Claude Opus 5.5 --- server/providers/cctv/catalog.js | 21 +++++++++++---------- src/data/cctvCatalogCap.test.mjs | 11 +++++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 396b62f8b..840ca9ce1 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -224,21 +224,22 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { _cctvSourceInflight = refreshCctvSources().finally(() => { _cctvSourceInflight = null; }); + // A background refresh has no caller waiting on it: report its failure + // here, once per refresh rather than once per request it serves. + if (_cctvSourceCache.length) { + _cctvSourceInflight.catch((error) => { + console.warn( + '[CCTV] background catalog refresh failed:', + error?.message || error, + ); + }); + } } // With a catalog in hand, answer with it while the refresh runs: the // refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up // to its 30 s timeout), and every /api/cctv request would otherwise wait // for it. Only the first load, with nothing to serve yet, waits. - if (_cctvSourceCache.length) { - _cctvSourceInflight.catch((error) => { - console.warn( - '[CCTV] background catalog refresh failed:', - error?.message || error, - ); - }); - return _cctvSourceCache; - } - return _cctvSourceInflight; + return _cctvSourceCache.length ? _cctvSourceCache : _cctvSourceInflight; } /** diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 46faa43de..26775cb48 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -217,7 +217,18 @@ test('an expired catalog is served at once while a slow pack refreshes in the ba ['old-0', 'old-1', 'old-2'], ); + // Requests during the refresh share it rather than starting another. + const pause = () => new Promise((resolve) => setTimeout(resolve, 50)); + await pause(); + const inFlight = hung.length; + assert.ok(inFlight > 0, 'the refresh reached the hanging upstreams'); + await getSources(); + await getSources(); + await pause(); + assert.equal(hung.length, inFlight, 'only one refresh ran'); + // When the slow pack gives up, the refreshed catalog replaces the old one. + hang = false; for (const giveUp of hung) giveUp(); for (let i = 0; i < 50; i += 1) { const ids = (await getSources()).map((s) => s.id);