From 167aa947ce338d90af77eb22af04d08be25eea47 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:11:31 +0100 Subject: [PATCH 1/7] fix(cctv): admit and bound HLS leases by peer --- docs/CURRENT-STATE.md | 7 ++++ scripts/package-boundaries.json | 2 + server/providers/cctv.js | 23 +++++++++-- server/providers/cctv/stream.js | 59 ++++++++++++++++++++++------- src/data/cctvHlsStream.test.mjs | 37 ++++++++++++++++++ src/data/cctvMediaFeedType.test.mjs | 43 +++++++++++++++++++++ 6 files changed, 154 insertions(+), 17 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 55b7a63ab1..9d674d1055 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,12 @@ # God's Eye View Current State +CCTV live-media requests are same-site gated before catalog or media work, including +HLS renewal, segment reads and DELETE cleanup. HLS leases belong to their socket +peer (IPv4-mapped IPv6 is normalized), with eight leases per peer across the two +global sessions. Peers behind the same NAT share that budget; this is resource +admission, not authentication. Native same-origin video and range requests remain +supported. + Shared distance checks use `src/geoDistance.js` with `{ lat, lon }` points and the IUGG mean spherical Earth radius, 6371.0088 km. `greatCircleKm` is canonical; `greatCircleMeters` is its exact metre conversion. Missing, diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index ca5dc0c7fa..fc9e4b5e98 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -187,6 +187,8 @@ "exports": ["./server/providers/cctv"], "modules": [ "server/providers/cctv.js", + "server/providers/common/same-site.js", + "src/localRequestGate.mjs", "server/providers/cctv/cap.js", "server/providers/cctv/groundHeights.js", "server/providers/cctv/catalog.js", diff --git a/server/providers/cctv.js b/server/providers/cctv.js index e549753e82..fc8f4f692d 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -17,6 +17,7 @@ import { CCTV_MAX_SOURCES_CEILING, } from './cctv/constants.js'; import { sanitizeCctvRangeHeader } from './cctv/range.js'; +import { admitSameSite } from './common/same-site.js'; import { createHlsPuller } from './cctv/stream.js'; import { googleServerApiKey } from './places/google-key.js'; export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream }; @@ -135,11 +136,18 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { }); server.middlewares.use('/api/cctv', async (req, res) => { try { + const url = new URL(req.url || '/', 'http://localhost'); + // Refuse before catalog I/O, session creation, renewal or release. + if (url.pathname.startsWith('/media/') && admitSameSite(req, res)) + return; + const clientId = String(req.socket?.remoteAddress || 'unknown').replace( + /^::ffff:/, + '', + ); const sources = await getCctvSources(); const sourceById = new Map( sources.map((source) => [source.id, source]), ); - const url = new URL(req.url || '/', 'http://localhost'); if (url.pathname === '/sources') { const body = { @@ -238,7 +246,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { return; } if (req.method === 'DELETE') { - if (leaseId) puller.release(cameraId, leaseId); + if (leaseId) puller.release(cameraId, leaseId, clientId); res.writeHead(204); res.end(); return; @@ -268,6 +276,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { url.searchParams.get('session'), Number(match[3]), leaseId, + clientId, ); res.writeHead(body ? 200 : 404, { 'Content-Type': 'video/mp2t', @@ -279,10 +288,15 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { const downstream = watchDownstreamClose(res); let entry; const cancelPending = () => { - if (entry) puller.release(cameraId, leaseId); + if (entry) puller.release(cameraId, leaseId, clientId); }; try { - entry = await puller.ensure(cameraId, mediaUrl, leaseId); + entry = await puller.ensure( + cameraId, + mediaUrl, + leaseId, + clientId, + ); if (downstream.closed) { cancelPending(); return; @@ -296,6 +310,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { entry, cameraId, leaseId, + clientId, ); if (downstream.closed) return; if (!playlist) throw new Error('Stream unavailable'); diff --git a/server/providers/cctv/stream.js b/server/providers/cctv/stream.js index 14d1743bb8..46608f294e 100644 --- a/server/providers/cctv/stream.js +++ b/server/providers/cctv/stream.js @@ -4,6 +4,7 @@ import { randomUUID } from 'node:crypto'; export const HLS_LIMITS = Object.freeze({ sessions: 2, leasesPerSession: 8, + leasesPerClient: 8, playlistBytes: 256 * 1024, segmentBytes: 4 * 1024 * 1024, sessionBytes: 24 * 1024 * 1024, @@ -138,25 +139,37 @@ export function createHlsPuller({ entry.upstream.clear(); entry.bytes = 0; }; - const release = (cameraId, leaseId) => { + const release = (cameraId, leaseId, clientId = 'legacy') => { const entry = active.get(cameraId); const lease = entry?.leases.get(leaseId); - if (!lease) return; + if (!lease || lease.clientId !== clientId) return; clearTimeout(lease.timer); entry.leases.delete(leaseId); if (!entry.leases.size) stop(cameraId, entry.token); }; - const touch = (entry, leaseId, create = false) => { + const touch = (entry, leaseId, create = false, clientId = 'legacy') => { let lease = entry.leases.get(leaseId); + if (lease && lease.clientId !== clientId) + throw new Error('HLS lease belongs to another client'); if (!lease) { + const clientLeases = [...active.values()].reduce( + (count, session) => + count + + [...session.leases.values()].filter( + (item) => item.clientId === clientId, + ).length, + 0, + ); + if (clientLeases >= limits.leasesPerClient) + throw new Error('HLS client lease capacity reached'); if (!create || entry.leases.size >= limits.leasesPerSession) throw new Error('HLS lease capacity reached'); - lease = {}; + lease = { clientId }; entry.leases.set(leaseId, lease); } clearTimeout(lease.timer); lease.timer = setTimeout( - () => release(entry.cameraId, leaseId), + () => release(entry.cameraId, leaseId, clientId), limits.idleMs, ); lease.timer.unref?.(); @@ -248,7 +261,12 @@ export function createHlsPuller({ } } }; - const ensure = async (cameraId, url, leaseId = 'legacy') => { + const ensure = async ( + cameraId, + url, + leaseId = 'legacy', + clientId = 'legacy', + ) => { if (closed) throw new Error('HLS service closed'); sameOriginHlsUrl(url, url); let entry = active.get(cameraId); @@ -257,7 +275,7 @@ export function createHlsPuller({ entry = null; } if (entry) { - touch(entry, leaseId, true); + touch(entry, leaseId, true, clientId); return entry; } // Reserve synchronously, before any await: concurrent creation is singleflight. @@ -279,8 +297,8 @@ export function createHlsPuller({ pendingDiscontinuity: false, discontinuitiesRemoved: 0, }; + touch(entry, leaseId, true, clientId); active.set(cameraId, entry); - touch(entry, leaseId, true); entry.polling = poll(entry); return entry; }; @@ -292,9 +310,14 @@ export function createHlsPuller({ } return false; }; - const buildPlaylist = async (entry, cameraId, leaseId = 'legacy') => { + const buildPlaylist = async ( + entry, + cameraId, + leaseId = 'legacy', + clientId = 'legacy', + ) => { if (entry.stopping || entry.segments.size < 2) return null; - touch(entry, leaseId); + touch(entry, leaseId, false, clientId); const segments = [...entry.segments.values()].sort((a, b) => a.seq - b.seq); const lines = [ '#EXTM3U', @@ -318,12 +341,22 @@ export function createHlsPuller({ } return lines.join('\n') + '\n'; }; - const getSegment = (cameraId, token, seq, leaseId = 'legacy') => { + const getSegment = ( + cameraId, + token, + seq, + leaseId = 'legacy', + clientId = 'legacy', + ) => { const entry = active.get(cameraId); - if (!entry || entry.token !== token || !entry.leases.has(leaseId)) + if ( + !entry || + entry.token !== token || + entry.leases.get(leaseId)?.clientId !== clientId + ) return null; const body = entry.segments.get(seq)?.body; - if (body) touch(entry, leaseId); + if (body) touch(entry, leaseId, false, clientId); return body || null; }; const shutdown = async () => { diff --git a/src/data/cctvHlsStream.test.mjs b/src/data/cctvHlsStream.test.mjs index a087eebdf8..01f5b15e48 100644 --- a/src/data/cctvHlsStream.test.mjs +++ b/src/data/cctvHlsStream.test.mjs @@ -223,3 +223,40 @@ test('an abandoned consumer expires while a renewed consumer keeps the session', assert.equal(entry.controller.signal.aborted, false); await manager.shutdown(); }); + +test('client lease ownership covers refresh, segments and release, with a bounded client budget', async (t) => { + const manager = createHlsPuller({ + limits: { ...HLS_LIMITS, leasesPerClient: 1, pollMs: 100000 }, + fetchImpl: async (url) => + new Response(url.endsWith('.m3u8') ? playlist : 'abc'), + }); + t.after(() => manager.shutdown()); + const entry = await manager.ensure('a', base, 'lease-a', 'peer-a'); + await manager.waitReady(entry); + await assert.rejects( + manager.ensure('a', base, 'lease-a', 'peer-b'), + /another client/, + ); + assert.equal( + manager.getSegment('a', entry.token, 0, 'lease-a', 'peer-b'), + null, + ); + await assert.rejects( + manager.buildPlaylist(entry, 'a', 'lease-a', 'peer-b'), + /another client/, + ); + manager.release('a', 'lease-a', 'peer-b'); + assert.equal(entry.leases.size, 1); + await assert.rejects( + manager.ensure('b', base, 'lease-b', 'peer-a'), + /client lease capacity/, + ); + assert.equal( + manager.stats().sessions, + 1, + 'failed admission reserves no session', + ); + await manager.ensure('b', base, 'lease-b', 'peer-b'); + manager.release('a', 'lease-a', 'peer-a'); + assert.equal(entry.stopping, true); +}); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs index 5791e433b1..e6ba9e3eb9 100644 --- a/src/data/cctvMediaFeedType.test.mjs +++ b/src/data/cctvMediaFeedType.test.mjs @@ -268,3 +268,46 @@ test('/frame does not follow a still camera off its origin', async (t) => { [STILL.url], ); }); + +test('cross-site media creation, segment reads and release fail before any catalog or media fetch', async (t) => { + let handler; + const originalFetch = globalThis.fetch; + let calls = 0; + globalThis.fetch = async () => { + calls++; + throw new Error('must not fetch'); + }; + t.after(() => { + globalThis.fetch = originalFetch; + }); + cctvProxy().configureServer({ + middlewares: { + use: (_path, fn) => { + handler = fn; + }, + }, + }); + for (const headers of [ + { origin: 'https://foreign.example' }, + { origin: 'null' }, + { 'sec-fetch-site': 'cross-site' }, + { 'sec-fetch-site': 'same-site' }, + ]) { + for (const [method, url] of [ + ['GET', '/media/camera?lease=test'], + ['DELETE', '/media/camera?lease=test'], + ['GET', '/media/camera/seg_0.ts?lease=test'], + ]) { + const { res } = recordingResponse(); + res.setHeader = (name, value) => { + res.headers[name] = value; + }; + await handler( + { method, url, headers: { host: 'localhost:4173', ...headers } }, + res, + ); + assert.equal(res.statusCode, 403); + } + } + assert.equal(calls, 0); +}); From a7160c62be9794875ecdd757b0f3b8141fd37dd4 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:11:49 +0100 Subject: [PATCH 2/7] docs: record CCTV session admission fix --- changelog.d/1267-cctv-session-admission.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog.d/1267-cctv-session-admission.md diff --git a/changelog.d/1267-cctv-session-admission.md b/changelog.d/1267-cctv-session-admission.md new file mode 100644 index 0000000000..c331e210d5 --- /dev/null +++ b/changelog.d/1267-cctv-session-admission.md @@ -0,0 +1,2 @@ +- Gate CCTV HLS media requests before session work, bind leases to the direct + peer, and bound per-peer leases across shared sessions (asturksever, #1267). From 5e9c14db83b26242ce8052f96a5ffeae979349ce Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 14:10:29 +0100 Subject: [PATCH 3/7] test: keep headful CCTV gate rendering when occluded --- changelog.d/1267-cctv-session-admission.md | 1 + scripts/qa-cctv-v2.mjs | 1 + 2 files changed, 2 insertions(+) diff --git a/changelog.d/1267-cctv-session-admission.md b/changelog.d/1267-cctv-session-admission.md index c331e210d5..c331222e70 100644 --- a/changelog.d/1267-cctv-session-admission.md +++ b/changelog.d/1267-cctv-session-admission.md @@ -1,2 +1,3 @@ - Gate CCTV HLS media requests before session work, bind leases to the direct peer, and bound per-peer leases across shared sessions (asturksever, #1267). +- Keep headful CCTV QA rendering active when its window is occluded (asturksever, #1267). diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 88e0a5a2c4..a6abdffa34 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -454,6 +454,7 @@ async function main() { ...(HEADFUL ? [] : ['--use-gl=angle', '--use-angle=swiftshader']), '--disable-dev-shm-usage', '--disable-web-security', + '--disable-backgrounding-occluded-windows', '--disable-background-timer-throttling', '--disable-renderer-backgrounding', '--window-size=1280,800', From 0699876e4e23c4c50a4f46f194252f703b29827b Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Thu, 19 Nov 2026 19:59:01 +0400 Subject: [PATCH 4/7] fix(cctv): keep main's HLS leases and test them through the middleware Drop the per-peer lease budget and the lease-to-address binding. Main's keyless same-site gate already refuses cross-site media requests, while the budget capped a shared address (Docker gateway, reverse proxy, NAT) at 8 leases across both sessions and still let one address hold both. A lease now keeps working when its viewer reconnects from a new address. Add a middleware test that drives the HLS playlist, segment and DELETE for viewers on one shared address, on distinct addresses and after an address change, and fold the media cases (segment read, DELETE, no catalog request) into the cross-site table test. Remove the redundant puppeteer flag, the CURRENT-STATE paragraph and the changelog fragment, which no longer describe a change. --- changelog.d/1267-cctv-session-admission.md | 3 - docs/CURRENT-STATE.md | 7 - scripts/qa-cctv-v2.mjs | 1 - server/providers/cctv.js | 17 +- server/providers/cctv/stream.js | 59 ++----- src/data/cctvHlsMiddleware.test.mjs | 174 +++++++++++++++++++++ src/data/cctvHlsStream.test.mjs | 37 ----- src/data/cctvMediaFeedType.test.mjs | 76 +++------ 8 files changed, 212 insertions(+), 162 deletions(-) delete mode 100644 changelog.d/1267-cctv-session-admission.md create mode 100644 src/data/cctvHlsMiddleware.test.mjs diff --git a/changelog.d/1267-cctv-session-admission.md b/changelog.d/1267-cctv-session-admission.md deleted file mode 100644 index c331222e70..0000000000 --- a/changelog.d/1267-cctv-session-admission.md +++ /dev/null @@ -1,3 +0,0 @@ -- Gate CCTV HLS media requests before session work, bind leases to the direct - peer, and bound per-peer leases across shared sessions (asturksever, #1267). -- Keep headful CCTV QA rendering active when its window is occluded (asturksever, #1267). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index a8a24d54a9..51dd971fe0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -185,13 +185,6 @@ The browser continues to split larger workloads into sequential 64-point request Traffic holds continuous rendering only while enabled with shown animated dots, a pulsing jam heat-line, or road lines whose geometry is still building (replaced lines stay drawn until their successors are ready, so a rebuild never blinks; one set builds at a time, and of the rebuilds asked for meanwhile only the newest runs, when that build finishes; turning lines off removes them at once); hidden dots (`dots` off) and settled flow lines hold nothing. Empty and high-altitude views release the hold. A failed load leaves no hold only when no animated content is retained: if an earlier load succeeded, ingestion keeps its animated points and the hold stays. Population admission reacquires the hold and resets animation time. Clear, disable, and destroy also release it. Retained fading dots keep animation alive until they are removed. When the last point leaves mid height correction, the correction is settled at once, before the hold is released, so it cannot keep the layer reporting loading. -CCTV live-media requests are same-site gated before catalog or media work, including -HLS renewal, segment reads and DELETE cleanup. HLS leases belong to their socket -peer (IPv4-mapped IPv6 is normalized), with eight leases per peer across the two -global sessions. Peers behind the same NAT share that budget; this is resource -admission, not authentication. Native same-origin video and range requests remain -supported. - Shared distance checks use `src/geoDistance.js` with `{ lat, lon }` points and the IUGG mean spherical Earth radius, 6371.0088 km. `greatCircleKm` is canonical; `greatCircleMeters` is its exact metre conversion. Missing, diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 6758001ef7..84eb09f6c1 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -452,7 +452,6 @@ async function main() { ...(HEADFUL ? [] : ['--use-gl=angle', '--use-angle=swiftshader']), '--disable-dev-shm-usage', '--disable-web-security', - '--disable-backgrounding-occluded-windows', '--disable-background-timer-throttling', '--disable-renderer-backgrounding', '--window-size=1280,800', diff --git a/server/providers/cctv.js b/server/providers/cctv.js index 3153012b83..5b5222a058 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -264,10 +264,6 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { // keyless gate keeps the routes reachable behind a reverse proxy; the // key-bearing Street View fallback below has its own check. if (admitSameSiteKeyless(req, res)) return; - const clientId = String(req.socket?.remoteAddress || 'unknown').replace( - /^::ffff:/, - '', - ); const sources = await getCctvSources(); const sourceById = new Map( sources.map((source) => [source.id, source]), @@ -379,7 +375,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { return; } if (req.method === 'DELETE') { - if (leaseId) puller.release(cameraId, leaseId, clientId); + if (leaseId) puller.release(cameraId, leaseId); res.writeHead(204); res.end(); return; @@ -409,7 +405,6 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { url.searchParams.get('session'), Number(match[3]), leaseId, - clientId, ); res.writeHead(body ? 200 : 404, { 'Content-Type': 'video/mp2t', @@ -421,15 +416,10 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { const downstream = watchDownstreamClose(res); let entry; const cancelPending = () => { - if (entry) puller.release(cameraId, leaseId, clientId); + if (entry) puller.release(cameraId, leaseId); }; try { - entry = await puller.ensure( - cameraId, - mediaUrl, - leaseId, - clientId, - ); + entry = await puller.ensure(cameraId, mediaUrl, leaseId); if (downstream.closed) { cancelPending(); return; @@ -443,7 +433,6 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { entry, cameraId, leaseId, - clientId, ); if (downstream.closed) return; if (!playlist) throw new Error('Stream unavailable'); diff --git a/server/providers/cctv/stream.js b/server/providers/cctv/stream.js index eb5477f762..4e37cda208 100644 --- a/server/providers/cctv/stream.js +++ b/server/providers/cctv/stream.js @@ -4,7 +4,6 @@ import { randomUUID } from 'node:crypto'; export const HLS_LIMITS = Object.freeze({ sessions: 2, leasesPerSession: 8, - leasesPerClient: 8, playlistBytes: 256 * 1024, // High-resolution cameras can exceed 4 MiB in a single ten-second TS segment. // Retained data still shares the independent 24 MiB session budget below. @@ -141,37 +140,25 @@ export function createHlsPuller({ entry.upstream.clear(); entry.bytes = 0; }; - const release = (cameraId, leaseId, clientId = 'legacy') => { + const release = (cameraId, leaseId) => { const entry = active.get(cameraId); const lease = entry?.leases.get(leaseId); - if (!lease || lease.clientId !== clientId) return; + if (!lease) return; clearTimeout(lease.timer); entry.leases.delete(leaseId); if (!entry.leases.size) stop(cameraId, entry.token); }; - const touch = (entry, leaseId, create = false, clientId = 'legacy') => { + const touch = (entry, leaseId, create = false) => { let lease = entry.leases.get(leaseId); - if (lease && lease.clientId !== clientId) - throw new Error('HLS lease belongs to another client'); if (!lease) { - const clientLeases = [...active.values()].reduce( - (count, session) => - count + - [...session.leases.values()].filter( - (item) => item.clientId === clientId, - ).length, - 0, - ); - if (clientLeases >= limits.leasesPerClient) - throw new Error('HLS client lease capacity reached'); if (!create || entry.leases.size >= limits.leasesPerSession) throw new Error('HLS lease capacity reached'); - lease = { clientId }; + lease = {}; entry.leases.set(leaseId, lease); } clearTimeout(lease.timer); lease.timer = setTimeout( - () => release(entry.cameraId, leaseId, clientId), + () => release(entry.cameraId, leaseId), limits.idleMs, ); lease.timer.unref?.(); @@ -263,12 +250,7 @@ export function createHlsPuller({ } } }; - const ensure = async ( - cameraId, - url, - leaseId = 'legacy', - clientId = 'legacy', - ) => { + const ensure = async (cameraId, url, leaseId = 'legacy') => { if (closed) throw new Error('HLS service closed'); sameOriginHlsUrl(url, url); let entry = active.get(cameraId); @@ -277,7 +259,7 @@ export function createHlsPuller({ entry = null; } if (entry) { - touch(entry, leaseId, true, clientId); + touch(entry, leaseId, true); return entry; } // Reserve synchronously, before any await: concurrent creation is singleflight. @@ -299,8 +281,8 @@ export function createHlsPuller({ pendingDiscontinuity: false, discontinuitiesRemoved: 0, }; - touch(entry, leaseId, true, clientId); active.set(cameraId, entry); + touch(entry, leaseId, true); entry.polling = poll(entry); return entry; }; @@ -312,14 +294,9 @@ export function createHlsPuller({ } return false; }; - const buildPlaylist = async ( - entry, - cameraId, - leaseId = 'legacy', - clientId = 'legacy', - ) => { + const buildPlaylist = async (entry, cameraId, leaseId = 'legacy') => { if (entry.stopping || entry.segments.size < 2) return null; - touch(entry, leaseId, false, clientId); + touch(entry, leaseId); const segments = [...entry.segments.values()].sort((a, b) => a.seq - b.seq); const lines = [ '#EXTM3U', @@ -343,22 +320,12 @@ export function createHlsPuller({ } return lines.join('\n') + '\n'; }; - const getSegment = ( - cameraId, - token, - seq, - leaseId = 'legacy', - clientId = 'legacy', - ) => { + const getSegment = (cameraId, token, seq, leaseId = 'legacy') => { const entry = active.get(cameraId); - if ( - !entry || - entry.token !== token || - entry.leases.get(leaseId)?.clientId !== clientId - ) + if (!entry || entry.token !== token || !entry.leases.has(leaseId)) return null; const body = entry.segments.get(seq)?.body; - if (body) touch(entry, leaseId, false, clientId); + if (body) touch(entry, leaseId); return body || null; }; const shutdown = async () => { diff --git a/src/data/cctvHlsMiddleware.test.mjs b/src/data/cctvHlsMiddleware.test.mjs new file mode 100644 index 0000000000..301c9796e9 --- /dev/null +++ b/src/data/cctvHlsMiddleware.test.mjs @@ -0,0 +1,174 @@ +// Live HLS through the real /api/cctv middleware: playlist, segment and +// release. HLS leases are random per-playback ids, not bound to the socket +// address, so viewers who share one address (a Docker bridge gateway, a reverse +// proxy, NAT) get the whole per-session budget, and a viewer whose address +// changes mid-playback keeps its lease. +// +// Run with: npm test (node --test) +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { cctvProxy } from '../../server/providers/cctv.js'; + +const camera = (id) => ({ + id, + name: id, + lat: 30.27, + lon: -97.74, + feedType: 'hls', + url: `https://cams.upstream.invalid/${id}/list.m3u8`, +}); +const PLAYLIST = + '#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:12\n#EXTINF:2,\na.ts\n#EXTINF:2,\nb.ts\n#EXTINF:2,\nc.ts\n'; +const lease = (index) => + `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`; + +/** Collects what a handler wrote, the way a Node ServerResponse would. */ +function recordingResponse() { + const chunks = []; + let finished; + const done = new Promise((resolve) => { + finished = resolve; + }); + const res = { + statusCode: 0, + headers: {}, + writableEnded: false, + body: '', + writeHead(status, headers) { + res.statusCode = status; + res.headers = { ...res.headers, ...headers }; + }, + setHeader(name, value) { + res.headers[name] = value; + }, + write(chunk) { + chunks.push(Buffer.from(chunk)); + return true; + }, + end(chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + res.writableEnded = true; + res.body = Buffer.concat(chunks).toString('utf8'); + finished(); + }, + on() {}, + once() {}, + emit() {}, + destroy() {}, + }; + return { res, done }; +} + +/** Mount the provider over two HLS cameras whose upstream always answers. */ +function mount(t) { + const before = { + json: process.env.CCTV_SOURCES_JSON, + file: process.env.CCTV_SOURCES_FILE, + austin: process.env.CCTV_FORCE_AUSTIN, + }; + process.env.CCTV_SOURCES_JSON = JSON.stringify([ + camera('cam-a'), + camera('cam-b'), + ]); + process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; + process.env.CCTV_FORCE_AUSTIN = '0'; + const nativeFetch = globalThis.fetch; + // The puller binds fetch when cctvProxy() creates it, so mock first. + globalThis.fetch = async (url) => + new Response(String(url).endsWith('.m3u8') ? PLAYLIST : 'abc'); + let handler; + const closers = []; + cctvProxy().configureServer({ + httpServer: { on: (_event, fn) => closers.push(fn) }, + middlewares: { use: (_route, fn) => (handler = fn) }, + }); + t.after(() => { + for (const fn of closers) fn(); + globalThis.fetch = nativeFetch; + for (const [name, value] of [ + ['CCTV_SOURCES_JSON', before.json], + ['CCTV_SOURCES_FILE', before.file], + ['CCTV_FORCE_AUSTIN', before.austin], + ]) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + return async (url, remoteAddress, method = 'GET') => { + const { res, done } = recordingResponse(); + await handler( + { + url, + method, + headers: { host: 'localhost:4173', 'sec-fetch-site': 'same-origin' }, + socket: { remoteAddress }, + on() {}, + }, + res, + ); + await done; + return res; + }; +} + +/** Open a playlist and return the first segment path it lists. */ +async function openPlaylist(call, cameraId, leaseId, remoteAddress) { + const res = await call(`/media/${cameraId}?lease=${leaseId}`, remoteAddress); + assert.equal(res.statusCode, 200, `playlist ${cameraId} ${leaseId}`); + const segment = res.body + .split('\n') + .find((line) => line.startsWith('/api/cctv/media/')); + assert.ok(segment, 'the playlist lists a segment'); + return segment.replace('/api/cctv', ''); +} + +test('viewers on distinct addresses open, read and release their own leases', async (t) => { + const call = mount(t); + const viewers = Array.from({ length: 9 }, (_, index) => ({ + camera: index < 5 ? 'cam-a' : 'cam-b', + lease: lease(index), + // IPv4-mapped form, as a dual-stack listener reports it. + peer: `::ffff:192.168.1.${20 + index}`, + })); + for (const viewer of viewers) + viewer.segment = await openPlaylist( + call, + viewer.camera, + viewer.lease, + viewer.peer, + ); + for (const viewer of viewers) + assert.equal((await call(viewer.segment, viewer.peer)).statusCode, 200); + for (const viewer of viewers) { + const release = await call( + `/media/${viewer.camera}?lease=${viewer.lease}`, + viewer.peer, + 'DELETE', + ); + assert.equal(release.statusCode, 204); + } +}); + +test('viewers behind one shared address get every session slot', async (t) => { + const call = mount(t); + const shared = '::ffff:172.17.0.1'; + for (let index = 0; index < 8; index++) + await openPlaylist(call, 'cam-a', lease(index), shared); + const segment = await openPlaylist(call, 'cam-b', lease(8), shared); + assert.equal((await call(segment, shared)).statusCode, 200); +}); + +test('a lease keeps working after its viewer reconnects from another address', async (t) => { + const call = mount(t); + await openPlaylist(call, 'cam-a', lease(0), '192.168.1.20'); + const segment = await openPlaylist(call, 'cam-a', lease(0), '192.168.1.21'); + assert.equal((await call(segment, '192.168.1.21')).statusCode, 200); + const release = await call( + `/media/cam-a?lease=${lease(0)}`, + '192.168.1.21', + 'DELETE', + ); + assert.equal(release.statusCode, 204); + // Released from the new address: the same lease can no longer read segments. + assert.equal((await call(segment, '192.168.1.21')).statusCode, 404); +}); diff --git a/src/data/cctvHlsStream.test.mjs b/src/data/cctvHlsStream.test.mjs index 1dc7eb8785..4a5f4afbc9 100644 --- a/src/data/cctvHlsStream.test.mjs +++ b/src/data/cctvHlsStream.test.mjs @@ -294,40 +294,3 @@ test('an abandoned consumer expires while a renewed consumer keeps the session', assert.equal(entry.controller.signal.aborted, false); await manager.shutdown(); }); - -test('client lease ownership covers refresh, segments and release, with a bounded client budget', async (t) => { - const manager = createHlsPuller({ - limits: { ...HLS_LIMITS, leasesPerClient: 1, pollMs: 100000 }, - fetchImpl: async (url) => - new Response(url.endsWith('.m3u8') ? playlist : 'abc'), - }); - t.after(() => manager.shutdown()); - const entry = await manager.ensure('a', base, 'lease-a', 'peer-a'); - await manager.waitReady(entry); - await assert.rejects( - manager.ensure('a', base, 'lease-a', 'peer-b'), - /another client/, - ); - assert.equal( - manager.getSegment('a', entry.token, 0, 'lease-a', 'peer-b'), - null, - ); - await assert.rejects( - manager.buildPlaylist(entry, 'a', 'lease-a', 'peer-b'), - /another client/, - ); - manager.release('a', 'lease-a', 'peer-b'); - assert.equal(entry.leases.size, 1); - await assert.rejects( - manager.ensure('b', base, 'lease-b', 'peer-a'), - /client lease capacity/, - ); - assert.equal( - manager.stats().sessions, - 1, - 'failed admission reserves no session', - ); - await manager.ensure('b', base, 'lease-b', 'peer-b'); - manager.release('a', 'lease-a', 'peer-a'); - assert.equal(entry.stopping, true); -}); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs index dff7779699..1ff221cb65 100644 --- a/src/data/cctvMediaFeedType.test.mjs +++ b/src/data/cctvMediaFeedType.test.mjs @@ -155,9 +155,9 @@ function mount(t, respond) { }); assert.ok(handler, 'the CCTV provider must mount a middleware'); - const call = async (path, headers = {}) => { + const call = async (path, headers = {}, method = 'GET') => { const { res, done } = recordingResponse(); - await handler({ url: path, headers, method: 'GET', on() {} }, res); + await handler({ url: path, headers, method, on() {} }, res); await done; return res; }; @@ -401,24 +401,35 @@ test('every CCTV route refuses cross-site requests and admits the app itself', a }), ); const host = 'localhost:5173'; - const paths = [ - '/sources', - '/health', - `/stream/${STILL.id}`, - `/frame/${STILL.id}`, - `/media/${VIDEO.id}`, + const lease = '00000000-0000-4000-8000-000000000000'; + const routes = [ + ['GET', '/sources'], + ['GET', '/health'], + ['GET', `/stream/${STILL.id}`], + ['GET', `/frame/${STILL.id}`], + ['GET', `/media/${VIDEO.id}`], + ['GET', `/media/${VIDEO.id}/seg_0.ts?lease=${lease}`], + ['DELETE', `/media/${VIDEO.id}?lease=${lease}`], ]; + // A catalog refresh would fetch the live packs: a refused request must not + // start one. + process.env.CCTV_FORCE_AUSTIN = '1'; for (const headers of [ { host, 'sec-fetch-site': 'cross-site' }, { host, 'sec-fetch-site': 'same-site' }, { host, origin: 'https://elsewhere.example' }, { host, origin: 'null' }, ]) - for (const path of paths) { - const res = await app.call(path, headers); - assert.equal(res.statusCode, 403, `${path} ${JSON.stringify(headers)}`); + for (const [method, path] of routes) { + const res = await app.call(path, headers, method); + assert.equal( + res.statusCode, + 403, + `${method} ${path} ${JSON.stringify(headers)}`, + ); } assert.deepEqual(app.requests, [], 'a refused request reached an upstream'); + process.env.CCTV_FORCE_AUSTIN = '0'; for (const headers of [ { host, 'sec-fetch-site': 'same-origin' }, { host, origin: `http://${host}` }, @@ -528,46 +539,3 @@ test('a forwarded same-origin request reaches every CCTV route but not the Stree assert.equal(direct.body, 'street-view'); assert.equal(streetViewCalls(), 1); }); - -test('cross-site media creation, segment reads and release fail before any catalog or media fetch', async (t) => { - let handler; - const originalFetch = globalThis.fetch; - let calls = 0; - globalThis.fetch = async () => { - calls++; - throw new Error('must not fetch'); - }; - t.after(() => { - globalThis.fetch = originalFetch; - }); - cctvProxy().configureServer({ - middlewares: { - use: (_path, fn) => { - handler = fn; - }, - }, - }); - for (const headers of [ - { origin: 'https://foreign.example' }, - { origin: 'null' }, - { 'sec-fetch-site': 'cross-site' }, - { 'sec-fetch-site': 'same-site' }, - ]) { - for (const [method, url] of [ - ['GET', '/media/camera?lease=test'], - ['DELETE', '/media/camera?lease=test'], - ['GET', '/media/camera/seg_0.ts?lease=test'], - ]) { - const { res } = recordingResponse(); - res.setHeader = (name, value) => { - res.headers[name] = value; - }; - await handler( - { method, url, headers: { host: 'localhost:4173', ...headers } }, - res, - ); - assert.equal(res.statusCode, 403); - } - } - assert.equal(calls, 0); -}); From 4e1f004a83760b795e904c53ecb74ae47eefe848 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 19 Nov 2026 17:45:15 +0000 Subject: [PATCH 5/7] fix(cctv): keep a camera's health when an HLS viewer is refused for capacity A full lease or session budget fell into the general catch and marked the camera degraded, although its upstream was fine and other viewers were watching it. Capacity refusals now throw HlsCapacityError and still answer 503 with Retry-After, without touching health. Tests: fill all 16 lease slots from one shared address and pin the refusal of a ninth viewer and a third session, keep a second lease alive so the released-lease 404 is the lease check, and share recordingResponse between the two middleware test files. --- changelog.d/1267-cctv-hls-capacity-health.md | 3 + server/providers/cctv.js | 19 +++-- server/providers/cctv/stream.js | 7 +- src/data/cctvHlsMiddleware.test.mjs | 82 +++++++++----------- src/data/cctvMediaFeedType.test.mjs | 38 +-------- src/data/cctvRecordingResponse.mjs | 38 +++++++++ 6 files changed, 93 insertions(+), 94 deletions(-) create mode 100644 changelog.d/1267-cctv-hls-capacity-health.md create mode 100644 src/data/cctvRecordingResponse.mjs diff --git a/changelog.d/1267-cctv-hls-capacity-health.md b/changelog.d/1267-cctv-hls-capacity-health.md new file mode 100644 index 0000000000..0b79ea13c5 --- /dev/null +++ b/changelog.d/1267-cctv-hls-capacity-health.md @@ -0,0 +1,3 @@ +- Stop reporting a live HLS camera as degraded when a viewer is refused because + every lease or session slot is taken; the refusal still answers 503 with + `Retry-After` (Said Turksever, German Khokhlov, #1267). diff --git a/server/providers/cctv.js b/server/providers/cctv.js index 5b5222a058..57589ee787 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -18,7 +18,7 @@ import { import { sanitizeCctvRangeHeader } from './cctv/range.js'; import { admitSameSiteKeyless } from './common/same-site.js'; import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs'; -import { createHlsPuller } from './cctv/stream.js'; +import { createHlsPuller, HlsCapacityError } from './cctv/stream.js'; import { isMlitRiverAdjustmentImage } from './cctv/worldCameras.js'; import { ottawaAuthorizedFrameUrl } from './cctv/ottawa.js'; import { googleServerApiKey } from './places/google-key.js'; @@ -449,13 +449,16 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { 'X-CCTV-Session': entry.token, }); res.end(playlist); - } catch { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'fallback', - label: source?.provider || 'Configured source', - message: 'Live HLS unavailable', - }); + } catch (error) { + // A full lease or session budget says nothing about the source: + // leave the health its other viewers see untouched. + if (!(error instanceof HlsCapacityError)) + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'fallback', + label: source?.provider || 'Configured source', + message: 'Live HLS unavailable', + }); if (!downstream.closed) { res.writeHead(503, { 'Content-Type': 'application/json', diff --git a/server/providers/cctv/stream.js b/server/providers/cctv/stream.js index 4e37cda208..8b4b046d6e 100644 --- a/server/providers/cctv/stream.js +++ b/server/providers/cctv/stream.js @@ -16,6 +16,9 @@ export const HLS_LIMITS = Object.freeze({ readyMs: 12000, }); +/** Every lease or session slot is taken: the source itself has not failed. */ +export class HlsCapacityError extends Error {} + /** Refuse redirects; count actual streamed bytes, including chunked responses. */ export async function fetchHlsBytes( url, @@ -152,7 +155,7 @@ export function createHlsPuller({ let lease = entry.leases.get(leaseId); if (!lease) { if (!create || entry.leases.size >= limits.leasesPerSession) - throw new Error('HLS lease capacity reached'); + throw new HlsCapacityError('HLS lease capacity reached'); lease = {}; entry.leases.set(leaseId, lease); } @@ -264,7 +267,7 @@ export function createHlsPuller({ } // Reserve synchronously, before any await: concurrent creation is singleflight. if (active.size >= limits.sessions) - throw new Error('HLS session capacity reached'); + throw new HlsCapacityError('HLS session capacity reached'); entry = { cameraId, url, diff --git a/src/data/cctvHlsMiddleware.test.mjs b/src/data/cctvHlsMiddleware.test.mjs index 301c9796e9..de446b4a47 100644 --- a/src/data/cctvHlsMiddleware.test.mjs +++ b/src/data/cctvHlsMiddleware.test.mjs @@ -8,6 +8,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { cctvProxy } from '../../server/providers/cctv.js'; +import { recordingResponse } from './cctvRecordingResponse.mjs'; const camera = (id) => ({ id, @@ -22,54 +23,16 @@ const PLAYLIST = const lease = (index) => `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`; -/** Collects what a handler wrote, the way a Node ServerResponse would. */ -function recordingResponse() { - const chunks = []; - let finished; - const done = new Promise((resolve) => { - finished = resolve; - }); - const res = { - statusCode: 0, - headers: {}, - writableEnded: false, - body: '', - writeHead(status, headers) { - res.statusCode = status; - res.headers = { ...res.headers, ...headers }; - }, - setHeader(name, value) { - res.headers[name] = value; - }, - write(chunk) { - chunks.push(Buffer.from(chunk)); - return true; - }, - end(chunk) { - if (chunk) chunks.push(Buffer.from(chunk)); - res.writableEnded = true; - res.body = Buffer.concat(chunks).toString('utf8'); - finished(); - }, - on() {}, - once() {}, - emit() {}, - destroy() {}, - }; - return { res, done }; -} - -/** Mount the provider over two HLS cameras whose upstream always answers. */ +/** Mount the provider over three HLS cameras whose upstream always answers. */ function mount(t) { const before = { json: process.env.CCTV_SOURCES_JSON, file: process.env.CCTV_SOURCES_FILE, austin: process.env.CCTV_FORCE_AUSTIN, }; - process.env.CCTV_SOURCES_JSON = JSON.stringify([ - camera('cam-a'), - camera('cam-b'), - ]); + process.env.CCTV_SOURCES_JSON = JSON.stringify( + ['cam-a', 'cam-b', 'cam-c'].map(camera), + ); process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; process.env.CCTV_FORCE_AUSTIN = '0'; const nativeFetch = globalThis.fetch; @@ -82,6 +45,7 @@ function mount(t) { httpServer: { on: (_event, fn) => closers.push(fn) }, middlewares: { use: (_route, fn) => (handler = fn) }, }); + assert.ok(handler, 'the CCTV provider must mount a middleware'); t.after(() => { for (const fn of closers) fn(); globalThis.fetch = nativeFetch; @@ -149,13 +113,34 @@ test('viewers on distinct addresses open, read and release their own leases', as } }); -test('viewers behind one shared address get every session slot', async (t) => { +test('viewers behind one shared address get every slot, and no more', async (t) => { const call = mount(t); const shared = '::ffff:172.17.0.1'; - for (let index = 0; index < 8; index++) - await openPlaylist(call, 'cam-a', lease(index), shared); - const segment = await openPlaylist(call, 'cam-b', lease(8), shared); + const health = async (id) => + JSON.parse((await call('/health', shared)).body).cameras.find( + (entry) => entry.id === id, + ); + let segment; + for (let index = 0; index < 16; index++) + segment = await openPlaylist( + call, + index < 8 ? 'cam-a' : 'cam-b', + lease(index), + shared, + ); assert.equal((await call(segment, shared)).statusCode, 200); + // A ninth viewer of a full session and a third session are both refused, + // without reporting a camera whose upstream is fine as degraded. + for (const cameraId of ['cam-a', 'cam-c']) { + const refused = await call(`/media/${cameraId}?lease=${lease(16)}`, shared); + assert.equal(refused.statusCode, 503, cameraId); + assert.equal(refused.headers['Retry-After'], '2'); + } + assert.equal((await health('cam-a')).status, 'ok'); + assert.equal(await health('cam-c'), undefined); + // A released slot is free for the next viewer. + await call(`/media/cam-a?lease=${lease(0)}`, shared, 'DELETE'); + await openPlaylist(call, 'cam-a', lease(16), shared); }); test('a lease keeps working after its viewer reconnects from another address', async (t) => { @@ -163,12 +148,15 @@ test('a lease keeps working after its viewer reconnects from another address', a await openPlaylist(call, 'cam-a', lease(0), '192.168.1.20'); const segment = await openPlaylist(call, 'cam-a', lease(0), '192.168.1.21'); assert.equal((await call(segment, '192.168.1.21')).statusCode, 200); + // A second viewer keeps the session alive, so the 404 below is the released + // lease and not a stopped session. + const other = await openPlaylist(call, 'cam-a', lease(1), '192.168.1.30'); const release = await call( `/media/cam-a?lease=${lease(0)}`, '192.168.1.21', 'DELETE', ); assert.equal(release.statusCode, 204); - // Released from the new address: the same lease can no longer read segments. assert.equal((await call(segment, '192.168.1.21')).statusCode, 404); + assert.equal((await call(other, '192.168.1.30')).statusCode, 200); }); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs index 1ff221cb65..e3dfc60064 100644 --- a/src/data/cctvMediaFeedType.test.mjs +++ b/src/data/cctvMediaFeedType.test.mjs @@ -9,6 +9,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { Readable } from 'node:stream'; import { cctvProxy } from '../../server/providers/cctv.js'; +import { recordingResponse } from './cctvRecordingResponse.mjs'; const STILL = { id: 'still-fixture', @@ -63,43 +64,6 @@ function upstreamResponse({ status = 200, headers = {}, body = 'payload' }) { }; } -/** Collects what a handler wrote, the way a Node ServerResponse would. */ -function recordingResponse() { - const chunks = []; - let finished; - const done = new Promise((resolve) => { - finished = resolve; - }); - const res = { - statusCode: 0, - headers: {}, - writableEnded: false, - body: '', - writeHead(status, headers) { - res.statusCode = status; - res.headers = { ...res.headers, ...headers }; - }, - setHeader(name, value) { - res.headers[name] = value; - }, - write(chunk) { - chunks.push(Buffer.from(chunk)); - return true; - }, - end(chunk) { - if (chunk) chunks.push(Buffer.from(chunk)); - res.writableEnded = true; - res.body = Buffer.concat(chunks).toString('utf8'); - finished(); - }, - on() {}, - once() {}, - emit() {}, - destroy() {}, - }; - return { res, done }; -} - /** * Mount the real provider over the fixture cameras and answer every upstream * request from the supplied function. diff --git a/src/data/cctvRecordingResponse.mjs b/src/data/cctvRecordingResponse.mjs new file mode 100644 index 0000000000..5d45e38a8f --- /dev/null +++ b/src/data/cctvRecordingResponse.mjs @@ -0,0 +1,38 @@ +// Test support for the CCTV middleware tests: no test file of its own. + +/** Collects what a handler wrote, the way a Node ServerResponse would. */ +export function recordingResponse() { + const chunks = []; + let finished; + const done = new Promise((resolve) => { + finished = resolve; + }); + const res = { + statusCode: 0, + headers: {}, + writableEnded: false, + body: '', + writeHead(status, headers) { + res.statusCode = status; + res.headers = { ...res.headers, ...headers }; + }, + setHeader(name, value) { + res.headers[name] = value; + }, + write(chunk) { + chunks.push(Buffer.from(chunk)); + return true; + }, + end(chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + res.writableEnded = true; + res.body = Buffer.concat(chunks).toString('utf8'); + finished(); + }, + on() {}, + once() {}, + emit() {}, + destroy() {}, + }; + return { res, done }; +} From ed38a1ab3fddfd0a62af4663e461910ab591a713 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 20 Nov 2026 11:48:22 +0400 Subject: [PATCH 6/7] fix(cctv): keep an HLS camera's health through the refused viewer's frame A viewer refused for capacity falls back to /frame after hls.js gives up. For a pullable HLS camera without a snapshot, /frame only has a placeholder or a Street View frame, which says nothing about the stream, yet it wrote degraded over the health that /media owns. /frame now leaves that health as it is; a dead upstream is still reported through /media. Tests: the middleware suite covers both fallbacks, a real upstream failure that still degrades, and cross-site segment and release refusals on a live HLS lease; capacity checks use HLS_LIMITS and HlsCapacityError; the three CCTV suites share one provider harness in src/testSupport. CURRENT-STATE and the changelog fragment describe the refusal. --- changelog.d/1267-cctv-hls-capacity-health.md | 5 +- docs/CURRENT-STATE.md | 5 +- server/providers/cctv.js | 50 ++-- server/providers/cctv/stream.js | 2 +- src/data/cctvHlsMiddleware.test.mjs | 231 ++++++++++--------- src/data/cctvHlsStream.test.mjs | 8 +- src/data/cctvMediaFeedType.test.mjs | 72 +----- src/data/cctvMediaRange.test.mjs | 102 +------- src/data/cctvRecordingResponse.mjs | 38 --- src/testSupport/cctvProviderHarness.mjs | 91 ++++++++ 10 files changed, 283 insertions(+), 321 deletions(-) delete mode 100644 src/data/cctvRecordingResponse.mjs create mode 100644 src/testSupport/cctvProviderHarness.mjs diff --git a/changelog.d/1267-cctv-hls-capacity-health.md b/changelog.d/1267-cctv-hls-capacity-health.md index 0b79ea13c5..a86c90e82a 100644 --- a/changelog.d/1267-cctv-hls-capacity-health.md +++ b/changelog.d/1267-cctv-hls-capacity-health.md @@ -1,3 +1,4 @@ - Stop reporting a live HLS camera as degraded when a viewer is refused because - every lease or session slot is taken; the refusal still answers 503 with - `Retry-After` (Said Turksever, German Khokhlov, #1267). + every viewer or session slot is taken: the refusal answers 503 with + `Retry-After`, and the frame shown in its place leaves the camera's health + as it was (asturksever, gekh, #1267). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 51dd971fe0..9537e97727 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -9582,7 +9582,10 @@ its paint loop. Camera deactivation destroys the decoder, and fatal playback errors revert to the labeled frame fallback. Camera changes clear the old panel frame. hls.js is imported only when an HLS camera starts. Two live sessions may coexist with at most eight independent consumer leases each; no-access lease -expiry is 15 seconds. Closing one consumer leaves other consumers running. Lease +expiry is 15 seconds. A viewer refused because every lease or session slot is +taken gets 503 with `Retry-After: 2` and leaves the camera's health unchanged; +so does its frame fallback, since an HLS camera without a snapshot has no frame +of its own. Closing one consumer leaves other consumers running. Lease ids use `crypto.randomUUID` where the page is a secure context and an equivalent v4 id from `crypto.getRandomValues` on plain-HTTP LAN pages. Encrypted/fMP4/byte-range playlists fail closed. Performance and live agency reliability require recorded validation. diff --git a/server/providers/cctv.js b/server/providers/cctv.js index 57589ee787..d5ad30ba80 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -23,6 +23,11 @@ import { isMlitRiverAdjustmentImage } from './cctv/worldCameras.js'; import { ottawaAuthorizedFrameUrl } from './cctv/ottawa.js'; import { googleServerApiKey } from './places/google-key.js'; export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream }; + +/** An http(s) .m3u8 URL: what the HLS puller behind /media can play. */ +const isPullableHlsUrl = (url) => + /^https?:\/\//i.test(url) && /\.m3u8(?:\?|$)/i.test(url); + /** * Vite plugin: CCTV camera proxy with source registry, frame/media serving, * fallback chain (upstream -> Street View -> synthetic SVG), and health tracking. @@ -386,10 +391,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { return; } if (feedType === 'hls') { - if ( - !/^https?:\/\//i.test(mediaUrl) || - !/\.m3u8(?:\?|$)/i.test(mediaUrl) - ) { + if (!isPullableHlsUrl(mediaUrl)) { res.writeHead(503, { 'Content-Type': 'application/json' }); res.end( JSON.stringify({ @@ -450,8 +452,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { }); res.end(playlist); } catch (error) { - // A full lease or session budget says nothing about the source: - // leave the health its other viewers see untouched. + // A capacity refusal says nothing about the source: keep its health. if (!(error instanceof HlsCapacityError)) setHealth(cameraId, { status: 'degraded', @@ -653,6 +654,13 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { (!isVideoFeedType(normalizeFeedType(source?.feedType)) ? source?.url : ''); + // /media reports a pullable HLS camera's health. Without a snapshot of + // its own, the placeholder or Street View frame here says nothing about + // its stream, so it leaves that health as it is. + const keepHealth = + normalizeFeedType(source?.feedType) === 'hls' && + !source?.snapshotUrl && + isPullableHlsUrl(source?.url || ''); const upstreamImage = source?.pack === 'world-cameras' @@ -747,12 +755,13 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { pitch, }); if (sv?.ok) { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'streetview', - label: 'Google Street View', - message: 'Fallback Street View frame', - }); + if (!keepHealth) + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'streetview', + label: 'Google Street View', + message: 'Fallback Street View frame', + }); res.writeHead(200, { 'Content-Type': sv.contentType, 'Cache-Control': 'no-store', @@ -771,14 +780,15 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { : 'NO UPSTREAM CONFIGURED', }); - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'synthetic', - label: source?.provider || 'Synthetic fallback', - message: source?.url - ? 'Upstream unavailable' - : 'No source configured', - }); + if (!keepHealth) + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'synthetic', + label: source?.provider || 'Synthetic fallback', + message: source?.url + ? 'Upstream unavailable' + : 'No source configured', + }); res.writeHead(200, { 'Content-Type': 'image/svg+xml', diff --git a/server/providers/cctv/stream.js b/server/providers/cctv/stream.js index 8b4b046d6e..e1d22ae5f7 100644 --- a/server/providers/cctv/stream.js +++ b/server/providers/cctv/stream.js @@ -16,7 +16,7 @@ export const HLS_LIMITS = Object.freeze({ readyMs: 12000, }); -/** Every lease or session slot is taken: the source itself has not failed. */ +/** Every lease or session slot is taken, or the viewer's lease ended mid-request: the source itself has not failed. */ export class HlsCapacityError extends Error {} /** Refuse redirects; count actual streamed bytes, including chunked responses. */ diff --git a/src/data/cctvHlsMiddleware.test.mjs b/src/data/cctvHlsMiddleware.test.mjs index de446b4a47..4be55b7783 100644 --- a/src/data/cctvHlsMiddleware.test.mjs +++ b/src/data/cctvHlsMiddleware.test.mjs @@ -1,15 +1,13 @@ -// Live HLS through the real /api/cctv middleware: playlist, segment and -// release. HLS leases are random per-playback ids, not bound to the socket -// address, so viewers who share one address (a Docker bridge gateway, a reverse -// proxy, NAT) get the whole per-session budget, and a viewer whose address -// changes mid-playback keeps its lease. -// -// Run with: npm test (node --test) +// Live HLS through the real /api/cctv middleware. Leases are random +// per-playback ids, not bound to the socket address, and a viewer refused for +// capacity leaves the camera's health as it was. import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { cctvProxy } from '../../server/providers/cctv.js'; -import { recordingResponse } from './cctvRecordingResponse.mjs'; +import { HLS_LIMITS } from '../../server/providers/cctv/stream.js'; +import { mountCctvProvider } from '../testSupport/cctvProviderHarness.mjs'; +const perSession = HLS_LIMITS.leasesPerSession; +const SAME_ORIGIN = { host: 'localhost:4173', 'sec-fetch-site': 'same-origin' }; const camera = (id) => ({ id, name: id, @@ -23,61 +21,29 @@ const PLAYLIST = const lease = (index) => `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`; -/** Mount the provider over three HLS cameras whose upstream always answers. */ -function mount(t) { - const before = { - json: process.env.CCTV_SOURCES_JSON, - file: process.env.CCTV_SOURCES_FILE, - austin: process.env.CCTV_FORCE_AUSTIN, - }; - process.env.CCTV_SOURCES_JSON = JSON.stringify( - ['cam-a', 'cam-b', 'cam-c'].map(camera), - ); - process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; - process.env.CCTV_FORCE_AUSTIN = '0'; - const nativeFetch = globalThis.fetch; - // The puller binds fetch when cctvProxy() creates it, so mock first. - globalThis.fetch = async (url) => - new Response(String(url).endsWith('.m3u8') ? PLAYLIST : 'abc'); - let handler; - const closers = []; - cctvProxy().configureServer({ - httpServer: { on: (_event, fn) => closers.push(fn) }, - middlewares: { use: (_route, fn) => (handler = fn) }, +/** Mount the provider over three HLS cameras; `failing` upstreams answer 500. */ +function mount(t, failing = []) { + const upstream = []; + const app = mountCctvProvider(t, { + cameras: ['cam-a', 'cam-b', 'cam-c'].map(camera), + fetch: async (url) => { + upstream.push(String(url)); + if (failing.some((id) => String(url).includes(`/${id}/`))) + return new Response('', { status: 500 }); + return new Response(String(url).endsWith('.m3u8') ? PLAYLIST : 'abc'); + }, }); - assert.ok(handler, 'the CCTV provider must mount a middleware'); - t.after(() => { - for (const fn of closers) fn(); - globalThis.fetch = nativeFetch; - for (const [name, value] of [ - ['CCTV_SOURCES_JSON', before.json], - ['CCTV_SOURCES_FILE', before.file], - ['CCTV_FORCE_AUSTIN', before.austin], - ]) { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; - } - }); - return async (url, remoteAddress, method = 'GET') => { - const { res, done } = recordingResponse(); - await handler( - { - url, - method, - headers: { host: 'localhost:4173', 'sec-fetch-site': 'same-origin' }, - socket: { remoteAddress }, - on() {}, - }, - res, - ); - await done; - return res; - }; + return { ...app, upstream }; } /** Open a playlist and return the first segment path it lists. */ async function openPlaylist(call, cameraId, leaseId, remoteAddress) { - const res = await call(`/media/${cameraId}?lease=${leaseId}`, remoteAddress); + const res = await call( + `/media/${cameraId}?lease=${leaseId}`, + SAME_ORIGIN, + 'GET', + remoteAddress, + ); assert.equal(res.statusCode, 200, `playlist ${cameraId} ${leaseId}`); const segment = res.body .split('\n') @@ -86,77 +52,136 @@ async function openPlaylist(call, cameraId, leaseId, remoteAddress) { return segment.replace('/api/cctv', ''); } -test('viewers on distinct addresses open, read and release their own leases', async (t) => { - const call = mount(t); - const viewers = Array.from({ length: 9 }, (_, index) => ({ - camera: index < 5 ? 'cam-a' : 'cam-b', - lease: lease(index), - // IPv4-mapped form, as a dual-stack listener reports it. - peer: `::ffff:192.168.1.${20 + index}`, - })); - for (const viewer of viewers) - viewer.segment = await openPlaylist( - call, - viewer.camera, - viewer.lease, - viewer.peer, - ); - for (const viewer of viewers) - assert.equal((await call(viewer.segment, viewer.peer)).statusCode, 200); - for (const viewer of viewers) { - const release = await call( - `/media/${viewer.camera}?lease=${viewer.lease}`, - viewer.peer, - 'DELETE', - ); - assert.equal(release.statusCode, 204); - } -}); - test('viewers behind one shared address get every slot, and no more', async (t) => { - const call = mount(t); + assert.equal(HLS_LIMITS.sessions, 2, 'the test fills both sessions'); + const { call, health } = mount(t); const shared = '::ffff:172.17.0.1'; - const health = async (id) => - JSON.parse((await call('/health', shared)).body).cameras.find( - (entry) => entry.id === id, - ); let segment; - for (let index = 0; index < 16; index++) + for (let index = 0; index < 2 * perSession; index++) segment = await openPlaylist( call, - index < 8 ? 'cam-a' : 'cam-b', + index < perSession ? 'cam-a' : 'cam-b', lease(index), shared, ); - assert.equal((await call(segment, shared)).statusCode, 200); + assert.equal( + (await call(segment, SAME_ORIGIN, 'GET', shared)).statusCode, + 200, + ); + const before = await health('cam-a'); // A ninth viewer of a full session and a third session are both refused, // without reporting a camera whose upstream is fine as degraded. for (const cameraId of ['cam-a', 'cam-c']) { - const refused = await call(`/media/${cameraId}?lease=${lease(16)}`, shared); + const refused = await call( + `/media/${cameraId}?lease=${lease(2 * perSession)}`, + SAME_ORIGIN, + 'GET', + shared, + ); assert.equal(refused.statusCode, 503, cameraId); assert.equal(refused.headers['Retry-After'], '2'); } - assert.equal((await health('cam-a')).status, 'ok'); + assert.deepEqual(await health('cam-a'), before); assert.equal(await health('cam-c'), undefined); // A released slot is free for the next viewer. - await call(`/media/cam-a?lease=${lease(0)}`, shared, 'DELETE'); - await openPlaylist(call, 'cam-a', lease(16), shared); + await call(`/media/cam-a?lease=${lease(0)}`, SAME_ORIGIN, 'DELETE', shared); + await openPlaylist(call, 'cam-a', lease(2 * perSession), shared); }); test('a lease keeps working after its viewer reconnects from another address', async (t) => { - const call = mount(t); + const { call } = mount(t); await openPlaylist(call, 'cam-a', lease(0), '192.168.1.20'); const segment = await openPlaylist(call, 'cam-a', lease(0), '192.168.1.21'); - assert.equal((await call(segment, '192.168.1.21')).statusCode, 200); + assert.equal( + (await call(segment, SAME_ORIGIN, 'GET', '192.168.1.21')).statusCode, + 200, + ); // A second viewer keeps the session alive, so the 404 below is the released // lease and not a stopped session. const other = await openPlaylist(call, 'cam-a', lease(1), '192.168.1.30'); const release = await call( `/media/cam-a?lease=${lease(0)}`, - '192.168.1.21', + SAME_ORIGIN, 'DELETE', + '192.168.1.21', ); assert.equal(release.statusCode, 204); - assert.equal((await call(segment, '192.168.1.21')).statusCode, 404); - assert.equal((await call(other, '192.168.1.30')).statusCode, 200); + assert.equal( + (await call(segment, SAME_ORIGIN, 'GET', '192.168.1.21')).statusCode, + 404, + ); + assert.equal( + (await call(other, SAME_ORIGIN, 'GET', '192.168.1.30')).statusCode, + 200, + ); +}); + +test('ninth viewer frame fallback preserves active camera health', async (t) => { + const { call, health } = mount(t); + let segment; + for (let index = 0; index < perSession; index++) + segment = await openPlaylist(call, 'cam-a', lease(index)); + const before = await health('cam-a'); + assert.equal(before.status, 'ok'); + const refused = await call( + `/media/cam-a?lease=${lease(perSession)}`, + SAME_ORIGIN, + ); + assert.equal(refused.statusCode, 503); + const frame = await call( + '/frame/cam-a?label=cam-a&lat=30.27&lon=-97.74', + SAME_ORIGIN, + ); + assert.equal(frame.statusCode, 200); + assert.equal( + (await call(segment, SAME_ORIGIN)).statusCode, + 200, + 'admitted viewer still plays', + ); + assert.deepEqual(await health('cam-a'), before); +}); + +test('third camera frame fallback keeps absent health', async (t) => { + const { call, health, upstream } = mount(t); + await openPlaylist(call, 'cam-a', lease(0)); + await openPlaylist(call, 'cam-b', lease(1)); + const refused = await call(`/media/cam-c?lease=${lease(2)}`, SAME_ORIGIN); + assert.equal(refused.statusCode, 503); + assert.equal(await health('cam-c'), undefined); + const frame = await call( + '/frame/cam-c?label=cam-c&lat=30.27&lon=-97.74', + SAME_ORIGIN, + ); + assert.equal(frame.statusCode, 200); + assert.equal(upstream.filter((url) => url.includes('/cam-c/')).length, 0); + assert.equal( + await health('cam-c'), + undefined, + 'the fallback frame wrote cam-c health', + ); +}); + +test('a failing HLS upstream with free slots is still reported degraded', async (t) => { + const { call, health } = mount(t, ['cam-c']); + const res = await call(`/media/cam-c?lease=${lease(0)}`, SAME_ORIGIN); + assert.equal(res.statusCode, 503); + assert.equal(res.headers['Retry-After'], '2'); + const row = await health('cam-c'); + assert.deepEqual( + { status: row.status, message: row.message }, + { status: 'degraded', message: 'Live HLS unavailable' }, + ); +}); + +test('a cross-site request can neither read nor release a live HLS lease', async (t) => { + const { call } = mount(t); + const segment = await openPlaylist(call, 'cam-a', lease(0)); + const crossSite = { host: 'localhost:4173', 'sec-fetch-site': 'cross-site' }; + assert.equal((await call(segment, crossSite)).statusCode, 403); + assert.equal( + (await call(`/media/cam-a?lease=${lease(0)}`, crossSite, 'DELETE')) + .statusCode, + 403, + ); + assert.equal((await call(segment, SAME_ORIGIN)).statusCode, 200); }); diff --git a/src/data/cctvHlsStream.test.mjs b/src/data/cctvHlsStream.test.mjs index 4a5f4afbc9..5041a5c163 100644 --- a/src/data/cctvHlsStream.test.mjs +++ b/src/data/cctvHlsStream.test.mjs @@ -5,6 +5,7 @@ import { fetchHlsBytes, parseHlsMedia, HLS_LIMITS, + HlsCapacityError, } from '../../server/providers/cctv/stream.js'; const playlist = '#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:12\n#EXTINF:2,\na.ts\n#EXTINF:2,\nb.ts\n#EXTINF:2,\nc.ts\n'; @@ -144,7 +145,7 @@ test('one session is reserved before await; disk-free cache and capacity remain manager.ensure('a', base), ]); assert.equal(a, b); - await assert.rejects(manager.ensure('b', base)); + await assert.rejects(manager.ensure('b', base), HlsCapacityError); assert.equal(await manager.waitReady(a), true); assert.deepEqual(manager.stats(), { sessions: 1, bytes: 6 }); const text = await manager.buildPlaylist(a, 'a'); @@ -261,7 +262,10 @@ test('two consumers share downloads but release and abandoned expiry are indepen await manager.waitReady(a); assert.equal(downloads, 4); // One manifest and three segments, not per consumer. assert.equal(a.leases.size, 2); - await assert.rejects(manager.ensure('camera', base, 'viewer-c')); + await assert.rejects( + manager.ensure('camera', base, 'viewer-c'), + HlsCapacityError, + ); manager.release('camera', 'viewer-a'); assert.equal(manager.stats().sessions, 1); assert.equal(manager.getSegment('camera', a.token, 0, 'viewer-a'), null); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs index e3dfc60064..d75610a272 100644 --- a/src/data/cctvMediaFeedType.test.mjs +++ b/src/data/cctvMediaFeedType.test.mjs @@ -8,8 +8,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { Readable } from 'node:stream'; -import { cctvProxy } from '../../server/providers/cctv.js'; -import { recordingResponse } from './cctvRecordingResponse.mjs'; +import { mountCctvProvider } from '../testSupport/cctvProviderHarness.mjs'; const STILL = { id: 'still-fixture', @@ -64,72 +63,17 @@ function upstreamResponse({ status = 200, headers = {}, body = 'payload' }) { }; } -/** - * Mount the real provider over the fixture cameras and answer every upstream - * request from the supplied function. - */ +/** Mount the real provider over the fixture cameras, recording upstream requests. */ function mount(t, respond) { - const before = { - json: process.env.CCTV_SOURCES_JSON, - file: process.env.CCTV_SOURCES_FILE, - austin: process.env.CCTV_FORCE_AUSTIN, - serverKey: process.env.GOOGLE_MAPS_SERVER_API_KEY, - browserKey: process.env.GOOGLE_MAPS_API_KEY, - }; - process.env.CCTV_SOURCES_JSON = JSON.stringify([ - STILL, - STILL_ALIAS, - VIDEO, - WEBM, - MJPEG, - ]); - process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; - process.env.CCTV_FORCE_AUSTIN = '0'; - // No Street View key: a /frame miss lands on the synthetic billboard rather - // than a network call this file does not model. - delete process.env.GOOGLE_MAPS_SERVER_API_KEY; - delete process.env.GOOGLE_MAPS_API_KEY; - const nativeFetch = globalThis.fetch; const requests = []; - globalThis.fetch = async (url, init = {}) => { - requests.push({ url: String(url), redirect: init.redirect }); - return respond({ url: String(url), init }); - }; - t.after(() => { - globalThis.fetch = nativeFetch; - for (const [name, value] of [ - ['CCTV_SOURCES_JSON', before.json], - ['CCTV_SOURCES_FILE', before.file], - ['CCTV_FORCE_AUSTIN', before.austin], - ['GOOGLE_MAPS_SERVER_API_KEY', before.serverKey], - ['GOOGLE_MAPS_API_KEY', before.browserKey], - ]) { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; - } - }); - - let handler = null; - cctvProxy().configureServer({ - middlewares: { - use: (_route, fn) => { - handler = fn; - }, + const app = mountCctvProvider(t, { + cameras: [STILL, STILL_ALIAS, VIDEO, WEBM, MJPEG], + fetch: async (url, init = {}) => { + requests.push({ url: String(url), redirect: init.redirect }); + return respond({ url: String(url), init }); }, }); - assert.ok(handler, 'the CCTV provider must mount a middleware'); - - const call = async (path, headers = {}, method = 'GET') => { - const { res, done } = recordingResponse(); - await handler({ url: path, headers, method, on() {} }, res); - await done; - return res; - }; - const health = async (id) => { - const res = await call('/health'); - return JSON.parse(res.body).cameras.find((entry) => entry.id === id); - }; - return { call, health, requests }; + return { ...app, requests }; } test('a still camera is refused by /media without any upstream request', async (t) => { diff --git a/src/data/cctvMediaRange.test.mjs b/src/data/cctvMediaRange.test.mjs index 9f279bdc77..42021d3c28 100644 --- a/src/data/cctvMediaRange.test.mjs +++ b/src/data/cctvMediaRange.test.mjs @@ -9,7 +9,7 @@ import assert from 'node:assert/strict'; import http from 'node:http'; import net from 'node:net'; import { Readable, Writable } from 'node:stream'; -import { cctvProxy } from '../../server/providers/cctv.js'; +import { mountCctvProvider } from '../testSupport/cctvProviderHarness.mjs'; import { proxyMediaResponse } from '../../server/providers/cctv/media.js'; import { sanitizeCctvRangeHeader } from '../../server/providers/cctv/range.js'; import { CCTV_MEDIA_MAX_BODY_BYTES as CAP } from '../../server/providers/cctv/constants.js'; @@ -131,99 +131,21 @@ function upstreamResponse({ status = 206, headers = {}, body = 'payload' }) { }; } -/** Collects what a handler wrote, the way a Node ServerResponse would. */ -function recordingResponse() { - const chunks = []; - let finished; - const done = new Promise((resolve) => { - finished = resolve; - }); - const res = { - statusCode: 0, - headers: {}, - writableEnded: false, - body: '', - writeHead(status, headers) { - res.statusCode = status; - res.headers = headers || {}; - }, - write(chunk) { - chunks.push(Buffer.from(chunk)); - return true; - }, - end(chunk) { - if (chunk) chunks.push(Buffer.from(chunk)); - res.writableEnded = true; - res.body = Buffer.concat(chunks).toString('utf8'); - finished(); - }, - on() {}, - once() {}, - emit() {}, - destroy() {}, - }; - return { res, done }; -} - -/** - * Mount the real provider, serve one camera, and answer its upstream from the - * supplied function. - */ +/** Mount the real provider over the fixture cameras, recording upstream requests. */ function mount(t, respond) { - const before = { - json: process.env.CCTV_SOURCES_JSON, - file: process.env.CCTV_SOURCES_FILE, - austin: process.env.CCTV_FORCE_AUSTIN, - }; - process.env.CCTV_SOURCES_JSON = JSON.stringify([CAMERA]); - process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; - process.env.CCTV_FORCE_AUSTIN = '0'; - const nativeFetch = globalThis.fetch; const requests = []; - globalThis.fetch = async (url, init = {}) => { - requests.push({ - url: String(url), - headers: init.headers || {}, - signal: init.signal || null, - }); - return respond({ url: String(url), init }); - }; - t.after(() => { - globalThis.fetch = nativeFetch; - for (const [name, value] of [ - ['CCTV_SOURCES_JSON', before.json], - ['CCTV_SOURCES_FILE', before.file], - ['CCTV_FORCE_AUSTIN', before.austin], - ]) { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; - } - }); - - let handler = null; - const plugin = cctvProxy(); - plugin.configureServer({ - middlewares: { - use: (_route, fn) => { - handler = fn; - }, + const app = mountCctvProvider(t, { + cameras: [CAMERA], + fetch: async (url, init = {}) => { + requests.push({ + url: String(url), + headers: init.headers || {}, + signal: init.signal || null, + }); + return respond({ url: String(url), init }); }, }); - assert.ok(handler, 'the CCTV provider must mount a middleware'); - - /** Drive one request through the mounted handler. */ - const call = async (path, headers = {}) => { - const { res, done } = recordingResponse(); - await handler({ url: path, headers, method: 'GET', on() {} }, res); - await done; - return res; - }; - /** Read the camera's current health entry. */ - const health = async () => { - const res = await call('/health'); - return JSON.parse(res.body).cameras.find(({ id }) => id === CAMERA.id); - }; - return { call, health, requests, handler: (req, res) => handler(req, res) }; + return { ...app, requests, health: () => app.health(CAMERA.id) }; } test('a canonical range reaches the upstream and its 206 is passed through', async (t) => { diff --git a/src/data/cctvRecordingResponse.mjs b/src/data/cctvRecordingResponse.mjs deleted file mode 100644 index 5d45e38a8f..0000000000 --- a/src/data/cctvRecordingResponse.mjs +++ /dev/null @@ -1,38 +0,0 @@ -// Test support for the CCTV middleware tests: no test file of its own. - -/** Collects what a handler wrote, the way a Node ServerResponse would. */ -export function recordingResponse() { - const chunks = []; - let finished; - const done = new Promise((resolve) => { - finished = resolve; - }); - const res = { - statusCode: 0, - headers: {}, - writableEnded: false, - body: '', - writeHead(status, headers) { - res.statusCode = status; - res.headers = { ...res.headers, ...headers }; - }, - setHeader(name, value) { - res.headers[name] = value; - }, - write(chunk) { - chunks.push(Buffer.from(chunk)); - return true; - }, - end(chunk) { - if (chunk) chunks.push(Buffer.from(chunk)); - res.writableEnded = true; - res.body = Buffer.concat(chunks).toString('utf8'); - finished(); - }, - on() {}, - once() {}, - emit() {}, - destroy() {}, - }; - return { res, done }; -} diff --git a/src/testSupport/cctvProviderHarness.mjs b/src/testSupport/cctvProviderHarness.mjs new file mode 100644 index 0000000000..8b7369470e --- /dev/null +++ b/src/testSupport/cctvProviderHarness.mjs @@ -0,0 +1,91 @@ +import { cctvProxy } from '../../server/providers/cctv.js'; + +/** Collects what a handler wrote, the way a Node ServerResponse would. */ +export function recordingResponse() { + const chunks = []; + let finished; + const done = new Promise((resolve) => { + finished = resolve; + }); + const res = { + statusCode: 0, + headers: {}, + writableEnded: false, + body: '', + writeHead(status, headers) { + res.statusCode = status; + res.headers = { ...res.headers, ...headers }; + }, + setHeader(name, value) { + res.headers[name] = value; + }, + write(chunk) { + chunks.push(Buffer.from(chunk)); + return true; + }, + end(chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + res.writableEnded = true; + res.body = Buffer.concat(chunks).toString('utf8'); + finished(); + }, + on() {}, + once() {}, + emit() {}, + destroy() {}, + }; + return { res, done }; +} + +export function mountCctvProvider(t, { cameras, fetch }) { + const saved = {}; + for (const name of [ + 'CCTV_SOURCES_JSON', + 'CCTV_SOURCES_FILE', + 'CCTV_FORCE_AUSTIN', + 'GOOGLE_MAPS_SERVER_API_KEY', + 'GOOGLE_MAPS_API_KEY', + ]) + saved[name] = process.env[name]; + + const nativeFetch = globalThis.fetch; + const closers = []; + t.after(() => { + for (const close of closers) close(); + globalThis.fetch = nativeFetch; + for (const [name, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + process.env.CCTV_SOURCES_JSON = JSON.stringify(cameras); + process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; + process.env.CCTV_FORCE_AUSTIN = '0'; + // No Street View key: a /frame miss lands on the synthetic billboard. + delete process.env.GOOGLE_MAPS_SERVER_API_KEY; + delete process.env.GOOGLE_MAPS_API_KEY; + // The HLS puller binds fetch when cctvProxy() creates it, so mock first. + globalThis.fetch = fetch; + let handler; + cctvProxy().configureServer({ + httpServer: { on: (_event, fn) => closers.push(fn) }, + middlewares: { use: (_route, fn) => (handler = fn) }, + }); + if (!handler) throw new Error('the CCTV provider must mount a middleware'); + + const call = async (path, headers = {}, method = 'GET', remoteAddress) => { + const { res, done } = recordingResponse(); + await handler( + { url: path, headers, method, socket: { remoteAddress }, on() {} }, + res, + ); + await done; + return res; + }; + const health = async (id) => { + const res = await call('/health'); + return JSON.parse(res.body).cameras.find((entry) => entry.id === id); + }; + return { call, health, handler }; +} From 58d2c23860b4bfe19ab676b9d4ec40cee1a67c63 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 20 Nov 2026 12:29:50 +0400 Subject: [PATCH 7/7] fix(cctv): label a refused viewer's frame fallback instead of the live row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /frame now keeps a pullable HLS camera's shared health, so a viewer refused for capacity saw its placeholder or Street View frame under the live row's LIVE · OK badge and "Live HLS connected". When a video camera is shown on its frame fallback while the row says live, the panel badge now reads FALLBACK · DEGRADED and the meta line drops the live message; the shared row stays as it is for the other viewers. --- changelog.d/1267-cctv-hls-capacity-health.md | 5 +- docs/CURRENT-STATE.md | 4 +- src/data/cctvRefusedViewerBadge.test.mjs | 85 ++++++++++++++++++++ src/ui/cctvFrames.js | 13 +++ src/ui/cctvPresentation.js | 13 ++- 5 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 src/data/cctvRefusedViewerBadge.test.mjs diff --git a/changelog.d/1267-cctv-hls-capacity-health.md b/changelog.d/1267-cctv-hls-capacity-health.md index a86c90e82a..e40c0762e1 100644 --- a/changelog.d/1267-cctv-hls-capacity-health.md +++ b/changelog.d/1267-cctv-hls-capacity-health.md @@ -1,4 +1,5 @@ - Stop reporting a live HLS camera as degraded when a viewer is refused because every viewer or session slot is taken: the refusal answers 503 with - `Retry-After`, and the frame shown in its place leaves the camera's health - as it was (asturksever, gekh, #1267). + `Retry-After`, the frame shown in its place leaves the camera's health + as it was, and that viewer's panel labels the frame `FALLBACK · DEGRADED` + rather than live (asturksever, gekh, #1267). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index a05ed3c4ae..ee9c7e6d95 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -9658,7 +9658,9 @@ coexist with at most eight independent consumer leases each; no-access lease expiry is 15 seconds. A viewer refused because every lease or session slot is taken gets 503 with `Retry-After: 2` and leaves the camera's health unchanged; so does its frame fallback, since an HLS camera without a snapshot has no frame -of its own. Closing one consumer leaves other consumers running. Lease +of its own. That viewer's panel badge reads FALLBACK · DEGRADED and its meta +line drops the live health message, since the shared live health does not +describe the frame it shows. Closing one consumer leaves other consumers running. Lease ids use `crypto.randomUUID` where the page is a secure context and an equivalent v4 id from `crypto.getRandomValues` on plain-HTTP LAN pages. Encrypted/fMP4/byte-range playlists fail closed. Performance and live agency reliability require recorded validation. diff --git a/src/data/cctvRefusedViewerBadge.test.mjs b/src/data/cctvRefusedViewerBadge.test.mjs new file mode 100644 index 0000000000..8231eb0e83 --- /dev/null +++ b/src/data/cctvRefusedViewerBadge.test.mjs @@ -0,0 +1,85 @@ +// Reproduction: a viewer refused for HLS capacity is shown its fallback frame +// under a badge that still reads the shared live health ("LIVE · OK"). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { HLS_LIMITS } from '../../server/providers/cctv/stream.js'; +import { mountCctvProvider } from '../testSupport/cctvProviderHarness.mjs'; +import { presentCctvSourceBadge } from '../ui/cctvFrames.js'; +import { presentCctvPanel } from '../ui/cctvPresentation.js'; + +const SAME_ORIGIN = { host: 'localhost:4173', 'sec-fetch-site': 'same-origin' }; +const PLAYLIST = + '#EXTM3U\n#EXT-X-TARGETDURATION:2\n#EXT-X-MEDIA-SEQUENCE:12\n#EXTINF:2,\na.ts\n#EXTINF:2,\nb.ts\n#EXTINF:2,\nc.ts\n'; +const lease = (index) => + `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`; +const camera = (id) => ({ + id, + name: id, + lat: 30.27, + lon: -97.74, + feedType: 'hls', + url: `https://cams.upstream.invalid/${id}/list.m3u8`, +}); + +test('a capacity-refused viewer is not shown its fallback frame as live and ok', async (t) => { + assert.equal(HLS_LIMITS.sessions, 2); + const { call, health } = mountCctvProvider(t, { + cameras: ['cam-a', 'cam-b', 'cam-c'].map(camera), + fetch: async (url) => + new Response(String(url).endsWith('.m3u8') ? PLAYLIST : 'abc'), + }); + const open = (id, index) => + call(`/media/${id}?lease=${lease(index)}`, SAME_ORIGIN); + // cam-c played earlier and was released: its row is live/ok. + assert.equal((await open('cam-c', 0)).statusCode, 200); + assert.equal( + (await call(`/media/cam-c?lease=${lease(0)}`, SAME_ORIGIN, 'DELETE')) + .statusCode, + 204, + ); + // Two other viewers now hold both camera sessions. + assert.equal((await open('cam-a', 1)).statusCode, 200); + assert.equal((await open('cam-b', 2)).statusCode, 200); + // The cam-c viewer is refused and falls back to the frame. + const refused = await open('cam-c', 3); + assert.equal(refused.statusCode, 503); + const frame = await call('/frame/cam-c', SAME_ORIGIN); + assert.equal(frame.statusCode, 200); + assert.equal(frame.headers['X-CCTV-Source'], 'synthetic'); + assert.match(frame.body, /UPSTREAM UNAVAILABLE/); + const row = await health('cam-c'); + // The badge the panel derives for this viewer, in image (fallback) mode. + const shown = { + feedType: 'hls', + isVideo: false, + sourceKind: row?.sourceKind || 'configured', + sourceStatus: row?.status || 'unknown', + sourceMessage: row?.message || '', + }; + const badge = presentCctvSourceBadge(shown, true, { + loading: false, + error: false, + hasFrame: true, + }); + assert.notEqual( + badge.label, + 'LIVE · OK', + `fallback frame (${frame.headers['X-CCTV-Source']}) labelled ${badge.label}; health message "${row?.message}"`, + ); + const { meta } = presentCctvPanel( + { + enabled: true, + activeCameraId: 'cam-c', + activeCamera: { + ...shown, + id: 'cam-c', + city: 'Austin', + headingDeg: 0, + fovDeg: 60, + rangeM: 100, + }, + }, + true, + ); + assert.doesNotMatch(meta, /Live HLS connected/); +}); diff --git a/src/ui/cctvFrames.js b/src/ui/cctvFrames.js index 7617111748..171a67fb34 100644 --- a/src/ui/cctvFrames.js +++ b/src/ui/cctvFrames.js @@ -186,6 +186,14 @@ export function _settleCctvFrame(token, src, ok) { ); } +/** Shared live health describes the stream even when this viewer shows a frame. */ +export function isCctvLiveFrameFallback(activeCamera) { + return ( + activeCamera?.isVideo === false && + String(activeCamera.sourceKind || '').toLowerCase() === 'live' + ); +} + /** The source badge's label and frame state for the camera and preview. */ export function presentCctvSourceBadge(activeCamera, enabled, frame) { if (!enabled || !activeCamera) @@ -194,6 +202,11 @@ export function presentCctvSourceBadge(activeCamera, enabled, frame) { return { label: t('FRAME · LOADING'), state: 'loading' }; if (frame.error && !frame.hasFrame) return { label: t('FRAME · UNAVAILABLE'), state: 'error' }; + if (isCctvLiveFrameFallback(activeCamera)) + return { + label: `${cctvSourceKindLabel('fallback')} · ${cctvSourceStatusLabel('degraded')}`, + state: 'ready', + }; const kind = cctvSourceKindLabel( activeCamera.sourceKind || activeCamera.feedType || 'unknown', ); diff --git a/src/ui/cctvPresentation.js b/src/ui/cctvPresentation.js index 2f4cb8584e..92653e0ad7 100644 --- a/src/ui/cctvPresentation.js +++ b/src/ui/cctvPresentation.js @@ -1,5 +1,9 @@ import { getLocale, keepCodesWithValues, msg, t, tn } from '../i18n/index.js'; -import { cctvSourceLabelText, cctvSourceMessageText } from './cctvFrames.js'; +import { + cctvSourceLabelText, + cctvSourceMessageText, + isCctvLiveFrameFallback, +} from './cctvFrames.js'; import { createCctvVideoSurface } from './cctvVideo.js'; import { presentCctvCalibration } from './cctvCalibration.js'; @@ -96,9 +100,10 @@ function metaText(state, enabled, activeCamera, catalogCount) { const provider = cctvSourceLabelText( activeCamera.sourceLabel || activeCamera.provider || 'Configured Source', ); - const statusMsg = activeCamera.sourceMessage - ? ` · ${cctvSourceMessageText(activeCamera.sourceMessage)}` - : ''; + const statusMsg = + activeCamera.sourceMessage && !isCctvLiveFrameFallback(activeCamera) + ? ` · ${cctvSourceMessageText(activeCamera.sourceMessage)}` + : ''; // A partner-supplied feed inside a pack names its owner here. const credit = activeCamera.credit ? ` · ${activeCamera.credit}` : ''; const calBadge = activeCamera.calBadge