From 4e1f004a83760b795e904c53ecb74ae47eefe848 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 19 Nov 2026 17:45:15 +0000 Subject: [PATCH] fix(cctv): keep a camera's health when an HLS viewer is refused for capacity A full lease or session budget fell into the general catch and marked the camera degraded, although its upstream was fine and other viewers were watching it. Capacity refusals now throw HlsCapacityError and still answer 503 with Retry-After, without touching health. Tests: fill all 16 lease slots from one shared address and pin the refusal of a ninth viewer and a third session, keep a second lease alive so the released-lease 404 is the lease check, and share recordingResponse between the two middleware test files. --- changelog.d/1267-cctv-hls-capacity-health.md | 3 + server/providers/cctv.js | 19 +++-- server/providers/cctv/stream.js | 7 +- src/data/cctvHlsMiddleware.test.mjs | 82 +++++++++----------- src/data/cctvMediaFeedType.test.mjs | 38 +-------- src/data/cctvRecordingResponse.mjs | 38 +++++++++ 6 files changed, 93 insertions(+), 94 deletions(-) create mode 100644 changelog.d/1267-cctv-hls-capacity-health.md create mode 100644 src/data/cctvRecordingResponse.mjs diff --git a/changelog.d/1267-cctv-hls-capacity-health.md b/changelog.d/1267-cctv-hls-capacity-health.md new file mode 100644 index 0000000000..0b79ea13c5 --- /dev/null +++ b/changelog.d/1267-cctv-hls-capacity-health.md @@ -0,0 +1,3 @@ +- Stop reporting a live HLS camera as degraded when a viewer is refused because + every lease or session slot is taken; the refusal still answers 503 with + `Retry-After` (Said Turksever, German Khokhlov, #1267). diff --git a/server/providers/cctv.js b/server/providers/cctv.js index 5b5222a058..57589ee787 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -18,7 +18,7 @@ import { import { sanitizeCctvRangeHeader } from './cctv/range.js'; import { admitSameSiteKeyless } from './common/same-site.js'; import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs'; -import { createHlsPuller } from './cctv/stream.js'; +import { createHlsPuller, HlsCapacityError } from './cctv/stream.js'; import { isMlitRiverAdjustmentImage } from './cctv/worldCameras.js'; import { ottawaAuthorizedFrameUrl } from './cctv/ottawa.js'; import { googleServerApiKey } from './places/google-key.js'; @@ -449,13 +449,16 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { 'X-CCTV-Session': entry.token, }); res.end(playlist); - } catch { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'fallback', - label: source?.provider || 'Configured source', - message: 'Live HLS unavailable', - }); + } catch (error) { + // A full lease or session budget says nothing about the source: + // leave the health its other viewers see untouched. + if (!(error instanceof HlsCapacityError)) + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'fallback', + label: source?.provider || 'Configured source', + message: 'Live HLS unavailable', + }); if (!downstream.closed) { res.writeHead(503, { 'Content-Type': 'application/json', diff --git a/server/providers/cctv/stream.js b/server/providers/cctv/stream.js index 4e37cda208..8b4b046d6e 100644 --- a/server/providers/cctv/stream.js +++ b/server/providers/cctv/stream.js @@ -16,6 +16,9 @@ export const HLS_LIMITS = Object.freeze({ readyMs: 12000, }); +/** Every lease or session slot is taken: the source itself has not failed. */ +export class HlsCapacityError extends Error {} + /** Refuse redirects; count actual streamed bytes, including chunked responses. */ export async function fetchHlsBytes( url, @@ -152,7 +155,7 @@ export function createHlsPuller({ let lease = entry.leases.get(leaseId); if (!lease) { if (!create || entry.leases.size >= limits.leasesPerSession) - throw new Error('HLS lease capacity reached'); + throw new HlsCapacityError('HLS lease capacity reached'); lease = {}; entry.leases.set(leaseId, lease); } @@ -264,7 +267,7 @@ export function createHlsPuller({ } // Reserve synchronously, before any await: concurrent creation is singleflight. if (active.size >= limits.sessions) - throw new Error('HLS session capacity reached'); + throw new HlsCapacityError('HLS session capacity reached'); entry = { cameraId, url, diff --git a/src/data/cctvHlsMiddleware.test.mjs b/src/data/cctvHlsMiddleware.test.mjs index 301c9796e9..de446b4a47 100644 --- a/src/data/cctvHlsMiddleware.test.mjs +++ b/src/data/cctvHlsMiddleware.test.mjs @@ -8,6 +8,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { cctvProxy } from '../../server/providers/cctv.js'; +import { recordingResponse } from './cctvRecordingResponse.mjs'; const camera = (id) => ({ id, @@ -22,54 +23,16 @@ const PLAYLIST = const lease = (index) => `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`; -/** Collects what a handler wrote, the way a Node ServerResponse would. */ -function recordingResponse() { - const chunks = []; - let finished; - const done = new Promise((resolve) => { - finished = resolve; - }); - const res = { - statusCode: 0, - headers: {}, - writableEnded: false, - body: '', - writeHead(status, headers) { - res.statusCode = status; - res.headers = { ...res.headers, ...headers }; - }, - setHeader(name, value) { - res.headers[name] = value; - }, - write(chunk) { - chunks.push(Buffer.from(chunk)); - return true; - }, - end(chunk) { - if (chunk) chunks.push(Buffer.from(chunk)); - res.writableEnded = true; - res.body = Buffer.concat(chunks).toString('utf8'); - finished(); - }, - on() {}, - once() {}, - emit() {}, - destroy() {}, - }; - return { res, done }; -} - -/** Mount the provider over two HLS cameras whose upstream always answers. */ +/** Mount the provider over three HLS cameras whose upstream always answers. */ function mount(t) { const before = { json: process.env.CCTV_SOURCES_JSON, file: process.env.CCTV_SOURCES_FILE, austin: process.env.CCTV_FORCE_AUSTIN, }; - process.env.CCTV_SOURCES_JSON = JSON.stringify([ - camera('cam-a'), - camera('cam-b'), - ]); + process.env.CCTV_SOURCES_JSON = JSON.stringify( + ['cam-a', 'cam-b', 'cam-c'].map(camera), + ); process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; process.env.CCTV_FORCE_AUSTIN = '0'; const nativeFetch = globalThis.fetch; @@ -82,6 +45,7 @@ function mount(t) { httpServer: { on: (_event, fn) => closers.push(fn) }, middlewares: { use: (_route, fn) => (handler = fn) }, }); + assert.ok(handler, 'the CCTV provider must mount a middleware'); t.after(() => { for (const fn of closers) fn(); globalThis.fetch = nativeFetch; @@ -149,13 +113,34 @@ test('viewers on distinct addresses open, read and release their own leases', as } }); -test('viewers behind one shared address get every session slot', async (t) => { +test('viewers behind one shared address get every slot, and no more', async (t) => { const call = mount(t); const shared = '::ffff:172.17.0.1'; - for (let index = 0; index < 8; index++) - await openPlaylist(call, 'cam-a', lease(index), shared); - const segment = await openPlaylist(call, 'cam-b', lease(8), shared); + const health = async (id) => + JSON.parse((await call('/health', shared)).body).cameras.find( + (entry) => entry.id === id, + ); + let segment; + for (let index = 0; index < 16; index++) + segment = await openPlaylist( + call, + index < 8 ? 'cam-a' : 'cam-b', + lease(index), + shared, + ); assert.equal((await call(segment, shared)).statusCode, 200); + // A ninth viewer of a full session and a third session are both refused, + // without reporting a camera whose upstream is fine as degraded. + for (const cameraId of ['cam-a', 'cam-c']) { + const refused = await call(`/media/${cameraId}?lease=${lease(16)}`, shared); + assert.equal(refused.statusCode, 503, cameraId); + assert.equal(refused.headers['Retry-After'], '2'); + } + assert.equal((await health('cam-a')).status, 'ok'); + assert.equal(await health('cam-c'), undefined); + // A released slot is free for the next viewer. + await call(`/media/cam-a?lease=${lease(0)}`, shared, 'DELETE'); + await openPlaylist(call, 'cam-a', lease(16), shared); }); test('a lease keeps working after its viewer reconnects from another address', async (t) => { @@ -163,12 +148,15 @@ test('a lease keeps working after its viewer reconnects from another address', a await openPlaylist(call, 'cam-a', lease(0), '192.168.1.20'); const segment = await openPlaylist(call, 'cam-a', lease(0), '192.168.1.21'); assert.equal((await call(segment, '192.168.1.21')).statusCode, 200); + // A second viewer keeps the session alive, so the 404 below is the released + // lease and not a stopped session. + const other = await openPlaylist(call, 'cam-a', lease(1), '192.168.1.30'); const release = await call( `/media/cam-a?lease=${lease(0)}`, '192.168.1.21', 'DELETE', ); assert.equal(release.statusCode, 204); - // Released from the new address: the same lease can no longer read segments. assert.equal((await call(segment, '192.168.1.21')).statusCode, 404); + assert.equal((await call(other, '192.168.1.30')).statusCode, 200); }); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs index 1ff221cb65..e3dfc60064 100644 --- a/src/data/cctvMediaFeedType.test.mjs +++ b/src/data/cctvMediaFeedType.test.mjs @@ -9,6 +9,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { Readable } from 'node:stream'; import { cctvProxy } from '../../server/providers/cctv.js'; +import { recordingResponse } from './cctvRecordingResponse.mjs'; const STILL = { id: 'still-fixture', @@ -63,43 +64,6 @@ function upstreamResponse({ status = 200, headers = {}, body = 'payload' }) { }; } -/** Collects what a handler wrote, the way a Node ServerResponse would. */ -function recordingResponse() { - const chunks = []; - let finished; - const done = new Promise((resolve) => { - finished = resolve; - }); - const res = { - statusCode: 0, - headers: {}, - writableEnded: false, - body: '', - writeHead(status, headers) { - res.statusCode = status; - res.headers = { ...res.headers, ...headers }; - }, - setHeader(name, value) { - res.headers[name] = value; - }, - write(chunk) { - chunks.push(Buffer.from(chunk)); - return true; - }, - end(chunk) { - if (chunk) chunks.push(Buffer.from(chunk)); - res.writableEnded = true; - res.body = Buffer.concat(chunks).toString('utf8'); - finished(); - }, - on() {}, - once() {}, - emit() {}, - destroy() {}, - }; - return { res, done }; -} - /** * Mount the real provider over the fixture cameras and answer every upstream * request from the supplied function. diff --git a/src/data/cctvRecordingResponse.mjs b/src/data/cctvRecordingResponse.mjs new file mode 100644 index 0000000000..5d45e38a8f --- /dev/null +++ b/src/data/cctvRecordingResponse.mjs @@ -0,0 +1,38 @@ +// Test support for the CCTV middleware tests: no test file of its own. + +/** Collects what a handler wrote, the way a Node ServerResponse would. */ +export function recordingResponse() { + const chunks = []; + let finished; + const done = new Promise((resolve) => { + finished = resolve; + }); + const res = { + statusCode: 0, + headers: {}, + writableEnded: false, + body: '', + writeHead(status, headers) { + res.statusCode = status; + res.headers = { ...res.headers, ...headers }; + }, + setHeader(name, value) { + res.headers[name] = value; + }, + write(chunk) { + chunks.push(Buffer.from(chunk)); + return true; + }, + end(chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + res.writableEnded = true; + res.body = Buffer.concat(chunks).toString('utf8'); + finished(); + }, + on() {}, + once() {}, + emit() {}, + destroy() {}, + }; + return { res, done }; +}