diff --git a/changelog.d/1267-cctv-hls-capacity-health.md b/changelog.d/1267-cctv-hls-capacity-health.md new file mode 100644 index 0000000000..0b79ea13c5 --- /dev/null +++ b/changelog.d/1267-cctv-hls-capacity-health.md @@ -0,0 +1,3 @@ +- Stop reporting a live HLS camera as degraded when a viewer is refused because + every lease or session slot is taken; the refusal still answers 503 with + `Retry-After` (Said Turksever, German Khokhlov, #1267). diff --git a/server/providers/cctv.js b/server/providers/cctv.js index 5b5222a058..57589ee787 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -18,7 +18,7 @@ import { import { sanitizeCctvRangeHeader } from './cctv/range.js'; import { admitSameSiteKeyless } from './common/same-site.js'; import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs'; -import { createHlsPuller } from './cctv/stream.js'; +import { createHlsPuller, HlsCapacityError } from './cctv/stream.js'; import { isMlitRiverAdjustmentImage } from './cctv/worldCameras.js'; import { ottawaAuthorizedFrameUrl } from './cctv/ottawa.js'; import { googleServerApiKey } from './places/google-key.js'; @@ -449,13 +449,16 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { 'X-CCTV-Session': entry.token, }); res.end(playlist); - } catch { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'fallback', - label: source?.provider || 'Configured source', - message: 'Live HLS unavailable', - }); + } catch (error) { + // A full lease or session budget says nothing about the source: + // leave the health its other viewers see untouched. + if (!(error instanceof HlsCapacityError)) + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'fallback', + label: source?.provider || 'Configured source', + message: 'Live HLS unavailable', + }); if (!downstream.closed) { res.writeHead(503, { 'Content-Type': 'application/json', diff --git a/server/providers/cctv/stream.js b/server/providers/cctv/stream.js index 4e37cda208..8b4b046d6e 100644 --- a/server/providers/cctv/stream.js +++ b/server/providers/cctv/stream.js @@ -16,6 +16,9 @@ export const HLS_LIMITS = Object.freeze({ readyMs: 12000, }); +/** Every lease or session slot is taken: the source itself has not failed. */ +export class HlsCapacityError extends Error {} + /** Refuse redirects; count actual streamed bytes, including chunked responses. */ export async function fetchHlsBytes( url, @@ -152,7 +155,7 @@ export function createHlsPuller({ let lease = entry.leases.get(leaseId); if (!lease) { if (!create || entry.leases.size >= limits.leasesPerSession) - throw new Error('HLS lease capacity reached'); + throw new HlsCapacityError('HLS lease capacity reached'); lease = {}; entry.leases.set(leaseId, lease); } @@ -264,7 +267,7 @@ export function createHlsPuller({ } // Reserve synchronously, before any await: concurrent creation is singleflight. if (active.size >= limits.sessions) - throw new Error('HLS session capacity reached'); + throw new HlsCapacityError('HLS session capacity reached'); entry = { cameraId, url, diff --git a/src/data/cctvHlsMiddleware.test.mjs b/src/data/cctvHlsMiddleware.test.mjs index 301c9796e9..de446b4a47 100644 --- a/src/data/cctvHlsMiddleware.test.mjs +++ b/src/data/cctvHlsMiddleware.test.mjs @@ -8,6 +8,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { cctvProxy } from '../../server/providers/cctv.js'; +import { recordingResponse } from './cctvRecordingResponse.mjs'; const camera = (id) => ({ id, @@ -22,54 +23,16 @@ const PLAYLIST = const lease = (index) => `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`; -/** Collects what a handler wrote, the way a Node ServerResponse would. */ -function recordingResponse() { - const chunks = []; - let finished; - const done = new Promise((resolve) => { - finished = resolve; - }); - const res = { - statusCode: 0, - headers: {}, - writableEnded: false, - body: '', - writeHead(status, headers) { - res.statusCode = status; - res.headers = { ...res.headers, ...headers }; - }, - setHeader(name, value) { - res.headers[name] = value; - }, - write(chunk) { - chunks.push(Buffer.from(chunk)); - return true; - }, - end(chunk) { - if (chunk) chunks.push(Buffer.from(chunk)); - res.writableEnded = true; - res.body = Buffer.concat(chunks).toString('utf8'); - finished(); - }, - on() {}, - once() {}, - emit() {}, - destroy() {}, - }; - return { res, done }; -} - -/** Mount the provider over two HLS cameras whose upstream always answers. */ +/** Mount the provider over three HLS cameras whose upstream always answers. */ function mount(t) { const before = { json: process.env.CCTV_SOURCES_JSON, file: process.env.CCTV_SOURCES_FILE, austin: process.env.CCTV_FORCE_AUSTIN, }; - process.env.CCTV_SOURCES_JSON = JSON.stringify([ - camera('cam-a'), - camera('cam-b'), - ]); + process.env.CCTV_SOURCES_JSON = JSON.stringify( + ['cam-a', 'cam-b', 'cam-c'].map(camera), + ); process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; process.env.CCTV_FORCE_AUSTIN = '0'; const nativeFetch = globalThis.fetch; @@ -82,6 +45,7 @@ function mount(t) { httpServer: { on: (_event, fn) => closers.push(fn) }, middlewares: { use: (_route, fn) => (handler = fn) }, }); + assert.ok(handler, 'the CCTV provider must mount a middleware'); t.after(() => { for (const fn of closers) fn(); globalThis.fetch = nativeFetch; @@ -149,13 +113,34 @@ test('viewers on distinct addresses open, read and release their own leases', as } }); -test('viewers behind one shared address get every session slot', async (t) => { +test('viewers behind one shared address get every slot, and no more', async (t) => { const call = mount(t); const shared = '::ffff:172.17.0.1'; - for (let index = 0; index < 8; index++) - await openPlaylist(call, 'cam-a', lease(index), shared); - const segment = await openPlaylist(call, 'cam-b', lease(8), shared); + const health = async (id) => + JSON.parse((await call('/health', shared)).body).cameras.find( + (entry) => entry.id === id, + ); + let segment; + for (let index = 0; index < 16; index++) + segment = await openPlaylist( + call, + index < 8 ? 'cam-a' : 'cam-b', + lease(index), + shared, + ); assert.equal((await call(segment, shared)).statusCode, 200); + // A ninth viewer of a full session and a third session are both refused, + // without reporting a camera whose upstream is fine as degraded. + for (const cameraId of ['cam-a', 'cam-c']) { + const refused = await call(`/media/${cameraId}?lease=${lease(16)}`, shared); + assert.equal(refused.statusCode, 503, cameraId); + assert.equal(refused.headers['Retry-After'], '2'); + } + assert.equal((await health('cam-a')).status, 'ok'); + assert.equal(await health('cam-c'), undefined); + // A released slot is free for the next viewer. + await call(`/media/cam-a?lease=${lease(0)}`, shared, 'DELETE'); + await openPlaylist(call, 'cam-a', lease(16), shared); }); test('a lease keeps working after its viewer reconnects from another address', async (t) => { @@ -163,12 +148,15 @@ test('a lease keeps working after its viewer reconnects from another address', a await openPlaylist(call, 'cam-a', lease(0), '192.168.1.20'); const segment = await openPlaylist(call, 'cam-a', lease(0), '192.168.1.21'); assert.equal((await call(segment, '192.168.1.21')).statusCode, 200); + // A second viewer keeps the session alive, so the 404 below is the released + // lease and not a stopped session. + const other = await openPlaylist(call, 'cam-a', lease(1), '192.168.1.30'); const release = await call( `/media/cam-a?lease=${lease(0)}`, '192.168.1.21', 'DELETE', ); assert.equal(release.statusCode, 204); - // Released from the new address: the same lease can no longer read segments. assert.equal((await call(segment, '192.168.1.21')).statusCode, 404); + assert.equal((await call(other, '192.168.1.30')).statusCode, 200); }); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs index 1ff221cb65..e3dfc60064 100644 --- a/src/data/cctvMediaFeedType.test.mjs +++ b/src/data/cctvMediaFeedType.test.mjs @@ -9,6 +9,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { Readable } from 'node:stream'; import { cctvProxy } from '../../server/providers/cctv.js'; +import { recordingResponse } from './cctvRecordingResponse.mjs'; const STILL = { id: 'still-fixture', @@ -63,43 +64,6 @@ function upstreamResponse({ status = 200, headers = {}, body = 'payload' }) { }; } -/** Collects what a handler wrote, the way a Node ServerResponse would. */ -function recordingResponse() { - const chunks = []; - let finished; - const done = new Promise((resolve) => { - finished = resolve; - }); - const res = { - statusCode: 0, - headers: {}, - writableEnded: false, - body: '', - writeHead(status, headers) { - res.statusCode = status; - res.headers = { ...res.headers, ...headers }; - }, - setHeader(name, value) { - res.headers[name] = value; - }, - write(chunk) { - chunks.push(Buffer.from(chunk)); - return true; - }, - end(chunk) { - if (chunk) chunks.push(Buffer.from(chunk)); - res.writableEnded = true; - res.body = Buffer.concat(chunks).toString('utf8'); - finished(); - }, - on() {}, - once() {}, - emit() {}, - destroy() {}, - }; - return { res, done }; -} - /** * Mount the real provider over the fixture cameras and answer every upstream * request from the supplied function. diff --git a/src/data/cctvRecordingResponse.mjs b/src/data/cctvRecordingResponse.mjs new file mode 100644 index 0000000000..5d45e38a8f --- /dev/null +++ b/src/data/cctvRecordingResponse.mjs @@ -0,0 +1,38 @@ +// Test support for the CCTV middleware tests: no test file of its own. + +/** Collects what a handler wrote, the way a Node ServerResponse would. */ +export function recordingResponse() { + const chunks = []; + let finished; + const done = new Promise((resolve) => { + finished = resolve; + }); + const res = { + statusCode: 0, + headers: {}, + writableEnded: false, + body: '', + writeHead(status, headers) { + res.statusCode = status; + res.headers = { ...res.headers, ...headers }; + }, + setHeader(name, value) { + res.headers[name] = value; + }, + write(chunk) { + chunks.push(Buffer.from(chunk)); + return true; + }, + end(chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + res.writableEnded = true; + res.body = Buffer.concat(chunks).toString('utf8'); + finished(); + }, + on() {}, + once() {}, + emit() {}, + destroy() {}, + }; + return { res, done }; +}