fix(cctv): keep an HLS camera's health through the refused viewer's frame

A viewer refused for capacity falls back to /frame after hls.js gives up.
For a pullable HLS camera without a snapshot, /frame only has a
placeholder or a Street View frame, which says nothing about the stream,
yet it wrote degraded over the health that /media owns. /frame now
leaves that health as it is; a dead upstream is still reported through
/media.

Tests: the middleware suite covers both fallbacks, a real upstream
failure that still degrades, and cross-site segment and release refusals
on a live HLS lease; capacity checks use HLS_LIMITS and HlsCapacityError;
the three CCTV suites share one provider harness in src/testSupport.
CURRENT-STATE and the changelog fragment describe the refusal.
This commit is contained in:
German Khokhlov committed 2026-11-20 11:48:22 +04:00
1 parent f5f5f04b5d
commit ed38a1ab3f
10 files changed
+283 -321

No files matched your search

+128 -103
View File
@@ -1,15 +1,13 @@
// Live HLS through the real /api/cctv middleware: playlist, segment and
// release. HLS leases are random per-playback ids, not bound to the socket
// address, so viewers who share one address (a Docker bridge gateway, a reverse
// proxy, NAT) get the whole per-session budget, and a viewer whose address
// changes mid-playback keeps its lease.
//
// Run with: npm test (node --test)
// Live HLS through the real /api/cctv middleware. Leases are random
// per-playback ids, not bound to the socket address, and a viewer refused for
// capacity leaves the camera's health as it was.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { cctvProxy } from '../../server/providers/cctv.js';
import { recordingResponse } from './cctvRecordingResponse.mjs';
import { HLS_LIMITS } from '../../server/providers/cctv/stream.js';
import { mountCctvProvider } from '../testSupport/cctvProviderHarness.mjs';
const perSession = HLS_LIMITS.leasesPerSession;
const SAME_ORIGIN = { host: 'localhost:4173', 'sec-fetch-site': 'same-origin' };
const camera = (id) => ({
id,
name: id,
@@ -23,61 +21,29 @@ const PLAYLIST =
const lease = (index) =>
`00000000-0000-4000-8000-${String(index).padStart(12, '0')}`;
/** Mount the provider over three HLS cameras whose upstream always answers. */
function mount(t) {
const before = {
json: process.env.CCTV_SOURCES_JSON,
file: process.env.CCTV_SOURCES_FILE,
austin: process.env.CCTV_FORCE_AUSTIN,
};
process.env.CCTV_SOURCES_JSON = JSON.stringify(
['cam-a', 'cam-b', 'cam-c'].map(camera),
);
process.env.CCTV_SOURCES_FILE = 'absent-source-file.json';
process.env.CCTV_FORCE_AUSTIN = '0';
const nativeFetch = globalThis.fetch;
// The puller binds fetch when cctvProxy() creates it, so mock first.
globalThis.fetch = async (url) =>
new Response(String(url).endsWith('.m3u8') ? PLAYLIST : 'abc');
let handler;
const closers = [];
cctvProxy().configureServer({
httpServer: { on: (_event, fn) => closers.push(fn) },
middlewares: { use: (_route, fn) => (handler = fn) },
/** Mount the provider over three HLS cameras; `failing` upstreams answer 500. */
function mount(t, failing = []) {
const upstream = [];
const app = mountCctvProvider(t, {
cameras: ['cam-a', 'cam-b', 'cam-c'].map(camera),
fetch: async (url) => {
upstream.push(String(url));
if (failing.some((id) => String(url).includes(`/${id}/`)))
return new Response('', { status: 500 });
return new Response(String(url).endsWith('.m3u8') ? PLAYLIST : 'abc');
},
});
assert.ok(handler, 'the CCTV provider must mount a middleware');
t.after(() => {
for (const fn of closers) fn();
globalThis.fetch = nativeFetch;
for (const [name, value] of [
['CCTV_SOURCES_JSON', before.json],
['CCTV_SOURCES_FILE', before.file],
['CCTV_FORCE_AUSTIN', before.austin],
]) {
if (value === undefined) delete process.env[name];
else process.env[name] = value;
}
});
return async (url, remoteAddress, method = 'GET') => {
const { res, done } = recordingResponse();
await handler(
{
url,
method,
headers: { host: 'localhost:4173', 'sec-fetch-site': 'same-origin' },
socket: { remoteAddress },
on() {},
},
res,
);
await done;
return res;
};
return { ...app, upstream };
}
/** Open a playlist and return the first segment path it lists. */
async function openPlaylist(call, cameraId, leaseId, remoteAddress) {
const res = await call(`/media/${cameraId}?lease=${leaseId}`, remoteAddress);
const res = await call(
`/media/${cameraId}?lease=${leaseId}`,
SAME_ORIGIN,
'GET',
remoteAddress,
);
assert.equal(res.statusCode, 200, `playlist ${cameraId} ${leaseId}`);
const segment = res.body
.split('\n')
@@ -86,77 +52,136 @@ async function openPlaylist(call, cameraId, leaseId, remoteAddress) {
return segment.replace('/api/cctv', '');
}
test('viewers on distinct addresses open, read and release their own leases', async (t) => {
const call = mount(t);
const viewers = Array.from({ length: 9 }, (_, index) => ({
camera: index < 5 ? 'cam-a' : 'cam-b',
lease: lease(index),
// IPv4-mapped form, as a dual-stack listener reports it.
peer: `::ffff:192.168.1.${20 + index}`,
}));
for (const viewer of viewers)
viewer.segment = await openPlaylist(
call,
viewer.camera,
viewer.lease,
viewer.peer,
);
for (const viewer of viewers)
assert.equal((await call(viewer.segment, viewer.peer)).statusCode, 200);
for (const viewer of viewers) {
const release = await call(
`/media/${viewer.camera}?lease=${viewer.lease}`,
viewer.peer,
'DELETE',
);
assert.equal(release.statusCode, 204);
}
});
test('viewers behind one shared address get every slot, and no more', async (t) => {
const call = mount(t);
assert.equal(HLS_LIMITS.sessions, 2, 'the test fills both sessions');
const { call, health } = mount(t);
const shared = '::ffff:172.17.0.1';
const health = async (id) =>
JSON.parse((await call('/health', shared)).body).cameras.find(
(entry) => entry.id === id,
);
let segment;
for (let index = 0; index < 16; index++)
for (let index = 0; index < 2 * perSession; index++)
segment = await openPlaylist(
call,
index < 8 ? 'cam-a' : 'cam-b',
index < perSession ? 'cam-a' : 'cam-b',
lease(index),
shared,
);
assert.equal((await call(segment, shared)).statusCode, 200);
assert.equal(
(await call(segment, SAME_ORIGIN, 'GET', shared)).statusCode,
200,
);
const before = await health('cam-a');
// A ninth viewer of a full session and a third session are both refused,
// without reporting a camera whose upstream is fine as degraded.
for (const cameraId of ['cam-a', 'cam-c']) {
const refused = await call(`/media/${cameraId}?lease=${lease(16)}`, shared);
const refused = await call(
`/media/${cameraId}?lease=${lease(2 * perSession)}`,
SAME_ORIGIN,
'GET',
shared,
);
assert.equal(refused.statusCode, 503, cameraId);
assert.equal(refused.headers['Retry-After'], '2');
}
assert.equal((await health('cam-a')).status, 'ok');
assert.deepEqual(await health('cam-a'), before);
assert.equal(await health('cam-c'), undefined);
// A released slot is free for the next viewer.
await call(`/media/cam-a?lease=${lease(0)}`, shared, 'DELETE');
await openPlaylist(call, 'cam-a', lease(16), shared);
await call(`/media/cam-a?lease=${lease(0)}`, SAME_ORIGIN, 'DELETE', shared);
await openPlaylist(call, 'cam-a', lease(2 * perSession), shared);
});
test('a lease keeps working after its viewer reconnects from another address', async (t) => {
const call = mount(t);
const { call } = mount(t);
await openPlaylist(call, 'cam-a', lease(0), '192.168.1.20');
const segment = await openPlaylist(call, 'cam-a', lease(0), '192.168.1.21');
assert.equal((await call(segment, '192.168.1.21')).statusCode, 200);
assert.equal(
(await call(segment, SAME_ORIGIN, 'GET', '192.168.1.21')).statusCode,
200,
);
// A second viewer keeps the session alive, so the 404 below is the released
// lease and not a stopped session.
const other = await openPlaylist(call, 'cam-a', lease(1), '192.168.1.30');
const release = await call(
`/media/cam-a?lease=${lease(0)}`,
'192.168.1.21',
SAME_ORIGIN,
'DELETE',
'192.168.1.21',
);
assert.equal(release.statusCode, 204);
assert.equal((await call(segment, '192.168.1.21')).statusCode, 404);
assert.equal((await call(other, '192.168.1.30')).statusCode, 200);
assert.equal(
(await call(segment, SAME_ORIGIN, 'GET', '192.168.1.21')).statusCode,
404,
);
assert.equal(
(await call(other, SAME_ORIGIN, 'GET', '192.168.1.30')).statusCode,
200,
);
});
test('ninth viewer frame fallback preserves active camera health', async (t) => {
const { call, health } = mount(t);
let segment;
for (let index = 0; index < perSession; index++)
segment = await openPlaylist(call, 'cam-a', lease(index));
const before = await health('cam-a');
assert.equal(before.status, 'ok');
const refused = await call(
`/media/cam-a?lease=${lease(perSession)}`,
SAME_ORIGIN,
);
assert.equal(refused.statusCode, 503);
const frame = await call(
'/frame/cam-a?label=cam-a&lat=30.27&lon=-97.74',
SAME_ORIGIN,
);
assert.equal(frame.statusCode, 200);
assert.equal(
(await call(segment, SAME_ORIGIN)).statusCode,
200,
'admitted viewer still plays',
);
assert.deepEqual(await health('cam-a'), before);
});
test('third camera frame fallback keeps absent health', async (t) => {
const { call, health, upstream } = mount(t);
await openPlaylist(call, 'cam-a', lease(0));
await openPlaylist(call, 'cam-b', lease(1));
const refused = await call(`/media/cam-c?lease=${lease(2)}`, SAME_ORIGIN);
assert.equal(refused.statusCode, 503);
assert.equal(await health('cam-c'), undefined);
const frame = await call(
'/frame/cam-c?label=cam-c&lat=30.27&lon=-97.74',
SAME_ORIGIN,
);
assert.equal(frame.statusCode, 200);
assert.equal(upstream.filter((url) => url.includes('/cam-c/')).length, 0);
assert.equal(
await health('cam-c'),
undefined,
'the fallback frame wrote cam-c health',
);
});
test('a failing HLS upstream with free slots is still reported degraded', async (t) => {
const { call, health } = mount(t, ['cam-c']);
const res = await call(`/media/cam-c?lease=${lease(0)}`, SAME_ORIGIN);
assert.equal(res.statusCode, 503);
assert.equal(res.headers['Retry-After'], '2');
const row = await health('cam-c');
assert.deepEqual(
{ status: row.status, message: row.message },
{ status: 'degraded', message: 'Live HLS unavailable' },
);
});
test('a cross-site request can neither read nor release a live HLS lease', async (t) => {
const { call } = mount(t);
const segment = await openPlaylist(call, 'cam-a', lease(0));
const crossSite = { host: 'localhost:4173', 'sec-fetch-site': 'cross-site' };
assert.equal((await call(segment, crossSite)).statusCode, 403);
assert.equal(
(await call(`/media/cam-a?lease=${lease(0)}`, crossSite, 'DELETE'))
.statusCode,
403,
);
assert.equal((await call(segment, SAME_ORIGIN)).statusCode, 200);
});
+6 -2
View File
@@ -5,6 +5,7 @@ import {
fetchHlsBytes,
parseHlsMedia,
HLS_LIMITS,
HlsCapacityError,
} from '../../server/providers/cctv/stream.js';
const playlist =
'#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:12\n#EXTINF:2,\na.ts\n#EXTINF:2,\nb.ts\n#EXTINF:2,\nc.ts\n';
@@ -144,7 +145,7 @@ test('one session is reserved before await; disk-free cache and capacity remain
manager.ensure('a', base),
]);
assert.equal(a, b);
await assert.rejects(manager.ensure('b', base));
await assert.rejects(manager.ensure('b', base), HlsCapacityError);
assert.equal(await manager.waitReady(a), true);
assert.deepEqual(manager.stats(), { sessions: 1, bytes: 6 });
const text = await manager.buildPlaylist(a, 'a');
@@ -261,7 +262,10 @@ test('two consumers share downloads but release and abandoned expiry are indepen
await manager.waitReady(a);
assert.equal(downloads, 4); // One manifest and three segments, not per consumer.
assert.equal(a.leases.size, 2);
await assert.rejects(manager.ensure('camera', base, 'viewer-c'));
await assert.rejects(
manager.ensure('camera', base, 'viewer-c'),
HlsCapacityError,
);
manager.release('camera', 'viewer-a');
assert.equal(manager.stats().sessions, 1);
assert.equal(manager.getSegment('camera', a.token, 0, 'viewer-a'), null);
+8 -64
View File
@@ -8,8 +8,7 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { Readable } from 'node:stream';
import { cctvProxy } from '../../server/providers/cctv.js';
import { recordingResponse } from './cctvRecordingResponse.mjs';
import { mountCctvProvider } from '../testSupport/cctvProviderHarness.mjs';
const STILL = {
id: 'still-fixture',
@@ -64,72 +63,17 @@ function upstreamResponse({ status = 200, headers = {}, body = 'payload' }) {
};
}
/**
* Mount the real provider over the fixture cameras and answer every upstream
* request from the supplied function.
*/
/** Mount the real provider over the fixture cameras, recording upstream requests. */
function mount(t, respond) {
const before = {
json: process.env.CCTV_SOURCES_JSON,
file: process.env.CCTV_SOURCES_FILE,
austin: process.env.CCTV_FORCE_AUSTIN,
serverKey: process.env.GOOGLE_MAPS_SERVER_API_KEY,
browserKey: process.env.GOOGLE_MAPS_API_KEY,
};
process.env.CCTV_SOURCES_JSON = JSON.stringify([
STILL,
STILL_ALIAS,
VIDEO,
WEBM,
MJPEG,
]);
process.env.CCTV_SOURCES_FILE = 'absent-source-file.json';
process.env.CCTV_FORCE_AUSTIN = '0';
// No Street View key: a /frame miss lands on the synthetic billboard rather
// than a network call this file does not model.
delete process.env.GOOGLE_MAPS_SERVER_API_KEY;
delete process.env.GOOGLE_MAPS_API_KEY;
const nativeFetch = globalThis.fetch;
const requests = [];
globalThis.fetch = async (url, init = {}) => {
requests.push({ url: String(url), redirect: init.redirect });
return respond({ url: String(url), init });
};
t.after(() => {
globalThis.fetch = nativeFetch;
for (const [name, value] of [
['CCTV_SOURCES_JSON', before.json],
['CCTV_SOURCES_FILE', before.file],
['CCTV_FORCE_AUSTIN', before.austin],
['GOOGLE_MAPS_SERVER_API_KEY', before.serverKey],
['GOOGLE_MAPS_API_KEY', before.browserKey],
]) {
if (value === undefined) delete process.env[name];
else process.env[name] = value;
}
});
let handler = null;
cctvProxy().configureServer({
middlewares: {
use: (_route, fn) => {
handler = fn;
},
const app = mountCctvProvider(t, {
cameras: [STILL, STILL_ALIAS, VIDEO, WEBM, MJPEG],
fetch: async (url, init = {}) => {
requests.push({ url: String(url), redirect: init.redirect });
return respond({ url: String(url), init });
},
});
assert.ok(handler, 'the CCTV provider must mount a middleware');
const call = async (path, headers = {}, method = 'GET') => {
const { res, done } = recordingResponse();
await handler({ url: path, headers, method, on() {} }, res);
await done;
return res;
};
const health = async (id) => {
const res = await call('/health');
return JSON.parse(res.body).cameras.find((entry) => entry.id === id);
};
return { call, health, requests };
return { ...app, requests };
}
test('a still camera is refused by /media without any upstream request', async (t) => {
+12 -90
View File
@@ -9,7 +9,7 @@ import assert from 'node:assert/strict';
import http from 'node:http';
import net from 'node:net';
import { Readable, Writable } from 'node:stream';
import { cctvProxy } from '../../server/providers/cctv.js';
import { mountCctvProvider } from '../testSupport/cctvProviderHarness.mjs';
import { proxyMediaResponse } from '../../server/providers/cctv/media.js';
import { sanitizeCctvRangeHeader } from '../../server/providers/cctv/range.js';
import { CCTV_MEDIA_MAX_BODY_BYTES as CAP } from '../../server/providers/cctv/constants.js';
@@ -131,99 +131,21 @@ function upstreamResponse({ status = 206, headers = {}, body = 'payload' }) {
};
}
/** Collects what a handler wrote, the way a Node ServerResponse would. */
function recordingResponse() {
const chunks = [];
let finished;
const done = new Promise((resolve) => {
finished = resolve;
});
const res = {
statusCode: 0,
headers: {},
writableEnded: false,
body: '',
writeHead(status, headers) {
res.statusCode = status;
res.headers = headers || {};
},
write(chunk) {
chunks.push(Buffer.from(chunk));
return true;
},
end(chunk) {
if (chunk) chunks.push(Buffer.from(chunk));
res.writableEnded = true;
res.body = Buffer.concat(chunks).toString('utf8');
finished();
},
on() {},
once() {},
emit() {},
destroy() {},
};
return { res, done };
}
/**
* Mount the real provider, serve one camera, and answer its upstream from the
* supplied function.
*/
/** Mount the real provider over the fixture cameras, recording upstream requests. */
function mount(t, respond) {
const before = {
json: process.env.CCTV_SOURCES_JSON,
file: process.env.CCTV_SOURCES_FILE,
austin: process.env.CCTV_FORCE_AUSTIN,
};
process.env.CCTV_SOURCES_JSON = JSON.stringify([CAMERA]);
process.env.CCTV_SOURCES_FILE = 'absent-source-file.json';
process.env.CCTV_FORCE_AUSTIN = '0';
const nativeFetch = globalThis.fetch;
const requests = [];
globalThis.fetch = async (url, init = {}) => {
requests.push({
url: String(url),
headers: init.headers || {},
signal: init.signal || null,
});
return respond({ url: String(url), init });
};
t.after(() => {
globalThis.fetch = nativeFetch;
for (const [name, value] of [
['CCTV_SOURCES_JSON', before.json],
['CCTV_SOURCES_FILE', before.file],
['CCTV_FORCE_AUSTIN', before.austin],
]) {
if (value === undefined) delete process.env[name];
else process.env[name] = value;
}
});
let handler = null;
const plugin = cctvProxy();
plugin.configureServer({
middlewares: {
use: (_route, fn) => {
handler = fn;
},
const app = mountCctvProvider(t, {
cameras: [CAMERA],
fetch: async (url, init = {}) => {
requests.push({
url: String(url),
headers: init.headers || {},
signal: init.signal || null,
});
return respond({ url: String(url), init });
},
});
assert.ok(handler, 'the CCTV provider must mount a middleware');
/** Drive one request through the mounted handler. */
const call = async (path, headers = {}) => {
const { res, done } = recordingResponse();
await handler({ url: path, headers, method: 'GET', on() {} }, res);
await done;
return res;
};
/** Read the camera's current health entry. */
const health = async () => {
const res = await call('/health');
return JSON.parse(res.body).cameras.find(({ id }) => id === CAMERA.id);
};
return { call, health, requests, handler: (req, res) => handler(req, res) };
return { ...app, requests, health: () => app.health(CAMERA.id) };
}
test('a canonical range reaches the upstream and its 206 is passed through', async (t) => {
-38
View File
@@ -1,38 +0,0 @@
// Test support for the CCTV middleware tests: no test file of its own.
/** Collects what a handler wrote, the way a Node ServerResponse would. */
export function recordingResponse() {
const chunks = [];
let finished;
const done = new Promise((resolve) => {
finished = resolve;
});
const res = {
statusCode: 0,
headers: {},
writableEnded: false,
body: '',
writeHead(status, headers) {
res.statusCode = status;
res.headers = { ...res.headers, ...headers };
},
setHeader(name, value) {
res.headers[name] = value;
},
write(chunk) {
chunks.push(Buffer.from(chunk));
return true;
},
end(chunk) {
if (chunk) chunks.push(Buffer.from(chunk));
res.writableEnded = true;
res.body = Buffer.concat(chunks).toString('utf8');
finished();
},
on() {},
once() {},
emit() {},
destroy() {},
};
return { res, done };
}
+91
View File
@@ -0,0 +1,91 @@
import { cctvProxy } from '../../server/providers/cctv.js';
/** Collects what a handler wrote, the way a Node ServerResponse would. */
export function recordingResponse() {
const chunks = [];
let finished;
const done = new Promise((resolve) => {
finished = resolve;
});
const res = {
statusCode: 0,
headers: {},
writableEnded: false,
body: '',
writeHead(status, headers) {
res.statusCode = status;
res.headers = { ...res.headers, ...headers };
},
setHeader(name, value) {
res.headers[name] = value;
},
write(chunk) {
chunks.push(Buffer.from(chunk));
return true;
},
end(chunk) {
if (chunk) chunks.push(Buffer.from(chunk));
res.writableEnded = true;
res.body = Buffer.concat(chunks).toString('utf8');
finished();
},
on() {},
once() {},
emit() {},
destroy() {},
};
return { res, done };
}
export function mountCctvProvider(t, { cameras, fetch }) {
const saved = {};
for (const name of [
'CCTV_SOURCES_JSON',
'CCTV_SOURCES_FILE',
'CCTV_FORCE_AUSTIN',
'GOOGLE_MAPS_SERVER_API_KEY',
'GOOGLE_MAPS_API_KEY',
])
saved[name] = process.env[name];
const nativeFetch = globalThis.fetch;
const closers = [];
t.after(() => {
for (const close of closers) close();
globalThis.fetch = nativeFetch;
for (const [name, value] of Object.entries(saved)) {
if (value === undefined) delete process.env[name];
else process.env[name] = value;
}
});
process.env.CCTV_SOURCES_JSON = JSON.stringify(cameras);
process.env.CCTV_SOURCES_FILE = 'absent-source-file.json';
process.env.CCTV_FORCE_AUSTIN = '0';
// No Street View key: a /frame miss lands on the synthetic billboard.
delete process.env.GOOGLE_MAPS_SERVER_API_KEY;
delete process.env.GOOGLE_MAPS_API_KEY;
// The HLS puller binds fetch when cctvProxy() creates it, so mock first.
globalThis.fetch = fetch;
let handler;
cctvProxy().configureServer({
httpServer: { on: (_event, fn) => closers.push(fn) },
middlewares: { use: (_route, fn) => (handler = fn) },
});
if (!handler) throw new Error('the CCTV provider must mount a middleware');
const call = async (path, headers = {}, method = 'GET', remoteAddress) => {
const { res, done } = recordingResponse();
await handler(
{ url: path, headers, method, socket: { remoteAddress }, on() {} },
res,
);
await done;
return res;
};
const health = async (id) => {
const res = await call('/health');
return JSON.parse(res.body).cameras.find((entry) => entry.id === id);
};
return { call, health, handler };
}