benmichael/gods-eye-view
1
0 Fork 0
mirror of https://github.com/bilawalsidhu/gods-eye-view.git synced 2026-11-15 15:52:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cctv): key-gate Ottawa, opt-in Québec video, keep shared slots

Browse Source 
Review fixes for #1506.

- Ottawa: use the documented camera list and the city's Traffic Camera
  Images API. The pack is skipped with no request unless
  OTTAWA_CAMERA_CERTIFICATE is set; the certificate is added only at
  fetch time inside the pinned /opendata/ path, and each camera is
  fetched at most once a minute through the shared frame cache.
- Québec 511 video is opt-in (CCTV_QUEBEC_VIDEO_ENABLED=1) and loads
  only after the viewer presses Play; selection, auto-hop, scenes and
  voice no longer contact Québec 511. Credit splits locations (MTQ,
  CC BY 4.0) from video (Québec 511). Interrupted play() is no longer
  reported as an unavailable source; the per-frame listener runs only
  while a Québec camera is selected and the canvas rect is cached.
- Catalog cap: reserved Québec and World slots come only from capacity
  above the 4,000 shared slots, so a lower CCTV_MAX_SOURCES thins every
  pack instead of starving the existing ones. dev-fresh and
  .env.example default to 7,516.
- Throttled frames (World, Ottawa): inside the refresh floor with no
  cached frame the proxy serves the placeholder, never Street View; a
  failed fetch backs off for a minute; the cache is bounded by bytes.
- MJPEG first-frame extraction runs only for sources registered as
  MJPEG (Taiwan Highway Bureau), caps a frame at 2 MB, reads in linear
  time, accepts a CRLF preamble and a boundary that already starts with
  "--", and ends a part without Content-Length at the next delimiter.
  THB hosts are registered in the redirect pin list.
- Tests: World loader policy, Québec loader and opt-in, proxy cooldown
  without Street View, Ottawa signing, cap reservations at 7,516, 4,000
  and 48, overlay Play gate, refresh, visibility and teardown. Guard
  tests check the 4,000 shared slots again.
- Remove the unused standalone Québec preview module and public page.
- Docs: DATA_SOURCES, SECURITY, CURRENT-STATE, CCTV_INTEGRATION, README,
  .env.example, changelog fragment renamed to its full scope.
This commit is contained in:
Said Turksever committed 2026-11-03 15:31:08 +00:00
1 parent fa4ce4b2a5
commit db8afbae81
43 files changed
+1327 -1110
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/providers/cctv/media.js
+157 -85
View File
@@ -18,6 +18,7 @@ import {
OTTAWA_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
TFL_IMAGE_ORIGIN,
WARENDORF_IMAGE_ORIGINS,
WSDOT_IMAGE_ORIGIN,
@@ -538,6 +539,7 @@ for (const prefix of [
VEGAGERDIN_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
MLIT_RIVER_IMAGE_ORIGIN,
...TAIWAN_HIGHWAY_IMAGE_ORIGINS,
]) {
const host = new URL(prefix).hostname;
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
@@ -629,18 +631,32 @@ export function cctvUpstreamUserAgent(url) {
}
}
/** Read one bounded JPEG part from an open MJPEG response, then stop the stream. */
/** Largest single JPEG the MJPEG first-frame reader will buffer. */
export const CCTV_MJPEG_FRAME_MAX_BYTES = 2 * 1024 * 1024;
/**
* Read one bounded JPEG part from an open MJPEG response, then stop the stream.
*
* The first part may follow a CRLF preamble, and its delimiter may be written
* as `--<boundary>` or, when the declared boundary already starts with `--`,
* as the boundary itself. A part with Content-Length is read to that length;
* a part without one ends at the next delimiter, so a JPEG end marker inside
* the image (an EXIF thumbnail) never truncates it. Bytes are copied once
* into a preallocated buffer and every search resumes where it stopped, so a
* trickling upstream costs linear time.
*/
async function readFirstMjpegFrame(upstream, maxBytes) {
const mediaType = cctvLiveMediaType(
'mjpeg',
upstream.headers.get('content-type'),
);
const boundary = parseMediaType(mediaType)?.params.get('boundary') || '';
const frameMax = Math.min(maxBytes, CCTV_MJPEG_FRAME_MAX_BYTES);
if (
!boundary ||
!upstream.body ||
!Number.isSafeInteger(maxBytes) ||
maxBytes < 4
!Number.isSafeInteger(frameMax) ||
frameMax < 4
) {
try {
await upstream.body?.cancel();
@@ -650,19 +666,91 @@ async function readFirstMjpegFrame(upstream, maxBytes) {
return null;
}
const nextBoundary = Buffer.from('\r\n--' + boundary, 'latin1');
const opening = Buffer.from('--' + boundary + '\r\n', 'latin1');
const delimiters = [Buffer.from('--' + boundary, 'latin1')];
if (boundary.startsWith('--') && boundary.length > 2)
delimiters.push(Buffer.from(boundary, 'latin1'));
const lineEnd = Buffer.from('\r\n', 'ascii');
const headerTerminator = Buffer.from('\r\n\r\n', 'ascii');
const jpegStart = Buffer.from([0xff, 0xd8]);
const jpegEnd = Buffer.from([0xff, 0xd9]);
const maxPreambleBytes = 64;
const maxHeaderBytes = 8 * 1024;
const maxBufferedBytes =
opening.length + headerTerminator.length + maxHeaderBytes + maxBytes;
maxPreambleBytes +
delimiters[0].length +
lineEnd.length +
maxHeaderBytes +
headerTerminator.length +
frameMax +
lineEnd.length +
delimiters[0].length;
const buffer = Buffer.allocUnsafe(maxBufferedBytes);
let length = 0;
let reader;
let buffer = Buffer.alloc(0);
let parsedHeaders = false;
let headerStart = -1;
let delimiter = null;
let headerScanFrom = 0;
let bodyStart = -1;
let declaredLength = null;
let boundaryScanFrom = -1;
/** Match the opening delimiter after an optional CRLF preamble. */
function findOpening() {
let at = 0;
while (at < length && (buffer[at] === 0x0d || buffer[at] === 0x0a)) at++;
if (at > maxPreambleBytes) return { failed: true };
for (const candidate of delimiters) {
const line = Buffer.concat([candidate, lineEnd]);
const need = at + line.length;
if (length < need) {
const seen = buffer.subarray(at, length);
if (line.subarray(0, seen.length).equals(seen))
return { pending: true };
continue;
}
if (buffer.subarray(at, need).equals(line))
return { start: need, delimiter: candidate };
}
return { failed: true };
}
/** Parse the part headers; false rejects the stream. */
function parseHeaders(headerEnd) {
const headerText = buffer
.subarray(headerStart, headerEnd)
.toString('latin1');
const headers = new Map();
for (const line of headerText.split('\r\n')) {
const colon = line.indexOf(':');
if (colon <= 0) return false;
const name = line.slice(0, colon).trim().toLowerCase();
if (!name || headers.has(name)) return false;
headers.set(name, line.slice(colon + 1).trim());
}
if (cctvRasterImageType(headers.get('content-type')) !== 'image/jpeg')
return false;
const declared = headers.get('content-length');
if (declared !== undefined) {
if (!/^\d+$/.test(declared)) return false;
declaredLength = Number(declared);
if (
!Number.isSafeInteger(declaredLength) ||
declaredLength < jpegStart.length + jpegEnd.length ||
declaredLength > frameMax
)
return false;
}
return true;
}
/** A complete JPEG body, or null. */
function asJpeg(frame) {
return frame.length >= jpegStart.length + jpegEnd.length &&
frame.subarray(0, jpegStart.length).equals(jpegStart) &&
frame.subarray(-jpegEnd.length).equals(jpegEnd)
? Buffer.from(frame)
: null;
}
try {
reader = upstream.body.getReader();
@@ -671,92 +759,71 @@ async function readFirstMjpegFrame(upstream, maxBytes) {
if (done) return null;
const chunk = Buffer.from(value);
const room = Math.max(0, maxBufferedBytes - buffer.length);
const included = chunk.subarray(0, room);
const overflow = included.length < chunk.length;
buffer = Buffer.concat(
[buffer, included],
buffer.length + included.length,
);
const room = maxBufferedBytes - length;
const included = Math.min(room, chunk.length);
chunk.copy(buffer, length, 0, included);
length += included;
const overflow = included < chunk.length;
if (buffer.length < opening.length) {
if (overflow) return null;
continue;
}
if (!buffer.subarray(0, opening.length).equals(opening)) return null;
if (!parsedHeaders) {
const headerEnd = buffer.indexOf(headerTerminator, opening.length);
if (headerEnd < 0) {
if (buffer.length - opening.length > maxHeaderBytes || overflow)
return null;
if (headerStart < 0) {
const opening = findOpening();
if (opening.failed) return null;
if (opening.pending) {
if (overflow) return null;
continue;
}
if (headerEnd - opening.length > maxHeaderBytes) return null;
const headerText = buffer
.subarray(opening.length, headerEnd)
.toString('latin1');
const headers = new Map();
for (const line of headerText.split('\r\n')) {
const colon = line.indexOf(':');
if (colon <= 0) return null;
const name = line.slice(0, colon).trim().toLowerCase();
if (!name || headers.has(name)) return null;
headers.set(name, line.slice(colon + 1).trim());
}
if (cctvRasterImageType(headers.get('content-type')) !== 'image/jpeg')
return null;
const length = headers.get('content-length');
if (length !== undefined) {
if (!/^\d+$/.test(length)) return null;
declaredLength = Number(length);
if (
!Number.isSafeInteger(declaredLength) ||
declaredLength < jpegStart.length + jpegEnd.length ||
declaredLength > maxBytes
)
return null;
}
bodyStart = headerEnd + headerTerminator.length;
parsedHeaders = true;
headerStart = opening.start;
delimiter = opening.delimiter;
headerScanFrom = headerStart;
}
const available = buffer.length - bodyStart;
if (declaredLength !== null) {
const nextPart = buffer.indexOf(nextBoundary, bodyStart);
if (nextPart >= bodyStart && nextPart - bodyStart < declaredLength)
return null;
if (available >= declaredLength) {
const frame = buffer.subarray(bodyStart, bodyStart + declaredLength);
if (
frame.subarray(0, jpegStart.length).equals(jpegStart) &&
frame.subarray(-jpegEnd.length).equals(jpegEnd)
)
return Buffer.from(frame);
return null;
if (bodyStart < 0) {
const headerEnd = buffer
.subarray(0, length)
.indexOf(headerTerminator, headerScanFrom);
if (headerEnd < 0) {
if (length - headerStart > maxHeaderBytes || overflow) return null;
headerScanFrom = Math.max(
headerStart,
length - headerTerminator.length + 1,
);
continue;
}
} else if (available >= jpegStart.length) {
const frameStart = buffer.subarray(
bodyStart,
bodyStart + jpegStart.length,
);
if (!frameStart.equals(jpegStart)) return null;
const nextPart = buffer.indexOf(nextBoundary, bodyStart);
const end = buffer.indexOf(jpegEnd, bodyStart + jpegStart.length);
if (headerEnd - headerStart > maxHeaderBytes) return null;
if (!parseHeaders(headerEnd)) return null;
bodyStart = headerEnd + headerTerminator.length;
boundaryScanFrom = bodyStart;
}
const available = length - bodyStart;
if (available >= jpegStart.length) {
if (
nextPart >= bodyStart &&
(end < 0 || end + jpegEnd.length > nextPart)
!buffer
.subarray(bodyStart, bodyStart + jpegStart.length)
.equals(jpegStart)
)
return null;
if (end >= 0) {
const frameLength = end + jpegEnd.length - bodyStart;
if (frameLength > maxBytes) return null;
return Buffer.from(buffer.subarray(bodyStart, end + jpegEnd.length));
}
if (available > maxBytes) return null;
}
const nextDelimiter = Buffer.concat([lineEnd, delimiter]);
const nextPart = buffer
.subarray(0, length)
.indexOf(nextDelimiter, boundaryScanFrom);
if (declaredLength !== null) {
if (nextPart >= bodyStart && nextPart - bodyStart < declaredLength)
return null;
if (available >= declaredLength)
return asJpeg(buffer.subarray(bodyStart, bodyStart + declaredLength));
} else if (nextPart >= bodyStart) {
const frame = buffer.subarray(bodyStart, nextPart);
return frame.length > frameMax ? null : asJpeg(frame);
} else if (available > frameMax) {
return null;
}
if (nextPart < 0)
boundaryScanFrom = Math.max(
bodyStart,
length - nextDelimiter.length + 1,
);
if (overflow) return null;
}
} finally {
@@ -786,6 +853,10 @@ async function readFirstMjpegFrame(upstream, maxBytes) {
* @param {typeof fetch} [options.fetchImpl=fetch] - Fetch implementation.
* @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout.
* @param {number} [options.maxBytes=CCTV_FRAME_MAX_BODY_BYTES] - Snapshot byte cap.
* @param {boolean} [options.allowMjpeg=false] - Accept a multipart MJPEG
* answer and keep its first JPEG. Only sources registered as MJPEG stills
* (the Taiwan Highway Bureau feeds) set it; any other host answering
* multipart falls through to the next fallback.
* @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>}
*/
export async function fetchCctvImageFromUpstream(
@@ -794,6 +865,7 @@ export async function fetchCctvImageFromUpstream(
fetchImpl = fetch,
timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS,
maxBytes = CCTV_FRAME_MAX_BODY_BYTES,
allowMjpeg = false,
} = {},
) {
if (!url || !/^https?:\/\//i.test(url)) return null;
@@ -828,7 +900,7 @@ export async function fetchCctvImageFromUpstream(
if (!body) return null;
return { ok: true, body, contentType };
}
if (!cctvLiveMediaType('mjpeg', upstreamType)) {
if (!allowMjpeg || !cctvLiveMediaType('mjpeg', upstreamType)) {
try {
await upstream.body?.cancel();
} catch {