Merge remote-tracking branch 'origin/main' into fix/cctv-streetview-fallback-catalog

# Conflicts:
#	SECURITY.md
This commit is contained in:
German Khokhlov committed 2026-11-19 19:12:56 +04:00
commit 01368b95bf
3201 files changed
+430431 -53001

No files matched your search

+187 -5
View File
@@ -1,4 +1,5 @@
import { createCctvCatalog } from './cctv/catalog.js';
import { quebec511VideoUrl, quebecBrowserVideoEnabled } from './cctv/quebec.js';
import { normalizeFeedType, isVideoFeedType } from './cctv/normalize.js';
import {
buildSyntheticCctvSvg,
@@ -18,6 +19,8 @@ import { sanitizeCctvRangeHeader } from './cctv/range.js';
import { admitSameSiteKeyless } from './common/same-site.js';
import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs';
import { createHlsPuller } from './cctv/stream.js';
import { isMlitRiverAdjustmentImage } from './cctv/worldCameras.js';
import { ottawaAuthorizedFrameUrl } from './cctv/ottawa.js';
import { googleServerApiKey } from './places/google-key.js';
import { clientKey, makeCostRateLimiter } from './common/rate-limit.js';
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
@@ -45,6 +48,120 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
const HEALTH_MAX_ENTRIES = CCTV_MAX_SOURCES_CEILING;
/** Live HLS strategies (see ./cctv/stream.js). Shared across dev and preview. */
const puller = createHlsPuller();
/**
* Bounded transient image reuse keeps cadence-limited providers (World
* stills, Ottawa's once-a-minute API) inside their refresh floor. The byte
* cap is the real bound; the entry cap only limits map overhead.
*/
const WORLD_FRAME_CACHE_MAX_ENTRIES = 1024;
/** Retry a failed fetch sooner than a full refresh window, never under a minute. */
const THROTTLED_FRAME_FAILURE_BACKOFF_MS = 60 * 1000;
const WORLD_FRAME_CACHE_MAX_ENTRY_BYTES = 2 * 1024 * 1024;
const WORLD_FRAME_CACHE_MAX_BYTES = 32 * 1024 * 1024;
const worldFrameCache = new Map();
const worldFrameInflight = new Map();
const worldFrameNextAllowedAt = new Map();
let worldFrameCacheBytes = 0;
function getWorldFrameCache(cameraId) {
const entry = worldFrameCache.get(cameraId);
if (!entry) return null;
worldFrameCache.delete(cameraId);
worldFrameCache.set(cameraId, entry);
return entry;
}
function storeWorldFrame(cameraId, image) {
if (
!Buffer.isBuffer(image?.body) ||
image.body.length > WORLD_FRAME_CACHE_MAX_ENTRY_BYTES
)
return;
const previous = worldFrameCache.get(cameraId);
if (previous) worldFrameCacheBytes -= previous.body.length;
const entry = {
body: image.body,
contentType: image.contentType,
fetchedAt: Date.now(),
};
worldFrameCache.delete(cameraId);
worldFrameCache.set(cameraId, entry);
worldFrameCacheBytes += entry.body.length;
while (
worldFrameCache.size > WORLD_FRAME_CACHE_MAX_ENTRIES ||
worldFrameCacheBytes > WORLD_FRAME_CACHE_MAX_BYTES
) {
const oldestId = worldFrameCache.keys().next().value;
const oldest = worldFrameCache.get(oldestId);
worldFrameCache.delete(oldestId);
worldFrameCacheBytes -= oldest.body.length;
}
}
function setNextAllowedAt(cameraId, at) {
worldFrameNextAllowedAt.delete(cameraId);
worldFrameNextAllowedAt.set(cameraId, at);
while (worldFrameNextAllowedAt.size > 4096) {
worldFrameNextAllowedAt.delete(
worldFrameNextAllowedAt.keys().next().value,
);
}
}
/**
* Fetch a frame from a provider with a per-camera refresh floor. Inside the
* floor the last frame is reused; with no frame to reuse the caller gets
* `{ throttled: true }` and serves a placeholder, so a cadence-limited camera
* never falls through to the key-bearing Street View fallback. A failed
* fetch backs off for a minute (capped at the floor) instead of a full
* window.
*/
async function fetchThrottledFrame(source, cameraId, frameUrl) {
const cached = getWorldFrameCache(cameraId);
const rawRefreshMs = Number(source?.frameRefreshMs);
const refreshMs =
Number.isFinite(rawRefreshMs) && rawRefreshMs > 0
? rawRefreshMs
: 5 * 60 * 1000;
const now = Date.now();
if (cached && now - cached.fetchedAt < refreshMs) {
return { ok: true, ...cached, cached: true };
}
const inflight = worldFrameInflight.get(cameraId);
if (inflight) return inflight;
const stale = cached
? {
ok: true,
body: cached.body,
contentType: cached.contentType,
stale: true,
}
: null;
const nextAllowedAt = worldFrameNextAllowedAt.get(cameraId) || 0;
if (now < nextAllowedAt) return stale || { ok: false, throttled: true };
if (!frameUrl) return stale;
setNextAllowedAt(cameraId, now + refreshMs);
let request;
request = fetchCctvImageFromUpstream(frameUrl)
.then((image) => {
if (image?.ok) {
storeWorldFrame(cameraId, image);
return image;
}
setNextAllowedAt(
cameraId,
Date.now() + Math.min(refreshMs, THROTTLED_FRAME_FAILURE_BACKOFF_MS),
);
return stale || image;
})
.finally(() => {
if (worldFrameInflight.get(cameraId) === request)
worldFrameInflight.delete(cameraId);
});
worldFrameInflight.set(cameraId, request);
return request;
}
/** Update the health entry for a camera, evicting the oldest entry if at capacity. */
const setHealth = (cameraId, patch) => {
@@ -171,6 +288,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
);
if (url.pathname === '/sources') {
const quebecVideo = quebecBrowserVideoEnabled();
const body = {
trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [],
sources: sources.map((source) => ({
@@ -182,6 +300,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
provider: source.provider,
lat: source.lat,
lon: source.lon,
frameRefreshMs: source.frameRefreshMs,
headingDeg: source.headingDeg,
headingConfidence: source.headingConfidence || '',
pitchDeg: source.pitchDeg,
@@ -192,6 +311,13 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
feedType: normalizeFeedType(source.feedType),
sourceKind:
source.sourceKind || (source.url ? 'configured' : 'fallback'),
sourceUrl: source.sourceUrl || '',
browserMediaUrl:
quebecVideo && source.sourceKind === 'quebec-511-browser'
? quebec511VideoUrl(source.id)
: '',
accuracyM: source.accuracyM,
positionSource: source.positionSource || '',
poseSource: source.poseSource,
model: source.model || '',
manufacturer: source.manufacturer || '',
@@ -544,15 +670,71 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
: '');
const upstreamImage =
source?.sourceKind === 'txdot-its'
? await fetchTxdotSnapshot(upstreamCandidate)
: await fetchCctvImageFromUpstream(upstreamCandidate);
source?.pack === 'world-cameras'
? await fetchThrottledFrame(source, cameraId, upstreamCandidate)
: source?.sourceKind === 'ottawa-traffic'
? await fetchThrottledFrame(
source,
cameraId,
ottawaAuthorizedFrameUrl(upstreamCandidate),
)
: source?.sourceKind === 'txdot-its'
? await fetchTxdotSnapshot(upstreamCandidate)
: await fetchCctvImageFromUpstream(upstreamCandidate, {
allowMjpeg: source?.sourceKind === 'taiwan-highway-motc',
});
if (
source?.sourceKind === 'mlit-river' &&
isMlitRiverAdjustmentImage(upstreamImage)
) {
const svg = buildSyntheticCctvSvg({
cameraId,
label,
city,
status: 'CAMERA UNDER ADJUSTMENT',
});
setHealth(cameraId, {
status: 'degraded',
sourceKind: 'snapshot',
label: source.provider || 'MLIT river cameras',
message: 'Provider camera under adjustment',
});
res.writeHead(200, {
'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-store',
'X-CCTV-Source': 'provider-maintenance',
});
res.end(svg);
return;
}
if (upstreamImage?.throttled) {
// Inside the provider's refresh floor with no frame to reuse: a
// placeholder, not Street View, and no health change.
res.writeHead(200, {
'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-store',
'X-CCTV-Source': 'synthetic',
});
res.end(
buildSyntheticCctvSvg({
cameraId,
label,
city,
status: 'WAITING FOR NEXT FRAME',
}),
);
return;
}
if (upstreamImage?.ok) {
setHealth(cameraId, {
status: 'ok',
status: upstreamImage.stale ? 'degraded' : 'ok',
sourceKind: 'snapshot',
label: source?.provider || 'Configured source',
message: 'Upstream snapshot active',
message: upstreamImage.stale
? 'Serving last snapshot; refresh failed'
: upstreamImage.cached
? 'Cached snapshot active'
: 'Upstream snapshot active',
});
res.writeHead(200, {
'Content-Type': upstreamImage.contentType,