feat(cctv): seven camera packs, semantic labels, terrain-clear monitor planes (#433)

Seven contributed CCTV camera packs (Ontario 511, Fintraffic Finland, DriveBC, TxDOT Texas, Estonia Tallinn + Tarktee, Warendorf, Live Traffic NSW) on a fair per-pack catalog cap; semantic camera labels; monitor planes lifted clear of the terrain from a precomputed, bundled ground-height dataset. Default catalog 800 → 3,446 cameras. Full context in PR #433.

Co-authored-by: sendmebits <[email protected]>
Co-authored-by: avocadoAgent <[email protected]>
Co-authored-by: Alex Harvey <[email protected]>
Co-authored-by: Jason Meredith <[email protected]>
Co-authored-by: Daniel Smith <[email protected]>
Co-authored-by: Vladimir Sobolev <[email protected]>
Co-authored-by: Matthias Schneider <[email protected]>
Co-authored-by: Cyber Knight <[email protected]>
Co-authored-by: NovaSoft Australia <[email protected]>
This commit is contained in:
51 files changed
+78682 -939

No files matched your search

+39 -4
View File
@@ -11,9 +11,11 @@ PORT="${PORT:-4173}"
HOST="${HOST:-localhost}"
# CCTV source packs (all keyless): Austin (~815 live upstream), Caltrans
# districts 4,7,11,3 = SF/LA/San Diego/Sacramento (~1,860 live upstream),
# TfL London JamCams (~870 live upstream). Caps keep the densest cores per
# pack; override per-run for lighter/heavier loads. Kill switches:
# CCTV_CALTRANS_DISTRICTS='' and CCTV_TFL_ENABLED=0.
# TfL London JamCams (~870 live upstream), Ontario 511 (~944 live upstream,
# including Kitchener-area highways), and Fintraffic Finland weathercams
# (~2,260 live presets). Caps keep the densest cores per pack; override
# per-run for lighter/heavier loads. Kill switches: CCTV_CALTRANS_DISTRICTS='',
# CCTV_TFL_ENABLED=0, CCTV_ONTARIO_ENABLED=0 and CCTV_FINTRAFFIC_ENABLED=0.
CCTV_AUSTIN_MAX_SOURCES="${CCTV_AUSTIN_MAX_SOURCES:-250}"
# Use `-` not `:-` so an explicit empty string (the documented kill switch)
# is preserved rather than replaced by the default. Still set-u-safe when unset.
@@ -21,7 +23,24 @@ CCTV_CALTRANS_DISTRICTS="${CCTV_CALTRANS_DISTRICTS-4,7,11,3}"
CCTV_CALTRANS_MAX_SOURCES="${CCTV_CALTRANS_MAX_SOURCES:-300}"
CCTV_TFL_ENABLED="${CCTV_TFL_ENABLED:-1}"
CCTV_TFL_MAX_SOURCES="${CCTV_TFL_MAX_SOURCES:-250}"
CCTV_MAX_SOURCES="${CCTV_MAX_SOURCES:-900}"
CCTV_ONTARIO_ENABLED="${CCTV_ONTARIO_ENABLED:-1}"
CCTV_ONTARIO_MAX_SOURCES="${CCTV_ONTARIO_MAX_SOURCES:-1000}"
CCTV_FINTRAFFIC_ENABLED="${CCTV_FINTRAFFIC_ENABLED:-1}"
CCTV_FINTRAFFIC_MAX_SOURCES="${CCTV_FINTRAFFIC_MAX_SOURCES:-300}"
CCTV_DRIVEBC_ENABLED="${CCTV_DRIVEBC_ENABLED:-1}"
CCTV_DRIVEBC_MAX_SOURCES="${CCTV_DRIVEBC_MAX_SOURCES:-250}"
CCTV_TXDOT_ENABLED="${CCTV_TXDOT_ENABLED:-1}"
# Use `-` so an explicit empty string (the documented kill switch) is preserved.
CCTV_TXDOT_DISTRICTS="${CCTV_TXDOT_DISTRICTS-AUS,SAT}"
CCTV_TXDOT_MAX_SOURCES="${CCTV_TXDOT_MAX_SOURCES:-500}"
CCTV_TALLINN_ENABLED="${CCTV_TALLINN_ENABLED:-1}"
CCTV_TALLINN_MAX_SOURCES="${CCTV_TALLINN_MAX_SOURCES:-255}"
CCTV_TARKTEE_ENABLED="${CCTV_TARKTEE_ENABLED:-1}"
CCTV_TARKTEE_MAX_SOURCES="${CCTV_TARKTEE_MAX_SOURCES:-179}"
CCTV_WARENDORF_ENABLED="${CCTV_WARENDORF_ENABLED:-1}"
CCTV_NSW_ENABLED="${CCTV_NSW_ENABLED:-1}"
CCTV_NSW_MAX_SOURCES="${CCTV_NSW_MAX_SOURCES:-250}"
CCTV_MAX_SOURCES="${CCTV_MAX_SOURCES:-4000}"
# Capture which provider credentials genuinely came from the parent shell
# before this launcher resolves dotenv and Keychain fallbacks. Only names are
@@ -364,7 +383,23 @@ put_env CCTV_CALTRANS_DISTRICTS "${CCTV_CALTRANS_DISTRICTS}"
put_env CCTV_CALTRANS_MAX_SOURCES "${CCTV_CALTRANS_MAX_SOURCES}"
put_env CCTV_TFL_ENABLED "${CCTV_TFL_ENABLED}"
put_env CCTV_TFL_MAX_SOURCES "${CCTV_TFL_MAX_SOURCES}"
put_env CCTV_ONTARIO_ENABLED "${CCTV_ONTARIO_ENABLED}"
put_env CCTV_ONTARIO_MAX_SOURCES "${CCTV_ONTARIO_MAX_SOURCES}"
put_env_if_set TFL_APP_KEY "${TFL_APP_KEY:-}"
put_env CCTV_FINTRAFFIC_ENABLED "${CCTV_FINTRAFFIC_ENABLED}"
put_env CCTV_FINTRAFFIC_MAX_SOURCES "${CCTV_FINTRAFFIC_MAX_SOURCES}"
put_env CCTV_DRIVEBC_ENABLED "${CCTV_DRIVEBC_ENABLED}"
put_env CCTV_DRIVEBC_MAX_SOURCES "${CCTV_DRIVEBC_MAX_SOURCES}"
put_env CCTV_TXDOT_ENABLED "${CCTV_TXDOT_ENABLED}"
put_env CCTV_TXDOT_DISTRICTS "${CCTV_TXDOT_DISTRICTS}"
put_env CCTV_TXDOT_MAX_SOURCES "${CCTV_TXDOT_MAX_SOURCES}"
put_env CCTV_TALLINN_ENABLED "${CCTV_TALLINN_ENABLED}"
put_env CCTV_TALLINN_MAX_SOURCES "${CCTV_TALLINN_MAX_SOURCES}"
put_env CCTV_TARKTEE_ENABLED "${CCTV_TARKTEE_ENABLED}"
put_env CCTV_TARKTEE_MAX_SOURCES "${CCTV_TARKTEE_MAX_SOURCES}"
put_env CCTV_WARENDORF_ENABLED "${CCTV_WARENDORF_ENABLED}"
put_env CCTV_NSW_ENABLED "${CCTV_NSW_ENABLED}"
put_env CCTV_NSW_MAX_SOURCES "${CCTV_NSW_MAX_SOURCES}"
put_env CCTV_MAX_SOURCES "${CCTV_MAX_SOURCES}"
put_env OPENSKY_AUTH_MODE "${OPENSKY_AUTH_MODE}"
put_env_if_set OPENSKY_CREDENTIALS_FILE "${OPENSKY_CREDENTIALS_FILE}"
+13
View File
@@ -5,6 +5,8 @@
"scripts/package-boundaries.json",
"scripts/format.mjs",
"scripts/check-package-boundaries.mjs",
"scripts/precompute-cctv-heights.mjs",
"src/tooling/precomputeCctvHeights.test.mjs",
"src/tooling/format.test.mjs",
"src/tooling/packageBoundaries.test.mjs",
"src/data/infrastructure.js",
@@ -71,6 +73,8 @@
"src/data/firmsCsv.test.mjs",
"src/tooling/environmentProviders.test.mjs",
"server/providers/cctv.js",
"server/providers/cctv/cap.js",
"server/providers/cctv/groundHeights.js",
"server/providers/cctv/catalog.js",
"server/providers/cctv/constants.js",
"server/providers/cctv/media.js",
@@ -455,6 +459,15 @@
"src/layers/cctv/ground.js",
"src/layers/cctv/health.js",
"src/data/cctv.js",
"src/data/cctvCatalogCap.test.mjs",
"src/data/cctvDisplayCode.test.mjs",
"src/data/cctvFootprint.js",
"src/data/cctvGroundHeights.test.mjs",
"src/data/cctvFootprint.test.mjs",
"src/data/cctvEstonia.test.mjs",
"src/data/cctvWarendorf.test.mjs",
"src/data/cctvNswSource.test.mjs",
"src/data/cctvTxdotSource.test.mjs",
"src/layers/cctv/source.test.mjs",
"src/layers/radio/catalogModel.js",
"src/layers/radio/categories.js",
+6 -2
View File
@@ -141,6 +141,8 @@
"exports": ["./server/providers/cctv"],
"modules": [
"server/providers/cctv.js",
"server/providers/cctv/cap.js",
"server/providers/cctv/groundHeights.js",
"server/providers/cctv/catalog.js",
"server/providers/cctv/constants.js",
"server/providers/cctv/media.js",
@@ -149,7 +151,8 @@
"src/data/directionText.js",
"server/providers/common/geo.js",
"server/providers/places/google-key.js",
"scripts/google-server-key.mjs"
"scripts/google-server-key.mjs",
"src/data/cctvFootprint.js"
],
"external": []
},
@@ -890,7 +893,8 @@
"src/layers/cctv/source.js",
"src/layers/cctv/state.js",
"src/layers/cctv/testing.js",
"src/overlays/worldOverlayTokens.js"
"src/overlays/worldOverlayTokens.js",
"src/data/cctvFootprint.js"
],
"external": ["cesium"]
},
+487
View File
@@ -0,0 +1,487 @@
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import {
planeSupportPoints,
poseHash,
SUPPORT_KEYS,
} from '../src/data/cctvFootprint.js';
const ROOT = fileURLToPath(new URL('../', import.meta.url));
const SAMPLE_KEYS = ['mount', ...SUPPORT_KEYS];
const BATCH_SIZES = [16, 4, 1];
const DEADLINE_MS = 90_000;
/** Extract the served nominal pose (the values the catalog ships). */
export function nominalPose(source) {
const pose = Object.fromEntries(
[
'lat',
'lon',
'headingDeg',
'pitchDeg',
'fovDeg',
'rangeM',
'mountHeightM',
].map((key) => {
if (typeof source[key] !== 'number' || !Number.isFinite(source[key])) {
throw new Error(`Camera ${source.id}: invalid ${key}`);
}
return [key, source[key]];
}),
);
if (Math.abs(pose.lat) > 90 || Math.abs(pose.lon) > 180) {
throw new Error(`Camera ${source.id}: invalid coordinates`);
}
// Hashed exactly as served: the server join (groundHeights.js) and the
// client (camera.servedPose) hash the same raw values, so no clamp here.
return pose;
}
/** Return a roughly 2 km grid key, with longitude width fixed per row. */
export function cellKey({ lat, lon }) {
if (
!Number.isFinite(lat) ||
!Number.isFinite(lon) ||
Math.abs(lat) > 90 ||
Math.abs(lon) > 180
) {
throw new Error('Invalid cell coordinates');
}
const row = Math.min(8999, Math.floor((lat + 90) / 0.02));
const rowLat = -90 + (row + 0.5) * 0.02;
const lonWidth = 0.02 / Math.cos((rowLat * Math.PI) / 180);
const col = Math.floor(((lon === 180 ? -180 : lon) + 180) / lonWidth);
return `${row},${col}`;
}
/** Split an ordered array into bounded batches without mutating it. */
export function batches(items, size = 16) {
if (!Number.isSafeInteger(size) || size < 1)
throw new Error('Invalid batch size');
const result = [];
for (let i = 0; i < items.length; i += size)
result.push(items.slice(i, i + size));
return result;
}
/** Group cameras by cell and order cells numerically by row, then column. */
export function planCells(cameras) {
const cells = new Map();
for (const camera of cameras) {
const key = cellKey(camera.pose);
if (!cells.has(key)) cells.set(key, { key, cameras: [] });
cells.get(key).cameras.push(camera);
}
return [...cells.values()].sort((a, b) => {
const [ar, ac] = a.key.split(',').map(Number);
const [br, bc] = b.key.split(',').map(Number);
return ar - br || ac - bc;
});
}
/** Preserve ellipsoidal metres; missing, non-finite and implausible values are null. */
export function plausibleHeight(height) {
return Number.isFinite(height) && height >= -150 ? height : null;
}
/**
* Only a complete, successful entry for exactly the current nominal pose is
* reusable; an entry with missing supports is resampled (valid heights are
* kept by mergeCameraSamples).
*/
export function needsSampling(entry, pose) {
return (
entry?.status !== 'ok' ||
entry.poseHash !== poseHash(pose) ||
(Array.isArray(entry.misses) && entry.misses.length > 0)
);
}
/** Merge retry samples without losing valid heights; a missing mount stores no heights. */
export function mergeCameraSamples(
pose,
heights,
previous,
attempts,
sampledAt,
) {
const hash = poseHash(pose);
const prior = previous?.poseHash === hash ? previous : null;
const values = SAMPLE_KEYS.map(
(key, i) =>
plausibleHeight(heights[i]) ??
plausibleHeight(
key === 'mount' ? prior?.mountGroundM : prior?.supports?.[key],
),
);
const entry = {
poseHash: hash,
status: values[0] === null ? 'miss' : 'ok',
misses: SAMPLE_KEYS.filter((_, i) => values[i] === null),
sampledAt,
attempts,
};
if (entry.status === 'ok') {
entry.mountGroundM = values[0];
entry.supports = Object.fromEntries(
SUPPORT_KEYS.map((key, i) => [key, values[i + 1]]),
);
}
return entry;
}
/** Build a sidecar while retaining entries outside this run's selection. */
export function mergeSidecar(existing, updates, generatedAt) {
const cameras = { ...existing?.cameras, ...updates };
return {
schemaVersion: 1,
provider: 'google-photorealistic-3d',
heightReference: 'WGS84-ellipsoid',
generatedAt,
cameras: Object.fromEntries(
Object.keys(cameras)
.sort()
.map((id) => [id, cameras[id]]),
),
};
}
/** Parse pilot filters; the limit applies after kinds, before resume filtering. */
export function parseArgs(argv) {
const options = { limit: Infinity, kinds: null, dryRun: false };
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--dry-run') options.dryRun = true;
else if (arg === '--limit') {
const value = argv[++i];
if (!/^\d+$/.test(value ?? '') || !Number.isSafeInteger(Number(value))) {
throw new Error('--limit requires a non-negative integer');
}
options.limit = Number(value);
} else if (arg === '--kinds') {
const value = argv[++i];
if (
!value ||
value.startsWith('--') ||
value.split(',').some((kind) => !kind.trim())
) {
throw new Error('--kinds requires comma-separated sourceKinds');
}
options.kinds = value.split(',').map((kind) => kind.trim());
} else throw new Error(`Unknown argument: ${arg}`);
}
return options;
}
/** Use the shared footprint's mount and nine supports in the sampling wire order. */
export function samplingPoints(pose) {
const footprint = planeSupportPoints(pose);
return SAMPLE_KEYS.map((key) => {
const point = key === 'mount' ? footprint.mount : footprint.supports[key];
if (!Number.isFinite(point?.lat) || !Number.isFinite(point?.lon)) {
throw new Error(
`Shared planeSupportPoints returned invalid coordinates for ${key}`,
);
}
return { lat: point.lat, lon: point.lon };
});
}
/** Sample one cell serially, retrying failed cameras at most three times (16/4/1). */
export async function sampleCell(
cameras,
sampler,
now = () => new Date().toISOString(),
existing = null,
) {
// Seed with the prior entries so a resumed camera keeps every valid height
// it already has (mergeCameraSamples only replaces misses).
const entries = Object.create(null);
for (const camera of cameras) {
if (existing?.[camera.id]) entries[camera.id] = existing[camera.id];
}
let pending = cameras;
for (const [index, size] of BATCH_SIZES.entries()) {
const retry = [];
for (const batch of batches(pending, size)) {
let samples;
try {
samples = await sampler.sample(batch);
} catch (error) {
sampler.warn?.(
`Attempt ${index + 1}, ${batch.length} cameras: ${error.message}`,
);
// A timed-out evaluate can still be sampling: destroy its page before retrying.
await sampler.recycle();
samples = batch.map(() => []);
}
const sampledAt = now();
for (let i = 0; i < batch.length; i += 1) {
const camera = batch[i];
const entry = mergeCameraSamples(
camera.pose,
samples[i] ?? [],
entries[camera.id],
index + 1,
sampledAt,
);
entries[camera.id] = entry;
if (entry.misses.length) retry.push(camera);
}
}
pending = retry;
if (!pending.length) break;
}
return entries;
}
/** Bound a sampling operation; callers must recycle its page after rejection. */
export async function withDeadline(operation, timeoutMs = DEADLINE_MS) {
let timer;
try {
return await Promise.race([
operation,
new Promise((_, reject) => {
timer = setTimeout(
() =>
reject(
new Error(
`Height sampling timed out after ${timeoutMs / 1000} s`,
),
),
timeoutMs,
);
}),
]);
} finally {
clearTimeout(timer);
}
}
async function openPage(browser, base) {
const page = await browser.newPage();
try {
await page.goto(
`${base}/#v=2&lat=30&lon=-95&alt=17000000&heading=0&pitch=-90&roll=0`,
);
await page.waitForFunction(() => !!window.__godsEyeView?.viewer);
return page;
} catch (error) {
await page.close();
throw error;
}
}
async function loadSidecar(output) {
let existing;
try {
existing = JSON.parse(await readFile(output, 'utf8'));
} catch (error) {
if (error.code === 'ENOENT') return null;
throw error;
}
if (
existing?.schemaVersion !== 1 ||
existing.provider !== 'google-photorealistic-3d' ||
existing.heightReference !== 'WGS84-ellipsoid' ||
!existing.cameras ||
typeof existing.cameras !== 'object' ||
Array.isArray(existing.cameras)
) {
throw new Error(`Incompatible sidecar: ${output}`);
}
return existing;
}
async function writeSidecar(output, sidecar) {
await mkdir(path.dirname(output), { recursive: true });
const temporary = `${output}.${process.pid}.tmp`;
try {
await writeFile(temporary, `${JSON.stringify(sidecar, null, 2)}\n`);
await rename(temporary, output);
} finally {
await rm(temporary, { force: true });
}
}
function counts(cameras, entries) {
const ok = cameras.filter(
(camera) => entries[camera.id]?.status === 'ok',
).length;
return { ok, miss: cameras.length - ok };
}
async function main() {
const started = Date.now();
const options = parseArgs(process.argv.slice(2));
const base = (process.env.GEV_BASE || 'http://localhost:4173').replace(
/\/+$/,
'',
);
const output = path.resolve(
ROOT,
process.env.OUT ||
'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json',
);
let sidecar = await loadSidecar(output);
const response = await fetch(`${base}/api/cctv/sources`, {
signal: AbortSignal.timeout(30_000),
});
if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`);
const catalog = await response.json();
if (!Array.isArray(catalog.sources))
throw new Error('Catalog must contain sources[]');
const ids = new Set();
const selected = catalog.sources
.filter(
(source) => !options.kinds || options.kinds.includes(source.sourceKind),
)
.slice(0, options.limit)
.flatMap((source) => {
if (typeof source.id !== 'string' || !source.id || ids.has(source.id)) {
throw new Error('Catalog has an invalid or duplicate camera id');
}
ids.add(source.id);
let pose;
try {
pose = nominalPose(source);
} catch (error) {
// One camera with a malformed pose must not abort the whole run.
console.warn(`skip ${source.id}: ${error.message}`);
return [];
}
return [
{
id: source.id,
sourceKind: source.sourceKind || 'unknown',
pose,
},
];
});
const pending = selected.filter((camera) =>
needsSampling(sidecar?.cameras?.[camera.id], camera.pose),
);
const cells = planCells(pending);
console.log(
`${selected.length} cameras selected, ${selected.length - pending.length} resumed, ${pending.length} to sample in ${cells.length} cells`,
);
if (options.dryRun) {
for (const cell of cells) {
const sizes = batches(cell.cameras).map(
(batch) => `${batch.length} cameras/${batch.length * 10} points`,
);
console.log(
`cell ${cell.key}: ${cell.cameras.length} cameras; batches: ${sizes.join(', ')}`,
);
}
console.log(
'Retries: failed cameras in batches of 4, then singly; 3 attempts, 90 s per call.',
);
return;
}
// Validate shared geometry before starting an expensive browser session.
for (const camera of pending) camera.points = samplingPoints(camera.pose);
let browser;
try {
if (pending.length) {
const { default: puppeteer } = await import('puppeteer');
browser = await puppeteer.launch({
headless: false,
executablePath:
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
protocolTimeout: 600000,
args: [
'--no-sandbox',
'--disable-background-timer-throttling',
'--disable-renderer-backgrounding',
'--window-size=1200,800',
],
});
let page = await openPage(browser, base);
const sampler = {
warn: (message) => console.error(message),
recycle: async () => {
await page.close();
page = await openPage(browser, base);
},
sample: (batch) =>
withDeadline(
page.evaluate(async (cameras) => {
const gev = window.__godsEyeView;
const scene = gev.viewer.scene;
const Cartesian3 = gev.viewer.camera.position.constructor;
const Cartographic =
gev.viewer.camera.positionCartographic.constructor;
const lat =
cameras.reduce((sum, camera) => sum + camera.pose.lat, 0) /
cameras.length;
const lon =
cameras.reduce((sum, camera) => sum + camera.pose.lon, 0) /
cameras.length;
gev.viewer.camera.setView({
destination: Cartesian3.fromDegrees(lon, lat, 3000),
orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 },
});
const cartographics = cameras.flatMap((camera) =>
camera.points.map((point) =>
Cartographic.fromDegrees(point.lon, point.lat),
),
);
const sampled =
await scene.sampleHeightMostDetailed(cartographics);
return cameras.map((_, i) =>
sampled
.slice(i * 10, (i + 1) * 10)
.map((point) =>
Number.isFinite(point.height) ? point.height : null,
),
);
}, batch),
),
};
for (const cell of cells) {
const cellStarted = Date.now();
const entries = await sampleCell(
cell.cameras,
sampler,
undefined,
sidecar?.cameras,
);
sidecar = mergeSidecar(sidecar, entries, new Date().toISOString());
await writeSidecar(output, sidecar);
const tally = counts(cell.cameras, entries);
console.log(
`cell ${cell.key}: ${cell.cameras.length} cameras, ${tally.ok} ok/${tally.miss} miss, ${((Date.now() - cellStarted) / 1000).toFixed(1)} s`,
);
}
}
} finally {
if (browser) await browser.close();
}
const entries = sidecar?.cameras || {};
const tally = counts(selected, entries);
console.log(
`Total: ${selected.length} cameras, ${tally.ok} ok/${tally.miss} miss, ${((Date.now() - started) / 1000).toFixed(1)} s`,
);
for (const kind of [
...new Set(selected.map((camera) => camera.sourceKind)),
].sort()) {
const tally = counts(
selected.filter((camera) => camera.sourceKind === kind),
entries,
);
console.log(` ${kind}: ${tally.ok} ok/${tally.miss} miss`);
}
}
if (
process.argv[1] &&
import.meta.url === pathToFileURL(process.argv[1]).href
) {
main().catch((error) => {
console.error(error.message);
process.exitCode = 1;
});
}
+814 -323
View File
File diff suppressed because it is too large. Load diff