Merge origin/main into german/navara

Main's work since the last merge runs on the globe contract where this
branch draws through it:

- Satellites take main's classes, sensors, ground-track bearings and
  world time through contract points and rings; launches take the world
  timeline. The satellite's cockpit subject dims, copies positions and
  refocuses through the contract.
- Flights, military and vessels keep main's photos, retry cool-downs,
  world-clock motion and the ship bridge. Trail seeding in the shared
  aircraft tracking reads each layer's clock.
- Traffic's flow lines and single build draw as ground sets; transit
  keeps dated credits and the new cities; CCTV opens the nearest camera,
  and the Quebec overlay projects and tests the horizon through the
  contract. Bikeshare's manager cleanup moved into its current lifecycle.
- Draw keeps main's mark actions across engine switches.
- The world clock in src/time is plain numbers again: Julian dates come
  from src/geo, the frame loop is injected, and the binding to Cesium's
  viewer clock lives in src/globe/cesium/worldTime.js. Navara's sky and
  sun follow the world time; live feeds keep the real time.
- The cockpit's orbital view, with its sensor zoom, ground focus and
  satellite switching, stays Cesium-only for now: Navara hides its entry.
  Its Cesium calls live in src/globe/cesium/orbital.js. The ship bridge's
  finest-detail waterline sampling is Cesium-only too; Navara samples
  loaded geometry and detailed terrain. Both are in KNOWN-ISSUES.
- Main's tests that built a Cesium scene by hand run on the fake engine.
- The boundary manifest is regenerated, the coupling baseline records
  the orbital and bridge code, and the main-chunk baseline is raised to
  the merged size: main itself grew its main chunk by 93,959 bytes
  (gzip -9) since the previous merge base.
This commit is contained in:
German Khokhlov committed 2026-11-10 13:52:52 +04:00
commit 2a761aa5eb
452 files changed
+73437 -1752

No files matched your search

+9
View File
@@ -83,6 +83,15 @@ export function adsbLolProxy() {
// This mount also matches its /track route, served by its own handler.
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
return next();
// After the /track hand-off, so /track keeps its own method handling.
if (req.method !== 'GET') {
res.writeHead(405, {
'Content-Type': 'application/json',
Allow: 'GET',
});
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
const now = Date.now();
if (_cache && now - _cacheAt < CACHE_MS) {
serve(res, 200, _cache, 'HIT');
+69 -22
View File
@@ -1,6 +1,7 @@
import fs from 'node:fs';
import os from 'node:os';
import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback.js';
import { encodedBody, gzipBody } from '../common/compress.js';
import {
coalesceProxyRequest,
readResponseJsonCapped,
@@ -18,6 +19,11 @@ let _openskyTokenExpiry = 0;
let _openskyTokenPromise = null;
/** @type {string|null} Cached upstream response body (JSON text). */
let _openskyCacheBody = null;
/**
* @type {Buffer|null} Gzip of `_openskyCacheBody`, written together with it.
* Null when the body is not worth compressing.
*/
let _openskyCacheGzip = null;
/** @type {number} HTTP status of the cached response. */
let _openskyCacheStatus = 0;
/** @type {number} Epoch-ms when the response was cached. */
@@ -399,8 +405,12 @@ async function fetchAdsbLolPointFallback(req) {
ADSBLOL_POINT_MAX_RESPONSE_BYTES,
);
const normalized = normalizeAdsbLolPointResponse(payload);
const body = JSON.stringify(normalized);
const record = {
body: JSON.stringify(normalized),
body,
// Compressed once per entry; ADSBLOL_POINT_CACHE_MAX bounds the
// extra memory.
gzip: await gzipBody(body),
cachedAt: Date.now(),
count: normalized.states.length,
};
@@ -429,6 +439,7 @@ async function fetchAdsbLolPointFallback(req) {
async function serveAdsbLolPointFallback(req, res, requestedMode, reason) {
const fallback = await fetchAdsbLolPointFallback(req);
if (!fallback) return false;
const encoded = encodedBody(req, fallback.body, fallback.gzip);
res.writeHead(200, {
...buildOpenSkyHeaders({
cacheStatus: fallback.cacheStatus,
@@ -439,8 +450,9 @@ async function serveAdsbLolPointFallback(req, res, requestedMode, reason) {
'X-Flight-Source': 'adsb.lol',
'X-Flight-Coverage': `${ADSBLOL_POINT_RADIUS_NM}nm regional fallback`,
'X-Flight-Count': String(fallback.count),
...encoded.headers,
});
res.end(fallback.body);
res.end(encoded.body);
return true;
}
@@ -484,6 +496,15 @@ export function openSkyProxy() {
// This mount also matches its /track route, served by its own handler.
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
return next();
// After the /track hand-off, so /track keeps its own method handling.
if (req.method !== 'GET') {
res.writeHead(405, {
'Content-Type': 'application/json',
Allow: 'GET',
});
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
res.setHeader('X-Feed-Source', 'OpenSky Network');
const requestedMode = normalizeOpenSkyAuthMode(
process.env.OPENSKY_AUTH_MODE,
@@ -514,9 +535,13 @@ export function openSkyProxy() {
reason: 'cached',
};
const isStale = now - _openskyCacheTime >= _openskyTtlMs;
res.writeHead(
_openskyCacheStatus || 200,
buildOpenSkyHeaders({
const encoded = encodedBody(
req,
_openskyCacheBody,
_openskyCacheGzip,
);
res.writeHead(_openskyCacheStatus || 200, {
...buildOpenSkyHeaders({
cacheStatus: isStale ? 'STALE' : 'HIT',
requestedMode: cachedMeta.requestedMode || requestedMode,
usedMode: cachedMeta.usedMode || 'unknown',
@@ -532,8 +557,9 @@ export function openSkyProxy() {
? (_openskyCooldownUntil - now) / 1000
: undefined,
}),
);
res.end(_openskyCacheBody);
...encoded.headers,
});
res.end(encoded.body);
return;
}
// Cooling down with nothing cached (cold start into a rate limit):
@@ -628,7 +654,11 @@ export function openSkyProxy() {
// Only cache successful responses — error responses (401/403/429/5xx)
// should not be served from cache on subsequent requests
if (upstream.ok) {
// Compress before writing either slot, so a request arriving
// mid-refresh still reads a body with its own frame.
const frame = await gzipBody(body);
_openskyCacheBody = body;
_openskyCacheGzip = frame;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
@@ -681,9 +711,13 @@ export function openSkyProxy() {
// Serve the last-good body instead of the 429 when we have one —
// the layer keeps rendering (STALE-cued) instead of dying.
if (_openskyCacheBody && _openskyCacheStatus === 200) {
res.writeHead(
200,
buildOpenSkyHeaders({
const encoded = encodedBody(
req,
_openskyCacheBody,
_openskyCacheGzip,
);
res.writeHead(200, {
...buildOpenSkyHeaders({
cacheStatus: 'STALE',
requestedMode,
usedMode,
@@ -691,8 +725,9 @@ export function openSkyProxy() {
staleSeconds: (now - _openskyCacheTime) / 1000,
retryAfterSeconds: cooldownMs / 1000,
}),
);
res.end(_openskyCacheBody);
...encoded.headers,
});
res.end(encoded.body);
return;
}
}
@@ -734,16 +769,23 @@ export function openSkyProxy() {
}
}
res.writeHead(
upstream.status,
buildOpenSkyHeaders({
// The frame belongs to this body only if this body was cached. An
// uncached error must not borrow the previous snapshot's frame.
const encoded = encodedBody(
req,
body,
_openskyCacheBody === body ? _openskyCacheGzip : null,
);
res.writeHead(upstream.status, {
...buildOpenSkyHeaders({
cacheStatus: 'MISS',
requestedMode,
usedMode,
reason,
}),
);
res.end(body);
...encoded.headers,
});
res.end(encoded.body);
} catch (e) {
console.error('[OpenSky Proxy]', e.message);
if (_openskyCacheBody) {
@@ -754,17 +796,22 @@ export function openSkyProxy() {
usedMode: 'unknown',
reason: 'cached_stale',
};
res.writeHead(
_openskyCacheStatus || 200,
buildOpenSkyHeaders({
const encoded = encodedBody(
req,
_openskyCacheBody,
_openskyCacheGzip,
);
res.writeHead(_openskyCacheStatus || 200, {
...buildOpenSkyHeaders({
cacheStatus: 'STALE',
requestedMode:
cachedMeta.requestedMode || OPENSKY_AUTH_MODE_DEFAULT,
usedMode: cachedMeta.usedMode || 'unknown',
reason: cachedMeta.reason || 'cached_stale',
}),
);
res.end(_openskyCacheBody);
...encoded.headers,
});
res.end(encoded.body);
return;
}
const requestedMode = normalizeOpenSkyAuthMode(
+187 -5
View File
@@ -1,4 +1,5 @@
import { createCctvCatalog } from './cctv/catalog.js';
import { quebec511VideoUrl, quebecBrowserVideoEnabled } from './cctv/quebec.js';
import { normalizeFeedType, isVideoFeedType } from './cctv/normalize.js';
import {
buildSyntheticCctvSvg,
@@ -18,6 +19,8 @@ import { sanitizeCctvRangeHeader } from './cctv/range.js';
import { admitSameSiteKeyless } from './common/same-site.js';
import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs';
import { createHlsPuller } from './cctv/stream.js';
import { isMlitRiverAdjustmentImage } from './cctv/worldCameras.js';
import { ottawaAuthorizedFrameUrl } from './cctv/ottawa.js';
import { googleServerApiKey } from './places/google-key.js';
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
/**
@@ -43,6 +46,120 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
const HEALTH_MAX_ENTRIES = CCTV_MAX_SOURCES_CEILING;
/** Live HLS strategies (see ./cctv/stream.js). Shared across dev and preview. */
const puller = createHlsPuller();
/**
* Bounded transient image reuse keeps cadence-limited providers (World
* stills, Ottawa's once-a-minute API) inside their refresh floor. The byte
* cap is the real bound; the entry cap only limits map overhead.
*/
const WORLD_FRAME_CACHE_MAX_ENTRIES = 1024;
/** Retry a failed fetch sooner than a full refresh window, never under a minute. */
const THROTTLED_FRAME_FAILURE_BACKOFF_MS = 60 * 1000;
const WORLD_FRAME_CACHE_MAX_ENTRY_BYTES = 2 * 1024 * 1024;
const WORLD_FRAME_CACHE_MAX_BYTES = 32 * 1024 * 1024;
const worldFrameCache = new Map();
const worldFrameInflight = new Map();
const worldFrameNextAllowedAt = new Map();
let worldFrameCacheBytes = 0;
function getWorldFrameCache(cameraId) {
const entry = worldFrameCache.get(cameraId);
if (!entry) return null;
worldFrameCache.delete(cameraId);
worldFrameCache.set(cameraId, entry);
return entry;
}
function storeWorldFrame(cameraId, image) {
if (
!Buffer.isBuffer(image?.body) ||
image.body.length > WORLD_FRAME_CACHE_MAX_ENTRY_BYTES
)
return;
const previous = worldFrameCache.get(cameraId);
if (previous) worldFrameCacheBytes -= previous.body.length;
const entry = {
body: image.body,
contentType: image.contentType,
fetchedAt: Date.now(),
};
worldFrameCache.delete(cameraId);
worldFrameCache.set(cameraId, entry);
worldFrameCacheBytes += entry.body.length;
while (
worldFrameCache.size > WORLD_FRAME_CACHE_MAX_ENTRIES ||
worldFrameCacheBytes > WORLD_FRAME_CACHE_MAX_BYTES
) {
const oldestId = worldFrameCache.keys().next().value;
const oldest = worldFrameCache.get(oldestId);
worldFrameCache.delete(oldestId);
worldFrameCacheBytes -= oldest.body.length;
}
}
function setNextAllowedAt(cameraId, at) {
worldFrameNextAllowedAt.delete(cameraId);
worldFrameNextAllowedAt.set(cameraId, at);
while (worldFrameNextAllowedAt.size > 4096) {
worldFrameNextAllowedAt.delete(
worldFrameNextAllowedAt.keys().next().value,
);
}
}
/**
* Fetch a frame from a provider with a per-camera refresh floor. Inside the
* floor the last frame is reused; with no frame to reuse the caller gets
* `{ throttled: true }` and serves a placeholder, so a cadence-limited camera
* never falls through to the key-bearing Street View fallback. A failed
* fetch backs off for a minute (capped at the floor) instead of a full
* window.
*/
async function fetchThrottledFrame(source, cameraId, frameUrl) {
const cached = getWorldFrameCache(cameraId);
const rawRefreshMs = Number(source?.frameRefreshMs);
const refreshMs =
Number.isFinite(rawRefreshMs) && rawRefreshMs > 0
? rawRefreshMs
: 5 * 60 * 1000;
const now = Date.now();
if (cached && now - cached.fetchedAt < refreshMs) {
return { ok: true, ...cached, cached: true };
}
const inflight = worldFrameInflight.get(cameraId);
if (inflight) return inflight;
const stale = cached
? {
ok: true,
body: cached.body,
contentType: cached.contentType,
stale: true,
}
: null;
const nextAllowedAt = worldFrameNextAllowedAt.get(cameraId) || 0;
if (now < nextAllowedAt) return stale || { ok: false, throttled: true };
if (!frameUrl) return stale;
setNextAllowedAt(cameraId, now + refreshMs);
let request;
request = fetchCctvImageFromUpstream(frameUrl)
.then((image) => {
if (image?.ok) {
storeWorldFrame(cameraId, image);
return image;
}
setNextAllowedAt(
cameraId,
Date.now() + Math.min(refreshMs, THROTTLED_FRAME_FAILURE_BACKOFF_MS),
);
return stale || image;
})
.finally(() => {
if (worldFrameInflight.get(cameraId) === request)
worldFrameInflight.delete(cameraId);
});
worldFrameInflight.set(cameraId, request);
return request;
}
/** Update the health entry for a camera, evicting the oldest entry if at capacity. */
const setHealth = (cameraId, patch) => {
@@ -153,6 +270,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
);
if (url.pathname === '/sources') {
const quebecVideo = quebecBrowserVideoEnabled();
const body = {
trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [],
sources: sources.map((source) => ({
@@ -164,6 +282,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
provider: source.provider,
lat: source.lat,
lon: source.lon,
frameRefreshMs: source.frameRefreshMs,
headingDeg: source.headingDeg,
headingConfidence: source.headingConfidence || '',
pitchDeg: source.pitchDeg,
@@ -174,6 +293,13 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
feedType: normalizeFeedType(source.feedType),
sourceKind:
source.sourceKind || (source.url ? 'configured' : 'fallback'),
sourceUrl: source.sourceUrl || '',
browserMediaUrl:
quebecVideo && source.sourceKind === 'quebec-511-browser'
? quebec511VideoUrl(source.id)
: '',
accuracyM: source.accuracyM,
positionSource: source.positionSource || '',
poseSource: source.poseSource,
model: source.model || '',
manufacturer: source.manufacturer || '',
@@ -526,15 +652,71 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
: '');
const upstreamImage =
source?.sourceKind === 'txdot-its'
? await fetchTxdotSnapshot(upstreamCandidate)
: await fetchCctvImageFromUpstream(upstreamCandidate);
source?.pack === 'world-cameras'
? await fetchThrottledFrame(source, cameraId, upstreamCandidate)
: source?.sourceKind === 'ottawa-traffic'
? await fetchThrottledFrame(
source,
cameraId,
ottawaAuthorizedFrameUrl(upstreamCandidate),
)
: source?.sourceKind === 'txdot-its'
? await fetchTxdotSnapshot(upstreamCandidate)
: await fetchCctvImageFromUpstream(upstreamCandidate, {
allowMjpeg: source?.sourceKind === 'taiwan-highway-motc',
});
if (
source?.sourceKind === 'mlit-river' &&
isMlitRiverAdjustmentImage(upstreamImage)
) {
const svg = buildSyntheticCctvSvg({
cameraId,
label,
city,
status: 'CAMERA UNDER ADJUSTMENT',
});
setHealth(cameraId, {
status: 'degraded',
sourceKind: 'snapshot',
label: source.provider || 'MLIT river cameras',
message: 'Provider camera under adjustment',
});
res.writeHead(200, {
'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-store',
'X-CCTV-Source': 'provider-maintenance',
});
res.end(svg);
return;
}
if (upstreamImage?.throttled) {
// Inside the provider's refresh floor with no frame to reuse: a
// placeholder, not Street View, and no health change.
res.writeHead(200, {
'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-store',
'X-CCTV-Source': 'synthetic',
});
res.end(
buildSyntheticCctvSvg({
cameraId,
label,
city,
status: 'WAITING FOR NEXT FRAME',
}),
);
return;
}
if (upstreamImage?.ok) {
setHealth(cameraId, {
status: 'ok',
status: upstreamImage.stale ? 'degraded' : 'ok',
sourceKind: 'snapshot',
label: source?.provider || 'Configured source',
message: 'Upstream snapshot active',
message: upstreamImage.stale
? 'Serving last snapshot; refresh failed'
: upstreamImage.cached
? 'Cached snapshot active'
: 'Upstream snapshot active',
});
res.writeHead(200, {
'Content-Type': upstreamImage.contentType,
+49 -12
View File
@@ -1,5 +1,6 @@
import {
CCTV_MAX_SOURCES_CEILING,
CCTV_SHARED_SOURCE_SLOTS,
DEFAULT_CCTV_MAX_SOURCES,
} from './constants.js';
@@ -23,16 +24,27 @@ export function resolveCatalogCap(raw) {
* exceed the catalog cap, the cap is filled ROUND-ROBIN — one camera from
* each pack per turn, in each pack's own order — so a lowered
* CCTV_MAX_SOURCES thins every region a little instead of deleting whichever
* pack happened to be appended last. Duplicate ids resolve last-pack-wins so
* pack happened to be appended last. A pack with a `reservedCount` keeps up to
* that many rows only from the capacity ABOVE the shared floor, so a
* reservation adds room and never shrinks the other packs' round-robin
* share; at or below the floor reserved packs are ordinary lanes. Duplicate
* ids resolve last-pack-wins so
* file/env overrides keep replacing live records; the record is then counted
* against the pack that won it.
*
* @param {Array<{name:string, sources:Array<object>}>} packs - Normalized
* packs in merge order (earlier packs lose duplicate ids to later ones).
* @param {number} maxCount - Catalog cap.
* @param {object} [options]
* @param {number} [options.sharedFloor=CCTV_SHARED_SOURCE_SLOTS] - Slots kept
* for the round-robin before reservations apply.
* @returns {{sources:Array<object>, packs:Array<{name:string, offered:number, kept:number}>}}
*/
export function allocateSourceCap(packs, maxCount) {
export function allocateSourceCap(
packs,
maxCount,
{ sharedFloor = CCTV_SHARED_SOURCE_SLOTS } = {},
) {
const owner = new Map();
for (const pack of packs) {
for (const source of pack.sources) {
@@ -40,30 +52,53 @@ export function allocateSourceCap(packs, maxCount) {
owner.set(source.id, { pack: pack.name, source });
}
}
const lanes = packs.map((pack) => ({
name: pack.name,
const lanes = packs.map((pack) => {
// Only the record that WON its id (the last occurrence anywhere, so also
// the last within this pack) takes a lane slot; earlier duplicates never
// count against the cap.
queue: pack.sources.filter(
const queue = pack.sources.filter(
(source) => source?.id && owner.get(source.id)?.source === source,
),
next: 0,
kept: 0,
}));
);
return {
name: pack.name,
queue,
reserved: Math.max(
0,
Math.min(queue.length, Math.floor(Number(pack.reservedCount) || 0)),
),
next: 0,
kept: 0,
};
});
const total = lanes.reduce((sum, lane) => sum + lane.queue.length, 0);
const limit = Math.min(total, Math.max(0, Math.floor(maxCount)));
const sources = [];
const reservedSources = [];
if (limit >= total) {
for (const lane of lanes) {
sources.push(...lane.queue);
lane.kept = lane.queue.length;
}
} else {
while (sources.length < limit) {
// Reserved packs draw only on the capacity above the shared floor, then
// every lane (reserved ones included) shares the rest round-robin.
const reserveBudget = Math.max(0, limit - Math.max(0, sharedFloor));
while (reservedSources.length < reserveBudget) {
let tookReserved = false;
for (const lane of lanes) {
if (sources.length >= limit) break;
if (reservedSources.length >= reserveBudget) break;
if (lane.kept >= lane.reserved) continue;
reservedSources.push(lane.queue[lane.next]);
lane.next += 1;
lane.kept += 1;
tookReserved = true;
}
if (!tookReserved) break;
}
while (sources.length + reservedSources.length < limit) {
for (const lane of lanes) {
if (sources.length + reservedSources.length >= limit) break;
if (lane.next >= lane.queue.length) continue;
sources.push(lane.queue[lane.next]);
lane.next += 1;
@@ -72,7 +107,9 @@ export function allocateSourceCap(packs, maxCount) {
}
}
return {
sources,
// Keep the shared round-robin sequence first so the default active camera
// remains in its established region; guaranteed regional rows follow it.
sources: [...sources, ...reservedSources],
packs: lanes.map((lane) => ({
name: lane.name,
offered: lane.queue.length,
+34 -2
View File
@@ -1,11 +1,18 @@
import fs from 'node:fs';
import path from 'node:path';
import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
import {
DEFAULT_CCTV_SOURCE_FILE,
CCTV_SOURCE_CACHE_MS,
WORLD_CCTV_SOURCE_COUNT,
} from './constants.js';
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
import { cameraRegion, normalizeSourceItem } from './normalize.js';
import { loadLithuaniaSources } from './lithuania.js';
import { loadOttawaSourcesFromOpenData } from './ottawa.js';
import { loadQuebecSources, QUEBEC_CAMERA_SOURCE_COUNT } from './quebec.js';
import { loadTaiwanSources } from './taiwan.js';
import { loadWorldCameraSources } from './worldCameras.js';
import {
loadAustinSourcesFromOpenData,
loadCaltransSourcesFromOpenData,
@@ -20,6 +27,7 @@ import {
loadNswSourcesFromOpenData,
loadCalgarySourcesFromOpenData,
loadCataloniaSourcesFromOpenData,
loadIcelandSourcesFromOpenData,
loadHkSourcesFromOpenData,
loadDelDOTSourcesFromOpenData,
loadKingCountySourcesFromOpenData,
@@ -75,6 +83,11 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_DRIVEBC_ENABLED'),
load: loadDriveBcSourcesFromOpenData,
},
{
name: 'ottawa',
enabled: () => envEnabled('CCTV_OTTAWA_ENABLED'),
load: loadOttawaSourcesFromOpenData,
},
{
name: 'txdot',
enabled: () => envEnabled('CCTV_TXDOT_ENABLED'),
@@ -110,6 +123,11 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
load: loadCataloniaSourcesFromOpenData,
},
{
name: 'iceland',
enabled: () => envEnabled('CCTV_ICELAND_ENABLED'),
load: loadIcelandSourcesFromOpenData,
},
{
name: 'hk',
enabled: () => envEnabled('CCTV_HK_ENABLED'),
@@ -172,6 +190,18 @@ const LIVE_PACKS = [
enabled: () => envOptIn('CCTV_HOUSTON_ENABLED'),
load: loadHoustonSourcesFromOpenData,
},
{
name: 'quebec',
enabled: () => envEnabled('CCTV_QUEBEC_ENABLED'),
load: loadQuebecSources,
reservedCount: QUEBEC_CAMERA_SOURCE_COUNT,
},
{
name: 'world-cameras',
enabled: () => envEnabled('CCTV_WORLD_CAMERAS_ENABLED'),
load: loadWorldCameraSources,
reservedCount: WORLD_CCTV_SOURCE_COUNT,
},
{
name: 'a22',
enabled: () => envEnabled('CCTV_A22_ENABLED'),
@@ -316,7 +346,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
: [];
// Live packs first so file/env overrides win on duplicate IDs; each pack
// keeps its own priority order and the catalog cap is shared fairly.
const normalizePack = (name, items) => ({
const normalizePack = (name, items, reservedCount = 0) => ({
name,
sources: items
.filter((item) => item && typeof item === 'object')
@@ -326,6 +356,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
? items.available
: items.length,
region: items.region ?? null,
reservedCount,
});
const packs = [
...LIVE_PACKS.map((pack, index) =>
@@ -334,6 +365,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
liveResults[index]?.status === 'fulfilled'
? liveResults[index].value
: [],
pack.reservedCount,
),
),
normalizePack('file', fromFile),
+74 -7
View File
@@ -5,15 +5,18 @@ export const DEFAULT_AUSTIN_ROWS_URL =
/** Default cap on Austin cameras after distance-based prioritization. */
export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
/**
* Catalog-wide safety ceiling on served cameras. Each pack already caps
* itself (nearest-to-anchor first); this bound only matters when the packs
* together exceed it, and it is then filled round-robin across packs (see
* cap.js) so no region is silently dropped. The default per-pack caps
* together can exceed it, so a default install may be thinned evenly here.
* Slots every pack shares round-robin before any reservation applies. A
* reserved pack (Québec, World) only takes capacity above this floor, so a
* cap at or below it thins every region evenly (see cap.js).
*/
export const DEFAULT_CCTV_MAX_SOURCES = 4000;
export const CCTV_SHARED_SOURCE_SLOTS = 4000;
/**
* Catalog-wide default: the 4,000 shared slots plus room reserved for the
* 1,120 Québec and 2,230 World image locations.
*/
export const DEFAULT_CCTV_MAX_SOURCES = 7350;
/** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */
export const CCTV_MAX_SOURCES_CEILING = 5000;
export const CCTV_MAX_SOURCES_CEILING = 8000;
/** Reference point for Austin camera prioritization (Congress & 6th). */
export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 };
/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */
@@ -197,6 +200,39 @@ export const DRIVEBC_ANCHORS = [
{ lat: 49.2827, lon: -123.1207 }, // Vancouver
{ lat: 48.4284, lon: -123.3656 }, // Victoria
];
/** City of Ottawa traffic cameras: documented JSON list (keyless). */
export const OTTAWA_CAMERAS_URL =
'https://traffic.ottawa.ca/map/service/camera';
/**
* Keyless canonical frame URL for one camera number. The city's Traffic Camera
* Images API needs a certificate (OTTAWA_CAMERA_CERTIFICATE), which the frame
* proxy adds at request time so it never appears in a catalog response.
*/
export const OTTAWA_IMAGE_URL = (cameraNumber) =>
`https://traffic.ottawa.ca/opendata/camera?c=${encodeURIComponent(cameraNumber)}`;
/** Pinned frame directory, for redirect containment in the media proxy. */
export const OTTAWA_IMAGE_ORIGIN = 'https://traffic.ottawa.ca/opendata/';
/** The API lets one certificate and user id fetch a camera once a minute. */
export const OTTAWA_FRAME_REFRESH_MS = 60 * 1000;
/** Alphanumeric user id this server sends with its certificate. */
export const OTTAWA_API_USER_ID = 'gev';
export const DEFAULT_OTTAWA_MAX_SOURCES = 250;
/** Prioritization anchor: downtown Ottawa (Parliament Hill). */
export const OTTAWA_ANCHORS = [{ lat: 45.4215, lon: -75.6972 }];
/**
* Taiwan Highway Bureau MJPEG servers. Each camera is one root path segment,
* so the pin is the host root: a redirect must stay on the same server.
*/
export const TAIWAN_HIGHWAY_IMAGE_ORIGINS = Object.freeze(
[1, 2, 3, 4, 5, 6, 7, 8].map((n) => `https://cctv-ss0${n}.thb.gov.tw/`),
);
/** Direct still-image directories from the World cams add-on's eligible packs. */
export const TRIPCHECK_IMAGE_ORIGIN =
'https://www.tripcheck.com/roadcams/cams/';
export const MLIT_RIVER_IMAGE_ORIGIN = 'https://cam.river.go.jp/cam/now/';
export const DEFAULT_WORLD_CCTV_SOURCE_FILE =
'config/cctv_sources.world-image-cameras.json';
export const WORLD_CCTV_SOURCE_COUNT = 2230;
/** Tallinn intersection cameras: curated catalog + public stills on ristmikud.tallinn.ee. */
export const DEFAULT_TALLINN_SOURCE_FILE = 'config/cctv_sources.tallinn.json';
export const DEFAULT_TALLINN_MAX_SOURCES = 255;
@@ -309,6 +345,37 @@ export const CATALONIA_ANCHORS = [
{ lat: 41.5636, lon: 2.0111 }, // Terrassa
];
/**
* Icelandic Road and Coastal Administration (Vegagerðin, IRCA) road cameras:
* one keyless JSON list, one row per camera view (about 500 views at roughly
* 170 stations, verified 2026-11-06). Frames are stills on the IRCA host, each
* rewritten by the publisher every few minutes. Camera images are published
* under CC BY 4.0 (IRCA terms of use, "Open data" section).
*/
export const ICELAND_CAMERAS_URL =
'https://gagnaveita.vegagerdin.is/api/vefmyndavelar2014_1';
/** The only directory Iceland camera frames may come from. */
export const ICELAND_IMAGE_ORIGIN =
'https://www.vegagerdin.is/vgdata/vefmyndavelar/';
/** Byte ceiling for the Iceland catalog read (~150 KB observed 2026-11-06). */
export const ICELAND_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
/** Ground-elevation prior in metres: the feed carries none, and the country
* runs from sea level to highland passes. The client's ground snap corrects
* this on 3D-tile stacks. */
export const ICELAND_GROUND_ELEVATION_M = 60;
/** The feed lists about 500 views; the default keeps half so the shared
* catalog cap still leaves room for denser regions. */
export const DEFAULT_ICELAND_MAX_SOURCES = 250;
/** Prioritization anchors: Reykjavík and the regional centres, so a cap keeps
* every region's nearest cameras instead of only the capital's. */
export const ICELAND_ANCHORS = [
{ lat: 64.1466, lon: -21.9426 }, // Reykjavík
{ lat: 65.6885, lon: -18.1262 }, // Akureyri
{ lat: 65.2639, lon: -14.3948 }, // Egilsstaðir
{ lat: 66.0749, lon: -23.1351 }, // Ísafjörður
{ lat: 63.9331, lon: -20.9971 }, // Selfoss
];
/** Hong Kong Transport Department traffic snapshot cameras: keyless English
* locations XML on DATA.GOV.HK; frames are stills on the TD CCTV host. */
export const DEFAULT_HK_CAMERAS_URL =
+243 -6
View File
@@ -10,10 +10,15 @@ import {
CCTV_MEDIA_MAX_BODY_BYTES,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
MLIT_RIVER_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
NSW_IMAGE_ORIGIN,
NSW_IMAGE_USER_AGENT,
OTTAWA_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
ICELAND_IMAGE_ORIGIN,
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
TFL_IMAGE_ORIGIN,
WARENDORF_IMAGE_ORIGINS,
WSDOT_IMAGE_ORIGIN,
@@ -523,13 +528,18 @@ for (const prefix of [
TFL_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
ICELAND_IMAGE_ORIGIN,
...WARENDORF_IMAGE_ORIGINS,
NSW_IMAGE_ORIGIN,
OTTAWA_IMAGE_ORIGIN,
CALGARY_IMAGE_ORIGIN,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
WSDOT_IMAGE_ORIGIN,
A22_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
MLIT_RIVER_IMAGE_ORIGIN,
...TAIWAN_HIGHWAY_IMAGE_ORIGINS,
]) {
const host = new URL(prefix).hostname;
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
@@ -621,6 +631,216 @@ export function cctvUpstreamUserAgent(url) {
}
}
/** Largest single JPEG the MJPEG first-frame reader will buffer. */
export const CCTV_MJPEG_FRAME_MAX_BYTES = 2 * 1024 * 1024;
/**
* Read one bounded JPEG part from an open MJPEG response, then stop the stream.
*
* The first part may follow a CRLF preamble, and its delimiter may be written
* as `--<boundary>` or, when the declared boundary already starts with `--`,
* as the boundary itself. A part with Content-Length is read to that length;
* a part without one ends at the next delimiter, so a JPEG end marker inside
* the image (an EXIF thumbnail) never truncates it. Bytes are copied once
* into a preallocated buffer and every search resumes where it stopped, so a
* trickling upstream costs linear time.
*/
async function readFirstMjpegFrame(upstream, maxBytes) {
const mediaType = cctvLiveMediaType(
'mjpeg',
upstream.headers.get('content-type'),
);
const boundary = parseMediaType(mediaType)?.params.get('boundary') || '';
const frameMax = Math.min(maxBytes, CCTV_MJPEG_FRAME_MAX_BYTES);
if (
!boundary ||
!upstream.body ||
!Number.isSafeInteger(frameMax) ||
frameMax < 4
) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const delimiters = [Buffer.from('--' + boundary, 'latin1')];
if (boundary.startsWith('--') && boundary.length > 2)
delimiters.push(Buffer.from(boundary, 'latin1'));
const lineEnd = Buffer.from('\r\n', 'ascii');
const headerTerminator = Buffer.from('\r\n\r\n', 'ascii');
const jpegStart = Buffer.from([0xff, 0xd8]);
const jpegEnd = Buffer.from([0xff, 0xd9]);
const maxPreambleBytes = 64;
const maxHeaderBytes = 8 * 1024;
const maxBufferedBytes =
maxPreambleBytes +
delimiters[0].length +
lineEnd.length +
maxHeaderBytes +
headerTerminator.length +
frameMax +
lineEnd.length +
delimiters[0].length;
const buffer = Buffer.allocUnsafe(maxBufferedBytes);
let length = 0;
let reader;
let headerStart = -1;
let delimiter = null;
let headerScanFrom = 0;
let bodyStart = -1;
let declaredLength = null;
let boundaryScanFrom = -1;
/** Match the opening delimiter after an optional CRLF preamble. */
function findOpening() {
let at = 0;
while (at < length && (buffer[at] === 0x0d || buffer[at] === 0x0a)) at++;
if (at > maxPreambleBytes) return { failed: true };
for (const candidate of delimiters) {
const line = Buffer.concat([candidate, lineEnd]);
const need = at + line.length;
if (length < need) {
const seen = buffer.subarray(at, length);
if (line.subarray(0, seen.length).equals(seen))
return { pending: true };
continue;
}
if (buffer.subarray(at, need).equals(line))
return { start: need, delimiter: candidate };
}
return { failed: true };
}
/** Parse the part headers; false rejects the stream. */
function parseHeaders(headerEnd) {
const headerText = buffer
.subarray(headerStart, headerEnd)
.toString('latin1');
const headers = new Map();
for (const line of headerText.split('\r\n')) {
const colon = line.indexOf(':');
if (colon <= 0) return false;
const name = line.slice(0, colon).trim().toLowerCase();
if (!name || headers.has(name)) return false;
headers.set(name, line.slice(colon + 1).trim());
}
if (cctvRasterImageType(headers.get('content-type')) !== 'image/jpeg')
return false;
const declared = headers.get('content-length');
if (declared !== undefined) {
if (!/^\d+$/.test(declared)) return false;
declaredLength = Number(declared);
if (
!Number.isSafeInteger(declaredLength) ||
declaredLength < jpegStart.length + jpegEnd.length ||
declaredLength > frameMax
)
return false;
}
return true;
}
/** A complete JPEG body, or null. */
function asJpeg(frame) {
return frame.length >= jpegStart.length + jpegEnd.length &&
frame.subarray(0, jpegStart.length).equals(jpegStart) &&
frame.subarray(-jpegEnd.length).equals(jpegEnd)
? Buffer.from(frame)
: null;
}
try {
reader = upstream.body.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done) return null;
const chunk = Buffer.from(value);
const room = maxBufferedBytes - length;
const included = Math.min(room, chunk.length);
chunk.copy(buffer, length, 0, included);
length += included;
const overflow = included < chunk.length;
if (headerStart < 0) {
const opening = findOpening();
if (opening.failed) return null;
if (opening.pending) {
if (overflow) return null;
continue;
}
headerStart = opening.start;
delimiter = opening.delimiter;
headerScanFrom = headerStart;
}
if (bodyStart < 0) {
const headerEnd = buffer
.subarray(0, length)
.indexOf(headerTerminator, headerScanFrom);
if (headerEnd < 0) {
if (length - headerStart > maxHeaderBytes || overflow) return null;
headerScanFrom = Math.max(
headerStart,
length - headerTerminator.length + 1,
);
continue;
}
if (headerEnd - headerStart > maxHeaderBytes) return null;
if (!parseHeaders(headerEnd)) return null;
bodyStart = headerEnd + headerTerminator.length;
boundaryScanFrom = bodyStart;
}
const available = length - bodyStart;
if (available >= jpegStart.length) {
if (
!buffer
.subarray(bodyStart, bodyStart + jpegStart.length)
.equals(jpegStart)
)
return null;
}
const nextDelimiter = Buffer.concat([lineEnd, delimiter]);
const nextPart = buffer
.subarray(0, length)
.indexOf(nextDelimiter, boundaryScanFrom);
if (declaredLength !== null) {
if (nextPart >= bodyStart && nextPart - bodyStart < declaredLength)
return null;
if (available >= declaredLength)
return asJpeg(buffer.subarray(bodyStart, bodyStart + declaredLength));
} else if (nextPart >= bodyStart) {
const frame = buffer.subarray(bodyStart, nextPart);
return frame.length > frameMax ? null : asJpeg(frame);
} else if (available > frameMax) {
return null;
}
if (nextPart < 0)
boundaryScanFrom = Math.max(
bodyStart,
length - nextDelimiter.length + 1,
);
if (overflow) return null;
}
} finally {
if (reader) {
try {
await reader.cancel();
} catch {
/* already closed or aborted */
}
try {
reader.releaseLock();
} catch {
/* already released */
}
}
}
}
/**
* Fetch one upstream CCTV image within the frame-refresh budget.
*
@@ -633,6 +853,10 @@ export function cctvUpstreamUserAgent(url) {
* @param {typeof fetch} [options.fetchImpl=fetch] - Fetch implementation.
* @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout.
* @param {number} [options.maxBytes=CCTV_FRAME_MAX_BODY_BYTES] - Snapshot byte cap.
* @param {boolean} [options.allowMjpeg=false] - Accept a multipart MJPEG
* answer and keep its first JPEG. Only sources registered as MJPEG stills
* (the Taiwan Highway Bureau feeds) set it; any other host answering
* multipart falls through to the next fallback.
* @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>}
*/
export async function fetchCctvImageFromUpstream(
@@ -641,6 +865,7 @@ export async function fetchCctvImageFromUpstream(
fetchImpl = fetch,
timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS,
maxBytes = CCTV_FRAME_MAX_BODY_BYTES,
allowMjpeg = false,
} = {},
) {
if (!url || !/^https?:\/\//i.test(url)) return null;
@@ -660,10 +885,7 @@ export async function fetchCctvImageFromUpstream(
fetchImpl,
);
if (!upstream) return null;
const contentType = cctvRasterImageType(
upstream.headers.get('content-type'),
);
if (!upstream.ok || !contentType) {
if (!upstream.ok) {
try {
await upstream.body?.cancel();
} catch {
@@ -671,9 +893,24 @@ export async function fetchCctvImageFromUpstream(
}
return null;
}
const body = await readCappedResponseBytes(upstream, maxBytes);
const upstreamType = upstream.headers.get('content-type');
const contentType = cctvRasterImageType(upstreamType);
if (contentType) {
const body = await readCappedResponseBytes(upstream, maxBytes);
if (!body) return null;
return { ok: true, body, contentType };
}
if (!allowMjpeg || !cctvLiveMediaType('mjpeg', upstreamType)) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const body = await readFirstMjpegFrame(upstream, maxBytes);
if (!body) return null;
return { ok: true, body, contentType };
return { ok: true, body, contentType: 'image/jpeg' };
} catch {
return null;
} finally {
+15
View File
@@ -460,6 +460,17 @@ export function isLikelyCataloniaCoordinate(lat, lon) {
);
}
/** Iceland bounding box (mainland and nearby islands, with a margin). */
export function isLikelyIcelandCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 63.0 &&
lat <= 66.7 &&
lon >= -25.0 &&
lon <= -13.0
);
}
/**
* Decode numeric HTML character references, decimal ("&#237;") and hex
* ("&#xED;") alike, the entity form the Catalonia feed uses for accented
@@ -639,6 +650,10 @@ export function normalizeSourceItem(item) {
feedType: normalizeFeedType(item.feedType || item.type || ''),
url: typeof item.url === 'string' ? item.url : '',
snapshotUrl: typeof item.snapshotUrl === 'string' ? item.snapshotUrl : '',
frameRefreshMs: toFiniteNumber(item.frameRefreshMs),
sourceUrl: typeof item.sourceUrl === 'string' ? item.sourceUrl : '',
accuracyM: toFiniteNumber(item.accuracyM),
positionSource: String(item.positionSource || ''),
license: String(item.license || item.licenseNote || ''),
// Per-camera attribution for feeds a partner supplies inside a pack
// (DriveBC: TransLink, city cameras). Shown beside the provider.
+185
View File
@@ -0,0 +1,185 @@
import {
CCTV_SOURCE_FETCH_TIMEOUT_MS,
DEFAULT_OTTAWA_MAX_SOURCES,
OTTAWA_ANCHORS,
OTTAWA_API_USER_ID,
OTTAWA_CAMERAS_URL,
OTTAWA_FRAME_REFRESH_MS,
OTTAWA_IMAGE_ORIGIN,
OTTAWA_IMAGE_URL,
} from './constants.js';
import {
fallbackHeadingFromId,
pinnedCameraUrl,
prioritizeSources,
toFiniteNumber,
} from './normalize.js';
let ottawaCertificateMissingLogged = false;
/** Test hook: let the next missing-certificate skip log again. */
export function _resetOttawaCertificateWarningForTest() {
ottawaCertificateMissingLogged = false;
}
/** The operator's Traffic Camera Images API certificate, trimmed. */
function ottawaCertificate(env = process.env) {
return String(env.OTTAWA_CAMERA_CERTIFICATE ?? '').trim();
}
/**
* Add the operator's certificate to a registered keyless Ottawa frame URL.
* Only a URL inside the pinned `/opendata/` directory with a numeric camera
* number is signed, so the certificate never goes to any other path or host.
*
* @param {string} url - Canonical frame URL from the catalog.
* @param {Record<string,string|undefined>} [env]
* @returns {string} The signed URL, or '' when there is no certificate or the
* URL is not a registered Ottawa frame.
*/
export function ottawaAuthorizedFrameUrl(url, env = process.env) {
const certificate = ottawaCertificate(env);
const pinned = pinnedCameraUrl(url, OTTAWA_IMAGE_ORIGIN);
if (!certificate || !pinned) return '';
const parsed = new URL(pinned);
const cameraNumber = parsed.searchParams.get('c') || '';
if (parsed.pathname !== '/opendata/camera' || !/^\d+$/.test(cameraNumber))
return '';
const signed = new URL(OTTAWA_IMAGE_URL(cameraNumber));
signed.searchParams.set('certificate', certificate);
signed.searchParams.set('id', OTTAWA_API_USER_ID);
return signed.href;
}
/**
* City of Ottawa envelope: Kanata to Orléans, Barrhaven to the river, with
* slack for the MTO cameras on the 416/417 that the city feed also carries.
*/
const OTTAWA_BOUNDS = Object.freeze({
minLat: 45.1,
maxLat: 45.65,
minLon: -76.2,
maxLon: -75.1,
});
/** Whether a point is inside the city feed's service area. */
function isOttawaCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= OTTAWA_BOUNDS.minLat &&
lat <= OTTAWA_BOUNDS.maxLat &&
lon >= OTTAWA_BOUNDS.minLon &&
lon <= OTTAWA_BOUNDS.maxLon
);
}
/**
* Load the City of Ottawa's traffic cameras.
*
* Camera images come from the city's Traffic Camera Images API, which needs a
* certificate from the city's registration form (OTTAWA_CAMERA_CERTIFICATE)
* and allows one fetch per camera per minute for a certificate and user id.
* Without a certificate the pack is skipped, with no request and one log line.
* The camera list is the documented `map/service/camera` JSON: a camera
* number, English and French names, coordinates and the owner (CITY or MTO).
* The feed carries no bearing, no elevation and no status field, so headings
* use the same deterministic low-confidence fallback as the other
* heading-less packs and the client's ground snap refines the elevation
* prior.
*
* @param {object} [_options]
* @param {Record<string,string|undefined>} [env]
* @returns {Promise<Array<object>>} Cameras nearest downtown first, or [] on
* a missing certificate or any upstream or payload failure.
*/
export async function loadOttawaSourcesFromOpenData(
_options = {},
env = process.env,
) {
if (!ottawaCertificate(env)) {
if (!ottawaCertificateMissingLogged) {
ottawaCertificateMissingLogged = true;
console.warn(
'[CCTV] Ottawa cameras skipped: the city camera image API needs a certificate (set OTTAWA_CAMERA_CERTIFICATE)',
);
}
return [];
}
try {
const resp = await fetch(OTTAWA_CAMERAS_URL, {
headers: { Accept: 'application/json' },
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
if (!resp.ok) {
console.warn('[CCTV] Ottawa camera download failed:', resp.status);
return [];
}
const payload = await resp.json();
const rows = Array.isArray(payload?.cameras) ? payload.cameras : null;
if (!rows) return [];
const cameras = [];
const seen = new Set();
for (const row of rows) {
const number = toFiniteNumber(row?.camera_number);
if (!Number.isSafeInteger(number) || number <= 0 || seen.has(number))
continue;
const lat = toFiniteNumber(row?.latitude);
const lon = toFiniteNumber(row?.longitude);
if (!isOttawaCoordinate(lat, lon)) continue;
seen.add(number);
const cameraId = `ottawa-${number}`;
const imageUrl = OTTAWA_IMAGE_URL(number);
// The docs name the field camera_owner; the live feed sends cameraOwner.
const owner = String(row?.camera_owner || row?.cameraOwner || '')
.trim()
.toUpperCase();
cameras.push({
id: cameraId,
name: String(row?.name || '').trim() || `Ottawa camera ${number}`,
city: 'Ottawa',
cityId: 'ottawa',
provider: 'City of Ottawa',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: 0,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
frameRefreshMs: OTTAWA_FRAME_REFRESH_MS,
sourceKind: 'ottawa-traffic',
license: 'City of Ottawa Traffic Camera Images API (registered access)',
credit:
owner === 'MTO'
? 'Ministry of Transportation of Ontario (MTO) camera'
: '',
});
}
const maxRaw = Number(
process.env.CCTV_OTTAWA_MAX_SOURCES || DEFAULT_OTTAWA_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(500, Math.floor(maxRaw)))
: DEFAULT_OTTAWA_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, OTTAWA_ANCHORS);
console.log(
`[CCTV] Loaded Ottawa camera sources: ${cameras.length} published (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Ottawa camera download error:',
error?.message || error,
);
return [];
}
}
+138
View File
@@ -0,0 +1,138 @@
import fs from 'node:fs';
import path from 'node:path';
import { cameraDisplayCode, fallbackHeadingFromId } from './normalize.js';
const QUEBEC_CATALOG_PATH = 'public/quebec-cameras.json';
export const QUEBEC_CAMERA_SOURCE_COUNT = 1120;
/**
* Whether the operator opted in to the browser-direct Québec 511 video. Off by
* default: the camera locations are CC BY 4.0, but whether Québec 511's
* service terms allow another site to embed its clips is unconfirmed, so an
* unset flag lists the cameras with a link to the 511 viewer page only.
*/
export function quebecBrowserVideoEnabled(env = process.env) {
return String(env.CCTV_QUEBEC_VIDEO_ENABLED || '').trim() === '1';
}
/** Build the direct browser-only MP4 URL for one namespaced MTQ camera id. */
export function quebec511VideoUrl(cameraId) {
const match = /^qc-511-(\d+)$/.exec(String(cameraId || ''));
return match
? `https://www.quebec511.info/Carte/Fenetres/camera.ashx?id=${match[1]}&format=mp4`
: '';
}
function safeViewerUrl(value) {
try {
const url = new URL(String(value || ''));
return url.protocol === 'https:' && !url.username && !url.password
? url.href
: '';
} catch {
return '';
}
}
function sourceId(camera) {
const id = String(camera?.id || '').trim();
if (camera?.kind === 'traffic' && /^\d+$/.test(id)) return `qc-511-${id}`;
if (camera?.kind === 'montreal' && /^mtl-\d+(?:-\d+)*$/.test(id))
return `qc-mtl-${id.slice(4)}`;
if (camera?.kind === 'webcam' && /^web-\d+$/.test(id))
return `qc-web-${id.slice(4)}`;
return '';
}
/**
* Read the checked-in, no-key Québec location catalog and adapt it to the
* regular CCTV source schema. MTQ video stays browser-only; other rows link to
* their operator page because this dataset has no direct image URL for them.
*/
export function loadQuebecSources({ sourceRoot = process.cwd() } = {}) {
const filename = path.resolve(sourceRoot, QUEBEC_CATALOG_PATH);
let cameras;
try {
const payload = JSON.parse(fs.readFileSync(filename, 'utf8'));
cameras = Array.isArray(payload?.cameras) ? payload.cameras : [];
} catch (error) {
console.warn('[CCTV] Québec camera catalog unavailable:', error?.message);
return [];
}
const sources = [];
const seen = new Set();
for (const camera of cameras) {
const id = sourceId(camera);
const lat = Number(camera.lat);
const lon = Number(camera.lon);
if (
!id ||
seen.has(id) ||
!Number.isFinite(lat) ||
!Number.isFinite(lon) ||
lat < 44.9 ||
lat > 62.6 ||
lon < -79.9 ||
lon > -57.1
)
continue;
seen.add(id);
const name = String(camera.nameEn || camera.name || id).trim();
const isTraffic = camera.kind === 'traffic';
const isMontreal = camera.kind === 'montreal';
const sourceKind = isTraffic
? 'quebec-511-browser'
: isMontreal
? 'quebec-montreal-osm'
: 'quebec-webcam-link';
const provider = isTraffic
? 'Québec 511 / MTQ'
: isMontreal
? 'Ville de Montréal / OpenStreetMap'
: String(camera.operator || 'Québec webcam');
sources.push({
id,
name: name.slice(0, 180),
city: String(camera.region || 'Québec'),
cityId: 'quebec',
provider,
lat,
lon,
headingDeg: fallbackHeadingFromId(id),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 74,
rangeM: 400,
mountHeightM: 8,
groundElevationM: 0,
feedType: 'image',
// No frame URL is registered: the server never fetches 511 media. With
// CCTV_QUEBEC_VIDEO_ENABLED=1 the browser plays the 511 MP4 in the DOM
// overlay after a play click; otherwise the camera links to 511.
url: '',
snapshotUrl: '',
sourceKind,
sourceUrl: safeViewerUrl(camera.viewerUrl),
accuracyM: Number.isFinite(Number(camera.accuracyM))
? Number(camera.accuracyM)
: null,
positionSource: String(camera.positionSource || ''),
license: isTraffic
? 'Locations: Données Québec / MTQ, CC BY 4.0. Video: hosted by Québec 511 under its terms of use'
: isMontreal
? 'OpenStreetMap contributors, ODbL 1.0'
: 'Operator terms apply; location provenance shown per camera',
credit: isTraffic
? 'Locations: MTQ · CC BY 4.0 · Video: Québec 511'
: isMontreal
? 'OpenStreetMap contributors · ODbL'
: provider,
code: cameraDisplayCode(name.toUpperCase()),
});
}
return sources;
}
+175
View File
@@ -134,6 +134,12 @@ import {
HOUSTON_MAX_CATALOG_BYTES,
HOUSTON_GROUND_ELEVATION_M,
HOUSTON_ANCHORS,
ICELAND_CAMERAS_URL,
ICELAND_IMAGE_ORIGIN,
ICELAND_MAX_CATALOG_BYTES,
ICELAND_GROUND_ELEVATION_M,
DEFAULT_ICELAND_MAX_SOURCES,
ICELAND_ANCHORS,
} from './constants.js';
import {
toFiniteNumber,
@@ -152,6 +158,7 @@ import {
isLikelyCalgaryCoordinate,
isLikelyFloridaCoordinate,
isLikelyCataloniaCoordinate,
isLikelyIcelandCoordinate,
decodeNumericEntities,
isLikelyNycCoordinate,
isLikelyHoustonCoordinate,
@@ -2029,6 +2036,174 @@ export async function loadCataloniaSourcesFromOpenData() {
}
}
/**
* Compass headings for Icelandic direction words, as they appear after "til"
* (genitive: "til vesturs"), "í" (accusative: "í vestur") or "séð" ("seen",
* as in "séð austur").
*/
const ICELAND_DIRECTION_HEADINGS = {
norður: 0,
norðaustur: 45,
austur: 90,
suðaustur: 135,
suður: 180,
suðvestur: 225,
vestur: 270,
norðvestur: 315,
};
const ICELAND_DIRECTION_WORD =
'(norðaustur|norðvestur|suðaustur|suðvestur|norður|suður|austur|vestur)s?';
/** A facing, plus a second one joined by "og" ("til austurs og vesturs"). */
const ICELAND_DIRECTION_RE = new RegExp(
`(?<![\\p{L}])(?:til|í|séð)\\s+${ICELAND_DIRECTION_WORD}` +
`(?:\\s+og\\s+(?:til\\s+|í\\s+)?${ICELAND_DIRECTION_WORD})?(?![\\p{L}])`,
'gu',
);
/**
* The compass heading an Icelandic view description states, or NaN. Only an
* explicit "til <átt>", "í <átt>" or "séð <átt>" counts: a bare place name or
* a road direction ("Miklabraut austur") is not a facing. A description naming
* two different directions ("til norðurs og til suðurs", "til austurs og
* vesturs") is ambiguous and yields NaN, and "niður á veg" (down onto the
* road) states none.
*
* @param {string} text - The feed's `Skyring` field.
* @returns {number} Heading in degrees [0..360), or NaN.
*/
export function icelandHeadingFromDescription(text) {
const headings = new Set();
for (const match of String(text || '')
.toLowerCase()
.matchAll(ICELAND_DIRECTION_RE)) {
for (const word of [match[1], match[2]]) {
if (word) headings.add(ICELAND_DIRECTION_HEADINGS[word]);
}
}
return headings.size === 1 ? headings.values().next().value : NaN;
}
/**
* One Vegagerðin camera row -> one catalog source, or null. A row is one camera
* view; a station (`Maelist_nr`) carries several. The id comes from the frame
* file name, which is unique per view in the live feed.
*
* @param {object} row - Element of the `vefmyndavelar2014_1` array.
* @returns {?object}
*/
export function icelandCameraToSource(row) {
const url = pinnedCameraUrl(row?.Slod, ICELAND_IMAGE_ORIGIN);
if (!url || !/\.jpe?g$/i.test(new URL(url).pathname)) return null;
// Numbers only: Number(null) and Number('') are 0.
const lat = typeof row.Breidd === 'number' ? row.Breidd : NaN;
const lon = typeof row.Lengd === 'number' ? row.Lengd : NaN;
if (!isLikelyIcelandCoordinate(lat, lon)) return null;
const stem = new URL(url).pathname
.slice(new URL(ICELAND_IMAGE_ORIGIN).pathname.length)
.replace(/\.jpe?g$/i, '')
.toLowerCase();
if (!/^[a-z0-9_-]+$/.test(stem)) return null;
const cameraId = `is-vegagerdin-${stem}`;
const description = String(row.Skyring || '')
.replace(/\s+/g, ' ')
// Letter lookarounds, not \b: \b is ASCII-only and would join "Þverá
// Árnessýslu" while missing "séð séð".
.replace(/(?<!\p{L})(\p{L}+) \1(?!\p{L})/giu, '$1')
.trim();
const station = String(row.Myndavel || '').trim();
const heading = icelandHeadingFromDescription(description);
const hasHeading = Number.isFinite(heading);
return {
id: cameraId,
name: description || station || `Vegagerðin ${row.Maelist_nr ?? stem}`,
city: 'Iceland',
cityId: 'iceland',
provider: 'Vegagerðin (IRCA)',
lat,
lon,
headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId),
headingConfidence: hasHeading ? 'high' : 'low',
pitchDeg: hasHeading ? -24 : -18,
fovDeg: hasHeading ? 56 : 44,
rangeM: hasHeading ? 210 : 145,
mountHeightM: hasHeading ? 10 : 8,
groundElevationM: ICELAND_GROUND_ELEVATION_M,
feedType: 'image',
url,
snapshotUrl: url,
sourceKind: 'iceland-vegagerdin',
license:
'Icelandic Road and Coastal Administration (IRCA) — road camera images, CC BY 4.0',
code: cameraDisplayCode(station.toUpperCase() || description),
};
}
/**
* Fetch Icelandic road cameras (Vegagerðin / IRCA), keyless: one JSON list.
* Frames are stills on the IRCA host and the loader registers only links
* inside that host's camera directory (ICELAND_IMAGE_ORIGIN). The list decides
* ids, coordinates, labels and which frame path is proxied, so it is fetched
* over https, redirects are refused and the body read is capped at
* ICELAND_MAX_CATALOG_BYTES.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadIcelandSourcesFromOpenData() {
try {
const resp = await fetch(ICELAND_CAMERAS_URL, {
headers: {
Accept: 'application/json',
'User-Agent': 'gods-eye-view-cctv-proxy/1.0',
},
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
// A response this loader will not read still owns its transport until the
// body is released, so every rejection path cancels before returning.
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] Iceland catalog redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] Iceland camera download failed:', resp.status);
return discard();
}
const body = await readResponseJsonCapped(resp, ICELAND_MAX_CATALOG_BYTES);
const rows = Array.isArray(body) ? body : [];
const cameras = rows.map(icelandCameraToSource).filter(Boolean);
const unique = Array.from(
new Map(cameras.map((camera) => [camera.id, camera])).values(),
);
const maxRaw = Number(
process.env.CCTV_ICELAND_MAX_SOURCES || DEFAULT_ICELAND_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(600, Math.floor(maxRaw)))
: DEFAULT_ICELAND_MAX_SOURCES;
const prioritized = prioritizeSources(unique, maxCount, ICELAND_ANCHORS);
console.log(
`[CCTV] Loaded Iceland camera sources: ${unique.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Iceland camera download error:',
error?.message || error,
);
return [];
}
}
/**
* Read one XML child tag's text from an `<image>` block.
*
+3 -1
View File
@@ -5,7 +5,9 @@ export const HLS_LIMITS = Object.freeze({
sessions: 2,
leasesPerSession: 8,
playlistBytes: 256 * 1024,
segmentBytes: 4 * 1024 * 1024,
// High-resolution cameras can exceed 4 MiB in a single ten-second TS segment.
// Retained data still shares the independent 24 MiB session budget below.
segmentBytes: 8 * 1024 * 1024,
sessionBytes: 24 * 1024 * 1024,
segments: 12,
pollMs: 2000,
+158 -11
View File
@@ -1,4 +1,7 @@
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
import {
CCTV_SOURCE_FETCH_TIMEOUT_MS,
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
} from './constants.js';
import {
cameraDisplayCode,
fallbackHeadingFromId,
@@ -10,6 +13,8 @@ import { readResponseTextCapped } from '../common/http.js';
* standard, published keyless and refreshed daily (~1,800 cameras). */
export const TAIWAN_FREEWAY_CCTV_URL =
'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml';
export const TAIWAN_HIGHWAY_CCTV_URL =
'https://cctv-maintain.thb.gov.tw/opendataCCTVs.xml';
/** The whole network is ~1.2 MB. */
export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024;
const DEFAULT_TAIWAN_MAX_SOURCES = 600;
@@ -35,6 +40,9 @@ const STREAM_TO_FRAME = new Map([
]);
const DIRECTIONS = { N: 'N', S: 'S', E: 'E', W: 'W' };
const TAIWAN_HIGHWAY_CAMERA_HOSTS = new Set(
TAIWAN_HIGHWAY_IMAGE_ORIGINS.map((origin) => new URL(origin).hostname),
);
/** Prioritization anchors, north to south, so a cap keeps the whole island
* rather than only greater Taipei. */
@@ -211,6 +219,74 @@ export function taiwanCameraToSource(row) {
};
}
/**
* One Highway Bureau row -> a source whose first MJPEG frame is served as a
* snapshot by the existing camera-frame proxy.
*
* @param {object} row - Output of parseTaiwanCctvXml.
* @returns {?object}
*/
export function taiwanHighwayCameraToSource(row) {
if (!row || typeof row !== 'object') return null;
const { lat, lon } = row;
if (!isLikelyTaiwanCoordinate(lat, lon)) return null;
let url;
try {
url = new URL(String(row.streamUrl ?? ''));
} catch {
return null;
}
if (
url.protocol !== 'https:' ||
url.username ||
url.password ||
url.port ||
url.search ||
url.hash ||
!TAIWAN_HIGHWAY_CAMERA_HOSTS.has(url.hostname) ||
!/^\/[A-Za-z0-9._+~-]{1,100}$/.test(url.pathname) ||
url.pathname.includes('..')
)
return null;
const rawId = String(row.cctvId ?? '').trim();
if (!/^[A-Za-z0-9-]{1,80}$/.test(rawId)) return null;
const cameraId = 'tw-thb-' + rawId.toLowerCase();
const road = String(row.roadName ?? '').trim();
const direction = DIRECTIONS[String(row.direction ?? '').trim()] || '';
const mile = String(row.mile ?? '').trim();
const section = [row.sectionStart, row.sectionEnd]
.map((part) => String(part ?? '').trim())
.filter(Boolean)
.join('→');
const name =
(road || mile || section
? [road, direction, mile, section].filter(Boolean).join(' ')
: '') || rawId;
return {
id: cameraId,
name: name.slice(0, 160),
city: 'Taiwan / 臺灣',
cityId: 'taiwan',
provider: 'Highway Bureau, MOTC',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'mjpeg',
url: url.href,
snapshotUrl: url.href,
sourceKind: 'taiwan-highway-motc',
license: 'Highway Bureau, MOTC open data',
code: cameraDisplayCode((road + ' ' + mile).trim().toUpperCase() || rawId),
};
}
let lastGood = null;
let lastGoodAt = 0;
let retryAfter = 0;
@@ -237,6 +313,49 @@ function taiwanSourceCap() {
: DEFAULT_TAIWAN_MAX_SOURCES;
}
/** Fetch the public provincial-road CCTV table as a no-key fallback. */
async function loadTaiwanHighwaySources() {
const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS);
const response = await fetch(TAIWAN_HIGHWAY_CCTV_URL, {
headers: { Accept: 'application/xml,text/xml' },
redirect: 'manual',
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'Highway Bureau catalog redirected; redirects are not followed'
: 'Highway Bureau catalog HTTP ' + response.status,
);
}
const xml = await readResponseTextCapped(
response,
TAIWAN_MAX_CATALOG_BYTES,
signal,
);
const cameras = new Map();
for (const row of parseTaiwanCctvXml(xml)) {
const camera = taiwanHighwayCameraToSource(row);
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
}
if (!cameras.size)
throw new Error('no usable Highway Bureau cameras in the table');
const sources = prioritizeSources(
[...cameras.values()],
taiwanSourceCap(),
ANCHORS,
);
console.log(
'[CCTV] Loaded Taiwan Highway Bureau fallback: ' +
cameras.size +
' (using ' +
sources.length +
')',
);
return sources;
}
/**
* Fetch Taiwan Freeway Bureau cameras from the keyless MOTC 2.0 XML table.
* One bounded request per shared catalog refresh. A failed download serves the
@@ -262,7 +381,7 @@ export async function loadTaiwanSources() {
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `HTTP ${response.status}`,
: 'HTTP ' + response.status,
);
}
const xml = await readResponseTextCapped(
@@ -282,7 +401,11 @@ export async function loadTaiwanSources() {
ANCHORS,
);
console.log(
`[CCTV] Loaded Taiwan camera sources: ${cameras.size} (using ${sources.length})`,
'[CCTV] Loaded Taiwan camera sources: ' +
cameras.size +
' (using ' +
sources.length +
')',
);
lastGood = sources;
lastGoodAt = Date.now();
@@ -291,13 +414,37 @@ export async function loadTaiwanSources() {
} catch (error) {
retryAfter = Date.now() + TAIWAN_RETRY_AFTER_FAILURE_MS;
const fallback = fallbackSources();
console.warn(
`[CCTV] Taiwan camera download failed (${error?.message || error}); ` +
(fallback.length
? `serving the last good ${fallback.length} cameras`
: 'no Taiwan cameras') +
', next attempt in 1 h',
);
return fallback;
if (fallback.length) {
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); serving the last good ' +
fallback.length +
' cameras, next attempt in 1 h',
);
return fallback;
}
try {
const sources = await loadTaiwanHighwaySources();
lastGood = sources;
lastGoodAt = Date.now();
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); using ' +
sources.length +
' Highway Bureau cameras as fallback, next attempt in 1 h',
);
return sources;
} catch (fallbackError) {
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); Highway Bureau fallback failed (' +
(fallbackError?.message || fallbackError) +
'); no Taiwan cameras, next attempt in 1 h',
);
return [];
}
}
}
+132
View File
@@ -0,0 +1,132 @@
import { createHash } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import {
DEFAULT_WORLD_CCTV_SOURCE_FILE,
MLIT_RIVER_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
WORLD_CCTV_SOURCE_COUNT,
} from './constants.js';
import { pinnedCameraUrl } from './normalize.js';
const SOURCE_POLICIES = Object.freeze({
'tripcheck-odot': {
idPrefix: 'world-or-',
origin: TRIPCHECK_IMAGE_ORIGIN,
bounds: [41, 47, -125, -116],
filename: /^\/roadcams\/cams\/[^/]+\.jpe?g$/i,
frameRefreshMs: 5 * 60 * 1000,
},
'mlit-river': {
idPrefix: 'world-jp-river-',
origin: MLIT_RIVER_IMAGE_ORIGIN,
bounds: [24, 46, 122, 154],
filename: /^\/cam\/now\/\d+\.jpg$/i,
// The provider asks clients to avoid frequent repeated image collection.
frameRefreshMs: 10 * 60 * 1000,
},
});
// MLIT currently serves this fixed JPEG on cameras whose images are paused
// for maintenance (verified against two Yamagata cameras on 2026-11-02).
const MLIT_ADJUSTMENT_IMAGE_SHA256 =
'dbd2c3790eb75edf9ca44162f7979f2a3799090e23f2c5142f63211a035cb191';
export function isMlitRiverAdjustmentImage(image) {
return (
image?.ok === true &&
Buffer.isBuffer(image.body) &&
createHash('sha256').update(image.body).digest('hex') ===
MLIT_ADJUSTMENT_IMAGE_SHA256
);
}
function safeFrameUrl(value, policy) {
const normalized = pinnedCameraUrl(value, policy.origin);
if (!normalized) return '';
const url = new URL(normalized);
return !url.search && !url.hash && policy.filename.test(url.pathname)
? url.href
: '';
}
function inBounds(lat, lon, [minLat, maxLat, minLon, maxLon]) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= minLat &&
lat <= maxLat &&
lon >= minLon &&
lon <= maxLon
);
}
/**
* Load only the new image-backed networks from the add-on export. Existing
* Canadian networks, Québec entries, JMA placeholder images, and NZTA's
* scraped Journey Planner rows are deliberately excluded. Vegagerðin (Iceland)
* rows are excluded too: the Iceland pack loads the full live list of those
* cameras, so a World copy would draw each one twice.
*/
export function loadWorldCameraSources({ sourceRoot = process.cwd() } = {}) {
const filename = path.resolve(sourceRoot, DEFAULT_WORLD_CCTV_SOURCE_FILE);
let rows;
try {
const payload = JSON.parse(fs.readFileSync(filename, 'utf8'));
rows = Array.isArray(payload) ? payload : [];
} catch (error) {
console.warn('[CCTV] World camera catalog unavailable:', error?.message);
return [];
}
const seen = new Set();
const sources = [];
for (const row of rows) {
const sourceKind = String(row?.sourceKind || '');
const policy = SOURCE_POLICIES[sourceKind];
if (!policy) continue;
const id = String(row.id || '').trim();
const lat = Number(row.lat);
const lon = Number(row.lon);
const url = safeFrameUrl(row.url || row.snapshotUrl, policy);
const snapshotUrl = safeFrameUrl(row.snapshotUrl || row.url, policy);
if (
!id.startsWith(policy.idPrefix) ||
seen.has(id) ||
!inBounds(lat, lon, policy.bounds) ||
!url ||
!snapshotUrl ||
url !== snapshotUrl
)
continue;
seen.add(id);
sources.push({
id,
name: String(row.name || id)
.trim()
.slice(0, 180),
city: String(row.city || row.country || 'World'),
cityId: String(row.cityId || ''),
provider: String(row.provider || 'World traffic camera'),
lat,
lon,
headingDeg: Number.isFinite(Number(row.headingDeg))
? Number(row.headingDeg)
: undefined,
headingConfidence: String(row.headingConfidence || 'low'),
feedType: 'image',
sourceKind,
url,
snapshotUrl,
frameRefreshMs: policy.frameRefreshMs,
license: String(row.license || ''),
credit: String(row.credit || row.provider || ''),
});
}
console.log(
`[CCTV] Loaded World camera image sources: ${sources.length}/${WORLD_CCTV_SOURCE_COUNT}`,
);
return sources;
}
+81
View File
@@ -0,0 +1,81 @@
import { gzip } from 'node:zlib';
import { promisify } from 'node:util';
const gzipAsync = promisify(gzip);
/**
* Opt-in gzip for text bodies a proxy already holds in memory.
*
* Not a middleware: `cctv/media.js` streams bodies and answers range requests,
* and several providers forward binary upstream types that gzip only grows.
* `encodedBody` returns the body with its headers so the two cannot disagree.
*/
/** Skip compression for bodies smaller than 1 KiB. */
const MIN_COMPRESSED_BYTES = 1024;
/**
* RFC 9110 section 12.4.2 `qvalue`. Matching the whole token keeps `q=1oops`
* and `q=2` unreadable instead of rounding them into acceptance.
*/
const QVALUE = /^(?:0(?:\.\d{0,3})?|1(?:\.0{0,3})?)$/;
/**
* Whether `Accept-Encoding` offers gzip. Anything unreadable (no header, a bad
* q value, `x-gzip`) reads as no: every client can decode identity.
*
* @param {import('node:http').IncomingMessage} request
* @returns {boolean}
*/
export function acceptsGzip(request) {
const header = request?.headers?.['accept-encoding'];
if (typeof header !== 'string') return false;
return header.split(',').some((part) => {
const [token, ...parameters] = part.split(';');
if (token.trim().toLowerCase() !== 'gzip') return false;
const quality = parameters
.map((parameter) => parameter.trim().toLowerCase())
.find((parameter) => parameter.startsWith('q='));
if (quality === undefined) return true;
// `q=0` names gzip and refuses it.
const value = quality.slice(2);
return QVALUE.test(value) && Number(value) > 0;
});
}
/**
* Gzip a body once for a caller that caches it. Async, so the ~860 KB OpenSky
* snapshot does not block other requests while it compresses.
*
* Returns null, meaning "serve identity", when the body is under 1 KiB or gzip
* does not shrink it.
*
* @param {string|Buffer} body - the exact bytes the identity response sends.
* @returns {Promise<Buffer|null>}
*/
export async function gzipBody(body) {
const raw = Buffer.isBuffer(body) ? body : Buffer.from(body);
if (raw.byteLength < MIN_COMPRESSED_BYTES) return null;
const compressed = await gzipAsync(raw);
return compressed.byteLength < raw.byteLength ? compressed : null;
}
/**
* Pick the body and the headers that describe it. `Vary` is on both branches
* so a shared cache never serves one encoding to a client that asked for the
* other.
*
* @param {import('node:http').IncomingMessage} request
* @param {string|Buffer} identity - the uncompressed body.
* @param {Buffer|null} compressed - `gzipBody(identity)`, or null.
* @returns {{body: string|Buffer, headers: Record<string,string>}}
*/
export function encodedBody(request, identity, compressed) {
if (compressed && acceptsGzip(request)) {
return {
body: compressed,
headers: { 'Content-Encoding': 'gzip', Vary: 'Accept-Encoding' },
};
}
return { body: identity, headers: { Vary: 'Accept-Encoding' } };
}
+6 -9
View File
@@ -131,9 +131,9 @@ export async function fetchFirms(
* NASA FIRMS live active-fire proxy with a memory + disk cache.
* Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
*
* Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent
* satellites, no cross-source dedup) plus MODIS_NRT (combined Terra+Aqua,
* ~1 km, numeric 0-100 confidence), fetched sequentially (quota courtesy)
* Merges two VIIRS NRT sources (NOAA-20, NOAA-21 — independent satellites, no
* cross-source dedup) plus MODIS_NRT (combined Terra+Aqua, ~1 km, numeric
* 0-100 confidence), fetched sequentially (quota courtesy)
* with `days=2` (`days=1` means "current UTC day", nearly empty just after
* 00:00Z) and clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS
* quota is 5,000 transactions / 10 min per MAP_KEY, so the cache is the point:
@@ -141,6 +141,8 @@ export async function fetchFirms(
* fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream
* fetch across dev-server restarts. Pattern mirrors celestrakProxy.
*
* VIIRS_SNPP_NRT is left out: NASA stopped delivering Suomi NPP on 2026-11-02.
*
* Routes:
* GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires}
* GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions}
@@ -153,12 +155,7 @@ export async function fetchFirms(
export function firmsProxy() {
const TTL_MS = 30 * 60_000;
const STATUS_TTL_MS = 5 * 60_000;
const SOURCES = [
'VIIRS_NOAA20_NRT',
'VIIRS_NOAA21_NRT',
'VIIRS_SNPP_NRT',
'MODIS_NRT',
];
const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'MODIS_NRT'];
/**
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
+30 -1
View File
@@ -64,6 +64,30 @@ function validMessages(messages) {
return !pending.length;
}
/**
* A tool schema without the `pattern`s Chat Completions refuses: it answers
* 400 for a whole request whose tool uses Unicode property classes (`\p{L}`),
* as `resolve_area`'s area ids do. Every argument is still checked against
* the action's own contract before the tool runs, so the model loses a hint,
* not a rule. Realtime voice refuses them too now (#1618) and uses this as well.
*/
export function chatSchema(schema) {
if (Array.isArray(schema)) return schema.map(chatSchema);
if (!object(schema)) return schema;
return Object.fromEntries(
Object.entries(schema)
.filter(
([key, value]) =>
!(
key === 'pattern' &&
typeof value === 'string' &&
/\\[pP]\{/.test(value)
),
)
.map(([key, value]) => [key, chatSchema(value)]),
);
}
export function createOpenAiChatHandler({
annotationGuidance,
tools = GEV_REALTIME_TOOLS,
@@ -126,7 +150,12 @@ export function createOpenAiChatHandler({
tools: offeredTools(tools, { overpass }).map(
({ name, description, parameters }) => ({
type: 'function',
function: { name, description, parameters, strict: false },
function: {
name,
description,
parameters: chatSchema(parameters),
strict: false,
},
}),
),
},
+16 -1
View File
@@ -14,6 +14,7 @@ import {
} from './constants.js';
import { realtimeInstructions } from './instructions.js';
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
import { chatSchema } from './chat.js';
import { resolveOverpassUpstreams } from '../overpass/constants.js';
import {
preferCodexOAuth,
@@ -128,6 +129,19 @@ function createRealtimeOAuthLoginHandler({
};
}
/**
* The tools one Realtime session is offered, as OpenAI accepts them: it now
* refuses a whole session whose tool schema uses `\p{…}` patterns (#1618), as
* Chat Completions did (#1616), so voice drops the same pattern hints chat does.
*/
function realtimeSessionTools(tools, { overpass = false } = {}) {
return offeredTools(tools, { overpass }).map((tool) =>
tool.parameters
? { ...tool, parameters: chatSchema(tool.parameters) }
: tool,
);
}
function createRealtimeTokenHandler({
annotationGuidance,
endpoint = 'https://api.openai.com/v1/realtime/client_secrets',
@@ -303,7 +317,7 @@ function createRealtimeTokenHandler({
output: { voice },
},
instructions: realtimeInstructions(annotationGuidance, { overpass }),
tools: offeredTools(tools, { overpass }),
tools: realtimeSessionTools(tools, { overpass }),
tool_choice: 'auto',
},
};
@@ -370,4 +384,5 @@ export {
createRealtimeOAuthLoginHandler,
createRealtimeOAuthStatusHandler,
createRealtimeTokenHandler,
realtimeSessionTools,
};
+5
View File
@@ -227,6 +227,11 @@ export function installRouteMiddleware(
// only shape that asks the upstream for them. A response WITHOUT steps is
// byte for byte what this endpoint has always returned.
middlewares.use('/api/route', async (req, res) => {
if (req.method !== 'GET') {
res.writeHead(405, { 'Content-Type': 'application/json', Allow: 'GET' });
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
const fail = (msg) => {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ ok: false, error: msg }));
+130 -74
View File
@@ -1,6 +1,9 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
import {
celestrakSatcatUrl,
celestrakTleUrl,
} from '../../../src/data/spaceProviderRequests.js';
import { providerCooldownMs } from './cooldown.js';
import { admitSameSiteKeyless } from '../common/same-site.js';
import { readResponseTextCapped } from '../common/http.js';
@@ -22,84 +25,131 @@ import { readResponseTextCapped } from '../common/http.js';
* 6 h; on upstream failure the freshest stale copy is served (a stale TLE
* beats an empty satellites layer). Pattern mirrors openSkyProxy's
* cache+serve-stale. Adapted from skylight's TleStore (MIT).
*
* The same plugin serves CelesTrak SATCAT records for the satellites layer's
* groups at `/api/celestrak-satcat/<group>` (object type, operational status
* and owner, which file each object under a class). The catalog changes
* slowly, so SATCAT is cached for a day. CelesTrak throttles the client, not
* the endpoint, so a 403 or 429 holds both routes off; any other failure
* holds off only the route that failed, so a bad SATCAT answer never stops
* TLE refreshes.
*/
export function celestrakProxy() {
const TLE_TTL_MS = 6 * 3600_000;
const SATCAT_TTL_MS = 24 * 3600_000;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const groups = new Set([
/** The satellites layer's core groups (src/layers/satellites/policy.js). */
const coreGroups = [
'stations',
'visual',
'gps-ops',
'glo-ops',
'galileo',
'geo',
'starlink',
// Space Missions (`getActiveTle`) reads the full active catalog.
'active',
]);
let cooldownUntil = 0;
const mem = new Map(); // group -> { at: epochMs, body: string }
const inflight = new Map(); // group -> Promise<{at, body}|null>
const diskPath = (group) => path.join(CACHE_DIR, `celestrak-${group}.json`);
async function readDisk(group) {
try {
const parsed = JSON.parse(await fsp.readFile(diskPath(group), 'utf8'));
if (typeof parsed?.body === 'string' && Number.isFinite(parsed?.at))
return parsed;
} catch {
/* no disk cache yet */
}
return null;
}
async function writeDisk(group, entry) {
try {
await fsp.mkdir(CACHE_DIR, { recursive: true });
await fsp.writeFile(diskPath(group), JSON.stringify(entry), 'utf8');
} catch (err) {
console.warn('[celestrak-proxy] cache write failed');
}
}
'resource',
'weather',
'planet',
'sar',
'science',
];
const feeds = [
{
route: '/api/celestrak',
ttlMs: TLE_TTL_MS,
cacheHeader: 'x-tle-cache',
cachedAtHeader: 'X-TLE-Cached-At',
diskName: (group) => `celestrak-${group}.json`,
upstreamUrl: celestrakTleUrl,
groups: new Set([
...coreGroups,
'starlink',
// Space Missions (`getActiveTle`) reads the full active catalog.
'active',
]),
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
validate: (body) => /^1 /m.test(body),
invalid: 'no TLE lines in response',
},
{
route: '/api/celestrak-satcat',
ttlMs: SATCAT_TTL_MS,
cacheHeader: 'x-satcat-cache',
cachedAtHeader: 'X-SATCAT-Cached-At',
diskName: (group) => `celestrak-satcat-${group}.json`,
upstreamUrl: celestrakSatcatUrl,
groups: new Set(coreGroups),
// A real answer, even for an empty group, starts with the CSV header.
validate: (body) => /^OBJECT_NAME,/.test(body),
invalid: 'no SATCAT header in response',
},
];
// CelesTrak throttling (403/429) is per client, so it holds every feed
// off; any other failure holds off only the feed that failed, so a bad
// SATCAT answer never blocks the TLEs the layer cannot do without.
let hostCooldownUntil = 0;
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
async function fetchUpstream(group) {
const url = celestrakTleUrl(group);
const signal = AbortSignal.timeout(20000);
const res = await fetch(url.toString(), {
signal,
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
// descriptive User-Agent with a contact point.
headers: {
'User-Agent':
'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
},
});
if (!res.ok) {
// Group refreshes run in parallel: a later failure without Retry-After
// must never shorten a longer deadline an earlier one already set.
cooldownUntil = Math.max(
cooldownUntil,
Date.now() + providerCooldownMs(res),
);
void res.body?.cancel().catch(() => {});
throw new Error(`HTTP ${res.status}`);
}
const body = await readResponseTextCapped(
res,
CELESTRAK_MAX_RESPONSE_BYTES,
signal,
);
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
return { at: Date.now(), body };
}
function createFeedHandler(feed) {
let feedCooldownUntil = 0;
const cooldownUntil = () => Math.max(hostCooldownUntil, feedCooldownUntil);
const mem = new Map(); // group -> { at: epochMs, body: string }
const inflight = new Map(); // group -> Promise<{at, body}|null>
const diskPath = (group) => path.join(CACHE_DIR, feed.diskName(group));
const installMiddleware = (server) => {
server.middlewares.use('/api/celestrak', async (req, res) => {
async function readDisk(group) {
try {
const parsed = JSON.parse(await fsp.readFile(diskPath(group), 'utf8'));
if (typeof parsed?.body === 'string' && Number.isFinite(parsed?.at))
return parsed;
} catch {
/* no disk cache yet */
}
return null;
}
async function writeDisk(group, entry) {
try {
await fsp.mkdir(CACHE_DIR, { recursive: true });
await fsp.writeFile(diskPath(group), JSON.stringify(entry), 'utf8');
} catch (err) {
console.warn('[celestrak-proxy] cache write failed');
}
}
async function fetchUpstream(group) {
const url = feed.upstreamUrl(group);
const signal = AbortSignal.timeout(20000);
const res = await fetch(url.toString(), {
signal,
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
// descriptive User-Agent with a contact point.
headers: {
'User-Agent':
'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
},
});
if (!res.ok) {
// Group refreshes run in parallel: a later failure without Retry-After
// must never shorten a longer deadline an earlier one already set.
const until = Date.now() + providerCooldownMs(res);
if (res.status === 403 || res.status === 429)
hostCooldownUntil = Math.max(hostCooldownUntil, until);
else feedCooldownUntil = Math.max(feedCooldownUntil, until);
void res.body?.cancel().catch(() => {});
throw new Error(`HTTP ${res.status}`);
}
const body = await readResponseTextCapped(
res,
CELESTRAK_MAX_RESPONSE_BYTES,
signal,
);
if (!feed.validate(body)) throw new Error(feed.invalid);
return { at: Date.now(), body };
}
return async (req, res) => {
const group = String(req.url || '')
.replace(/^\//, '')
.split('?')[0];
@@ -111,7 +161,7 @@ export function celestrakProxy() {
res.end();
return;
}
if (!groups.has(group)) {
if (!feed.groups.has(group)) {
res.writeHead(400, { 'Content-Type': 'text/plain' });
res.end('invalid group');
return;
@@ -123,15 +173,15 @@ export function celestrakProxy() {
if (res.headersSent) return;
res.writeHead(status, {
'Content-Type': 'text/plain',
'x-tle-cache': cacheStatus,
[feed.cacheHeader]: cacheStatus,
'Cache-Control': 'no-store',
...(entry
? { 'X-TLE-Cached-At': new Date(entry.at).toISOString() }
? { [feed.cachedAtHeader]: new Date(entry.at).toISOString() }
: {}),
...(cooldownUntil > 0 && Date.now() < cooldownUntil
...(cooldownUntil() > 0 && Date.now() < cooldownUntil()
? {
'Retry-After': String(
Math.ceil((cooldownUntil - Date.now()) / 1000),
Math.ceil((cooldownUntil() - Date.now()) / 1000),
),
}
: {}),
@@ -145,11 +195,11 @@ export function celestrakProxy() {
entry = await readDisk(group);
if (entry) mem.set(group, entry);
}
if (entry && now - entry.at < TLE_TTL_MS) {
if (entry && now - entry.at < feed.ttlMs) {
send(200, entry.body, 'HIT', entry);
return;
}
if (Date.now() < cooldownUntil) {
if (Date.now() < cooldownUntil()) {
send(
entry ? 200 : 502,
entry?.body || 'celestrak upstream cooling down',
@@ -169,8 +219,8 @@ export function celestrakProxy() {
return fresh;
})
.catch((err) => {
if (Date.now() >= cooldownUntil)
cooldownUntil = Date.now() + 60000;
if (Date.now() >= feedCooldownUntil)
feedCooldownUntil = Date.now() + 60000;
console.warn(
'[celestrak-proxy] refresh failed — serving cache if any',
);
@@ -191,7 +241,13 @@ export function celestrakProxy() {
console.error('[celestrak-proxy] request failed');
send(500, 'celestrak proxy error', 'ERROR');
}
});
};
}
const handlers = feeds.map((feed) => [feed.route, createFeedHandler(feed)]);
const installMiddleware = (server) => {
for (const [route, handler] of handlers)
server.middlewares.use(route, handler);
};
return {
name: 'celestrak-proxy',