Merge branch 'main' into fix/radio-mirror-null-body

This commit is contained in:
Said Turksever authored and GitHub committed 2026-10-15 14:33:30 +01:00
commit cebd870b3b
229 files changed
+27782 -1699

No files matched your search

+12 -1
View File
@@ -19,17 +19,21 @@ import {
loadCalgarySourcesFromOpenData,
loadDelDOTSourcesFromOpenData,
loadVegvesenSourcesFromOpenData,
loadNycSourcesFromOpenData,
} from './sources.js';
/** Env kill switch: unset or anything but "0" means enabled. */
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
/** Env opt-in: only "1" enables. For packs whose feed access needs an
* operator agreement, so an unset flag never reaches the upstream. */
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
/**
* Live open-data packs, in merge order. Adding a region is one entry here
* plus its loader in sources.js; the catalog cap is shared across entries
* round-robin (cap.js), so a new pack never silently evicts an older one.
* Each pack fails independently (allSettled) and is gated by its own env
* kill switch.
* kill switch, or by an opt-in flag where access needs an agreement.
*/
const LIVE_PACKS = [
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
@@ -98,6 +102,13 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
load: loadVegvesenSourcesFromOpenData,
},
{
// NYC DOT requires a data-sharing agreement for this feed, so the pack is
// off unless the operator opts in (see DATA_SOURCES.md).
name: 'nyc',
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
load: loadNycSourcesFromOpenData,
},
];
/**
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
+30
View File
@@ -267,6 +267,36 @@ export const NORWAY_ANCHORS = [
{ lat: 69.6492, lon: 18.9553 }, // Tromsø
];
/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the
* five boroughs (~980 cameras). Frames are stills synthesized from each
* camera UUID on the city's webcam host — never taken from the payload. */
export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras';
/** The only origin NYC camera frames may come from. The catalog's `imageUrl`
* is ignored; the registered path is always this origin plus a validated
* UUID plus `/image`. */
export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/';
export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`;
export const NYC_CAMERA_ID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export const DEFAULT_NYC_MAX_SOURCES = 250;
/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB;
* this only exists so an upstream that streams an unbounded body cannot be
* buffered without limit. */
export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior;
* the client's one-shot ground snap corrects it wherever 3D tiles load. */
export const NYC_GROUND_ELEVATION_M = 10;
/** Prioritization anchors: one per borough so a cap keeps citywide coverage
* rather than Midtown alone. */
export const NYC_ANCHORS = [
{ lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square)
{ lat: 40.7128, lon: -74.006 }, // Lower Manhattan
{ lat: 40.6782, lon: -73.9442 }, // Brooklyn
{ lat: 40.7282, lon: -73.7949 }, // Queens
{ lat: 40.8448, lon: -73.8648 }, // Bronx
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
];
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
+11
View File
@@ -367,6 +367,17 @@ export function isLikelyCalgaryCoordinate(lat, lon) {
);
}
/** New York City's five boroughs, with slack for the harbor and city line. */
export function isLikelyNycCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 40.45 &&
lat <= 40.95 &&
lon >= -74.3 &&
lon <= -73.65
);
}
/** Mainland Norway, with slack for the coast and the Finnmark border. */
export function isLikelyNorwayCoordinate(lat, lon) {
return (
+153
View File
@@ -65,6 +65,13 @@ import {
DEFAULT_VEGVESEN_MAX_SOURCES,
VEGVESEN_MAX_CATALOG_BYTES,
NORWAY_ANCHORS,
NYC_CAMERAS_URL,
NYC_IMAGE_URL,
NYC_CAMERA_ID_RE,
DEFAULT_NYC_MAX_SOURCES,
NYC_MAX_CATALOG_BYTES,
NYC_GROUND_ELEVATION_M,
NYC_ANCHORS,
} from './constants.js';
import {
toFiniteNumber,
@@ -82,6 +89,7 @@ import {
isLikelyTexasCoordinate,
isLikelyNswCoordinate,
isLikelyCalgaryCoordinate,
isLikelyNycCoordinate,
isLikelyNorwayCoordinate,
cameraDisplayCode,
rowArrayToObject,
@@ -1863,3 +1871,148 @@ export async function loadVegvesenSourcesFromOpenData() {
return [];
}
}
const NYC_BOROUGHS = new Set([
'Manhattan',
'Brooklyn',
'Queens',
'Bronx',
'Staten Island',
]);
/** NYC TMC publishes `isOnline` as the strings "true"/"false". */
export function nycCameraIsOnline(value) {
return (
value === true ||
String(value ?? '')
.trim()
.toLowerCase() === 'true'
);
}
/**
* Label for one NYC camera: the intersection an operator recognises
* ("Central Park West @ 86 St"). The catalog's `area` is the borough, not
* a facing, and is not part of the name.
*
* @param {object} record - Raw TMC camera row.
* @param {string} cameraId - Derived stable id.
* @returns {string}
*/
export function nycCameraName(record, cameraId) {
const name = String(record?.name ?? '').trim();
if (name) return name;
const short = String(cameraId || '')
.replace(/^nyc-/, '')
.slice(0, 8);
return short ? `NYC Camera ${short}` : 'NYC Camera';
}
/**
* One NYC DOT TMC camera row -> one catalog source, or null.
*
* Frame URLs are synthesized from a validated UUID on the pinned TMC image
* origin. The payload's `imageUrl` is ignored so a catalog edit cannot steer
* the frame proxy off-host. There is no heading in the feed; headings use
* the shared id-hash fallback at low confidence, like Calgary and Fintraffic.
*
* @param {object} record - Raw TMC camera row.
* @returns {?object}
*/
export function nycCameraToSource(record) {
if (!record || typeof record !== 'object') return null;
if (!nycCameraIsOnline(record.isOnline)) return null;
const lat = toFiniteNumber(record.latitude);
const lon = toFiniteNumber(record.longitude);
if (!isLikelyNycCoordinate(lat, lon)) return null;
const uuid = String(record.id ?? '').trim();
if (!NYC_CAMERA_ID_RE.test(uuid)) return null;
const cameraId = `nyc-${uuid.toLowerCase()}`;
const imageUrl = NYC_IMAGE_URL(uuid);
const name = nycCameraName(record, cameraId);
const area = String(record.area ?? '').trim();
const city = NYC_BOROUGHS.has(area) ? area : 'New York';
return {
id: cameraId,
name,
city,
cityId: 'nyc',
provider: 'NYC DOT Traffic Management Center',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: NYC_GROUND_ELEVATION_M,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'nyc-dot-tmc',
license: 'Public NYC DOT Traffic Management Center traffic camera',
code: cameraDisplayCode(name.toUpperCase()),
};
}
/**
* Fetch NYC DOT Traffic Management Center cameras from the keyless
* `webcams.nyctmc.org` catalog. Frames are stills on that same host,
* registered from a validated UUID rather than from the payload URL.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadNycSourcesFromOpenData() {
try {
const endpoint = process.env.CCTV_NYC_CAMERAS_URL || NYC_CAMERAS_URL;
const resp = await fetch(endpoint, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn('[CCTV] NYC catalog redirected; redirects are not followed');
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] NYC camera download failed:', resp.status);
return discard();
}
const rows = await readResponseJsonCapped(resp, NYC_MAX_CATALOG_BYTES);
if (!Array.isArray(rows)) return [];
const cameras = [];
const seen = new Set();
for (const record of rows) {
const camera = nycCameraToSource(record);
if (!camera || seen.has(camera.id)) continue;
seen.add(camera.id);
cameras.push(camera);
}
const maxRaw = Number(
process.env.CCTV_NYC_MAX_SOURCES || DEFAULT_NYC_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
: DEFAULT_NYC_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, NYC_ANCHORS);
console.log(
`[CCTV] Loaded NYC camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn('[CCTV] NYC camera download error:', error?.message || error);
return [];
}
}
+238
View File
@@ -0,0 +1,238 @@
import {
GNSS_CLASSIFIER,
GNSS_WINDOW_MS,
normalizeGnssAircraft,
} from '../../src/layers/gnss/records.js';
import { adsbLolFallbackAnchor } from './aircraft/opensky.js';
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
// adsb.lol regional snapshot (keyless, ODbL). Only the integrity fields the
// GNSS layer needs leave this proxy, so the browser never sees the full feed.
// This is a separate read from the flights fallback on purpose: that point
// cache keeps only the fields the flights layer renders (no nic / nac_p), is
// keyed on 0.25° anchors and lives for 12 s, while integrity cells need
// whole-degree anchors held for a minute.
const RADIUS_NM = 250;
const CACHE_MS = 60_000;
const CACHE_MAX = 64;
/** Default 429 cooldown when upstream sends no usable Retry-After. */
const COOLDOWN_MS = 60_000;
/** Bound sane upstream deadlines to one hour; never shorten a deadline within that range. */
const COOLDOWN_MIN_MS = 5_000;
const COOLDOWN_MAX_MS = 60 * 60_000;
const FAILURE_COOLDOWN_MS = 15_000;
const MAX_IN_FLIGHT = 8;
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
const TIMEOUT_MS = 10_000;
/** Snap a view anchor to a whole degree so nearby views share one upstream read. */
export function gnssIntegrityAnchor(url) {
const anchor = adsbLolFallbackAnchor({ url });
if (!anchor) return null;
return {
lat: Math.round(anchor.latitude),
lon: Math.round(anchor.longitude),
};
}
/** Cooldown (ms) an upstream 429 earns, honouring Retry-After when sane. */
export function gnssRetryCooldownMs(raw, nowMs) {
if (raw) {
const seconds = /^\d+(?:\.\d+)?$/.test(raw.trim()) ? Number(raw) : NaN;
const ms = seconds * 1000;
if (Number.isFinite(seconds) && seconds > 0)
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms));
const at =
Number.isNaN(seconds) && /[a-z]/i.test(raw) ? Date.parse(raw) : NaN;
if (Number.isFinite(at) && at > nowMs)
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, at - nowMs));
}
return COOLDOWN_MS;
}
/** Fixed-origin, bounded adsb.lol integrity route for dev and preview. */
export function gnssIntegrityProxy({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
monotonicNow = () => performance.now(),
warn = (...args) => console.warn(...args),
} = {}) {
const cache = new Map();
const inFlight = new Map();
const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 });
let cooldownUntil = 0;
let cooldownStatus = 429;
async function load({ lat, lon }) {
if (monotonicNow() < cooldownUntil)
throw Object.assign(new Error('cooling_down'), {
status: cooldownStatus,
});
const signal = AbortSignal.timeout(TIMEOUT_MS);
try {
const response = await fetchImpl(
`https://api.adsb.lol/v2/lat/${lat}/lon/${lon}/dist/${RADIUS_NM}`,
{
signal,
redirect: 'error',
headers: {
Accept: 'application/json',
'User-Agent': 'gods-eye-view-gnss-integrity/1.0',
},
},
);
if (!response.ok) {
await response.body?.cancel();
if (response.status === 429) {
const at = monotonicNow();
const raw = response.headers.get('retry-after');
const delay = gnssRetryCooldownMs(raw, now());
if (delay === COOLDOWN_MAX_MS)
warn('[gnss-integrity] upstream cooldown capped at one hour');
cooldownStatus = 429;
cooldownUntil = Math.max(cooldownUntil, at + delay);
throw Object.assign(new Error('upstream_rate_limited'), {
status: 429,
});
}
throw new Error('upstream_unavailable');
}
const payload = await readResponseJsonCapped(
response,
MAX_RESPONSE_BYTES,
signal,
);
const rows = normalizeGnssAircraft(payload);
if (!rows) throw new Error('invalid_snapshot');
const sourceNow = payload.now;
if (!Number.isFinite(sourceNow) || sourceNow <= 0)
throw new Error('undated_snapshot');
const fetchedAt = Math.min(
now(),
sourceNow > 10_000_000_000 ? sourceNow : sourceNow * 1000,
);
if (now() - fetchedAt > GNSS_WINDOW_MS)
throw new Error('expired_snapshot');
return {
fetchedAt,
anchor: { lat, lon },
radiusNm: RADIUS_NM,
// `degraded` on each row is this classifier's verdict, not gpsjam's.
classifier: GNSS_CLASSIFIER.id,
rows,
};
} catch (error) {
if (error.status !== 429) {
const until = monotonicNow() + FAILURE_COOLDOWN_MS;
if (until > cooldownUntil) {
cooldownUntil = until;
cooldownStatus = 502;
}
}
throw error;
}
}
function age(entry) {
if (!entry) return Infinity;
const elapsed = monotonicNow() - entry.savedAt;
// A backwards injected clock cannot rejuvenate a snapshot.
return elapsed < 0 ? Infinity : entry.ageAtReceipt + elapsed;
}
async function acquire(anchor, peer) {
const key = `${anchor.lat},${anchor.lon}`;
const previous = cache.get(key);
if (age(previous) < CACHE_MS)
return { value: previous.value, ageMs: age(previous), stale: false };
try {
if (!inFlight.has(key) && inFlight.size >= MAX_IN_FLIGHT)
throw Object.assign(new Error('busy'), { status: 429, local: true });
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
if (monotonicNow() >= cooldownUntil && !allow(peer))
throw Object.assign(new Error('rate_limited'), {
status: 429,
local: true,
});
const value = await load(anchor);
const entry = {
value,
savedAt: monotonicNow(),
ageAtReceipt: Math.max(0, now() - value.fetchedAt),
};
cache.delete(key);
cache.set(key, entry);
while (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value);
return entry;
});
const entry = await promise;
const ageMs = age(entry);
return { value: entry.value, ageMs, stale: ageMs >= CACHE_MS };
} catch (error) {
if (age(previous) <= GNSS_WINDOW_MS)
return { value: previous.value, ageMs: age(previous), stale: true };
throw error;
}
}
async function handler(req, res) {
const json = (status, value, stale = false, retryAfterS = 60) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...([429, 502].includes(status)
? { 'Retry-After': String(retryAfterS) }
: {}),
...(stale ? { 'X-Data-Stale': 'true' } : {}),
});
res.end(JSON.stringify(value));
};
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
const path = (req.url || '/').split('?')[0];
if (path !== '/' && path !== '')
return json(404, { error: 'unknown_route' });
const anchor = gnssIntegrityAnchor(req.url);
if (!anchor) return json(400, { error: 'invalid_anchor' });
try {
const { value, stale, ageMs } = await acquire(anchor, clientKey(req));
// ageMs is measured on this server's clock, so the browser can date
// the rows without comparing its clock to ours.
json(200, { ...value, ageMs, ...(stale ? { stale: true } : {}) }, stale);
} catch (error) {
if (error.status === 429) {
// Tell the browser how long the shared upstream cooldown has left.
const retryAfterS = Math.max(
1,
error.local ? 60 : Math.ceil((cooldownUntil - monotonicNow()) / 1000),
);
return json(
429,
{
error: error.local ? error.message : 'gnss_integrity_rate_limited',
},
false,
retryAfterS,
);
}
json(
502,
{ error: 'gnss_integrity_unavailable' },
false,
Math.max(1, Math.ceil((cooldownUntil - monotonicNow()) / 1000)),
);
}
}
return {
name: 'gnss-integrity',
configureServer({ middlewares }) {
middlewares.use('/api/gnss-integrity', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/gnss-integrity', handler);
},
};
}
+2
View File
@@ -24,6 +24,7 @@ import { mapillaryProxy } from './mapillary.js';
import { keySetupEndpoint } from '../standalone/key-setup.js';
import { weatherProxy } from './weather.js';
import { firePerimetersProxy } from './firePerimeters.js';
import { gnssIntegrityProxy } from './gnssIntegrity.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
@@ -60,6 +61,7 @@ function localProviderPlugins({ realtime } = {}) {
cycloneProxy(),
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
keySetupEndpoint(),
];
}
+2 -2
View File
@@ -179,7 +179,7 @@ export const ACTION_DESCRIPTIONS = {
},
get_current_view_state: {
description:
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action.',
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action. Tracked aircraft emergency status is a crowd-sourced transponder broadcast, not a confirmed incident: quote emergencyBroadcast and never infer a hijack or an incident ending from a code change.',
$position: 1,
},
set_hud: {
@@ -543,7 +543,7 @@ export const ACTION_DESCRIPTIONS = {
},
analyst_query: {
description:
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values).',
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
$position: 1,
parameters: {
properties: {
+9 -7
View File
@@ -142,9 +142,11 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
const rows = Array.from({ length: 15000 }, (_, i) =>
row(now, `vehicle-${i}`),
);
const start = performance.now();
// Main-thread CPU: excludes scheduler waits and parallel GC helper threads.
const start = process.threadCpuUsage();
h.ingest(feed, rows, now);
times.push(performance.now() - start);
const usage = process.threadCpuUsage(start);
times.push((usage.user + usage.system) / 1000);
}
assert.equal(h.diagnostics().keys, 15000);
for (let i = 0; i < 15000; i++) {
@@ -154,12 +156,12 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
}
assert.ok(h.diagnostics().allocatedBytes < 16 * 1024 * 1024);
t.diagnostic(
`15000-vehicle ingestion ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
`15000-vehicle ingestion CPU ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
);
// A ceiling that says "not quadratic", not a benchmark: 15,000 rows in
// well under two seconds on a loaded laptop or a slow CI runner. The
// per-run figure above is the diagnostic to read.
assert.ok(Math.max(...times) < 2000, `ingestion ms: ${times}`);
// CPU time measures ingestion work without counting scheduler delays on
// loaded runners. Keep the two-second ceiling as a guard against quadratic
// work, not a benchmark; the per-poll figures above show any slowdown.
assert.ok(Math.max(...times) < 2000, `ingestion CPU ms: ${times}`);
});
test('snapshot capacity pressure preserves members not yet visited and discloses recreated history', (t) => {