mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
Merge branch 'main' into feat/bikeshare-norway
This commit is contained in:
260 files changed
+28882
-1788
No files matched your search
@@ -19,17 +19,21 @@ import {
|
||||
loadCalgarySourcesFromOpenData,
|
||||
loadDelDOTSourcesFromOpenData,
|
||||
loadVegvesenSourcesFromOpenData,
|
||||
loadNycSourcesFromOpenData,
|
||||
} from './sources.js';
|
||||
|
||||
/** Env kill switch: unset or anything but "0" means enabled. */
|
||||
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
|
||||
/** Env opt-in: only "1" enables. For packs whose feed access needs an
|
||||
* operator agreement, so an unset flag never reaches the upstream. */
|
||||
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
|
||||
|
||||
/**
|
||||
* Live open-data packs, in merge order. Adding a region is one entry here
|
||||
* plus its loader in sources.js; the catalog cap is shared across entries
|
||||
* round-robin (cap.js), so a new pack never silently evicts an older one.
|
||||
* Each pack fails independently (allSettled) and is gated by its own env
|
||||
* kill switch.
|
||||
* kill switch, or by an opt-in flag where access needs an agreement.
|
||||
*/
|
||||
const LIVE_PACKS = [
|
||||
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
|
||||
@@ -98,6 +102,13 @@ const LIVE_PACKS = [
|
||||
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
|
||||
load: loadVegvesenSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
// NYC DOT requires a data-sharing agreement for this feed, so the pack is
|
||||
// off unless the operator opts in (see DATA_SOURCES.md).
|
||||
name: 'nyc',
|
||||
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
|
||||
load: loadNycSourcesFromOpenData,
|
||||
},
|
||||
];
|
||||
/**
|
||||
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
|
||||
|
||||
@@ -267,6 +267,36 @@ export const NORWAY_ANCHORS = [
|
||||
{ lat: 69.6492, lon: 18.9553 }, // Tromsø
|
||||
];
|
||||
|
||||
/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the
|
||||
* five boroughs (~980 cameras). Frames are stills synthesized from each
|
||||
* camera UUID on the city's webcam host — never taken from the payload. */
|
||||
export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras';
|
||||
/** The only origin NYC camera frames may come from. The catalog's `imageUrl`
|
||||
* is ignored; the registered path is always this origin plus a validated
|
||||
* UUID plus `/image`. */
|
||||
export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/';
|
||||
export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`;
|
||||
export const NYC_CAMERA_ID_RE =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
export const DEFAULT_NYC_MAX_SOURCES = 250;
|
||||
/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB;
|
||||
* this only exists so an upstream that streams an unbounded body cannot be
|
||||
* buffered without limit. */
|
||||
export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior;
|
||||
* the client's one-shot ground snap corrects it wherever 3D tiles load. */
|
||||
export const NYC_GROUND_ELEVATION_M = 10;
|
||||
/** Prioritization anchors: one per borough so a cap keeps citywide coverage
|
||||
* rather than Midtown alone. */
|
||||
export const NYC_ANCHORS = [
|
||||
{ lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square)
|
||||
{ lat: 40.7128, lon: -74.006 }, // Lower Manhattan
|
||||
{ lat: 40.6782, lon: -73.9442 }, // Brooklyn
|
||||
{ lat: 40.7282, lon: -73.7949 }, // Queens
|
||||
{ lat: 40.8448, lon: -73.8648 }, // Bronx
|
||||
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
|
||||
];
|
||||
|
||||
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
|
||||
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
|
||||
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
|
||||
|
||||
@@ -367,6 +367,17 @@ export function isLikelyCalgaryCoordinate(lat, lon) {
|
||||
);
|
||||
}
|
||||
|
||||
/** New York City's five boroughs, with slack for the harbor and city line. */
|
||||
export function isLikelyNycCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 40.45 &&
|
||||
lat <= 40.95 &&
|
||||
lon >= -74.3 &&
|
||||
lon <= -73.65
|
||||
);
|
||||
}
|
||||
|
||||
/** Mainland Norway, with slack for the coast and the Finnmark border. */
|
||||
export function isLikelyNorwayCoordinate(lat, lon) {
|
||||
return (
|
||||
|
||||
@@ -65,6 +65,13 @@ import {
|
||||
DEFAULT_VEGVESEN_MAX_SOURCES,
|
||||
VEGVESEN_MAX_CATALOG_BYTES,
|
||||
NORWAY_ANCHORS,
|
||||
NYC_CAMERAS_URL,
|
||||
NYC_IMAGE_URL,
|
||||
NYC_CAMERA_ID_RE,
|
||||
DEFAULT_NYC_MAX_SOURCES,
|
||||
NYC_MAX_CATALOG_BYTES,
|
||||
NYC_GROUND_ELEVATION_M,
|
||||
NYC_ANCHORS,
|
||||
} from './constants.js';
|
||||
import {
|
||||
toFiniteNumber,
|
||||
@@ -82,6 +89,7 @@ import {
|
||||
isLikelyTexasCoordinate,
|
||||
isLikelyNswCoordinate,
|
||||
isLikelyCalgaryCoordinate,
|
||||
isLikelyNycCoordinate,
|
||||
isLikelyNorwayCoordinate,
|
||||
cameraDisplayCode,
|
||||
rowArrayToObject,
|
||||
@@ -1863,3 +1871,148 @@ export async function loadVegvesenSourcesFromOpenData() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const NYC_BOROUGHS = new Set([
|
||||
'Manhattan',
|
||||
'Brooklyn',
|
||||
'Queens',
|
||||
'Bronx',
|
||||
'Staten Island',
|
||||
]);
|
||||
|
||||
/** NYC TMC publishes `isOnline` as the strings "true"/"false". */
|
||||
export function nycCameraIsOnline(value) {
|
||||
return (
|
||||
value === true ||
|
||||
String(value ?? '')
|
||||
.trim()
|
||||
.toLowerCase() === 'true'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Label for one NYC camera: the intersection an operator recognises
|
||||
* ("Central Park West @ 86 St"). The catalog's `area` is the borough, not
|
||||
* a facing, and is not part of the name.
|
||||
*
|
||||
* @param {object} record - Raw TMC camera row.
|
||||
* @param {string} cameraId - Derived stable id.
|
||||
* @returns {string}
|
||||
*/
|
||||
export function nycCameraName(record, cameraId) {
|
||||
const name = String(record?.name ?? '').trim();
|
||||
if (name) return name;
|
||||
const short = String(cameraId || '')
|
||||
.replace(/^nyc-/, '')
|
||||
.slice(0, 8);
|
||||
return short ? `NYC Camera ${short}` : 'NYC Camera';
|
||||
}
|
||||
|
||||
/**
|
||||
* One NYC DOT TMC camera row -> one catalog source, or null.
|
||||
*
|
||||
* Frame URLs are synthesized from a validated UUID on the pinned TMC image
|
||||
* origin. The payload's `imageUrl` is ignored so a catalog edit cannot steer
|
||||
* the frame proxy off-host. There is no heading in the feed; headings use
|
||||
* the shared id-hash fallback at low confidence, like Calgary and Fintraffic.
|
||||
*
|
||||
* @param {object} record - Raw TMC camera row.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function nycCameraToSource(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
if (!nycCameraIsOnline(record.isOnline)) return null;
|
||||
|
||||
const lat = toFiniteNumber(record.latitude);
|
||||
const lon = toFiniteNumber(record.longitude);
|
||||
if (!isLikelyNycCoordinate(lat, lon)) return null;
|
||||
|
||||
const uuid = String(record.id ?? '').trim();
|
||||
if (!NYC_CAMERA_ID_RE.test(uuid)) return null;
|
||||
const cameraId = `nyc-${uuid.toLowerCase()}`;
|
||||
const imageUrl = NYC_IMAGE_URL(uuid);
|
||||
const name = nycCameraName(record, cameraId);
|
||||
const area = String(record.area ?? '').trim();
|
||||
const city = NYC_BOROUGHS.has(area) ? area : 'New York';
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name,
|
||||
city,
|
||||
cityId: 'nyc',
|
||||
provider: 'NYC DOT Traffic Management Center',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: NYC_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'nyc-dot-tmc',
|
||||
license: 'Public NYC DOT Traffic Management Center traffic camera',
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch NYC DOT Traffic Management Center cameras from the keyless
|
||||
* `webcams.nyctmc.org` catalog. Frames are stills on that same host,
|
||||
* registered from a validated UUID rather than from the payload URL.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadNycSourcesFromOpenData() {
|
||||
try {
|
||||
const endpoint = process.env.CCTV_NYC_CAMERAS_URL || NYC_CAMERAS_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn('[CCTV] NYC catalog redirected; redirects are not followed');
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] NYC camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const rows = await readResponseJsonCapped(resp, NYC_MAX_CATALOG_BYTES);
|
||||
if (!Array.isArray(rows)) return [];
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const record of rows) {
|
||||
const camera = nycCameraToSource(record);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_NYC_MAX_SOURCES || DEFAULT_NYC_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
|
||||
: DEFAULT_NYC_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, NYC_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded NYC camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn('[CCTV] NYC camera download error:', error?.message || error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
import {
|
||||
GNSS_CLASSIFIER,
|
||||
GNSS_WINDOW_MS,
|
||||
normalizeGnssAircraft,
|
||||
} from '../../src/layers/gnss/records.js';
|
||||
import { adsbLolFallbackAnchor } from './aircraft/opensky.js';
|
||||
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
|
||||
// adsb.lol regional snapshot (keyless, ODbL). Only the integrity fields the
|
||||
// GNSS layer needs leave this proxy, so the browser never sees the full feed.
|
||||
// This is a separate read from the flights fallback on purpose: that point
|
||||
// cache keeps only the fields the flights layer renders (no nic / nac_p), is
|
||||
// keyed on 0.25° anchors and lives for 12 s, while integrity cells need
|
||||
// whole-degree anchors held for a minute.
|
||||
const RADIUS_NM = 250;
|
||||
const CACHE_MS = 60_000;
|
||||
const CACHE_MAX = 64;
|
||||
/** Default 429 cooldown when upstream sends no usable Retry-After. */
|
||||
const COOLDOWN_MS = 60_000;
|
||||
/** Bound sane upstream deadlines to one hour; never shorten a deadline within that range. */
|
||||
const COOLDOWN_MIN_MS = 5_000;
|
||||
const COOLDOWN_MAX_MS = 60 * 60_000;
|
||||
const FAILURE_COOLDOWN_MS = 15_000;
|
||||
const MAX_IN_FLIGHT = 8;
|
||||
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Snap a view anchor to a whole degree so nearby views share one upstream read. */
|
||||
export function gnssIntegrityAnchor(url) {
|
||||
const anchor = adsbLolFallbackAnchor({ url });
|
||||
if (!anchor) return null;
|
||||
return {
|
||||
lat: Math.round(anchor.latitude),
|
||||
lon: Math.round(anchor.longitude),
|
||||
};
|
||||
}
|
||||
|
||||
/** Cooldown (ms) an upstream 429 earns, honouring Retry-After when sane. */
|
||||
export function gnssRetryCooldownMs(raw, nowMs) {
|
||||
if (raw) {
|
||||
const seconds = /^\d+(?:\.\d+)?$/.test(raw.trim()) ? Number(raw) : NaN;
|
||||
const ms = seconds * 1000;
|
||||
if (Number.isFinite(seconds) && seconds > 0)
|
||||
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms));
|
||||
const at =
|
||||
Number.isNaN(seconds) && /[a-z]/i.test(raw) ? Date.parse(raw) : NaN;
|
||||
if (Number.isFinite(at) && at > nowMs)
|
||||
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, at - nowMs));
|
||||
}
|
||||
return COOLDOWN_MS;
|
||||
}
|
||||
|
||||
/** Fixed-origin, bounded adsb.lol integrity route for dev and preview. */
|
||||
export function gnssIntegrityProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
monotonicNow = () => performance.now(),
|
||||
warn = (...args) => console.warn(...args),
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 });
|
||||
let cooldownUntil = 0;
|
||||
let cooldownStatus = 429;
|
||||
|
||||
async function load({ lat, lon }) {
|
||||
if (monotonicNow() < cooldownUntil)
|
||||
throw Object.assign(new Error('cooling_down'), {
|
||||
status: cooldownStatus,
|
||||
});
|
||||
const signal = AbortSignal.timeout(TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetchImpl(
|
||||
`https://api.adsb.lol/v2/lat/${lat}/lon/${lon}/dist/${RADIUS_NM}`,
|
||||
{
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': 'gods-eye-view-gnss-integrity/1.0',
|
||||
},
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
if (response.status === 429) {
|
||||
const at = monotonicNow();
|
||||
const raw = response.headers.get('retry-after');
|
||||
const delay = gnssRetryCooldownMs(raw, now());
|
||||
if (delay === COOLDOWN_MAX_MS)
|
||||
warn('[gnss-integrity] upstream cooldown capped at one hour');
|
||||
cooldownStatus = 429;
|
||||
cooldownUntil = Math.max(cooldownUntil, at + delay);
|
||||
throw Object.assign(new Error('upstream_rate_limited'), {
|
||||
status: 429,
|
||||
});
|
||||
}
|
||||
throw new Error('upstream_unavailable');
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
response,
|
||||
MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const rows = normalizeGnssAircraft(payload);
|
||||
if (!rows) throw new Error('invalid_snapshot');
|
||||
const sourceNow = payload.now;
|
||||
if (!Number.isFinite(sourceNow) || sourceNow <= 0)
|
||||
throw new Error('undated_snapshot');
|
||||
const fetchedAt = Math.min(
|
||||
now(),
|
||||
sourceNow > 10_000_000_000 ? sourceNow : sourceNow * 1000,
|
||||
);
|
||||
if (now() - fetchedAt > GNSS_WINDOW_MS)
|
||||
throw new Error('expired_snapshot');
|
||||
return {
|
||||
fetchedAt,
|
||||
anchor: { lat, lon },
|
||||
radiusNm: RADIUS_NM,
|
||||
// `degraded` on each row is this classifier's verdict, not gpsjam's.
|
||||
classifier: GNSS_CLASSIFIER.id,
|
||||
rows,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error.status !== 429) {
|
||||
const until = monotonicNow() + FAILURE_COOLDOWN_MS;
|
||||
if (until > cooldownUntil) {
|
||||
cooldownUntil = until;
|
||||
cooldownStatus = 502;
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function age(entry) {
|
||||
if (!entry) return Infinity;
|
||||
const elapsed = monotonicNow() - entry.savedAt;
|
||||
// A backwards injected clock cannot rejuvenate a snapshot.
|
||||
return elapsed < 0 ? Infinity : entry.ageAtReceipt + elapsed;
|
||||
}
|
||||
|
||||
async function acquire(anchor, peer) {
|
||||
const key = `${anchor.lat},${anchor.lon}`;
|
||||
const previous = cache.get(key);
|
||||
if (age(previous) < CACHE_MS)
|
||||
return { value: previous.value, ageMs: age(previous), stale: false };
|
||||
try {
|
||||
if (!inFlight.has(key) && inFlight.size >= MAX_IN_FLIGHT)
|
||||
throw Object.assign(new Error('busy'), { status: 429, local: true });
|
||||
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
|
||||
if (monotonicNow() >= cooldownUntil && !allow(peer))
|
||||
throw Object.assign(new Error('rate_limited'), {
|
||||
status: 429,
|
||||
local: true,
|
||||
});
|
||||
const value = await load(anchor);
|
||||
const entry = {
|
||||
value,
|
||||
savedAt: monotonicNow(),
|
||||
ageAtReceipt: Math.max(0, now() - value.fetchedAt),
|
||||
};
|
||||
cache.delete(key);
|
||||
cache.set(key, entry);
|
||||
while (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value);
|
||||
return entry;
|
||||
});
|
||||
const entry = await promise;
|
||||
const ageMs = age(entry);
|
||||
return { value: entry.value, ageMs, stale: ageMs >= CACHE_MS };
|
||||
} catch (error) {
|
||||
if (age(previous) <= GNSS_WINDOW_MS)
|
||||
return { value: previous.value, ageMs: age(previous), stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (status, value, stale = false, retryAfterS = 60) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...([429, 502].includes(status)
|
||||
? { 'Retry-After': String(retryAfterS) }
|
||||
: {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
if (path !== '/' && path !== '')
|
||||
return json(404, { error: 'unknown_route' });
|
||||
const anchor = gnssIntegrityAnchor(req.url);
|
||||
if (!anchor) return json(400, { error: 'invalid_anchor' });
|
||||
try {
|
||||
const { value, stale, ageMs } = await acquire(anchor, clientKey(req));
|
||||
// ageMs is measured on this server's clock, so the browser can date
|
||||
// the rows without comparing its clock to ours.
|
||||
json(200, { ...value, ageMs, ...(stale ? { stale: true } : {}) }, stale);
|
||||
} catch (error) {
|
||||
if (error.status === 429) {
|
||||
// Tell the browser how long the shared upstream cooldown has left.
|
||||
const retryAfterS = Math.max(
|
||||
1,
|
||||
error.local ? 60 : Math.ceil((cooldownUntil - monotonicNow()) / 1000),
|
||||
);
|
||||
return json(
|
||||
429,
|
||||
{
|
||||
error: error.local ? error.message : 'gnss_integrity_rate_limited',
|
||||
},
|
||||
false,
|
||||
retryAfterS,
|
||||
);
|
||||
}
|
||||
json(
|
||||
502,
|
||||
{ error: 'gnss_integrity_unavailable' },
|
||||
false,
|
||||
Math.max(1, Math.ceil((cooldownUntil - monotonicNow()) / 1000)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'gnss-integrity',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/gnss-integrity', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/gnss-integrity', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import { mapillaryProxy } from './mapillary.js';
|
||||
import { keySetupEndpoint } from '../standalone/key-setup.js';
|
||||
import { weatherProxy } from './weather.js';
|
||||
import { firePerimetersProxy } from './firePerimeters.js';
|
||||
import { gnssIntegrityProxy } from './gnssIntegrity.js';
|
||||
import { cycloneProxy } from './cyclones.js';
|
||||
import { windProxy } from './wind.js';
|
||||
|
||||
@@ -60,6 +61,7 @@ function localProviderPlugins({ realtime } = {}) {
|
||||
cycloneProxy(),
|
||||
mapillaryProxy(),
|
||||
firePerimetersProxy(),
|
||||
gnssIntegrityProxy(),
|
||||
keySetupEndpoint(),
|
||||
];
|
||||
}
|
||||
@@ -81,6 +83,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js';
|
||||
export { googlePlacesContextProxy } from './places.js';
|
||||
export { googleServerApiKey } from './places.js';
|
||||
export { keylessGooglePlacesResponse } from './places.js';
|
||||
export { keylessGoogleGeocodeResponse } from './places.js';
|
||||
export { validateGeocodeAddress } from './places.js';
|
||||
export { geocodeBounds } from './places.js';
|
||||
export { adsbLolFallbackAnchor } from './aircraft/opensky.js';
|
||||
export { readResponseTextCapped } from './common/http.js';
|
||||
export { readResponseJsonCapped } from './common/http.js';
|
||||
|
||||
@@ -179,7 +179,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
get_current_view_state: {
|
||||
description:
|
||||
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action.',
|
||||
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action. Tracked aircraft emergency status is a crowd-sourced transponder broadcast, not a confirmed incident: quote emergencyBroadcast and never infer a hijack or an incident ending from a code change.',
|
||||
$position: 1,
|
||||
},
|
||||
set_hud: {
|
||||
@@ -543,7 +543,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
analyst_query: {
|
||||
description:
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values).',
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
/** Place-search and route middleware for Node development servers. */
|
||||
export { googlePlacesContextProxy } from './places/google.js';
|
||||
export {
|
||||
googlePlacesContextProxy,
|
||||
validateGeocodeAddress,
|
||||
geocodeBounds,
|
||||
} from './places/google.js';
|
||||
export {
|
||||
googleServerApiKey,
|
||||
keylessGooglePlacesResponse,
|
||||
keylessGoogleGeocodeResponse,
|
||||
} from './places/google-key.js';
|
||||
export { installRouteMiddleware } from './places/routes.js';
|
||||
export {
|
||||
|
||||
@@ -14,12 +14,34 @@ export function keylessGooglePlacesResponse(apiKey) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Google API key for the SERVER-SIDE calls (Places nearby/text search, the
|
||||
* CCTV Street View fallback). These never reach the browser, so this key can
|
||||
* be restricted by server IP and scoped to Places API + Street View Static
|
||||
* API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles +
|
||||
* Geocoding for the browser (#33). Splitting them is opt-in: unset, this
|
||||
* falls back to the shared browser key and nothing changes.
|
||||
* Geocoding speaks Google's own response shape rather than the `places: []`
|
||||
* contract its two siblings use, because the browser adapters read `status`
|
||||
* and `results` directly. The keyless capability reads the same way: a status
|
||||
* that is not `OK` and no results, which the adapters already treat as "this
|
||||
* provider did not answer" and pass to the next geocoder in the chain.
|
||||
*/
|
||||
export function keylessGoogleGeocodeResponse(apiKey) {
|
||||
if (String(apiKey ?? '').trim()) return null;
|
||||
return {
|
||||
statusCode: 200,
|
||||
payload: {
|
||||
configured: false,
|
||||
status: 'REQUEST_DENIED',
|
||||
results: [],
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Google API key for the SERVER-SIDE calls (Places nearby/text search,
|
||||
* geocoding, the CCTV Street View fallback). These never reach the browser, so
|
||||
* this key can be restricted by server IP and scoped to Places API + Geocoding
|
||||
* API + Street View Static API — while GOOGLE_MAPS_API_KEY stays
|
||||
* referrer-restricted to Map Tiles + Maps JavaScript API for the browser
|
||||
* (#33, #363). Splitting them is opt-in: unset, this falls back to the shared
|
||||
* browser key, which then has to carry every API both sides use and cannot
|
||||
* take a referrer restriction.
|
||||
*/
|
||||
export function googleServerApiKey() {
|
||||
return resolveGoogleServerKey(process.env);
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import {
|
||||
googleServerApiKey,
|
||||
keylessGooglePlacesResponse,
|
||||
keylessGoogleGeocodeResponse,
|
||||
} from './google-key.js';
|
||||
import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { admitSameSite } from '../common/same-site.js';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
import {
|
||||
projectNearbyPlaces,
|
||||
projectTextSearchPlaces,
|
||||
@@ -25,7 +27,7 @@ let _googleRateLimiter;
|
||||
*/
|
||||
export const GOOGLE_DEFAULT_PER_MIN = 120;
|
||||
|
||||
/** Google cost endpoints (nearby-places + text-search). Null only when set to 0. */
|
||||
/** Google cost endpoints (nearby-places, text-search, geocoding). Null only when set to 0. */
|
||||
function googleRateLimiter() {
|
||||
if (_googleRateLimiter === undefined)
|
||||
_googleRateLimiter = makeCostRateLimiter(
|
||||
@@ -56,6 +58,51 @@ export function validatePlacesCoordinates(searchParams) {
|
||||
return { ok: true, latitude, longitude };
|
||||
}
|
||||
|
||||
/** Longest accepted geocoder query. Real place names are far shorter. */
|
||||
const GEOCODE_MAX_QUERY = 200;
|
||||
|
||||
/**
|
||||
* Upstream deadline and body ceiling for the geocode routes. The deadline stays
|
||||
* armed through body consumption, so a stalled upstream cannot hold a handler
|
||||
* open, and the cap bounds a chunked or length-omitted reply. A geocode answer
|
||||
* is a handful of results; 256 KiB is far above any honest one.
|
||||
*/
|
||||
const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000;
|
||||
|
||||
const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024;
|
||||
|
||||
/** Validate a free-text geocoder query before consuming request quota. */
|
||||
export function validateGeocodeAddress(searchParams) {
|
||||
const address = String(searchParams.get('address') || '').trim();
|
||||
if (!address) return { ok: false, error: 'address is required' };
|
||||
if (address.length > GEOCODE_MAX_QUERY)
|
||||
return { ok: false, error: 'address is too long' };
|
||||
return { ok: true, address };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the optional viewport bias, which Google reads as
|
||||
* `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than
|
||||
* refused: the bias only ranks results, so a malformed one costs relevance,
|
||||
* not an answer.
|
||||
*/
|
||||
export function geocodeBounds(searchParams) {
|
||||
const raw = String(searchParams.get('bounds') || '').trim();
|
||||
if (!raw) return null;
|
||||
const corners = raw.split('|');
|
||||
if (corners.length !== 2) return null;
|
||||
const parts = corners.flatMap((corner) => corner.split(','));
|
||||
// A blank component must be refused before Number() turns it into 0 and a
|
||||
// half-written corner like `30.4,` reads as the equator.
|
||||
if (parts.length !== 4 || parts.some((part) => !part.trim())) return null;
|
||||
const numbers = parts.map(Number);
|
||||
if (!numbers.every(Number.isFinite)) return null;
|
||||
const [swLat, swLon, neLat, neLon] = numbers;
|
||||
if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null;
|
||||
if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null;
|
||||
return raw;
|
||||
}
|
||||
|
||||
/** Nearby place labels and view-biased text search, with request-time key resolution. */
|
||||
export function googlePlacesContextProxy({
|
||||
resolveApiKey = googleServerApiKey,
|
||||
@@ -293,6 +340,149 @@ export function googlePlacesContextProxy({
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Geocoding belongs on the server, not in the page. Google's Geocoding web
|
||||
* service refuses referrer-restricted keys, so calling it from the browser
|
||||
* forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be
|
||||
* left unrestricted (#363). Here the request carries the server key, and
|
||||
* the browser key no longer needs the Geocoding API at all.
|
||||
*
|
||||
* Unlike the two Places routes above, these answer in Google's own shape
|
||||
* (`status` + `results`) rather than the `places: []` contract: the browser
|
||||
* normalizers read that shape directly, and keeping it means the proxy adds
|
||||
* a hop without adding a translation nobody asked for.
|
||||
*/
|
||||
function installGeocodeRoute(path, readQuery) {
|
||||
middlewares.use(path, async (req, res) => {
|
||||
const refuse = (statusCode, error) => {
|
||||
res.statusCode = statusCode;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }),
|
||||
);
|
||||
};
|
||||
|
||||
// Gate first, like the Places routes above: a cross-site caller learns
|
||||
// nothing about this endpoint's method surface.
|
||||
if (admitSameSite(req, res)) return;
|
||||
if (req.method !== 'GET') {
|
||||
refuse(405, 'Method not allowed');
|
||||
return;
|
||||
}
|
||||
|
||||
// A keyless geocode has no provider cost, so it resolves before the
|
||||
// paid-endpoint limiter can consume or exhaust quota (mirrors the
|
||||
// Places routes above).
|
||||
const apiKey = resolveApiKey();
|
||||
const keyless = keylessGoogleGeocodeResponse(apiKey);
|
||||
if (keyless) {
|
||||
res.statusCode = keyless.statusCode;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(JSON.stringify(keyless.payload));
|
||||
return;
|
||||
}
|
||||
|
||||
const requestUrl = new URL(req.url || '', 'http://localhost');
|
||||
const query = readQuery(requestUrl.searchParams);
|
||||
if (!query.ok) {
|
||||
refuse(400, query.error);
|
||||
return;
|
||||
}
|
||||
|
||||
// Per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with the
|
||||
// Places routes because both spend the same Google budget.
|
||||
const _grl = googleRateLimiter();
|
||||
if (_grl && !_grl(clientKey(req))) {
|
||||
res.statusCode = 429;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Retry-After', '5');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'OVER_QUERY_LIMIT',
|
||||
results: [],
|
||||
error: 'Rate limit exceeded',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const upstream = new URL(
|
||||
endpoints.geocode ||
|
||||
'https://maps.googleapis.com/maps/api/geocode/json',
|
||||
);
|
||||
for (const [name, value] of Object.entries(query.params))
|
||||
upstream.searchParams.set(name, value);
|
||||
upstream.searchParams.set('key', apiKey);
|
||||
const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS);
|
||||
const response = await fetchImpl(upstream.toString(), {
|
||||
redirect: 'error',
|
||||
signal: deadline,
|
||||
});
|
||||
// An unreadable, oversized or timed-out body is not an answer: it
|
||||
// falls through to the same shaped failure as a refusal below.
|
||||
const data = await readResponseJsonCapped(
|
||||
response,
|
||||
GEOCODE_MAX_RESPONSE_BYTES,
|
||||
deadline,
|
||||
).catch(() => null);
|
||||
|
||||
// A body we could not read is an upstream failure, not an answer with
|
||||
// a status of its own, so it does not borrow the upstream's code.
|
||||
if (!data) {
|
||||
res.statusCode = 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'UNKNOWN_ERROR',
|
||||
results: [],
|
||||
error: 'Google Geocoding request failed',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = response.ok ? 200 : response.status;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'private, max-age=300');
|
||||
res.end(JSON.stringify(data));
|
||||
} catch {
|
||||
// Fixed public text: a transport, URL or runtime message would
|
||||
// describe the server to the browser (SECURITY.md, sanitized errors).
|
||||
res.statusCode = 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'UNKNOWN_ERROR',
|
||||
results: [],
|
||||
error: 'Google Geocoding request failed',
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
installGeocodeRoute('/api/google/geocode', (searchParams) => {
|
||||
const address = validateGeocodeAddress(searchParams);
|
||||
if (!address.ok) return address;
|
||||
const bounds = geocodeBounds(searchParams);
|
||||
return {
|
||||
ok: true,
|
||||
params: {
|
||||
address: address.address,
|
||||
...(bounds ? { bounds } : {}),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => {
|
||||
const coordinates = validatePlacesCoordinates(searchParams);
|
||||
if (!coordinates.ok) return coordinates;
|
||||
const { latitude, longitude } = coordinates;
|
||||
return { ok: true, params: { latlng: `${latitude},${longitude}` } };
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -142,9 +142,11 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
|
||||
const rows = Array.from({ length: 15000 }, (_, i) =>
|
||||
row(now, `vehicle-${i}`),
|
||||
);
|
||||
const start = performance.now();
|
||||
// Main-thread CPU: excludes scheduler waits and parallel GC helper threads.
|
||||
const start = process.threadCpuUsage();
|
||||
h.ingest(feed, rows, now);
|
||||
times.push(performance.now() - start);
|
||||
const usage = process.threadCpuUsage(start);
|
||||
times.push((usage.user + usage.system) / 1000);
|
||||
}
|
||||
assert.equal(h.diagnostics().keys, 15000);
|
||||
for (let i = 0; i < 15000; i++) {
|
||||
@@ -154,12 +156,12 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
|
||||
}
|
||||
assert.ok(h.diagnostics().allocatedBytes < 16 * 1024 * 1024);
|
||||
t.diagnostic(
|
||||
`15000-vehicle ingestion ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
|
||||
`15000-vehicle ingestion CPU ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
|
||||
);
|
||||
// A ceiling that says "not quadratic", not a benchmark: 15,000 rows in
|
||||
// well under two seconds on a loaded laptop or a slow CI runner. The
|
||||
// per-run figure above is the diagnostic to read.
|
||||
assert.ok(Math.max(...times) < 2000, `ingestion ms: ${times}`);
|
||||
// CPU time measures ingestion work without counting scheduler delays on
|
||||
// loaded runners. Keep the two-second ceiling as a guard against quadratic
|
||||
// work, not a benchmark; the per-poll figures above show any slowdown.
|
||||
assert.ok(Math.max(...times) < 2000, `ingestion CPU ms: ${times}`);
|
||||
});
|
||||
|
||||
test('snapshot capacity pressure preserves members not yet visited and discloses recreated history', (t) => {
|
||||
|
||||
Reference in new issue
Block a user