Merge remote-tracking branch 'origin/main' into fix/cctv-streetview-fallback-catalog

# Conflicts:
#	SECURITY.md
This commit is contained in:
German Khokhlov committed 2026-11-19 19:12:56 +04:00
commit 01368b95bf
3201 files changed
+430431 -53001

No files matched your search

+9
View File
@@ -83,6 +83,15 @@ export function adsbLolProxy() {
// This mount also matches its /track route, served by its own handler.
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
return next();
// After the /track hand-off, so /track keeps its own method handling.
if (req.method !== 'GET') {
res.writeHead(405, {
'Content-Type': 'application/json',
Allow: 'GET',
});
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
const now = Date.now();
if (_cache && now - _cacheAt < CACHE_MS) {
serve(res, 200, _cache, 'HIT');
+69 -22
View File
@@ -1,6 +1,7 @@
import fs from 'node:fs';
import os from 'node:os';
import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback.js';
import { encodedBody, gzipBody } from '../common/compress.js';
import {
coalesceProxyRequest,
readResponseJsonCapped,
@@ -18,6 +19,11 @@ let _openskyTokenExpiry = 0;
let _openskyTokenPromise = null;
/** @type {string|null} Cached upstream response body (JSON text). */
let _openskyCacheBody = null;
/**
* @type {Buffer|null} Gzip of `_openskyCacheBody`, written together with it.
* Null when the body is not worth compressing.
*/
let _openskyCacheGzip = null;
/** @type {number} HTTP status of the cached response. */
let _openskyCacheStatus = 0;
/** @type {number} Epoch-ms when the response was cached. */
@@ -399,8 +405,12 @@ async function fetchAdsbLolPointFallback(req) {
ADSBLOL_POINT_MAX_RESPONSE_BYTES,
);
const normalized = normalizeAdsbLolPointResponse(payload);
const body = JSON.stringify(normalized);
const record = {
body: JSON.stringify(normalized),
body,
// Compressed once per entry; ADSBLOL_POINT_CACHE_MAX bounds the
// extra memory.
gzip: await gzipBody(body),
cachedAt: Date.now(),
count: normalized.states.length,
};
@@ -429,6 +439,7 @@ async function fetchAdsbLolPointFallback(req) {
async function serveAdsbLolPointFallback(req, res, requestedMode, reason) {
const fallback = await fetchAdsbLolPointFallback(req);
if (!fallback) return false;
const encoded = encodedBody(req, fallback.body, fallback.gzip);
res.writeHead(200, {
...buildOpenSkyHeaders({
cacheStatus: fallback.cacheStatus,
@@ -439,8 +450,9 @@ async function serveAdsbLolPointFallback(req, res, requestedMode, reason) {
'X-Flight-Source': 'adsb.lol',
'X-Flight-Coverage': `${ADSBLOL_POINT_RADIUS_NM}nm regional fallback`,
'X-Flight-Count': String(fallback.count),
...encoded.headers,
});
res.end(fallback.body);
res.end(encoded.body);
return true;
}
@@ -484,6 +496,15 @@ export function openSkyProxy() {
// This mount also matches its /track route, served by its own handler.
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
return next();
// After the /track hand-off, so /track keeps its own method handling.
if (req.method !== 'GET') {
res.writeHead(405, {
'Content-Type': 'application/json',
Allow: 'GET',
});
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
res.setHeader('X-Feed-Source', 'OpenSky Network');
const requestedMode = normalizeOpenSkyAuthMode(
process.env.OPENSKY_AUTH_MODE,
@@ -514,9 +535,13 @@ export function openSkyProxy() {
reason: 'cached',
};
const isStale = now - _openskyCacheTime >= _openskyTtlMs;
res.writeHead(
_openskyCacheStatus || 200,
buildOpenSkyHeaders({
const encoded = encodedBody(
req,
_openskyCacheBody,
_openskyCacheGzip,
);
res.writeHead(_openskyCacheStatus || 200, {
...buildOpenSkyHeaders({
cacheStatus: isStale ? 'STALE' : 'HIT',
requestedMode: cachedMeta.requestedMode || requestedMode,
usedMode: cachedMeta.usedMode || 'unknown',
@@ -532,8 +557,9 @@ export function openSkyProxy() {
? (_openskyCooldownUntil - now) / 1000
: undefined,
}),
);
res.end(_openskyCacheBody);
...encoded.headers,
});
res.end(encoded.body);
return;
}
// Cooling down with nothing cached (cold start into a rate limit):
@@ -628,7 +654,11 @@ export function openSkyProxy() {
// Only cache successful responses — error responses (401/403/429/5xx)
// should not be served from cache on subsequent requests
if (upstream.ok) {
// Compress before writing either slot, so a request arriving
// mid-refresh still reads a body with its own frame.
const frame = await gzipBody(body);
_openskyCacheBody = body;
_openskyCacheGzip = frame;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
@@ -681,9 +711,13 @@ export function openSkyProxy() {
// Serve the last-good body instead of the 429 when we have one —
// the layer keeps rendering (STALE-cued) instead of dying.
if (_openskyCacheBody && _openskyCacheStatus === 200) {
res.writeHead(
200,
buildOpenSkyHeaders({
const encoded = encodedBody(
req,
_openskyCacheBody,
_openskyCacheGzip,
);
res.writeHead(200, {
...buildOpenSkyHeaders({
cacheStatus: 'STALE',
requestedMode,
usedMode,
@@ -691,8 +725,9 @@ export function openSkyProxy() {
staleSeconds: (now - _openskyCacheTime) / 1000,
retryAfterSeconds: cooldownMs / 1000,
}),
);
res.end(_openskyCacheBody);
...encoded.headers,
});
res.end(encoded.body);
return;
}
}
@@ -734,16 +769,23 @@ export function openSkyProxy() {
}
}
res.writeHead(
upstream.status,
buildOpenSkyHeaders({
// The frame belongs to this body only if this body was cached. An
// uncached error must not borrow the previous snapshot's frame.
const encoded = encodedBody(
req,
body,
_openskyCacheBody === body ? _openskyCacheGzip : null,
);
res.writeHead(upstream.status, {
...buildOpenSkyHeaders({
cacheStatus: 'MISS',
requestedMode,
usedMode,
reason,
}),
);
res.end(body);
...encoded.headers,
});
res.end(encoded.body);
} catch (e) {
console.error('[OpenSky Proxy]', e.message);
if (_openskyCacheBody) {
@@ -754,17 +796,22 @@ export function openSkyProxy() {
usedMode: 'unknown',
reason: 'cached_stale',
};
res.writeHead(
_openskyCacheStatus || 200,
buildOpenSkyHeaders({
const encoded = encodedBody(
req,
_openskyCacheBody,
_openskyCacheGzip,
);
res.writeHead(_openskyCacheStatus || 200, {
...buildOpenSkyHeaders({
cacheStatus: 'STALE',
requestedMode:
cachedMeta.requestedMode || OPENSKY_AUTH_MODE_DEFAULT,
usedMode: cachedMeta.usedMode || 'unknown',
reason: cachedMeta.reason || 'cached_stale',
}),
);
res.end(_openskyCacheBody);
...encoded.headers,
});
res.end(encoded.body);
return;
}
const requestedMode = normalizeOpenSkyAuthMode(
+187 -5
View File
@@ -1,4 +1,5 @@
import { createCctvCatalog } from './cctv/catalog.js';
import { quebec511VideoUrl, quebecBrowserVideoEnabled } from './cctv/quebec.js';
import { normalizeFeedType, isVideoFeedType } from './cctv/normalize.js';
import {
buildSyntheticCctvSvg,
@@ -18,6 +19,8 @@ import { sanitizeCctvRangeHeader } from './cctv/range.js';
import { admitSameSiteKeyless } from './common/same-site.js';
import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs';
import { createHlsPuller } from './cctv/stream.js';
import { isMlitRiverAdjustmentImage } from './cctv/worldCameras.js';
import { ottawaAuthorizedFrameUrl } from './cctv/ottawa.js';
import { googleServerApiKey } from './places/google-key.js';
import { clientKey, makeCostRateLimiter } from './common/rate-limit.js';
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
@@ -45,6 +48,120 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
const HEALTH_MAX_ENTRIES = CCTV_MAX_SOURCES_CEILING;
/** Live HLS strategies (see ./cctv/stream.js). Shared across dev and preview. */
const puller = createHlsPuller();
/**
* Bounded transient image reuse keeps cadence-limited providers (World
* stills, Ottawa's once-a-minute API) inside their refresh floor. The byte
* cap is the real bound; the entry cap only limits map overhead.
*/
const WORLD_FRAME_CACHE_MAX_ENTRIES = 1024;
/** Retry a failed fetch sooner than a full refresh window, never under a minute. */
const THROTTLED_FRAME_FAILURE_BACKOFF_MS = 60 * 1000;
const WORLD_FRAME_CACHE_MAX_ENTRY_BYTES = 2 * 1024 * 1024;
const WORLD_FRAME_CACHE_MAX_BYTES = 32 * 1024 * 1024;
const worldFrameCache = new Map();
const worldFrameInflight = new Map();
const worldFrameNextAllowedAt = new Map();
let worldFrameCacheBytes = 0;
function getWorldFrameCache(cameraId) {
const entry = worldFrameCache.get(cameraId);
if (!entry) return null;
worldFrameCache.delete(cameraId);
worldFrameCache.set(cameraId, entry);
return entry;
}
function storeWorldFrame(cameraId, image) {
if (
!Buffer.isBuffer(image?.body) ||
image.body.length > WORLD_FRAME_CACHE_MAX_ENTRY_BYTES
)
return;
const previous = worldFrameCache.get(cameraId);
if (previous) worldFrameCacheBytes -= previous.body.length;
const entry = {
body: image.body,
contentType: image.contentType,
fetchedAt: Date.now(),
};
worldFrameCache.delete(cameraId);
worldFrameCache.set(cameraId, entry);
worldFrameCacheBytes += entry.body.length;
while (
worldFrameCache.size > WORLD_FRAME_CACHE_MAX_ENTRIES ||
worldFrameCacheBytes > WORLD_FRAME_CACHE_MAX_BYTES
) {
const oldestId = worldFrameCache.keys().next().value;
const oldest = worldFrameCache.get(oldestId);
worldFrameCache.delete(oldestId);
worldFrameCacheBytes -= oldest.body.length;
}
}
function setNextAllowedAt(cameraId, at) {
worldFrameNextAllowedAt.delete(cameraId);
worldFrameNextAllowedAt.set(cameraId, at);
while (worldFrameNextAllowedAt.size > 4096) {
worldFrameNextAllowedAt.delete(
worldFrameNextAllowedAt.keys().next().value,
);
}
}
/**
* Fetch a frame from a provider with a per-camera refresh floor. Inside the
* floor the last frame is reused; with no frame to reuse the caller gets
* `{ throttled: true }` and serves a placeholder, so a cadence-limited camera
* never falls through to the key-bearing Street View fallback. A failed
* fetch backs off for a minute (capped at the floor) instead of a full
* window.
*/
async function fetchThrottledFrame(source, cameraId, frameUrl) {
const cached = getWorldFrameCache(cameraId);
const rawRefreshMs = Number(source?.frameRefreshMs);
const refreshMs =
Number.isFinite(rawRefreshMs) && rawRefreshMs > 0
? rawRefreshMs
: 5 * 60 * 1000;
const now = Date.now();
if (cached && now - cached.fetchedAt < refreshMs) {
return { ok: true, ...cached, cached: true };
}
const inflight = worldFrameInflight.get(cameraId);
if (inflight) return inflight;
const stale = cached
? {
ok: true,
body: cached.body,
contentType: cached.contentType,
stale: true,
}
: null;
const nextAllowedAt = worldFrameNextAllowedAt.get(cameraId) || 0;
if (now < nextAllowedAt) return stale || { ok: false, throttled: true };
if (!frameUrl) return stale;
setNextAllowedAt(cameraId, now + refreshMs);
let request;
request = fetchCctvImageFromUpstream(frameUrl)
.then((image) => {
if (image?.ok) {
storeWorldFrame(cameraId, image);
return image;
}
setNextAllowedAt(
cameraId,
Date.now() + Math.min(refreshMs, THROTTLED_FRAME_FAILURE_BACKOFF_MS),
);
return stale || image;
})
.finally(() => {
if (worldFrameInflight.get(cameraId) === request)
worldFrameInflight.delete(cameraId);
});
worldFrameInflight.set(cameraId, request);
return request;
}
/** Update the health entry for a camera, evicting the oldest entry if at capacity. */
const setHealth = (cameraId, patch) => {
@@ -171,6 +288,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
);
if (url.pathname === '/sources') {
const quebecVideo = quebecBrowserVideoEnabled();
const body = {
trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [],
sources: sources.map((source) => ({
@@ -182,6 +300,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
provider: source.provider,
lat: source.lat,
lon: source.lon,
frameRefreshMs: source.frameRefreshMs,
headingDeg: source.headingDeg,
headingConfidence: source.headingConfidence || '',
pitchDeg: source.pitchDeg,
@@ -192,6 +311,13 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
feedType: normalizeFeedType(source.feedType),
sourceKind:
source.sourceKind || (source.url ? 'configured' : 'fallback'),
sourceUrl: source.sourceUrl || '',
browserMediaUrl:
quebecVideo && source.sourceKind === 'quebec-511-browser'
? quebec511VideoUrl(source.id)
: '',
accuracyM: source.accuracyM,
positionSource: source.positionSource || '',
poseSource: source.poseSource,
model: source.model || '',
manufacturer: source.manufacturer || '',
@@ -544,15 +670,71 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
: '');
const upstreamImage =
source?.sourceKind === 'txdot-its'
? await fetchTxdotSnapshot(upstreamCandidate)
: await fetchCctvImageFromUpstream(upstreamCandidate);
source?.pack === 'world-cameras'
? await fetchThrottledFrame(source, cameraId, upstreamCandidate)
: source?.sourceKind === 'ottawa-traffic'
? await fetchThrottledFrame(
source,
cameraId,
ottawaAuthorizedFrameUrl(upstreamCandidate),
)
: source?.sourceKind === 'txdot-its'
? await fetchTxdotSnapshot(upstreamCandidate)
: await fetchCctvImageFromUpstream(upstreamCandidate, {
allowMjpeg: source?.sourceKind === 'taiwan-highway-motc',
});
if (
source?.sourceKind === 'mlit-river' &&
isMlitRiverAdjustmentImage(upstreamImage)
) {
const svg = buildSyntheticCctvSvg({
cameraId,
label,
city,
status: 'CAMERA UNDER ADJUSTMENT',
});
setHealth(cameraId, {
status: 'degraded',
sourceKind: 'snapshot',
label: source.provider || 'MLIT river cameras',
message: 'Provider camera under adjustment',
});
res.writeHead(200, {
'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-store',
'X-CCTV-Source': 'provider-maintenance',
});
res.end(svg);
return;
}
if (upstreamImage?.throttled) {
// Inside the provider's refresh floor with no frame to reuse: a
// placeholder, not Street View, and no health change.
res.writeHead(200, {
'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-store',
'X-CCTV-Source': 'synthetic',
});
res.end(
buildSyntheticCctvSvg({
cameraId,
label,
city,
status: 'WAITING FOR NEXT FRAME',
}),
);
return;
}
if (upstreamImage?.ok) {
setHealth(cameraId, {
status: 'ok',
status: upstreamImage.stale ? 'degraded' : 'ok',
sourceKind: 'snapshot',
label: source?.provider || 'Configured source',
message: 'Upstream snapshot active',
message: upstreamImage.stale
? 'Serving last snapshot; refresh failed'
: upstreamImage.cached
? 'Cached snapshot active'
: 'Upstream snapshot active',
});
res.writeHead(200, {
'Content-Type': upstreamImage.contentType,
+49 -12
View File
@@ -1,5 +1,6 @@
import {
CCTV_MAX_SOURCES_CEILING,
CCTV_SHARED_SOURCE_SLOTS,
DEFAULT_CCTV_MAX_SOURCES,
} from './constants.js';
@@ -23,16 +24,27 @@ export function resolveCatalogCap(raw) {
* exceed the catalog cap, the cap is filled ROUND-ROBIN — one camera from
* each pack per turn, in each pack's own order — so a lowered
* CCTV_MAX_SOURCES thins every region a little instead of deleting whichever
* pack happened to be appended last. Duplicate ids resolve last-pack-wins so
* pack happened to be appended last. A pack with a `reservedCount` keeps up to
* that many rows only from the capacity ABOVE the shared floor, so a
* reservation adds room and never shrinks the other packs' round-robin
* share; at or below the floor reserved packs are ordinary lanes. Duplicate
* ids resolve last-pack-wins so
* file/env overrides keep replacing live records; the record is then counted
* against the pack that won it.
*
* @param {Array<{name:string, sources:Array<object>}>} packs - Normalized
* packs in merge order (earlier packs lose duplicate ids to later ones).
* @param {number} maxCount - Catalog cap.
* @param {object} [options]
* @param {number} [options.sharedFloor=CCTV_SHARED_SOURCE_SLOTS] - Slots kept
* for the round-robin before reservations apply.
* @returns {{sources:Array<object>, packs:Array<{name:string, offered:number, kept:number}>}}
*/
export function allocateSourceCap(packs, maxCount) {
export function allocateSourceCap(
packs,
maxCount,
{ sharedFloor = CCTV_SHARED_SOURCE_SLOTS } = {},
) {
const owner = new Map();
for (const pack of packs) {
for (const source of pack.sources) {
@@ -40,30 +52,53 @@ export function allocateSourceCap(packs, maxCount) {
owner.set(source.id, { pack: pack.name, source });
}
}
const lanes = packs.map((pack) => ({
name: pack.name,
const lanes = packs.map((pack) => {
// Only the record that WON its id (the last occurrence anywhere, so also
// the last within this pack) takes a lane slot; earlier duplicates never
// count against the cap.
queue: pack.sources.filter(
const queue = pack.sources.filter(
(source) => source?.id && owner.get(source.id)?.source === source,
),
next: 0,
kept: 0,
}));
);
return {
name: pack.name,
queue,
reserved: Math.max(
0,
Math.min(queue.length, Math.floor(Number(pack.reservedCount) || 0)),
),
next: 0,
kept: 0,
};
});
const total = lanes.reduce((sum, lane) => sum + lane.queue.length, 0);
const limit = Math.min(total, Math.max(0, Math.floor(maxCount)));
const sources = [];
const reservedSources = [];
if (limit >= total) {
for (const lane of lanes) {
sources.push(...lane.queue);
lane.kept = lane.queue.length;
}
} else {
while (sources.length < limit) {
// Reserved packs draw only on the capacity above the shared floor, then
// every lane (reserved ones included) shares the rest round-robin.
const reserveBudget = Math.max(0, limit - Math.max(0, sharedFloor));
while (reservedSources.length < reserveBudget) {
let tookReserved = false;
for (const lane of lanes) {
if (sources.length >= limit) break;
if (reservedSources.length >= reserveBudget) break;
if (lane.kept >= lane.reserved) continue;
reservedSources.push(lane.queue[lane.next]);
lane.next += 1;
lane.kept += 1;
tookReserved = true;
}
if (!tookReserved) break;
}
while (sources.length + reservedSources.length < limit) {
for (const lane of lanes) {
if (sources.length + reservedSources.length >= limit) break;
if (lane.next >= lane.queue.length) continue;
sources.push(lane.queue[lane.next]);
lane.next += 1;
@@ -72,7 +107,9 @@ export function allocateSourceCap(packs, maxCount) {
}
}
return {
sources,
// Keep the shared round-robin sequence first so the default active camera
// remains in its established region; guaranteed regional rows follow it.
sources: [...sources, ...reservedSources],
packs: lanes.map((lane) => ({
name: lane.name,
offered: lane.queue.length,
+34 -2
View File
@@ -1,11 +1,18 @@
import fs from 'node:fs';
import path from 'node:path';
import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
import {
DEFAULT_CCTV_SOURCE_FILE,
CCTV_SOURCE_CACHE_MS,
WORLD_CCTV_SOURCE_COUNT,
} from './constants.js';
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
import { cameraRegion, normalizeSourceItem } from './normalize.js';
import { loadLithuaniaSources } from './lithuania.js';
import { loadOttawaSourcesFromOpenData } from './ottawa.js';
import { loadQuebecSources, QUEBEC_CAMERA_SOURCE_COUNT } from './quebec.js';
import { loadTaiwanSources } from './taiwan.js';
import { loadWorldCameraSources } from './worldCameras.js';
import {
loadAustinSourcesFromOpenData,
loadCaltransSourcesFromOpenData,
@@ -20,6 +27,7 @@ import {
loadNswSourcesFromOpenData,
loadCalgarySourcesFromOpenData,
loadCataloniaSourcesFromOpenData,
loadIcelandSourcesFromOpenData,
loadHkSourcesFromOpenData,
loadDelDOTSourcesFromOpenData,
loadKingCountySourcesFromOpenData,
@@ -75,6 +83,11 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_DRIVEBC_ENABLED'),
load: loadDriveBcSourcesFromOpenData,
},
{
name: 'ottawa',
enabled: () => envEnabled('CCTV_OTTAWA_ENABLED'),
load: loadOttawaSourcesFromOpenData,
},
{
name: 'txdot',
enabled: () => envEnabled('CCTV_TXDOT_ENABLED'),
@@ -110,6 +123,11 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
load: loadCataloniaSourcesFromOpenData,
},
{
name: 'iceland',
enabled: () => envEnabled('CCTV_ICELAND_ENABLED'),
load: loadIcelandSourcesFromOpenData,
},
{
name: 'hk',
enabled: () => envEnabled('CCTV_HK_ENABLED'),
@@ -172,6 +190,18 @@ const LIVE_PACKS = [
enabled: () => envOptIn('CCTV_HOUSTON_ENABLED'),
load: loadHoustonSourcesFromOpenData,
},
{
name: 'quebec',
enabled: () => envEnabled('CCTV_QUEBEC_ENABLED'),
load: loadQuebecSources,
reservedCount: QUEBEC_CAMERA_SOURCE_COUNT,
},
{
name: 'world-cameras',
enabled: () => envEnabled('CCTV_WORLD_CAMERAS_ENABLED'),
load: loadWorldCameraSources,
reservedCount: WORLD_CCTV_SOURCE_COUNT,
},
{
name: 'a22',
enabled: () => envEnabled('CCTV_A22_ENABLED'),
@@ -316,7 +346,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
: [];
// Live packs first so file/env overrides win on duplicate IDs; each pack
// keeps its own priority order and the catalog cap is shared fairly.
const normalizePack = (name, items) => ({
const normalizePack = (name, items, reservedCount = 0) => ({
name,
sources: items
.filter((item) => item && typeof item === 'object')
@@ -326,6 +356,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
? items.available
: items.length,
region: items.region ?? null,
reservedCount,
});
const packs = [
...LIVE_PACKS.map((pack, index) =>
@@ -334,6 +365,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
liveResults[index]?.status === 'fulfilled'
? liveResults[index].value
: [],
pack.reservedCount,
),
),
normalizePack('file', fromFile),
+74 -7
View File
@@ -5,15 +5,18 @@ export const DEFAULT_AUSTIN_ROWS_URL =
/** Default cap on Austin cameras after distance-based prioritization. */
export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
/**
* Catalog-wide safety ceiling on served cameras. Each pack already caps
* itself (nearest-to-anchor first); this bound only matters when the packs
* together exceed it, and it is then filled round-robin across packs (see
* cap.js) so no region is silently dropped. The default per-pack caps
* together can exceed it, so a default install may be thinned evenly here.
* Slots every pack shares round-robin before any reservation applies. A
* reserved pack (Québec, World) only takes capacity above this floor, so a
* cap at or below it thins every region evenly (see cap.js).
*/
export const DEFAULT_CCTV_MAX_SOURCES = 4000;
export const CCTV_SHARED_SOURCE_SLOTS = 4000;
/**
* Catalog-wide default: the 4,000 shared slots plus room reserved for the
* 1,120 Québec and 2,230 World image locations.
*/
export const DEFAULT_CCTV_MAX_SOURCES = 7350;
/** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */
export const CCTV_MAX_SOURCES_CEILING = 5000;
export const CCTV_MAX_SOURCES_CEILING = 8000;
/** Reference point for Austin camera prioritization (Congress & 6th). */
export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 };
/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */
@@ -197,6 +200,39 @@ export const DRIVEBC_ANCHORS = [
{ lat: 49.2827, lon: -123.1207 }, // Vancouver
{ lat: 48.4284, lon: -123.3656 }, // Victoria
];
/** City of Ottawa traffic cameras: documented JSON list (keyless). */
export const OTTAWA_CAMERAS_URL =
'https://traffic.ottawa.ca/map/service/camera';
/**
* Keyless canonical frame URL for one camera number. The city's Traffic Camera
* Images API needs a certificate (OTTAWA_CAMERA_CERTIFICATE), which the frame
* proxy adds at request time so it never appears in a catalog response.
*/
export const OTTAWA_IMAGE_URL = (cameraNumber) =>
`https://traffic.ottawa.ca/opendata/camera?c=${encodeURIComponent(cameraNumber)}`;
/** Pinned frame directory, for redirect containment in the media proxy. */
export const OTTAWA_IMAGE_ORIGIN = 'https://traffic.ottawa.ca/opendata/';
/** The API lets one certificate and user id fetch a camera once a minute. */
export const OTTAWA_FRAME_REFRESH_MS = 60 * 1000;
/** Alphanumeric user id this server sends with its certificate. */
export const OTTAWA_API_USER_ID = 'gev';
export const DEFAULT_OTTAWA_MAX_SOURCES = 250;
/** Prioritization anchor: downtown Ottawa (Parliament Hill). */
export const OTTAWA_ANCHORS = [{ lat: 45.4215, lon: -75.6972 }];
/**
* Taiwan Highway Bureau MJPEG servers. Each camera is one root path segment,
* so the pin is the host root: a redirect must stay on the same server.
*/
export const TAIWAN_HIGHWAY_IMAGE_ORIGINS = Object.freeze(
[1, 2, 3, 4, 5, 6, 7, 8].map((n) => `https://cctv-ss0${n}.thb.gov.tw/`),
);
/** Direct still-image directories from the World cams add-on's eligible packs. */
export const TRIPCHECK_IMAGE_ORIGIN =
'https://www.tripcheck.com/roadcams/cams/';
export const MLIT_RIVER_IMAGE_ORIGIN = 'https://cam.river.go.jp/cam/now/';
export const DEFAULT_WORLD_CCTV_SOURCE_FILE =
'config/cctv_sources.world-image-cameras.json';
export const WORLD_CCTV_SOURCE_COUNT = 2230;
/** Tallinn intersection cameras: curated catalog + public stills on ristmikud.tallinn.ee. */
export const DEFAULT_TALLINN_SOURCE_FILE = 'config/cctv_sources.tallinn.json';
export const DEFAULT_TALLINN_MAX_SOURCES = 255;
@@ -309,6 +345,37 @@ export const CATALONIA_ANCHORS = [
{ lat: 41.5636, lon: 2.0111 }, // Terrassa
];
/**
* Icelandic Road and Coastal Administration (Vegagerðin, IRCA) road cameras:
* one keyless JSON list, one row per camera view (about 500 views at roughly
* 170 stations, verified 2026-11-06). Frames are stills on the IRCA host, each
* rewritten by the publisher every few minutes. Camera images are published
* under CC BY 4.0 (IRCA terms of use, "Open data" section).
*/
export const ICELAND_CAMERAS_URL =
'https://gagnaveita.vegagerdin.is/api/vefmyndavelar2014_1';
/** The only directory Iceland camera frames may come from. */
export const ICELAND_IMAGE_ORIGIN =
'https://www.vegagerdin.is/vgdata/vefmyndavelar/';
/** Byte ceiling for the Iceland catalog read (~150 KB observed 2026-11-06). */
export const ICELAND_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
/** Ground-elevation prior in metres: the feed carries none, and the country
* runs from sea level to highland passes. The client's ground snap corrects
* this on 3D-tile stacks. */
export const ICELAND_GROUND_ELEVATION_M = 60;
/** The feed lists about 500 views; the default keeps half so the shared
* catalog cap still leaves room for denser regions. */
export const DEFAULT_ICELAND_MAX_SOURCES = 250;
/** Prioritization anchors: Reykjavík and the regional centres, so a cap keeps
* every region's nearest cameras instead of only the capital's. */
export const ICELAND_ANCHORS = [
{ lat: 64.1466, lon: -21.9426 }, // Reykjavík
{ lat: 65.6885, lon: -18.1262 }, // Akureyri
{ lat: 65.2639, lon: -14.3948 }, // Egilsstaðir
{ lat: 66.0749, lon: -23.1351 }, // Ísafjörður
{ lat: 63.9331, lon: -20.9971 }, // Selfoss
];
/** Hong Kong Transport Department traffic snapshot cameras: keyless English
* locations XML on DATA.GOV.HK; frames are stills on the TD CCTV host. */
export const DEFAULT_HK_CAMERAS_URL =
+243 -6
View File
@@ -10,10 +10,15 @@ import {
CCTV_MEDIA_MAX_BODY_BYTES,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
MLIT_RIVER_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
NSW_IMAGE_ORIGIN,
NSW_IMAGE_USER_AGENT,
OTTAWA_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
ICELAND_IMAGE_ORIGIN,
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
TFL_IMAGE_ORIGIN,
WARENDORF_IMAGE_ORIGINS,
WSDOT_IMAGE_ORIGIN,
@@ -523,13 +528,18 @@ for (const prefix of [
TFL_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
ICELAND_IMAGE_ORIGIN,
...WARENDORF_IMAGE_ORIGINS,
NSW_IMAGE_ORIGIN,
OTTAWA_IMAGE_ORIGIN,
CALGARY_IMAGE_ORIGIN,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
WSDOT_IMAGE_ORIGIN,
A22_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
MLIT_RIVER_IMAGE_ORIGIN,
...TAIWAN_HIGHWAY_IMAGE_ORIGINS,
]) {
const host = new URL(prefix).hostname;
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
@@ -621,6 +631,216 @@ export function cctvUpstreamUserAgent(url) {
}
}
/** Largest single JPEG the MJPEG first-frame reader will buffer. */
export const CCTV_MJPEG_FRAME_MAX_BYTES = 2 * 1024 * 1024;
/**
* Read one bounded JPEG part from an open MJPEG response, then stop the stream.
*
* The first part may follow a CRLF preamble, and its delimiter may be written
* as `--<boundary>` or, when the declared boundary already starts with `--`,
* as the boundary itself. A part with Content-Length is read to that length;
* a part without one ends at the next delimiter, so a JPEG end marker inside
* the image (an EXIF thumbnail) never truncates it. Bytes are copied once
* into a preallocated buffer and every search resumes where it stopped, so a
* trickling upstream costs linear time.
*/
async function readFirstMjpegFrame(upstream, maxBytes) {
const mediaType = cctvLiveMediaType(
'mjpeg',
upstream.headers.get('content-type'),
);
const boundary = parseMediaType(mediaType)?.params.get('boundary') || '';
const frameMax = Math.min(maxBytes, CCTV_MJPEG_FRAME_MAX_BYTES);
if (
!boundary ||
!upstream.body ||
!Number.isSafeInteger(frameMax) ||
frameMax < 4
) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const delimiters = [Buffer.from('--' + boundary, 'latin1')];
if (boundary.startsWith('--') && boundary.length > 2)
delimiters.push(Buffer.from(boundary, 'latin1'));
const lineEnd = Buffer.from('\r\n', 'ascii');
const headerTerminator = Buffer.from('\r\n\r\n', 'ascii');
const jpegStart = Buffer.from([0xff, 0xd8]);
const jpegEnd = Buffer.from([0xff, 0xd9]);
const maxPreambleBytes = 64;
const maxHeaderBytes = 8 * 1024;
const maxBufferedBytes =
maxPreambleBytes +
delimiters[0].length +
lineEnd.length +
maxHeaderBytes +
headerTerminator.length +
frameMax +
lineEnd.length +
delimiters[0].length;
const buffer = Buffer.allocUnsafe(maxBufferedBytes);
let length = 0;
let reader;
let headerStart = -1;
let delimiter = null;
let headerScanFrom = 0;
let bodyStart = -1;
let declaredLength = null;
let boundaryScanFrom = -1;
/** Match the opening delimiter after an optional CRLF preamble. */
function findOpening() {
let at = 0;
while (at < length && (buffer[at] === 0x0d || buffer[at] === 0x0a)) at++;
if (at > maxPreambleBytes) return { failed: true };
for (const candidate of delimiters) {
const line = Buffer.concat([candidate, lineEnd]);
const need = at + line.length;
if (length < need) {
const seen = buffer.subarray(at, length);
if (line.subarray(0, seen.length).equals(seen))
return { pending: true };
continue;
}
if (buffer.subarray(at, need).equals(line))
return { start: need, delimiter: candidate };
}
return { failed: true };
}
/** Parse the part headers; false rejects the stream. */
function parseHeaders(headerEnd) {
const headerText = buffer
.subarray(headerStart, headerEnd)
.toString('latin1');
const headers = new Map();
for (const line of headerText.split('\r\n')) {
const colon = line.indexOf(':');
if (colon <= 0) return false;
const name = line.slice(0, colon).trim().toLowerCase();
if (!name || headers.has(name)) return false;
headers.set(name, line.slice(colon + 1).trim());
}
if (cctvRasterImageType(headers.get('content-type')) !== 'image/jpeg')
return false;
const declared = headers.get('content-length');
if (declared !== undefined) {
if (!/^\d+$/.test(declared)) return false;
declaredLength = Number(declared);
if (
!Number.isSafeInteger(declaredLength) ||
declaredLength < jpegStart.length + jpegEnd.length ||
declaredLength > frameMax
)
return false;
}
return true;
}
/** A complete JPEG body, or null. */
function asJpeg(frame) {
return frame.length >= jpegStart.length + jpegEnd.length &&
frame.subarray(0, jpegStart.length).equals(jpegStart) &&
frame.subarray(-jpegEnd.length).equals(jpegEnd)
? Buffer.from(frame)
: null;
}
try {
reader = upstream.body.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done) return null;
const chunk = Buffer.from(value);
const room = maxBufferedBytes - length;
const included = Math.min(room, chunk.length);
chunk.copy(buffer, length, 0, included);
length += included;
const overflow = included < chunk.length;
if (headerStart < 0) {
const opening = findOpening();
if (opening.failed) return null;
if (opening.pending) {
if (overflow) return null;
continue;
}
headerStart = opening.start;
delimiter = opening.delimiter;
headerScanFrom = headerStart;
}
if (bodyStart < 0) {
const headerEnd = buffer
.subarray(0, length)
.indexOf(headerTerminator, headerScanFrom);
if (headerEnd < 0) {
if (length - headerStart > maxHeaderBytes || overflow) return null;
headerScanFrom = Math.max(
headerStart,
length - headerTerminator.length + 1,
);
continue;
}
if (headerEnd - headerStart > maxHeaderBytes) return null;
if (!parseHeaders(headerEnd)) return null;
bodyStart = headerEnd + headerTerminator.length;
boundaryScanFrom = bodyStart;
}
const available = length - bodyStart;
if (available >= jpegStart.length) {
if (
!buffer
.subarray(bodyStart, bodyStart + jpegStart.length)
.equals(jpegStart)
)
return null;
}
const nextDelimiter = Buffer.concat([lineEnd, delimiter]);
const nextPart = buffer
.subarray(0, length)
.indexOf(nextDelimiter, boundaryScanFrom);
if (declaredLength !== null) {
if (nextPart >= bodyStart && nextPart - bodyStart < declaredLength)
return null;
if (available >= declaredLength)
return asJpeg(buffer.subarray(bodyStart, bodyStart + declaredLength));
} else if (nextPart >= bodyStart) {
const frame = buffer.subarray(bodyStart, nextPart);
return frame.length > frameMax ? null : asJpeg(frame);
} else if (available > frameMax) {
return null;
}
if (nextPart < 0)
boundaryScanFrom = Math.max(
bodyStart,
length - nextDelimiter.length + 1,
);
if (overflow) return null;
}
} finally {
if (reader) {
try {
await reader.cancel();
} catch {
/* already closed or aborted */
}
try {
reader.releaseLock();
} catch {
/* already released */
}
}
}
}
/**
* Fetch one upstream CCTV image within the frame-refresh budget.
*
@@ -633,6 +853,10 @@ export function cctvUpstreamUserAgent(url) {
* @param {typeof fetch} [options.fetchImpl=fetch] - Fetch implementation.
* @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout.
* @param {number} [options.maxBytes=CCTV_FRAME_MAX_BODY_BYTES] - Snapshot byte cap.
* @param {boolean} [options.allowMjpeg=false] - Accept a multipart MJPEG
* answer and keep its first JPEG. Only sources registered as MJPEG stills
* (the Taiwan Highway Bureau feeds) set it; any other host answering
* multipart falls through to the next fallback.
* @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>}
*/
export async function fetchCctvImageFromUpstream(
@@ -641,6 +865,7 @@ export async function fetchCctvImageFromUpstream(
fetchImpl = fetch,
timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS,
maxBytes = CCTV_FRAME_MAX_BODY_BYTES,
allowMjpeg = false,
} = {},
) {
if (!url || !/^https?:\/\//i.test(url)) return null;
@@ -660,10 +885,7 @@ export async function fetchCctvImageFromUpstream(
fetchImpl,
);
if (!upstream) return null;
const contentType = cctvRasterImageType(
upstream.headers.get('content-type'),
);
if (!upstream.ok || !contentType) {
if (!upstream.ok) {
try {
await upstream.body?.cancel();
} catch {
@@ -671,9 +893,24 @@ export async function fetchCctvImageFromUpstream(
}
return null;
}
const body = await readCappedResponseBytes(upstream, maxBytes);
const upstreamType = upstream.headers.get('content-type');
const contentType = cctvRasterImageType(upstreamType);
if (contentType) {
const body = await readCappedResponseBytes(upstream, maxBytes);
if (!body) return null;
return { ok: true, body, contentType };
}
if (!allowMjpeg || !cctvLiveMediaType('mjpeg', upstreamType)) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const body = await readFirstMjpegFrame(upstream, maxBytes);
if (!body) return null;
return { ok: true, body, contentType };
return { ok: true, body, contentType: 'image/jpeg' };
} catch {
return null;
} finally {
+15
View File
@@ -460,6 +460,17 @@ export function isLikelyCataloniaCoordinate(lat, lon) {
);
}
/** Iceland bounding box (mainland and nearby islands, with a margin). */
export function isLikelyIcelandCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 63.0 &&
lat <= 66.7 &&
lon >= -25.0 &&
lon <= -13.0
);
}
/**
* Decode numeric HTML character references, decimal ("&#237;") and hex
* ("&#xED;") alike, the entity form the Catalonia feed uses for accented
@@ -639,6 +650,10 @@ export function normalizeSourceItem(item) {
feedType: normalizeFeedType(item.feedType || item.type || ''),
url: typeof item.url === 'string' ? item.url : '',
snapshotUrl: typeof item.snapshotUrl === 'string' ? item.snapshotUrl : '',
frameRefreshMs: toFiniteNumber(item.frameRefreshMs),
sourceUrl: typeof item.sourceUrl === 'string' ? item.sourceUrl : '',
accuracyM: toFiniteNumber(item.accuracyM),
positionSource: String(item.positionSource || ''),
license: String(item.license || item.licenseNote || ''),
// Per-camera attribution for feeds a partner supplies inside a pack
// (DriveBC: TransLink, city cameras). Shown beside the provider.
+185
View File
@@ -0,0 +1,185 @@
import {
CCTV_SOURCE_FETCH_TIMEOUT_MS,
DEFAULT_OTTAWA_MAX_SOURCES,
OTTAWA_ANCHORS,
OTTAWA_API_USER_ID,
OTTAWA_CAMERAS_URL,
OTTAWA_FRAME_REFRESH_MS,
OTTAWA_IMAGE_ORIGIN,
OTTAWA_IMAGE_URL,
} from './constants.js';
import {
fallbackHeadingFromId,
pinnedCameraUrl,
prioritizeSources,
toFiniteNumber,
} from './normalize.js';
let ottawaCertificateMissingLogged = false;
/** Test hook: let the next missing-certificate skip log again. */
export function _resetOttawaCertificateWarningForTest() {
ottawaCertificateMissingLogged = false;
}
/** The operator's Traffic Camera Images API certificate, trimmed. */
function ottawaCertificate(env = process.env) {
return String(env.OTTAWA_CAMERA_CERTIFICATE ?? '').trim();
}
/**
* Add the operator's certificate to a registered keyless Ottawa frame URL.
* Only a URL inside the pinned `/opendata/` directory with a numeric camera
* number is signed, so the certificate never goes to any other path or host.
*
* @param {string} url - Canonical frame URL from the catalog.
* @param {Record<string,string|undefined>} [env]
* @returns {string} The signed URL, or '' when there is no certificate or the
* URL is not a registered Ottawa frame.
*/
export function ottawaAuthorizedFrameUrl(url, env = process.env) {
const certificate = ottawaCertificate(env);
const pinned = pinnedCameraUrl(url, OTTAWA_IMAGE_ORIGIN);
if (!certificate || !pinned) return '';
const parsed = new URL(pinned);
const cameraNumber = parsed.searchParams.get('c') || '';
if (parsed.pathname !== '/opendata/camera' || !/^\d+$/.test(cameraNumber))
return '';
const signed = new URL(OTTAWA_IMAGE_URL(cameraNumber));
signed.searchParams.set('certificate', certificate);
signed.searchParams.set('id', OTTAWA_API_USER_ID);
return signed.href;
}
/**
* City of Ottawa envelope: Kanata to Orléans, Barrhaven to the river, with
* slack for the MTO cameras on the 416/417 that the city feed also carries.
*/
const OTTAWA_BOUNDS = Object.freeze({
minLat: 45.1,
maxLat: 45.65,
minLon: -76.2,
maxLon: -75.1,
});
/** Whether a point is inside the city feed's service area. */
function isOttawaCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= OTTAWA_BOUNDS.minLat &&
lat <= OTTAWA_BOUNDS.maxLat &&
lon >= OTTAWA_BOUNDS.minLon &&
lon <= OTTAWA_BOUNDS.maxLon
);
}
/**
* Load the City of Ottawa's traffic cameras.
*
* Camera images come from the city's Traffic Camera Images API, which needs a
* certificate from the city's registration form (OTTAWA_CAMERA_CERTIFICATE)
* and allows one fetch per camera per minute for a certificate and user id.
* Without a certificate the pack is skipped, with no request and one log line.
* The camera list is the documented `map/service/camera` JSON: a camera
* number, English and French names, coordinates and the owner (CITY or MTO).
* The feed carries no bearing, no elevation and no status field, so headings
* use the same deterministic low-confidence fallback as the other
* heading-less packs and the client's ground snap refines the elevation
* prior.
*
* @param {object} [_options]
* @param {Record<string,string|undefined>} [env]
* @returns {Promise<Array<object>>} Cameras nearest downtown first, or [] on
* a missing certificate or any upstream or payload failure.
*/
export async function loadOttawaSourcesFromOpenData(
_options = {},
env = process.env,
) {
if (!ottawaCertificate(env)) {
if (!ottawaCertificateMissingLogged) {
ottawaCertificateMissingLogged = true;
console.warn(
'[CCTV] Ottawa cameras skipped: the city camera image API needs a certificate (set OTTAWA_CAMERA_CERTIFICATE)',
);
}
return [];
}
try {
const resp = await fetch(OTTAWA_CAMERAS_URL, {
headers: { Accept: 'application/json' },
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
if (!resp.ok) {
console.warn('[CCTV] Ottawa camera download failed:', resp.status);
return [];
}
const payload = await resp.json();
const rows = Array.isArray(payload?.cameras) ? payload.cameras : null;
if (!rows) return [];
const cameras = [];
const seen = new Set();
for (const row of rows) {
const number = toFiniteNumber(row?.camera_number);
if (!Number.isSafeInteger(number) || number <= 0 || seen.has(number))
continue;
const lat = toFiniteNumber(row?.latitude);
const lon = toFiniteNumber(row?.longitude);
if (!isOttawaCoordinate(lat, lon)) continue;
seen.add(number);
const cameraId = `ottawa-${number}`;
const imageUrl = OTTAWA_IMAGE_URL(number);
// The docs name the field camera_owner; the live feed sends cameraOwner.
const owner = String(row?.camera_owner || row?.cameraOwner || '')
.trim()
.toUpperCase();
cameras.push({
id: cameraId,
name: String(row?.name || '').trim() || `Ottawa camera ${number}`,
city: 'Ottawa',
cityId: 'ottawa',
provider: 'City of Ottawa',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: 0,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
frameRefreshMs: OTTAWA_FRAME_REFRESH_MS,
sourceKind: 'ottawa-traffic',
license: 'City of Ottawa Traffic Camera Images API (registered access)',
credit:
owner === 'MTO'
? 'Ministry of Transportation of Ontario (MTO) camera'
: '',
});
}
const maxRaw = Number(
process.env.CCTV_OTTAWA_MAX_SOURCES || DEFAULT_OTTAWA_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(500, Math.floor(maxRaw)))
: DEFAULT_OTTAWA_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, OTTAWA_ANCHORS);
console.log(
`[CCTV] Loaded Ottawa camera sources: ${cameras.length} published (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Ottawa camera download error:',
error?.message || error,
);
return [];
}
}
+138
View File
@@ -0,0 +1,138 @@
import fs from 'node:fs';
import path from 'node:path';
import { cameraDisplayCode, fallbackHeadingFromId } from './normalize.js';
const QUEBEC_CATALOG_PATH = 'public/quebec-cameras.json';
export const QUEBEC_CAMERA_SOURCE_COUNT = 1120;
/**
* Whether the operator opted in to the browser-direct Québec 511 video. Off by
* default: the camera locations are CC BY 4.0, but whether Québec 511's
* service terms allow another site to embed its clips is unconfirmed, so an
* unset flag lists the cameras with a link to the 511 viewer page only.
*/
export function quebecBrowserVideoEnabled(env = process.env) {
return String(env.CCTV_QUEBEC_VIDEO_ENABLED || '').trim() === '1';
}
/** Build the direct browser-only MP4 URL for one namespaced MTQ camera id. */
export function quebec511VideoUrl(cameraId) {
const match = /^qc-511-(\d+)$/.exec(String(cameraId || ''));
return match
? `https://www.quebec511.info/Carte/Fenetres/camera.ashx?id=${match[1]}&format=mp4`
: '';
}
function safeViewerUrl(value) {
try {
const url = new URL(String(value || ''));
return url.protocol === 'https:' && !url.username && !url.password
? url.href
: '';
} catch {
return '';
}
}
function sourceId(camera) {
const id = String(camera?.id || '').trim();
if (camera?.kind === 'traffic' && /^\d+$/.test(id)) return `qc-511-${id}`;
if (camera?.kind === 'montreal' && /^mtl-\d+(?:-\d+)*$/.test(id))
return `qc-mtl-${id.slice(4)}`;
if (camera?.kind === 'webcam' && /^web-\d+$/.test(id))
return `qc-web-${id.slice(4)}`;
return '';
}
/**
* Read the checked-in, no-key Québec location catalog and adapt it to the
* regular CCTV source schema. MTQ video stays browser-only; other rows link to
* their operator page because this dataset has no direct image URL for them.
*/
export function loadQuebecSources({ sourceRoot = process.cwd() } = {}) {
const filename = path.resolve(sourceRoot, QUEBEC_CATALOG_PATH);
let cameras;
try {
const payload = JSON.parse(fs.readFileSync(filename, 'utf8'));
cameras = Array.isArray(payload?.cameras) ? payload.cameras : [];
} catch (error) {
console.warn('[CCTV] Québec camera catalog unavailable:', error?.message);
return [];
}
const sources = [];
const seen = new Set();
for (const camera of cameras) {
const id = sourceId(camera);
const lat = Number(camera.lat);
const lon = Number(camera.lon);
if (
!id ||
seen.has(id) ||
!Number.isFinite(lat) ||
!Number.isFinite(lon) ||
lat < 44.9 ||
lat > 62.6 ||
lon < -79.9 ||
lon > -57.1
)
continue;
seen.add(id);
const name = String(camera.nameEn || camera.name || id).trim();
const isTraffic = camera.kind === 'traffic';
const isMontreal = camera.kind === 'montreal';
const sourceKind = isTraffic
? 'quebec-511-browser'
: isMontreal
? 'quebec-montreal-osm'
: 'quebec-webcam-link';
const provider = isTraffic
? 'Québec 511 / MTQ'
: isMontreal
? 'Ville de Montréal / OpenStreetMap'
: String(camera.operator || 'Québec webcam');
sources.push({
id,
name: name.slice(0, 180),
city: String(camera.region || 'Québec'),
cityId: 'quebec',
provider,
lat,
lon,
headingDeg: fallbackHeadingFromId(id),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 74,
rangeM: 400,
mountHeightM: 8,
groundElevationM: 0,
feedType: 'image',
// No frame URL is registered: the server never fetches 511 media. With
// CCTV_QUEBEC_VIDEO_ENABLED=1 the browser plays the 511 MP4 in the DOM
// overlay after a play click; otherwise the camera links to 511.
url: '',
snapshotUrl: '',
sourceKind,
sourceUrl: safeViewerUrl(camera.viewerUrl),
accuracyM: Number.isFinite(Number(camera.accuracyM))
? Number(camera.accuracyM)
: null,
positionSource: String(camera.positionSource || ''),
license: isTraffic
? 'Locations: Données Québec / MTQ, CC BY 4.0. Video: hosted by Québec 511 under its terms of use'
: isMontreal
? 'OpenStreetMap contributors, ODbL 1.0'
: 'Operator terms apply; location provenance shown per camera',
credit: isTraffic
? 'Locations: MTQ · CC BY 4.0 · Video: Québec 511'
: isMontreal
? 'OpenStreetMap contributors · ODbL'
: provider,
code: cameraDisplayCode(name.toUpperCase()),
});
}
return sources;
}
+175
View File
@@ -134,6 +134,12 @@ import {
HOUSTON_MAX_CATALOG_BYTES,
HOUSTON_GROUND_ELEVATION_M,
HOUSTON_ANCHORS,
ICELAND_CAMERAS_URL,
ICELAND_IMAGE_ORIGIN,
ICELAND_MAX_CATALOG_BYTES,
ICELAND_GROUND_ELEVATION_M,
DEFAULT_ICELAND_MAX_SOURCES,
ICELAND_ANCHORS,
} from './constants.js';
import {
toFiniteNumber,
@@ -152,6 +158,7 @@ import {
isLikelyCalgaryCoordinate,
isLikelyFloridaCoordinate,
isLikelyCataloniaCoordinate,
isLikelyIcelandCoordinate,
decodeNumericEntities,
isLikelyNycCoordinate,
isLikelyHoustonCoordinate,
@@ -2029,6 +2036,174 @@ export async function loadCataloniaSourcesFromOpenData() {
}
}
/**
* Compass headings for Icelandic direction words, as they appear after "til"
* (genitive: "til vesturs"), "í" (accusative: "í vestur") or "séð" ("seen",
* as in "séð austur").
*/
const ICELAND_DIRECTION_HEADINGS = {
norður: 0,
norðaustur: 45,
austur: 90,
suðaustur: 135,
suður: 180,
suðvestur: 225,
vestur: 270,
norðvestur: 315,
};
const ICELAND_DIRECTION_WORD =
'(norðaustur|norðvestur|suðaustur|suðvestur|norður|suður|austur|vestur)s?';
/** A facing, plus a second one joined by "og" ("til austurs og vesturs"). */
const ICELAND_DIRECTION_RE = new RegExp(
`(?<![\\p{L}])(?:til|í|séð)\\s+${ICELAND_DIRECTION_WORD}` +
`(?:\\s+og\\s+(?:til\\s+|í\\s+)?${ICELAND_DIRECTION_WORD})?(?![\\p{L}])`,
'gu',
);
/**
* The compass heading an Icelandic view description states, or NaN. Only an
* explicit "til <átt>", "í <átt>" or "séð <átt>" counts: a bare place name or
* a road direction ("Miklabraut austur") is not a facing. A description naming
* two different directions ("til norðurs og til suðurs", "til austurs og
* vesturs") is ambiguous and yields NaN, and "niður á veg" (down onto the
* road) states none.
*
* @param {string} text - The feed's `Skyring` field.
* @returns {number} Heading in degrees [0..360), or NaN.
*/
export function icelandHeadingFromDescription(text) {
const headings = new Set();
for (const match of String(text || '')
.toLowerCase()
.matchAll(ICELAND_DIRECTION_RE)) {
for (const word of [match[1], match[2]]) {
if (word) headings.add(ICELAND_DIRECTION_HEADINGS[word]);
}
}
return headings.size === 1 ? headings.values().next().value : NaN;
}
/**
* One Vegagerðin camera row -> one catalog source, or null. A row is one camera
* view; a station (`Maelist_nr`) carries several. The id comes from the frame
* file name, which is unique per view in the live feed.
*
* @param {object} row - Element of the `vefmyndavelar2014_1` array.
* @returns {?object}
*/
export function icelandCameraToSource(row) {
const url = pinnedCameraUrl(row?.Slod, ICELAND_IMAGE_ORIGIN);
if (!url || !/\.jpe?g$/i.test(new URL(url).pathname)) return null;
// Numbers only: Number(null) and Number('') are 0.
const lat = typeof row.Breidd === 'number' ? row.Breidd : NaN;
const lon = typeof row.Lengd === 'number' ? row.Lengd : NaN;
if (!isLikelyIcelandCoordinate(lat, lon)) return null;
const stem = new URL(url).pathname
.slice(new URL(ICELAND_IMAGE_ORIGIN).pathname.length)
.replace(/\.jpe?g$/i, '')
.toLowerCase();
if (!/^[a-z0-9_-]+$/.test(stem)) return null;
const cameraId = `is-vegagerdin-${stem}`;
const description = String(row.Skyring || '')
.replace(/\s+/g, ' ')
// Letter lookarounds, not \b: \b is ASCII-only and would join "Þverá
// Árnessýslu" while missing "séð séð".
.replace(/(?<!\p{L})(\p{L}+) \1(?!\p{L})/giu, '$1')
.trim();
const station = String(row.Myndavel || '').trim();
const heading = icelandHeadingFromDescription(description);
const hasHeading = Number.isFinite(heading);
return {
id: cameraId,
name: description || station || `Vegagerðin ${row.Maelist_nr ?? stem}`,
city: 'Iceland',
cityId: 'iceland',
provider: 'Vegagerðin (IRCA)',
lat,
lon,
headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId),
headingConfidence: hasHeading ? 'high' : 'low',
pitchDeg: hasHeading ? -24 : -18,
fovDeg: hasHeading ? 56 : 44,
rangeM: hasHeading ? 210 : 145,
mountHeightM: hasHeading ? 10 : 8,
groundElevationM: ICELAND_GROUND_ELEVATION_M,
feedType: 'image',
url,
snapshotUrl: url,
sourceKind: 'iceland-vegagerdin',
license:
'Icelandic Road and Coastal Administration (IRCA) — road camera images, CC BY 4.0',
code: cameraDisplayCode(station.toUpperCase() || description),
};
}
/**
* Fetch Icelandic road cameras (Vegagerðin / IRCA), keyless: one JSON list.
* Frames are stills on the IRCA host and the loader registers only links
* inside that host's camera directory (ICELAND_IMAGE_ORIGIN). The list decides
* ids, coordinates, labels and which frame path is proxied, so it is fetched
* over https, redirects are refused and the body read is capped at
* ICELAND_MAX_CATALOG_BYTES.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadIcelandSourcesFromOpenData() {
try {
const resp = await fetch(ICELAND_CAMERAS_URL, {
headers: {
Accept: 'application/json',
'User-Agent': 'gods-eye-view-cctv-proxy/1.0',
},
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
// A response this loader will not read still owns its transport until the
// body is released, so every rejection path cancels before returning.
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] Iceland catalog redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] Iceland camera download failed:', resp.status);
return discard();
}
const body = await readResponseJsonCapped(resp, ICELAND_MAX_CATALOG_BYTES);
const rows = Array.isArray(body) ? body : [];
const cameras = rows.map(icelandCameraToSource).filter(Boolean);
const unique = Array.from(
new Map(cameras.map((camera) => [camera.id, camera])).values(),
);
const maxRaw = Number(
process.env.CCTV_ICELAND_MAX_SOURCES || DEFAULT_ICELAND_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(600, Math.floor(maxRaw)))
: DEFAULT_ICELAND_MAX_SOURCES;
const prioritized = prioritizeSources(unique, maxCount, ICELAND_ANCHORS);
console.log(
`[CCTV] Loaded Iceland camera sources: ${unique.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Iceland camera download error:',
error?.message || error,
);
return [];
}
}
/**
* Read one XML child tag's text from an `<image>` block.
*
+3 -1
View File
@@ -5,7 +5,9 @@ export const HLS_LIMITS = Object.freeze({
sessions: 2,
leasesPerSession: 8,
playlistBytes: 256 * 1024,
segmentBytes: 4 * 1024 * 1024,
// High-resolution cameras can exceed 4 MiB in a single ten-second TS segment.
// Retained data still shares the independent 24 MiB session budget below.
segmentBytes: 8 * 1024 * 1024,
sessionBytes: 24 * 1024 * 1024,
segments: 12,
pollMs: 2000,
+158 -11
View File
@@ -1,4 +1,7 @@
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
import {
CCTV_SOURCE_FETCH_TIMEOUT_MS,
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
} from './constants.js';
import {
cameraDisplayCode,
fallbackHeadingFromId,
@@ -10,6 +13,8 @@ import { readResponseTextCapped } from '../common/http.js';
* standard, published keyless and refreshed daily (~1,800 cameras). */
export const TAIWAN_FREEWAY_CCTV_URL =
'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml';
export const TAIWAN_HIGHWAY_CCTV_URL =
'https://cctv-maintain.thb.gov.tw/opendataCCTVs.xml';
/** The whole network is ~1.2 MB. */
export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024;
const DEFAULT_TAIWAN_MAX_SOURCES = 600;
@@ -35,6 +40,9 @@ const STREAM_TO_FRAME = new Map([
]);
const DIRECTIONS = { N: 'N', S: 'S', E: 'E', W: 'W' };
const TAIWAN_HIGHWAY_CAMERA_HOSTS = new Set(
TAIWAN_HIGHWAY_IMAGE_ORIGINS.map((origin) => new URL(origin).hostname),
);
/** Prioritization anchors, north to south, so a cap keeps the whole island
* rather than only greater Taipei. */
@@ -211,6 +219,74 @@ export function taiwanCameraToSource(row) {
};
}
/**
* One Highway Bureau row -> a source whose first MJPEG frame is served as a
* snapshot by the existing camera-frame proxy.
*
* @param {object} row - Output of parseTaiwanCctvXml.
* @returns {?object}
*/
export function taiwanHighwayCameraToSource(row) {
if (!row || typeof row !== 'object') return null;
const { lat, lon } = row;
if (!isLikelyTaiwanCoordinate(lat, lon)) return null;
let url;
try {
url = new URL(String(row.streamUrl ?? ''));
} catch {
return null;
}
if (
url.protocol !== 'https:' ||
url.username ||
url.password ||
url.port ||
url.search ||
url.hash ||
!TAIWAN_HIGHWAY_CAMERA_HOSTS.has(url.hostname) ||
!/^\/[A-Za-z0-9._+~-]{1,100}$/.test(url.pathname) ||
url.pathname.includes('..')
)
return null;
const rawId = String(row.cctvId ?? '').trim();
if (!/^[A-Za-z0-9-]{1,80}$/.test(rawId)) return null;
const cameraId = 'tw-thb-' + rawId.toLowerCase();
const road = String(row.roadName ?? '').trim();
const direction = DIRECTIONS[String(row.direction ?? '').trim()] || '';
const mile = String(row.mile ?? '').trim();
const section = [row.sectionStart, row.sectionEnd]
.map((part) => String(part ?? '').trim())
.filter(Boolean)
.join('→');
const name =
(road || mile || section
? [road, direction, mile, section].filter(Boolean).join(' ')
: '') || rawId;
return {
id: cameraId,
name: name.slice(0, 160),
city: 'Taiwan / 臺灣',
cityId: 'taiwan',
provider: 'Highway Bureau, MOTC',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'mjpeg',
url: url.href,
snapshotUrl: url.href,
sourceKind: 'taiwan-highway-motc',
license: 'Highway Bureau, MOTC open data',
code: cameraDisplayCode((road + ' ' + mile).trim().toUpperCase() || rawId),
};
}
let lastGood = null;
let lastGoodAt = 0;
let retryAfter = 0;
@@ -237,6 +313,49 @@ function taiwanSourceCap() {
: DEFAULT_TAIWAN_MAX_SOURCES;
}
/** Fetch the public provincial-road CCTV table as a no-key fallback. */
async function loadTaiwanHighwaySources() {
const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS);
const response = await fetch(TAIWAN_HIGHWAY_CCTV_URL, {
headers: { Accept: 'application/xml,text/xml' },
redirect: 'manual',
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'Highway Bureau catalog redirected; redirects are not followed'
: 'Highway Bureau catalog HTTP ' + response.status,
);
}
const xml = await readResponseTextCapped(
response,
TAIWAN_MAX_CATALOG_BYTES,
signal,
);
const cameras = new Map();
for (const row of parseTaiwanCctvXml(xml)) {
const camera = taiwanHighwayCameraToSource(row);
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
}
if (!cameras.size)
throw new Error('no usable Highway Bureau cameras in the table');
const sources = prioritizeSources(
[...cameras.values()],
taiwanSourceCap(),
ANCHORS,
);
console.log(
'[CCTV] Loaded Taiwan Highway Bureau fallback: ' +
cameras.size +
' (using ' +
sources.length +
')',
);
return sources;
}
/**
* Fetch Taiwan Freeway Bureau cameras from the keyless MOTC 2.0 XML table.
* One bounded request per shared catalog refresh. A failed download serves the
@@ -262,7 +381,7 @@ export async function loadTaiwanSources() {
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `HTTP ${response.status}`,
: 'HTTP ' + response.status,
);
}
const xml = await readResponseTextCapped(
@@ -282,7 +401,11 @@ export async function loadTaiwanSources() {
ANCHORS,
);
console.log(
`[CCTV] Loaded Taiwan camera sources: ${cameras.size} (using ${sources.length})`,
'[CCTV] Loaded Taiwan camera sources: ' +
cameras.size +
' (using ' +
sources.length +
')',
);
lastGood = sources;
lastGoodAt = Date.now();
@@ -291,13 +414,37 @@ export async function loadTaiwanSources() {
} catch (error) {
retryAfter = Date.now() + TAIWAN_RETRY_AFTER_FAILURE_MS;
const fallback = fallbackSources();
console.warn(
`[CCTV] Taiwan camera download failed (${error?.message || error}); ` +
(fallback.length
? `serving the last good ${fallback.length} cameras`
: 'no Taiwan cameras') +
', next attempt in 1 h',
);
return fallback;
if (fallback.length) {
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); serving the last good ' +
fallback.length +
' cameras, next attempt in 1 h',
);
return fallback;
}
try {
const sources = await loadTaiwanHighwaySources();
lastGood = sources;
lastGoodAt = Date.now();
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); using ' +
sources.length +
' Highway Bureau cameras as fallback, next attempt in 1 h',
);
return sources;
} catch (fallbackError) {
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); Highway Bureau fallback failed (' +
(fallbackError?.message || fallbackError) +
'); no Taiwan cameras, next attempt in 1 h',
);
return [];
}
}
}
+132
View File
@@ -0,0 +1,132 @@
import { createHash } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import {
DEFAULT_WORLD_CCTV_SOURCE_FILE,
MLIT_RIVER_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
WORLD_CCTV_SOURCE_COUNT,
} from './constants.js';
import { pinnedCameraUrl } from './normalize.js';
const SOURCE_POLICIES = Object.freeze({
'tripcheck-odot': {
idPrefix: 'world-or-',
origin: TRIPCHECK_IMAGE_ORIGIN,
bounds: [41, 47, -125, -116],
filename: /^\/roadcams\/cams\/[^/]+\.jpe?g$/i,
frameRefreshMs: 5 * 60 * 1000,
},
'mlit-river': {
idPrefix: 'world-jp-river-',
origin: MLIT_RIVER_IMAGE_ORIGIN,
bounds: [24, 46, 122, 154],
filename: /^\/cam\/now\/\d+\.jpg$/i,
// The provider asks clients to avoid frequent repeated image collection.
frameRefreshMs: 10 * 60 * 1000,
},
});
// MLIT currently serves this fixed JPEG on cameras whose images are paused
// for maintenance (verified against two Yamagata cameras on 2026-11-02).
const MLIT_ADJUSTMENT_IMAGE_SHA256 =
'dbd2c3790eb75edf9ca44162f7979f2a3799090e23f2c5142f63211a035cb191';
export function isMlitRiverAdjustmentImage(image) {
return (
image?.ok === true &&
Buffer.isBuffer(image.body) &&
createHash('sha256').update(image.body).digest('hex') ===
MLIT_ADJUSTMENT_IMAGE_SHA256
);
}
function safeFrameUrl(value, policy) {
const normalized = pinnedCameraUrl(value, policy.origin);
if (!normalized) return '';
const url = new URL(normalized);
return !url.search && !url.hash && policy.filename.test(url.pathname)
? url.href
: '';
}
function inBounds(lat, lon, [minLat, maxLat, minLon, maxLon]) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= minLat &&
lat <= maxLat &&
lon >= minLon &&
lon <= maxLon
);
}
/**
* Load only the new image-backed networks from the add-on export. Existing
* Canadian networks, Québec entries, JMA placeholder images, and NZTA's
* scraped Journey Planner rows are deliberately excluded. Vegagerðin (Iceland)
* rows are excluded too: the Iceland pack loads the full live list of those
* cameras, so a World copy would draw each one twice.
*/
export function loadWorldCameraSources({ sourceRoot = process.cwd() } = {}) {
const filename = path.resolve(sourceRoot, DEFAULT_WORLD_CCTV_SOURCE_FILE);
let rows;
try {
const payload = JSON.parse(fs.readFileSync(filename, 'utf8'));
rows = Array.isArray(payload) ? payload : [];
} catch (error) {
console.warn('[CCTV] World camera catalog unavailable:', error?.message);
return [];
}
const seen = new Set();
const sources = [];
for (const row of rows) {
const sourceKind = String(row?.sourceKind || '');
const policy = SOURCE_POLICIES[sourceKind];
if (!policy) continue;
const id = String(row.id || '').trim();
const lat = Number(row.lat);
const lon = Number(row.lon);
const url = safeFrameUrl(row.url || row.snapshotUrl, policy);
const snapshotUrl = safeFrameUrl(row.snapshotUrl || row.url, policy);
if (
!id.startsWith(policy.idPrefix) ||
seen.has(id) ||
!inBounds(lat, lon, policy.bounds) ||
!url ||
!snapshotUrl ||
url !== snapshotUrl
)
continue;
seen.add(id);
sources.push({
id,
name: String(row.name || id)
.trim()
.slice(0, 180),
city: String(row.city || row.country || 'World'),
cityId: String(row.cityId || ''),
provider: String(row.provider || 'World traffic camera'),
lat,
lon,
headingDeg: Number.isFinite(Number(row.headingDeg))
? Number(row.headingDeg)
: undefined,
headingConfidence: String(row.headingConfidence || 'low'),
feedType: 'image',
sourceKind,
url,
snapshotUrl,
frameRefreshMs: policy.frameRefreshMs,
license: String(row.license || ''),
credit: String(row.credit || row.provider || ''),
});
}
console.log(
`[CCTV] Loaded World camera image sources: ${sources.length}/${WORLD_CCTV_SOURCE_COUNT}`,
);
return sources;
}
+81
View File
@@ -0,0 +1,81 @@
import { gzip } from 'node:zlib';
import { promisify } from 'node:util';
const gzipAsync = promisify(gzip);
/**
* Opt-in gzip for text bodies a proxy already holds in memory.
*
* Not a middleware: `cctv/media.js` streams bodies and answers range requests,
* and several providers forward binary upstream types that gzip only grows.
* `encodedBody` returns the body with its headers so the two cannot disagree.
*/
/** Skip compression for bodies smaller than 1 KiB. */
const MIN_COMPRESSED_BYTES = 1024;
/**
* RFC 9110 section 12.4.2 `qvalue`. Matching the whole token keeps `q=1oops`
* and `q=2` unreadable instead of rounding them into acceptance.
*/
const QVALUE = /^(?:0(?:\.\d{0,3})?|1(?:\.0{0,3})?)$/;
/**
* Whether `Accept-Encoding` offers gzip. Anything unreadable (no header, a bad
* q value, `x-gzip`) reads as no: every client can decode identity.
*
* @param {import('node:http').IncomingMessage} request
* @returns {boolean}
*/
export function acceptsGzip(request) {
const header = request?.headers?.['accept-encoding'];
if (typeof header !== 'string') return false;
return header.split(',').some((part) => {
const [token, ...parameters] = part.split(';');
if (token.trim().toLowerCase() !== 'gzip') return false;
const quality = parameters
.map((parameter) => parameter.trim().toLowerCase())
.find((parameter) => parameter.startsWith('q='));
if (quality === undefined) return true;
// `q=0` names gzip and refuses it.
const value = quality.slice(2);
return QVALUE.test(value) && Number(value) > 0;
});
}
/**
* Gzip a body once for a caller that caches it. Async, so the ~860 KB OpenSky
* snapshot does not block other requests while it compresses.
*
* Returns null, meaning "serve identity", when the body is under 1 KiB or gzip
* does not shrink it.
*
* @param {string|Buffer} body - the exact bytes the identity response sends.
* @returns {Promise<Buffer|null>}
*/
export async function gzipBody(body) {
const raw = Buffer.isBuffer(body) ? body : Buffer.from(body);
if (raw.byteLength < MIN_COMPRESSED_BYTES) return null;
const compressed = await gzipAsync(raw);
return compressed.byteLength < raw.byteLength ? compressed : null;
}
/**
* Pick the body and the headers that describe it. `Vary` is on both branches
* so a shared cache never serves one encoding to a client that asked for the
* other.
*
* @param {import('node:http').IncomingMessage} request
* @param {string|Buffer} identity - the uncompressed body.
* @param {Buffer|null} compressed - `gzipBody(identity)`, or null.
* @returns {{body: string|Buffer, headers: Record<string,string>}}
*/
export function encodedBody(request, identity, compressed) {
if (compressed && acceptsGzip(request)) {
return {
body: compressed,
headers: { 'Content-Encoding': 'gzip', Vary: 'Accept-Encoding' },
};
}
return { body: identity, headers: { Vary: 'Accept-Encoding' } };
}
+11 -9
View File
@@ -131,9 +131,9 @@ export async function fetchFirms(
* NASA FIRMS live active-fire proxy with a memory + disk cache.
* Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
*
* Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent
* satellites, no cross-source dedup) plus MODIS_NRT (combined Terra+Aqua,
* ~1 km, numeric 0-100 confidence), fetched sequentially (quota courtesy)
* Merges two VIIRS NRT sources (NOAA-20, NOAA-21 — independent satellites, no
* cross-source dedup) plus MODIS_NRT (combined Terra+Aqua, ~1 km, numeric
* 0-100 confidence), fetched sequentially (quota courtesy)
* with `days=2` (`days=1` means "current UTC day", nearly empty just after
* 00:00Z) and clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS
* quota is 5,000 transactions / 10 min per MAP_KEY, so the cache is the point:
@@ -141,6 +141,8 @@ export async function fetchFirms(
* fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream
* fetch across dev-server restarts. Pattern mirrors celestrakProxy.
*
* VIIRS_SNPP_NRT is left out: NASA stopped delivering Suomi NPP on 2026-11-02.
*
* Routes:
* GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires}
* GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions}
@@ -148,17 +150,17 @@ export async function fetchFirms(
* Keyless (no FIRMS_MAP_KEY): /api/firms → 503 {error:'no_key'}; status →
* {hasKey:false}. Upstream is never touched without a key.
*
* /api/firms answers the trailing 24 hours to now. A source may instead
* answer the 24 hours to a set time, naming its end as `at` (epoch ms); the
* layer's row then shows that time rather than LIVE. This proxy never sets
* `at`.
*
* @returns {import('vite').Plugin}
*/
export function firmsProxy() {
const TTL_MS = 30 * 60_000;
const STATUS_TTL_MS = 5 * 60_000;
const SOURCES = [
'VIIRS_NOAA20_NRT',
'VIIRS_NOAA21_NRT',
'VIIRS_SNPP_NRT',
'MODIS_NRT',
];
const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'MODIS_NRT'];
/**
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
+4 -1
View File
@@ -63,7 +63,10 @@ export function createGeminiTokenHandler({
return reply(403, 'Gemini voice requires a same-origin app request');
apiKey = String(resolveApiKey() || '').trim();
if (!apiKey)
return reply(503, 'Add a Gemini API key in Provider Settings');
return reply(503, {
error: 'Add a Gemini API key in Provider Settings',
code: 'API_KEY_MISSING',
});
model = normalizeGeminiModel(resolveModel());
if (!model)
return reply(503, 'The Gemini Live model configuration is invalid');
+103
View File
@@ -0,0 +1,103 @@
import { LIBRARY_ENTRIES } from '../../src/library/catalog.js';
import { createLibraryHandler } from '../../src/library/proxy.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSite, admitSameSiteKeyless } from './common/same-site.js';
export {
libraryAvailability,
libraryKeyValue,
parseLibraryTilePath,
} from '../../src/library/proxy.js';
/** `content-type` as `Content-Type`: a fetch Response's names, as Node writes them. */
const headerName = (name) =>
name.replace(
/(^|-)([a-z])/g,
(_, dash, letter) => dash + letter.toUpperCase(),
);
/**
* The Data Library's one server route on the dev and preview servers: the
* portable handler (`src/library/proxy.js`) behind this server's same-site
* gate and rate limits.
*/
export function libraryProxy({
entries = LIBRARY_ENTRIES,
env = process.env,
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
} = {}) {
const byId = new Map(entries.map((entry) => [entry.id, entry]));
const handle = createLibraryHandler({ entries, env, fetchImpl, now });
const allowData = makeRateLimiter({
windowMs: 60_000,
max: 30,
globalMax: 600,
});
const allowTiles = makeRateLimiter({
windowMs: 60_000,
max: 600,
globalMax: 6000,
});
function refuse(res, status, error) {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...(status === 429 ? { 'Retry-After': '60' } : {}),
});
res.end(JSON.stringify({ error }));
}
async function handler(req, res) {
if (req.method !== 'GET') return refuse(res, 405, 'method_not_allowed');
const path = (req.url || '/').split('?')[0];
const [route, id] = path.split('/').filter(Boolean);
const entry = byId.get(id);
if (route === 'catalog' || !entry) {
if (admitSameSiteKeyless(req, res)) return undefined;
} else {
// A keyed entry spends the operator's credential: refuse forwarded
// requests like the other cost-bearing routes. Keyless entries keep
// working behind a reverse proxy, like terrain heights.
if (
entry.source?.key
? admitSameSite(req, res)
: admitSameSiteKeyless(req, res)
)
return undefined;
const allow = route === 'tile' ? allowTiles : allowData;
if (!allow(clientKey(req))) return refuse(res, 429, 'rate_limited');
}
const response = await handle(
new Request(`http://library.invalid/api/library${req.url || '/'}`, {
headers: { Accept: String(req.headers?.accept || '*/*') },
}),
);
if (res.destroyed) return undefined;
const headers = {};
for (const [name, value] of response.headers)
headers[headerName(name)] = value;
const body = Buffer.from(await response.arrayBuffer());
res.writeHead(response.status, headers);
res.end(
String(headers['Content-Type']).startsWith('application/json')
? body.toString('utf8')
: body,
);
return undefined;
}
return {
name: 'data-library',
handler,
configureServer({ middlewares }) {
middlewares.use('/api/library', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/library', handler);
},
};
}
+381
View File
@@ -0,0 +1,381 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
libraryAvailability,
libraryProxy,
parseLibraryTilePath,
} from './library.js';
import { validateLibraryCatalog } from '../../src/library/manifest.js';
import { libraryEntry } from '../../src/library/catalog.js';
const base = {
family: 'events',
provider: 'Fixture',
description: 'Fixture entry.',
maintainer: 'test',
coverage: { global: true },
licence: { name: 'Public domain', url: 'https://example.org/licence' },
attribution: { text: 'Fixture' },
status: 'community',
};
const { entries, errors } = validateLibraryCatalog([
{
...base,
id: 'proxied-points',
kind: 'dataset',
title: 'Proxied points',
source: {
format: 'geojson',
url: 'https://data.example.org/points.json',
fetch: 'proxied',
maxBytes: 4096,
},
display: { label: 'title' },
},
{
...base,
id: 'direct-points',
kind: 'dataset',
title: 'Direct points',
source: {
format: 'geojson',
url: 'https://data.example.org/direct.json',
fetch: 'browser-direct',
},
},
{
...base,
id: 'keyed-points',
kind: 'dataset',
title: 'Keyed points',
source: {
format: 'geojson',
url: 'https://keyed.example.org/points.json',
fetch: 'proxied',
key: { id: 'firms', in: 'query', name: 'MAP_KEY' },
},
},
{
...base,
id: 'opt-in-points',
kind: 'dataset',
title: 'Opt-in points',
source: {
format: 'geojson',
url: 'https://data.example.org/opt.json',
fetch: 'proxied',
},
licence: { ...base.licence, operatorOptIn: true },
},
{
...base,
id: 'proxied-tiles',
kind: 'tiles',
title: 'Proxied tiles',
source: {
format: 'xyz',
url: 'https://tiles.example.org/{z}/{x}/{y}.png',
fetch: 'proxied',
minZoom: 2,
maxZoom: 5,
},
},
{
...base,
id: 'fixture-pack',
kind: 'pack',
family: 'cameras',
title: 'Fixture cameras',
source: { format: 'host', url: 'https://cams.example.org/' },
pack: { host: 'cctv', env: 'CCTV_FIXTURE_ENABLED' },
licence: { ...base.licence, operatorOptIn: true },
},
]);
assert.deepEqual(errors, []);
const collection = (features) =>
JSON.stringify({ type: 'FeatureCollection', features });
const point = (title) => ({
type: 'Feature',
geometry: { type: 'Point', coordinates: [1, 2] },
properties: { title },
});
function install({ env = {}, fetchImpl, now, catalog = entries } = {}) {
const plugin = libraryProxy({ entries: catalog, env, fetchImpl, now });
let handler;
plugin.configurePreviewServer({
middlewares: {
use(path, callback) {
assert.equal(path, '/api/library');
handler = callback;
},
},
});
return async (url, headers = {}) => {
const res = {
headers: {},
writeHead(status, values) {
this.status = status;
this.headers = values;
},
setHeader(name, value) {
this.headers[name] = value;
},
end(body) {
this.raw = body;
try {
this.body = JSON.parse(body);
} catch {
this.body = null;
}
},
set statusCode(value) {
this.status = value;
},
};
await handler(
{ url, method: 'GET', headers, socket: { remoteAddress: '127.0.0.1' } },
res,
);
return res;
};
}
test('availability is decided by the operator environment and names no secret', () => {
const none = libraryAvailability(entries, {});
assert.deepEqual(none.get('proxied-points'), {
available: true,
reason: null,
});
assert.equal(none.get('keyed-points').reason, 'key-missing');
assert.equal(none.get('opt-in-points').reason, 'operator-opt-in');
assert.equal(none.get('fixture-pack').reason, 'operator-opt-in');
const configured = libraryAvailability(entries, {
FIRMS_MAP_KEY: 'secret',
GEV_LIBRARY_ENABLE: 'opt-in-points',
GEV_LIBRARY_DISABLE: 'proxied-points',
CCTV_FIXTURE_ENABLED: '1',
});
assert.equal(configured.get('keyed-points').available, true);
assert.equal(configured.get('opt-in-points').available, true);
assert.equal(configured.get('proxied-points').reason, 'operator-disabled');
assert.equal(configured.get('fixture-pack').available, true);
assert.doesNotMatch(JSON.stringify([...configured]), /secret/);
});
test('the catalog route lists every entry with its availability', async () => {
const request = install();
const res = await request('/catalog');
assert.equal(res.status, 200);
assert.equal(res.body.entries.length, entries.length);
assert.deepEqual(
res.body.entries.find(({ id }) => id === 'opt-in-points'),
{ id: 'opt-in-points', available: false, reason: 'operator-opt-in' },
);
});
test('a proxied dataset reaches only its manifest URL, refuses redirects and is normalized', async () => {
const calls = [];
const request = install({
now: () => 42,
fetchImpl: async (url, options) => {
calls.push({ url, options });
return new Response(collection([point('Storm'), { geometry: null }]));
},
});
const res = await request('/data/proxied-points?url=https://evil.example');
assert.equal(res.status, 200);
assert.equal(calls.length, 1);
assert.equal(calls[0].url, 'https://data.example.org/points.json');
assert.equal(calls[0].options.redirect, 'error');
assert.ok(calls[0].options.signal instanceof AbortSignal);
assert.equal(res.body.features.length, 1);
assert.equal(res.body.features[0].properties.name, 'Storm');
assert.equal(res.body.dropped, 1);
assert.equal(res.body.fetchedAt, 42);
// A fresh cache answers without another upstream request.
await request('/data/proxied-points');
assert.equal(calls.length, 1);
});
test('browser-direct, unknown, opt-in and wrong-kind entries are not proxied', async () => {
let calls = 0;
const request = install({
fetchImpl: async () => {
calls++;
return new Response(collection([]));
},
});
assert.equal((await request('/data/direct-points')).status, 404);
assert.equal((await request('/data/nope-nope')).status, 404);
assert.equal((await request('/data/proxied-tiles')).status, 404);
assert.equal((await request('/tile/proxied-points/2/0/0')).status, 404);
const optIn = await request('/data/opt-in-points');
assert.equal(optIn.status, 403);
assert.equal(optIn.body.error, 'operator-opt-in');
assert.equal(calls, 0);
});
test('a keyed entry gets the operator key server-side and refuses forwarded requests', async () => {
const calls = [];
const request = install({
env: { FIRMS_MAP_KEY: 'k3y' },
fetchImpl: async (url) => {
calls.push(new URL(url));
return new Response(collection([point('Fire')]));
},
});
const forwarded = await request('/data/keyed-points', {
'x-forwarded-for': '203.0.113.9',
});
assert.equal(forwarded.status, 403);
assert.equal(calls.length, 0);
const res = await request('/data/keyed-points');
assert.equal(res.status, 200);
assert.equal(calls[0].searchParams.get('MAP_KEY'), 'k3y');
assert.doesNotMatch(res.raw, /k3y/);
});
test('an oversized or failing upstream answers 502 without its text, then serves stale', async () => {
let clock = 0;
let mode = 'ok';
const request = install({
now: () => clock,
fetchImpl: async () => {
if (mode === 'big') return new Response('x'.repeat(8192));
if (mode === 'fail')
return new Response('secret upstream text', { status: 500 });
return new Response(collection([point('A')]));
},
});
mode = 'big';
const big = await request('/data/proxied-points');
assert.equal(big.status, 502);
assert.doesNotMatch(big.raw, /x{20}/);
mode = 'ok';
assert.equal((await request('/data/proxied-points')).status, 200);
clock = 10 * 3600 * 1000;
mode = 'fail';
const stale = await request('/data/proxied-points');
assert.equal(stale.status, 200);
assert.equal(stale.body.stale, true);
assert.equal(stale.headers['X-Data-Stale'], 'true');
assert.doesNotMatch(stale.raw, /secret upstream text/);
});
test('cross-site browser requests are refused before any upstream request', async () => {
let calls = 0;
const request = install({
fetchImpl: async () => {
calls++;
return new Response(collection([]));
},
});
const res = await request('/data/proxied-points', {
host: 'localhost:5173',
origin: 'https://evil.example',
'sec-fetch-site': 'cross-site',
});
assert.equal(res.status, 403);
assert.equal(calls, 0);
});
test('tiles: bounded z/x/y, image types only, cached', async () => {
const calls = [];
const request = install({
fetchImpl: async (url) => {
calls.push(url);
if (url.includes('/3/'))
return new Response('<html>', {
headers: { 'content-type': 'text/html' },
});
return new Response(new Uint8Array([137, 80, 78, 71]), {
headers: { 'content-type': 'image/png' },
});
},
});
assert.equal((await request('/tile/proxied-tiles/1/0/0')).status, 400);
assert.equal((await request('/tile/proxied-tiles/2/4/0')).status, 400);
assert.equal((await request('/tile/proxied-tiles/2/a/0')).status, 400);
const ok = await request('/tile/proxied-tiles/2/1/3');
assert.equal(ok.status, 200);
assert.equal(ok.headers['Content-Type'], 'image/png');
assert.equal(calls[0], 'https://tiles.example.org/2/1/3.png');
await request('/tile/proxied-tiles/2/1/3');
assert.equal(calls.length, 1);
assert.equal((await request('/tile/proxied-tiles/3/1/1')).status, 502);
});
test('tile paths are parsed against the zoom range', () => {
const source = { minZoom: 0, maxZoom: 3 };
assert.deepEqual(parseLibraryTilePath(['3', '7', '7'], source), {
z: 3,
x: 7,
y: 7,
});
assert.equal(parseLibraryTilePath(['3', '8', '0'], source), null);
assert.equal(parseLibraryTilePath(['4', '0', '0'], source), null);
assert.equal(parseLibraryTilePath(['1', '0'], source), null);
});
test('only GET is served', async () => {
const plugin = libraryProxy({ entries, env: {} });
const res = {
writeHead(status, headers) {
this.status = status;
this.headers = headers;
},
end() {},
};
await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res);
assert.equal(res.status, 405);
});
test('a partition index is cached only when complete; a complete one outlives failed rebuilds', async () => {
const epc = libraryEntry('epc-england-wales');
let clock = 0;
let footers = 'ok';
let listings = 0;
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
const fetchImpl = async (url) => {
if (url.includes('list-type')) {
listings += 1;
return new Response(listing);
}
return footers === 'ok'
? new Response(new Uint8Array(100))
: new Response('', { status: 503 });
};
const request = install({ catalog: [epc], now: () => clock, fetchImpl });
const first = await request('/index/epc-england-wales');
assert.equal(first.status, 200);
assert.equal(first.body.complete, true);
const builtAt = first.body.builtAt;
// Past cacheSeconds the rebuild fails: the complete index is still served,
// and no rebuild runs again until the partial pause has passed.
clock += (epc.source.cacheSeconds + 1) * 1000;
footers = 'fail';
const kept = await request('/index/epc-england-wales');
assert.equal(kept.body.complete, true);
assert.equal(kept.body.builtAt, builtAt);
const listed = listings;
await request('/index/epc-england-wales');
assert.equal(listings, listed);
// A server that never had a complete index serves the partial one as such,
// and rebuilds after the pause.
const fresh = install({ catalog: [epc], now: () => clock, fetchImpl });
const partial = await fresh('/index/epc-england-wales');
assert.equal(partial.status, 200);
assert.equal(partial.body.complete, false);
const before = listings;
await fresh('/index/epc-england-wales');
assert.equal(listings, before);
clock += 6 * 60_000;
footers = 'ok';
const rebuilt = await fresh('/index/epc-england-wales');
assert.equal(rebuilt.body.complete, true);
assert.equal(listings, before + 1);
});
+8
View File
@@ -16,6 +16,7 @@ import { radioBrowserProxy } from './radio.js';
import { gbfsProxy } from './gbfs.js';
import { localReceiversProxy } from './local-receivers.js';
import { transitProxy } from './transit.js';
import { railProxy } from './rail.js';
import { adsbLolProxy } from './aircraft/adsb-lol.js';
import { aisLiveProxy } from './vessels/ais-live.js';
import { trackBackfillProxies } from './aircraft/tracks.js';
@@ -28,11 +29,14 @@ import { weatherProxy } from './weather.js';
import { firePerimetersProxy } from './firePerimeters.js';
import { gnssIntegrityProxy } from './gnssIntegrity.js';
import { ghostMapsProxy } from './ghostMaps.js';
import { shipwrecksProxy } from './shipwrecks.js';
import { droneAirspaceProxy } from './droneAirspace.js';
import { newsProxy } from './news.js';
import { inaturalistProxy } from './inaturalist/observations.js';
import { acartiaProxy } from './acartia.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
import { libraryProxy } from './library.js';
/**
* Construct the local provider plugins in their established order.
@@ -69,6 +73,7 @@ function localProviderPlugins({
gbfsProxy(),
localReceiversProxy(),
transitProxy(),
railProxy(),
adsbLolProxy(),
aisLiveProxy(),
trackBackfillProxies(),
@@ -81,8 +86,11 @@ function localProviderPlugins({
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
libraryProxy(),
ghostMapsProxy(),
shipwrecksProxy(),
droneAirspaceProxy(),
newsProxy(),
inaturalistProxy(),
acartiaProxy(),
keySetupEndpoint(),
+191
View File
@@ -0,0 +1,191 @@
import { newsQueryAllowed } from '../../src/layers/news/source.js';
import { readResponseTextCapped } from './common/http.js';
import { sameSiteGated } from './common/same-site.js';
import { createKeyedNews } from './news/keyed.js';
// News (/api/news/*). With GEV_NEWS_ORIGIN set (a news server such as
// https://news.example), this forwards exactly the GET requests the news
// layer sends there and relays the JSON. Without it (or with a value that
// is no plain origin, warned of once), Core answers the same routes itself
// (news/keyed.js): Conflicts from GDELT's event exports with no key, and
// Liveuamap's items beside them with LIVEUAMAP_API_KEY; Local from World
// News API, "key required" while WORLDNEWSAPI_KEY is unset. A host that
// serves news only from its own server mounts it with `builtIn: false`.
const TIMEOUT_MS = 15_000;
const MAX_BYTES = 4 * 1024 * 1024;
/**
* The configured news server's origin, or null: http(s) only, no
* credentials, query or fragment; a path is ignored.
* @param {string|undefined} raw
*/
export function newsOrigin(raw = process.env.GEV_NEWS_ORIGIN) {
const text = String(raw ?? '').trim();
if (!text) return null;
try {
const url = new URL(text);
if (
!['http:', 'https:'].includes(url.protocol) ||
url.username ||
url.password ||
url.search ||
url.hash
)
return null;
return url.origin;
} catch {
return null;
}
}
/** Text for the log: no address (a key may ride in one), one short line. */
const logText = (error) =>
`${error?.name || 'Error'}: ${String(error?.message ?? '')}`
.replace(/[a-z][a-z0-9+.-]*:\/\/\S*/gi, '<url>')
.replace(/\s+/g, ' ')
.slice(0, 200);
/** GEV_NEWS_ORIGIN as newsOrigin reads it; a value it refuses is warned of once. */
function configuredOrigin(env, log) {
let warned = null;
return () => {
const raw = String(env().GEV_NEWS_ORIGIN ?? '').trim();
const base = newsOrigin(raw);
if (!base && raw && warned !== raw) {
warned = raw;
// Not the value: it may carry credentials.
log(
'[news] GEV_NEWS_ORIGIN is not a plain http(s) origin; answering with the built-in news backend',
);
}
return base;
};
}
/** A 503's known key or configuration state, rebuilt from its fields; else null. */
async function upstreamState(upstream) {
let body = null;
try {
body = JSON.parse(await readResponseTextCapped(upstream, 16 * 1024));
} catch {
return null;
}
if (body?.error === 'news_not_configured')
return { error: body.error, unavailable: true, retryable: false };
if (
['no_key', 'key_rejected', 'key_quota'].includes(body?.error) &&
typeof body.key === 'string' &&
/^[a-z][a-z0-9-]{0,31}$/.test(body.key)
)
return { error: body.error, key: body.key };
return null;
}
/**
* A thin, validated GET proxy for the news layer's three read routes, or
* their keyed answers without a news server. A host that serves news only
* from its own server passes `builtIn: false`: without GEV_NEWS_ORIGIN the
* routes then answer "not configured" and Core reads nothing itself.
* @param {{env?: () => object, log?: (message: string) => void, origin?: () => string|null, fetchImpl?: typeof fetch, timeoutMs?: number, builtIn?: boolean, keyed?: {answer: (path: string, params: URLSearchParams) => Promise<{status: number, body: object}>}|null}} [options]
*/
export function newsProxy({
env = () => process.env,
log = (message) => console.warn(message),
origin = configuredOrigin(env, log),
fetchImpl = (...args) => globalThis.fetch(...args),
timeoutMs = TIMEOUT_MS,
builtIn = true,
keyed = builtIn ? createKeyedNews({ env, fetchImpl, log }) : null,
} = {}) {
const json = (res, status, value) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
});
res.end(typeof value === 'string' ? value : JSON.stringify(value));
};
async function handler(req, res) {
if (req.method !== 'GET')
return json(res, 405, { error: 'method_not_allowed' });
let local, target;
try {
// `//[` and the like are no URL: a 400, not an unhandled rejection.
local = new URL(req.url || '/', 'http://local');
target = new URL(
`/api/news${local.pathname}${local.search}`,
'http://local',
);
} catch {
return json(res, 400, { error: 'invalid_news_request' });
}
if (!newsQueryAllowed(target))
return json(res, 400, { error: 'invalid_news_request' });
const base = origin();
if (!base) {
if (!keyed)
return json(res, 503, {
error: 'news_not_configured',
unavailable: true,
retryable: false,
});
try {
const { status, body } = await keyed.answer(
local.pathname,
target.searchParams,
);
return json(res, status, body);
} catch (error) {
log(`[news] built-in news answer failed: ${logText(error)}`);
return json(res, 502, { error: 'news_unavailable' });
}
}
try {
const upstream = await fetchImpl(
new URL(`${target.pathname}${target.search}`, base),
{
headers: { Accept: 'application/json' },
redirect: 'error',
signal: AbortSignal.timeout(timeoutMs),
},
);
if (upstream.status === 503) {
// The news server's own key and configuration states reach the
// panel as they are, so it can say which key is missing.
const state = await upstreamState(upstream);
if (state) return json(res, 503, state);
}
if (!upstream.ok) {
await upstream.body?.cancel().catch(() => {});
return json(res, 502, {
error: 'news_upstream_error',
status: upstream.status,
});
}
const text = await readResponseTextCapped(upstream, MAX_BYTES);
// Relayed only as JSON; the browser validates every record.
JSON.parse(text);
return json(res, 200, text);
} catch (error) {
return json(res, error?.name === 'TimeoutError' ? 504 : 502, {
error: 'news_unavailable',
});
}
}
// The routes spend keys and upstream reads: no other site's page, and no
// proxied request, may drive them (SECURITY.md, cross-site gate).
const gated = sameSiteGated(handler);
return {
name: 'news',
configureServer({ middlewares }) {
middlewares.use('/api/news', gated);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/news', gated);
},
};
}
+294
View File
@@ -0,0 +1,294 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { newsOrigin, newsProxy } from './news.js';
import { createKeyedNews } from './news/keyed.js';
/** Mount the proxy on a fake middleware stack and call it like Connect does. */
function mount(options) {
let handler = null;
newsProxy(options).configureServer({
middlewares: {
use(prefix, fn) {
assert.equal(prefix, '/api/news');
handler = fn;
},
},
});
return async (url, method = 'GET', headers = {}) => {
const res = {
destroyed: false,
status: 0,
headers: {},
body: '',
writeHead(status, headers) {
this.status = status;
this.headers = headers;
},
setHeader(name, value) {
this.headers[name] = value;
},
end(body) {
if (this.statusCode) this.status = this.statusCode;
this.body = body;
},
};
await handler({ url, method, headers }, res);
return { ...res, json: res.body ? JSON.parse(res.body) : null };
};
}
test('the origin must be a plain http(s) origin', () => {
assert.equal(
newsOrigin('http://news.example:8080'),
'http://news.example:8080',
);
assert.equal(
newsOrigin('https://news.example/base/'),
'https://news.example',
);
for (const bad of [
undefined,
'',
' ',
'ftp://x',
'http://u:p@x',
'http://x/?a=1',
'http://x/#f',
'not a url',
])
assert.equal(newsOrigin(bad), null, String(bad));
});
test('without GEV_NEWS_ORIGIN Core answers itself: Conflicts from GDELT, Local saying it needs its key', async () => {
const hosts = [];
const fetchImpl = async (url) => {
hosts.push(new URL(String(url)).host);
throw new Error('offline');
};
const call = mount({
origin: () => null,
fetchImpl,
keyed: createKeyedNews({ env: () => ({}), fetchImpl, log: () => {} }),
});
// GDELT's exports need no key; offline, the read fails like any upstream.
const hotspots = await call('/hotspots');
assert.equal(hotspots.status, 502);
assert.deepEqual(hotspots.json, { error: 'news_upstream_error' });
const local = await call('/feed?hotspot=all&at=31.95,35.93');
assert.equal(local.status, 503);
assert.deepEqual(local.json, { error: 'no_key', key: 'worldnewsapi' });
// The area is read from the bundled countries, with no key.
const area = await call('/outline?at=31.95,35.93&v=4');
assert.equal(area.status, 200);
assert.equal(area.json.area.name, 'Jordan');
// The client's allowlist still holds.
assert.equal((await call('/feed?side=UA')).status, 400);
assert.ok(hosts.length > 0);
assert.ok(hosts.every((host) => host === 'data.gdeltproject.org'));
});
test('forwards only the client routes, by GET, and relays the JSON', async () => {
const calls = [];
const call = mount({
origin: () => 'https://news.example',
fetchImpl: async (url, init) => {
calls.push({ url: String(url), init });
return new Response('{"reports":[],"nextBefore":null}', { status: 200 });
},
});
const ok = await call('/feed?hotspot=all&limit=60&today=1&events=1');
assert.equal(ok.status, 200);
assert.deepEqual(ok.json, { reports: [], nextBefore: null });
assert.equal(ok.headers['Cache-Control'], 'no-store');
assert.equal(
calls[0].url,
'https://news.example/api/news/feed?hotspot=all&limit=60&today=1&events=1',
);
assert.equal(calls[0].init.redirect, 'error');
assert.ok(calls[0].init.signal instanceof AbortSignal);
for (const url of [
'/air-alerts',
'/feed?side=UA',
'/feed?limit=9999',
'/../admin',
'/hotspots?x=1',
]) {
const refused = await call(url);
assert.equal(refused.status, 400, url);
}
const post = await call('/hotspots', 'POST');
assert.equal(post.status, 405);
assert.equal(post.headers.Allow, 'GET');
assert.equal(calls.length, 1);
});
test('upstream failures, bad JSON and timeouts become bounded errors', async () => {
const failing = mount({
origin: () => 'https://news.example',
fetchImpl: async () => new Response('down', { status: 500 }),
});
const down = await failing('/hotspots');
assert.equal(down.status, 502);
assert.deepEqual(down.json, { error: 'news_upstream_error', status: 500 });
const garbled = mount({
origin: () => 'https://news.example',
fetchImpl: async () => new Response('<html>', { status: 200 }),
});
assert.equal((await garbled('/hotspots')).status, 502);
const slow = mount({
origin: () => 'https://news.example',
timeoutMs: 10,
// AbortSignal.timeout does not hold the event loop open: the hanging
// fetch does, until its signal aborts, so the run cannot end before it.
fetchImpl: (_url, { signal }) =>
new Promise((_, reject) => {
const alive = setTimeout(() => {}, 60_000);
signal.addEventListener('abort', () => {
clearTimeout(alive);
reject(signal.reason);
});
}),
});
const timedOut = await slow('/hotspots');
assert.equal(timedOut.status, 504);
assert.deepEqual(timedOut.json, { error: 'news_unavailable' });
});
test('a GEV_NEWS_ORIGIN that is no plain origin is warned of once, and Core answers itself', async () => {
const logs = [];
const asked = [];
const call = mount({
env: () => ({ GEV_NEWS_ORIGIN: 'https://user:[email protected]' }),
log: (line) => logs.push(line),
fetchImpl: async () => assert.fail('nothing is forwarded'),
keyed: {
answer: async (path) => {
asked.push(path);
return { status: 200, body: { reports: [], nextBefore: null } };
},
},
});
assert.equal((await call('/feed?hotspot=all')).status, 200);
assert.equal((await call('/feed?hotspot=all')).status, 200);
assert.deepEqual(asked, ['/feed', '/feed']);
assert.equal(logs.length, 1);
assert.doesNotMatch(logs[0], /secret|news\.example/);
});
test('a built-in answer that throws is a 502, logged in one line without addresses', async () => {
const logs = [];
const call = mount({
env: () => ({}),
log: (line) => logs.push(line),
keyed: {
answer: async () => {
throw new TypeError('bad https://api.example/?key=abc\nmore');
},
},
});
const failed = await call('/hotspots');
assert.equal(failed.status, 502);
assert.deepEqual(failed.json, { error: 'news_unavailable' });
assert.equal(logs.length, 1);
assert.match(logs[0], /TypeError/);
assert.doesNotMatch(logs[0], /key=abc|\n/);
});
test('a host that serves news only from its own server turns the built-in backend off', async () => {
let fetches = 0;
const call = mount({
origin: () => null,
builtIn: false,
fetchImpl: async () => {
fetches++;
throw new Error('unexpected');
},
});
for (const url of [
'/hotspots',
'/feed?hotspot=all',
'/outline?at=31.95,35.93&v=4',
]) {
const answer = await call(url);
assert.equal(answer.status, 503, url);
assert.deepEqual(answer.json, {
error: 'news_not_configured',
unavailable: true,
retryable: false,
});
}
assert.equal(fetches, 0);
});
test('a request line that is no URL is a 400, not a crash', async () => {
const call = mount({
origin: () => null,
keyed: { answer: async () => assert.fail('nothing is answered') },
});
// `GET /api/news//[`: Connect strips the mount prefix and leaves `//[`.
const bad = await call('//[');
assert.equal(bad.status, 400);
assert.deepEqual(bad.json, { error: 'invalid_news_request' });
});
test('cross-site and proxied requests are refused before anything is read', async () => {
let answered = 0;
const call = mount({
origin: () => null,
keyed: {
answer: async () => {
answered++;
return { status: 200, body: { reports: [], nextBefore: null } };
},
},
});
const host = 'localhost:5173';
for (const headers of [
{ host, origin: 'https://evil.example' },
{ host, origin: 'null' },
{ host, 'sec-fetch-site': 'cross-site' },
{ host, 'x-forwarded-for': '203.0.113.9' },
]) {
const refused = await call('/feed?hotspot=all', 'GET', headers);
assert.equal(refused.status, 403, JSON.stringify(headers));
}
assert.equal(answered, 0);
for (const headers of [
{ host },
{ host, origin: `http://${host}`, 'sec-fetch-site': 'same-origin' },
]) {
const passed = await call('/feed?hotspot=all', 'GET', headers);
assert.equal(passed.status, 200, JSON.stringify(headers));
}
assert.equal(answered, 2);
});
test("a news server's key and configuration states are relayed as they are", async () => {
for (const body of [
{ error: 'no_key', key: 'worldnewsapi' },
{ error: 'key_rejected', key: 'worldnewsapi' },
{ error: 'key_quota', key: 'worldnewsapi' },
{ error: 'news_not_configured', unavailable: true, retryable: false },
]) {
const call = mount({
origin: () => 'https://news.example',
fetchImpl: async () =>
new Response(JSON.stringify(body), { status: 503 }),
});
const relayed = await call('/feed?hotspot=all&at=31.95,35.93');
assert.equal(relayed.status, 503, body.error);
assert.deepEqual(relayed.json, body);
}
// Any other 503 stays a bounded upstream error.
const other = mount({
origin: () => 'https://news.example',
fetchImpl: async () =>
new Response('{"error":"maintenance","detail":"x"}', { status: 503 }),
});
const down = await other('/hotspots');
assert.equal(down.status, 502);
assert.deepEqual(down.json, { error: 'news_upstream_error', status: 503 });
});
+232
View File
@@ -0,0 +1,232 @@
// Areas for the News layer, from the bundled Natural Earth packs (public
// domain): the disputed area or else the country at a point, its outline in
// the /api/news/outline form, and a point inside a country to pin its
// national news at. A disputed area is checked first and named as itself,
// so no one country's name stands for it; so is a territory the countries
// pack gives no code (Somaliland), rather than read as sea. Read once, on
// first use.
import { readFileSync } from 'node:fs';
import {
containsAnchor,
geometryOf,
pointInRing,
} from '../../../src/data/adminBoundaries.js';
const PACKS = '../../../src/data/local_data/natural_earth/';
const PACK = new URL(`${PACKS}countries.json`, import.meta.url);
const DISPUTED = new URL(`${PACKS}disputed_areas.json`, import.meta.url);
const ISO2 = /^[A-Z]{2}$/;
let countries = null;
let territories = [];
/**
* One entry per ISO code: its name and every Country part's geometry. The
* Country features without a code ("-99": Somaliland, Northern Cyprus,
* sovereign base areas, no man's lands) are kept apart as territories.
*/
function loadCountries(read = () => JSON.parse(readFileSync(PACK, 'utf8'))) {
if (countries) return countries;
const pack = read();
const decimals = pack.meta?.decimals ?? 3;
const byCode = new Map();
const codeless = [];
for (const feature of pack.features || []) {
// The constituent countries of the United Kingdom repeat its part.
if (feature.type !== 'Country') continue;
if (!ISO2.test(feature.iso2 || '')) {
if (feature.name)
codeless.push({ name: feature.name, feature, decimals });
continue;
}
const entry = byCode.get(feature.iso2) || {
code: feature.iso2,
name: null,
size: 0,
parts: [],
};
entry.parts.push({ feature, decimals });
// Named after its largest feature: Kazakhstan, not the Baikonur lease.
const size = JSON.stringify(feature.polygons).length;
if (size > entry.size) {
entry.name = feature.name;
entry.size = size;
}
byCode.set(feature.iso2, entry);
}
countries = byCode;
territories = codeless;
return countries;
}
/** Test seam: read another pack (or the bundled one again). */
export function resetCountries(read) {
countries = null;
if (read) loadCountries(read);
}
const geometries = (entry) => entry.parts.map((part) => geometryOf(part));
let disputed = null;
const slug = (name) =>
name
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-|-$/g, '');
/** The disputed areas: `{key, name, kind, heldBy, claimedBy, rings}`. */
function loadDisputed() {
if (disputed) return disputed;
const pack = JSON.parse(readFileSync(DISPUTED, 'utf8'));
disputed = (pack.areas || []).map((area) => ({
key: `disputed-${slug(area.name)}`,
name: area.name,
kind: area.kind,
heldBy: area.heldBy ?? null,
claimedBy: area.claimedBy ?? null,
rings: area.rings,
}));
return disputed;
}
/** The disputed area at a point, or null. */
export function disputedAt(lat, lon) {
return (
loadDisputed().find((area) =>
area.rings.some((ring) => pointInRing(ring, lat, lon)),
) || null
);
}
/**
* The area a point is in, as the News layer names it: a disputed area, or
* a territory without a code, as itself (`kind: 'disputed'`, `code` null),
* else its country (`kind: 'country'`, with `country` the countryAt
* entry); null at sea.
*/
export function areaAt(lat, lon) {
const area = disputedAt(lat, lon);
if (area)
return {
...area,
kind: 'disputed',
held: area.kind,
code: null,
polygons: area.rings.map((outer) => ({ outer, holes: [] })),
};
const country = countryAt(lat, lon);
if (country)
return {
key: country.code,
name: country.name,
kind: 'country',
code: country.code,
country,
};
const territory = territoryAt(lat, lon);
return territory
? {
key: `disputed-${slug(territory.name)}`,
name: territory.name,
kind: 'disputed',
held: null,
code: null,
polygons: geometryOf(territory).polygons.map(([outer, ...holes]) => ({
outer,
holes,
})),
}
: null;
}
/** Whether a point lies in a geometry (geometryOf), across the antimeridian. */
function inGeometry(geometry, lat, lon) {
const [west, south, east, north] = geometry.bbox;
const x = lon - 360 * Math.round((lon - (west + east) / 2) / 360);
if (x < west || x > east || lat < south || lat > north) return false;
return containsAnchor(geometry.polygons, lat, lon);
}
/** The country at a point, or null at sea or in a territory without a code. */
export function countryAt(lat, lon) {
for (const entry of loadCountries().values())
for (const geometry of geometries(entry))
if (inGeometry(geometry, lat, lon)) return entry;
return null;
}
/** The territory without a code at a point (Somaliland), or null. */
function territoryAt(lat, lon) {
loadCountries();
return (
territories.find((part) => inGeometry(geometryOf(part), lat, lon)) || null
);
}
/** A country by its ISO code, or null. */
export const countryByCode = (code) => loadCountries().get(code) || null;
/**
* A country's outline as /api/news/outline answers it (`{outer, holes}`),
* the disputed areas inside it left out as holes (Russia's without Crimea).
*/
export function countryPolygons(entry) {
const areas = loadDisputed();
return geometries(entry).flatMap((geometry) =>
geometry.polygons.map(([outer, ...holes]) => ({
outer,
holes: [
...holes,
...areas
.flatMap((area) => area.rings)
// By a point inside it: its corners lie on the neighbours' borders.
.filter((ring) => {
const inside = ringInterior(ring);
return inside && pointInRing(outer, inside.lat, inside.lon);
}),
],
})),
);
}
/**
* A point inside the country's largest part, for news placed in the country
* alone: its label point when that lies inside, else the middle of the
* widest span across the part at its middle latitude.
*/
export function countryAnchor(entry) {
const largest = geometries(entry).sort((a, b) => b.areaKm2 - a.areaKm2)[0];
const { label, polygons, bbox } = largest;
if (containsAnchor(polygons, label.lat, label.lon)) return label;
const [outer] = polygons.reduce((a, b) =>
b[0].length > a[0].length ? b : a,
);
return ringInterior(outer, (bbox[1] + bbox[3]) / 2) || label;
}
/**
* A point inside a ring: the middle of its widest span at `lat` (its middle
* latitude by default); null when the ring is not crossed there.
*/
function ringInterior(outer, lat = middleLat(outer)) {
const crossings = [];
for (let i = 0, j = outer.length - 1; i < outer.length; j = i++) {
const [x1, y1] = outer[j];
const [x2, y2] = outer[i];
if (y1 > lat !== y2 > lat)
crossings.push(x1 + ((lat - y1) * (x2 - x1)) / (y2 - y1));
}
crossings.sort((a, b) => a - b);
let best = null;
for (let i = 0; i + 1 < crossings.length; i += 2)
if (!best || crossings[i + 1] - crossings[i] > best[1] - best[0])
best = [crossings[i], crossings[i + 1]];
return best ? { lat, lon: (best[0] + best[1]) / 2 } : null;
}
const middleLat = (ring) => {
const lats = ring.map(([, y]) => y);
return (Math.min(...lats) + Math.max(...lats)) / 2;
};
+872
View File
@@ -0,0 +1,872 @@
// Rule-based report categories, relevance and severity: the News layer's
// base reading, shared by every news server. Multilingual stems (en, uk, ru,
// ar, he and more) matched at a word start on normalised text; a trailing `$`
// makes a stem whole-word. A title hit counts double. Ties go to the earlier
// category in CATEGORY_ORDER, which runs from most to least specific.
//
// A news server may replace these readings with its own; the rules stay
// as the fallback.
/** Text as the rules match it: folded case, one apostrophe, no vowel marks. */
function normaliseForMatch(text) {
return String(text || '')
.normalize('NFKC')
.toLowerCase()
.replace(/ё/g, 'е')
.replace(/[’ʼ`´]/g, "'")
.replace(/[ً-ٰٟ֑-ׇ]/g, '')
.replace(/[أإآ]/g, 'ا');
}
export const CATEGORY_ORDER = Object.freeze([
'strike',
'alert',
'air-defence',
'ground',
'casualties',
'humanitarian',
'infrastructure',
'disaster',
'protest',
'political',
// Business and trade: the rules' reading, or World News API's business.
'economy',
]);
// [stem, weight]; weight defaults to 1.
const RULES = {
strike: [
'strike',
'airstrike',
'air strike',
'missile',
'rocket',
'drone',
'shelling',
'shelled',
'bomb',
'explosion',
'blast',
'artillery',
'shahed',
['attack', 0.5],
'удар',
'ракет',
'дрон',
'бпла',
'обстріл',
'обстрел',
'вибух',
'взрыв',
'шахед',
'бомб',
['атак', 0.5],
'غارة',
'غارات',
'قصف',
'صاروخ',
'صواريخ',
'مسيرة',
'انفجار',
['استهداف', 0.7],
['هجوم', 0.5],
'תקיפה',
'טיל',
'רקט',
'כטב"מ',
'פיצוץ',
'הפצצ',
],
'air-defence': [
'air defen',
'intercept',
'shot down',
'downed',
'air raid',
'ппо',
'збит',
'сбит',
'перехоп',
'перехват',
'повітряна тривога',
'воздушная тревога',
'الدفاع الجوي',
'اعتراض',
'יירוט',
'יורט',
'כיפת ברזל',
'אזעק',
],
ground: [
'advance$',
'advances$',
'advancing',
'offensive',
'captur',
'liberat',
'assault',
'clash',
'fighting',
'counteroffensive',
'frontline',
'front line',
'battle',
'encircl',
['troops', 0.5],
'наступ',
'штурм',
'звільн',
'освобо',
'бої$',
'бои$',
'контрнаступ',
'захоп',
'захват',
'оточ',
'окруж',
'продвиж',
'просуван',
'اشتباك',
'معارك',
'تقدم',
'سيطر',
'اقتحام',
'توغل',
'לחימה',
'קרב',
'פשיט',
'תמרון',
],
casualties: [
'killed',
'dead$',
'deaths',
'wounded',
'injured',
'casualt',
'death toll',
'загин',
'поранен',
'жертв',
'вбит',
'погиб',
'ранен',
'убит',
'قتلى',
'شهداء',
'شهيد',
'جرحى',
'مقتل',
'اصاب',
'הרוג',
'נהרג',
'פצוע',
'נפצע',
],
humanitarian: [
'evacuat',
'aid$',
'refugee',
'displac',
'humanitarian',
'famine',
'hunger',
'convoy',
'shelter',
'евакуац',
'эвакуац',
'гуманітар',
'гуманитар',
'біженц',
'беженц',
'انسانية',
'نازح',
'نزوح',
'مساعدات',
'اجلاء',
'مجاعة',
'הומניטרי',
'פינוי',
'סיוע',
],
infrastructure: [
'power outage',
'blackout',
'electricity',
'energy',
'grid$',
'substation',
'water supply',
'bridge',
'railway',
'refinery',
'port$',
'dam$',
'power plant',
'енерг',
'энерг',
'знеструм',
'обесточ',
'підстанц',
'подстанц',
'залізниц',
'железн',
'нпз',
'الكهرباء',
'محطة',
'مياه',
'جسر',
'חשמל',
'תשתית',
],
disaster: [
'earthquake',
'flood',
'wildfire',
'landslide',
'cyclone',
'hurricane',
'typhoon',
'tsunami',
'eruption',
'glof$',
'glacial lake',
'storm$',
'lightning',
'землетрус',
'землетрясен',
'повін',
'паводк',
'наводнен',
'زلزال',
'فيضان',
'سيول',
'רעידת אדמה',
'שיטפון',
],
protest: [
'protest',
'demonstrat',
'rally$',
'rallies',
'riot',
// Not a bare "протест": "протестировал" is "tested".
'протест$',
'протеста',
'протесту',
'протестом',
'протесте$',
'протесты',
'протестов',
'протестн',
'протестую',
'протестова',
'мітинг',
'митинг',
'احتجاج',
'مظاهر',
'تظاهر',
'הפגנ',
'מחאה',
],
political: [
'talks$',
'negotiat',
'sanction',
'summit',
'ceasefire',
'truce',
'peace',
'election',
'parliament',
['president', 0.5],
['minister', 0.5],
['statement', 0.5],
// A state visit ("Emir of Qatar arrives in Amman").
['visit', 0.5],
['delegation', 0.5],
['agreement', 0.5],
'memorandum',
'diplomat',
'bilateral',
'ambassador',
['визит', 0.5],
['делегац', 0.5],
'дипломат',
'переговор',
'санкц',
'перемир',
'вибори',
'выборы',
['президент', 0.5],
['міністр', 0.5],
['министр', 0.5],
'مفاوضات',
'عقوبات',
'هدنة',
'وقف اطلاق النار',
['وزير', 0.5],
['رئيس', 0.5],
'משא ומתן',
'הפסקת אש',
'סנקצי',
['ראש הממשלה', 0.5],
],
// Elections and party politics ("How Gina Hinojosa Captured Texas" is a
// governor's race, not a battle). They count toward political only when
// nothing physical happened (categorizeText): a strike in an election week
// is still a strike.
election: [
'midterm',
'electoral',
'elect$',
'ballot',
'voter',
'vote$',
'votes$',
'voting',
'polls$',
'candidate',
'senate race',
'referendum',
'inaugurat',
// Plurals only: "Democratic Republic of Congo", "Republican Guard".
'democrats$',
'republicans$',
'gop$',
'избира',
'голосован',
'виборц',
'голосуван',
'انتخاب',
'בחירות',
],
};
// Business, trade, industry and energy, counted only when nothing physical
// happened (categorizeText): an oil depot struck is a strike. English,
// Russian, Ukrainian, Turkish and the European stems.
RULES.economy = [
'econom',
'gdp$',
'inflation',
'investment',
'investor',
'export',
'imports$',
'import$',
'tariff',
'trade deal',
'trade turnover',
'central bank',
'stock market',
'budget',
'oil and gas',
'oilfield',
'oil field',
'gas field',
'exploration',
'drilling',
'mining',
'industr',
'manufactur',
'business',
'company',
'companies',
'revenue',
'profit',
'merger',
'tourism',
'agricultur',
'harvest',
'экономи',
'инвест',
'экспорт',
'импорт',
'инфляц',
'ввп$',
'бюджет',
'промышлен',
'бизнес',
'компани',
'товарооборот',
'економі',
'інвест',
'експорт',
'промисло',
'ekonomi',
'yatırım',
'ihracat',
'ithalat',
'enflasyon',
'sanayi',
'şirket',
'wirtschaft',
'konjunktur',
'unternehm',
'inversi',
'investiss',
'investiment',
'empresa',
'entreprise',
'impresa',
];
// Attack words beyond PHYSICAL's harm, which keep a report off economy.
const ATTACK = new RegExp(
String.raw`(?<![\p{L}])(?:attack|assault|target|hit(?![\p{L}])|fire|plot|war(?![\p{L}])|wartime|атак|удар|пораж|пожар|обстр|війн|войн|هجوم|استهداف|حرب|מתקפ|ataque|attaque|attacco|angriff|saldır)`,
'u',
);
function escapeRegex(text) {
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
// Spanish, French, Portuguese, Italian and German stems, for outlets that
// report in their own language (a train crash in Spain, in Spanish).
const EUROPEAN = {
strike: [
'bombarde',
'misil',
'míssil',
'dron$',
'explosión',
'explosão',
'esplosi',
'ataque',
'attaque',
'attacco',
'angriff',
'rakete',
'luftangriff',
],
'air-defence': ['antiaére', 'défense aérienne', 'flugabwehr', 'abatid'],
ground: [
'combate',
'combats',
'enfrentamiento',
'affrontement',
'scontri',
'gefecht',
],
casualties: [
'muert',
'fallecid',
'herid',
'morts$',
'tué',
'blessé',
'morti',
'ferit',
'mortos',
'ferido',
'muere',
'mueren',
'fallece',
'meurt',
'morre',
'tote$',
'verletzt',
'víctimas',
'victimes',
'vittime',
],
humanitarian: [
'evacua',
'évacu',
'refugiad',
'réfugi',
'flüchtling',
'desplazad',
],
infrastructure: [
'apagón',
'panne',
'blackout',
'descarril',
'déraill',
'deragliament',
'entgleis',
'accidente',
'accident',
'incidente',
'choque',
'collision',
'collisione',
'zusammenstoß',
'incendio',
'incendie',
],
disaster: [
'terremoto',
'séisme',
'tremblement',
'erdbeben',
'inundaci',
'inondation',
'alluvion',
'überschwemm',
'huracán',
'ouragan',
'uragano',
'tormenta',
'tempête',
'tempesta',
'sturm',
'cicl',
'erupci',
'éruption',
'eruzione',
],
protest: [
'protesta',
'manifesta',
'disturbio',
'émeute',
'huelga',
'grève',
'sciopero',
'streik',
'demonstration',
],
};
for (const [category, stems] of Object.entries(EUROPEAN))
RULES[category] = [...(RULES[category] || []), ...stems];
const COMPILED = Object.entries(RULES).map(([category, stems]) => ({
category,
rules: stems.map((entry) => {
const [stem, weight = 1] = Array.isArray(entry) ? entry : [entry];
const whole = stem.endsWith('$');
const body = escapeRegex(
normaliseForMatch(whole ? stem.slice(0, -1) : stem),
).replace(/\s+/g, '\\s+');
const arabic = /\p{Script=Arabic}/u.test(stem) ? '(?:[وفبلك]?(?:ال)?)' : '';
const hebrew = /\p{Script=Hebrew}/u.test(stem) ? '(?:[ובלמהשכ]{0,2})' : '';
return {
weight,
regex: new RegExp(
`(?<![\\p{L}\\p{N}])${arabic}${hebrew}${body}${whole ? '(?![\\p{L}\\p{N}])' : ''}`,
'u',
),
};
}),
}));
const EVENT_CATEGORIES = new Set([
'strike',
'air-defence',
'ground',
'casualties',
'disaster',
]);
// Words that tie a report to armed conflict or crisis, whatever its category
// (en, fr, ru, uk, ar, he, es).
const CONFLICT_TERMS = new RegExp(
[
'war',
'army',
'military',
'militar',
'troops',
'soldier',
'militant',
'jihad',
'terror',
'insurg',
'rebel',
'ceasefire',
'truce',
'hostage',
'sanction',
'shell',
'drone',
'missile',
'airstrike',
'frappe',
'armée',
'soldat',
'djihad',
'fama',
'jnim',
'houthi',
'hezbollah',
'hamas',
'irgc',
'rsf',
'wagner',
'africa corps',
'войн',
'війн',
'армі',
'арми',
'всу',
'зсу',
'бпла',
'обстр',
'фронт',
'حرب',
'الجيش',
'هجوم',
'قصف',
'الحوثي',
'مسلح',
'מלחמה',
'צה"ל',
'guerra',
'ejército',
// Violence, coercion and their aftermath.
'massacre',
'atrocit',
'genocide',
'arms',
'weapon',
'detain',
'abduct',
'kidnap',
'air raid',
'alert',
'jamming',
'тривог',
'тревог',
'відбій',
'отбой',
'occupied',
'occupation',
'territor',
'convoy',
'demining',
'forces',
'raid',
'conflict',
'putin',
'zelensk',
'окуп',
'оккуп',
// Diplomacy and strategic stakes of the conflict.
'ambassador',
'mediat',
'election',
'talks',
'negotiat',
'envoy',
'diplomat',
'foreign minister',
'condemn',
'nuclear',
'hormuz',
'blockade',
'négociation',
'переговор',
'перемир',
'مفاوضات',
'هدنة',
]
// Short words are whole words ("war" is not "warning" or "Warsaw").
.map((w) => `(?<![\\p{L}])${w}${w.length < 5 ? '(?![\\p{L}])' : ''}`)
.join('|'),
'iu',
);
/**
* How much a report matters to its hotspot, by rule: `core` for a physical conflict or crisis event, `related` for
* news tied to the conflict (by wording, or because the source covers
* nothing else), `minor` for the rest of what a broad outlet publishes
* (prices, sport, health, culture).
*/
const termPatterns = new Map();
/** A word-start pattern for a hotspot term ("pow" must not match "power"). */
function termPattern(term) {
if (!termPatterns.has(term))
termPatterns.set(
term,
new RegExp(
// Short terms are whole words ("pow" is not "power"); longer ones are
// stems ("mobiliz" → mobilized, mobilization).
`(?<![\\p{L}])${normaliseForMatch(term).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}${term.length < 5 ? '(?![\\p{L}])' : ''}`,
'u',
),
);
return termPatterns.get(term);
}
export function relevanceOf({
category,
tags = [],
title = '',
summary = '',
dedicated = false,
relatedTerms = [],
floor = 'minor',
}) {
const text = normaliseForMatch(`${title} ${summary}`);
// A hotspot's own vocabulary counts as conflict wording there (Iran:
// "Qalibaf", "IAEA"; Israel: "settler"; Ukraine: "prisoner exchange").
const conflict =
CONFLICT_TERMS.test(text) ||
relatedTerms.some((term) => termPattern(term).test(text));
// Any event reading counts, even when another category wins the title
// ("the accident toll rises to 74": infrastructure first, casualties too).
if (
EVENT_CATEGORIES.has(category) ||
tags.some((tag) => EVENT_CATEGORIES.has(tag))
)
return 'core';
if (conflict && ['humanitarian', 'infrastructure'].includes(category))
return 'core';
if (conflict || dedicated || category === 'humanitarian') return 'related';
return floor;
}
// Alerts: air-raid alerts and reports of drones or missiles on their way
// ("Реактивний БпЛА курсом на Полтаву", "КАБи на Донеччину", "drone heading
// for Kyiv", "➡️"). Something is in the air; nothing has hit yet. Most of an
// air force's channel is these, and as "strikes" they would merge into real
// strike events and inflate them.
// A direction counts only after a weapon ("БпЛА … з півдня"), so a convoy
// leaving "in the direction of Oleshky" is not an alert.
const WEAPON =
'(?:бпла|дрон|шахед|герань|ракет|каб|балістик|крилат|боєприпас|боеприпас|uav|drone|missile|shahed|ballistic|cruise|munition)';
const DIRECTION =
'(?:курсом на|у напрямку|в напрямку|в направлении|у бік|в сторону|повз |з (?:північ|півден|схід|сход|захід|заход)|с (?:север|юг|восток|запад)|на |над |heading |on course|toward)';
const ALERT = new RegExp(
`${WEAPON}[^.!?]{0,40}${DIRECTION}|^[^\\p{L}]*курсом на|тривог|тревог|загроза застосування|угроза применения|air[- ]raid|threat of (?:missile|drone|ballistic|uav)|[➡⬅⬆⬇↗↘↙↖]`,
'u',
);
// Wording that something did hit: then it is a strike (or air defence), not an alert.
const IMPACT =
/влуч|вибух|уражен|пошкодж|загин|поранен|зруйн|пожеж|атак|удар по|збит|знищ|попадан|взрыв|поражен|поврежд|погиб|ранен|разруш|сбит|уничтож|\bhit\b|\bhits\b|struck|explosion|blast|killed|injur|wounded|damage|destroy|fire\b|shot down|intercept|attack/u;
/** Whether a report's title is an alert: a threat on its way, nothing hit yet. */
function isAlertText(title = '', summary = '') {
const t = normaliseForMatch(`${title} ${summary}`.slice(0, 400));
return ALERT.test(t) && !IMPACT.test(t);
}
// "Hit" or "struck" says nothing on its own: what hit decides (a missile is a
// strike, a flood or lightning a disaster), and those words score by
// themselves. A few phrases use a category's word for another thing.
const PHRASES = [
[/storm shadow/gu, 'missile'],
[/lightning strikes?/gu, 'lightning'],
[
/(?:general|hunger|labou?r|workers?|nationwide|national|transport|rail|teachers?|doctors?|nurses?) strikes?|on strike|strike action/gu,
'protest',
],
];
const rephrase = (text) =>
PHRASES.reduce((out, [re, word]) => out.replace(re, word), text);
// An election story borrows the words of war and disaster ("‘An Earthquake’:
// A Democratic Rout", "captured Texas", "a bloodbath for Republicans"). With
// election wording and nothing physical (no weapon, no one hurt, no quake's
// magnitude), the event categories give way to political.
const ELECTION =
/(?<![\p{L}])(?:election|midterm|electoral|ballot|voters?(?![\p{L}])|votes?(?![\p{L}])|voting|polls(?![\p{L}])|senate race|referendum|democrats|republicans|gop(?![\p{L}])|[\p{L}]+-elect(?![\p{L}])|выбор|избира|вибор|голосув|голосов|انتخاب|בחירות)/u;
const PHYSICAL =
/(?<![\p{L}])(?:kill|dead(?![\p{L}])|fighting|clash|air ?strike|strikes?(?![\p{L}])|struck|destroy|downed|ablaze|on fire|catch(?:es)? fire|caught fire|naval mine|landmine|shot down|sunk|sank|deaths?|injur|wounded|casualt|troops|soldier|missile|drone|rocket|shell(?:ing|ed)|artiller|explosi|blast|bomb|gunfire|shot(?![\p{L}])|shooting|magnitude|richter|tremor|evacuat|flood(?:ed|ing|waters)|убит|погиб|ранен|ракет|дрон|обстрел|загин|поранен|обстріл|قتل|جرح|صاروخ|הרג|פצוע|טיל)/u;
/** The categories of a physical event (an alert is a threat, not yet one). */
const EVENT_KINDS = new Set([
'strike',
'air-defence',
'ground',
'casualties',
'disaster',
'infrastructure',
]);
/**
* Categorise a report from its title and summary.
* @returns {{category: string, tags: string[]}}
*/
export function categorizeText({ title = '', summary = '' } = {}) {
if (isAlertText(title)) return { category: 'alert', tags: ['alert'] };
const t = rephrase(normaliseForMatch(title));
const s = rephrase(normaliseForMatch(summary));
const scores = new Map();
for (const { category, rules } of COMPILED) {
let score = 0;
for (const { regex, weight } of rules) {
if (regex.test(t)) score += 2 * weight;
else if (regex.test(s)) score += weight;
}
if (score > 0) scores.set(category, score);
}
const all = `${t} ${s}`;
const election = scores.get('election') || 0;
scores.delete('election');
const physical = PHYSICAL.test(all);
// Business words and no harm or attack: "advance exploration" is no
// front-line advance; "attack an industrial facility" stays a strike.
if (physical || ATTACK.test(all)) scores.delete('economy');
else if (scores.has('economy'))
for (const kind of EVENT_KINDS) scores.delete(kind);
if (!physical) {
if (election)
scores.set('political', (scores.get('political') || 0) + election);
if (ELECTION.test(all)) {
for (const kind of EVENT_KINDS) scores.delete(kind);
scores.set('political', Math.max(scores.get('political') || 0, 1));
}
}
if (!scores.size) return { category: 'other', tags: [] };
const ranked = CATEGORY_ORDER.filter((c) => scores.has(c)).sort(
(a, b) => scores.get(b) - scores.get(a),
);
return { category: ranked[0], tags: ranked };
}
// Severity by rule, for reports without a severity of their own (0 none,
// 1 minor incident, 2 significant: deaths, injuries or real damage, 3 major:
// mass casualties). Wording in en, uk, ru, ar, he, es, fr, de.
const SEVERITY_EVENTS = new Set([...EVENT_CATEGORIES, 'infrastructure']);
const MAJOR_HARM =
/mass casualt|massacre|dozens (?:of \p{L}+ )?(?:killed|dead)|hundreds (?:of \p{L}+ )?(?:killed|dead)|масов\p{L}* (?:жертв|загибел|гибел)|десятки (?:загибл|погибш|убит|вбит|людей загин|человек погиб)|مجزرة|عشرات (?:القتلى|الشهداء)|טבח/u;
const HARM_WORDS =
'kill|dead|died|death|injur|wound|casualt|загин|загибл|погиб|убит|вбит|поранен|ранен|постражд|пострадал|травм|قتل|قتيل|شهيد|شهداء|جرح|جريح|إصاب|اصاب|נהרג|הרוג|פצוע|muert|herid|tué|mort|bless|getötet|tote|verletz';
const HARM = new RegExp(String.raw`(?<![\p{L}])(?:${HARM_WORDS})`, 'u');
// "No casualties", "без жертв": the harm word is there, the harm is not.
const NO_HARM =
/(?:no|without|zero) (?:casualt|injur|deaths|victims|one (?:was )?(?:hurt|injured))|без (?:жертв|постраждал|пострадавш|загибл|погибш)|не постраждал|не пострадал|لا (?:إصابات|اصابات)|אין נפגעים|sin (?:víctimas|heridos)|sans (?:victime|blessé)/u;
const DAMAGE =
/destroy|damag|ablaze|on fire|fire broke|explosion|collapse|зруйн|разруш|пошкодж|поврежд|пожеж|пожар|вибух|взрыв|دمار|تدمير|انفجار|حريق|הרס|פיצוץ|שריפה|destru|incendi|explos|zerstör/u;
// An age ("a 71-year-old woman", "71-річна", "71-летний") is not a count.
const NUMBER = String.raw`(\d{1,3}(?:[,.\s]\d{3})+|\d{1,5})(?![\d,.]*[-\s]*(?:year|yr|річн|рок|летн|год))`;
const COUNTS = [
// "14 killed", "three people dead" is left to the words; "14 civilians dead".
new RegExp(String.raw`${NUMBER}\s+(?:\p{L}+\s+){0,2}(?:${HARM_WORDS})`, 'gu'),
// "killing 14", "death toll rises to 74", "kills at least 20".
new RegExp(
String.raw`(?:kill\p{L}*|toll\p{L}*(?:\s+\p{L}+){0,3}|загинул\p{L}*|погибл\p{L}*|жертв\p{L}*)\s+(?:at least\s+|не менее\s+|щонайменше\s+|до\s+|to\s+)?${NUMBER}`,
'gu',
),
];
function largestCount(text) {
let most = 0;
for (const pattern of COUNTS)
for (const match of text.matchAll(pattern)) {
const n = Number(match[1].replace(/[,.\s]/g, ''));
// A year ("since 2022 killed…") is not a toll.
if (n < 1900 || n > 2100) most = Math.max(most, n);
}
return most;
}
/** A report's severity by rule, 0–3. */
export function severityByRules({
category,
tags = [],
title = '',
summary = '',
}) {
const text = normaliseForMatch(`${title} ${summary}`);
const harmed = HARM.test(text) && !NO_HARM.test(text);
if (MAJOR_HARM.test(text) || (harmed && largestCount(text) >= 10)) return 3;
const event =
SEVERITY_EVENTS.has(category) ||
tags.some((tag) => SEVERITY_EVENTS.has(tag));
if (harmed && (event || category === 'humanitarian')) return 2;
if (event && DAMAGE.test(text)) return 2;
return event ? 1 : 0;
}
+546
View File
@@ -0,0 +1,546 @@
// GDELT 2.0 events for the Conflicts section, from the 15-minute export
// files (data.gdeltproject.org, plain downloads; no key, and not the
// rate-limited search API). Each export is one zipped tab-separated table of
// machine-coded events, each placed by GDELT with a point and a place type.
//
// A story counts when GDELT coded physical violence in it (CAMEO roots 18
// assault, 19 fight, 20 mass violence); its place is read from all its
// placed events (choosePlace). GDELT's actor fields are machine-coded and
// often wrong, so no report says who did what: a report is the story (its
// article, and the copies other outlets ran under the same address words).
// The events carry no headline: a story's title is its page title from the
// GKG export of the same quarter hour (read for the newest hours only, it
// runs to 6 MB), else the words of its address. All of it is rules over
// GDELT's own fields; nothing reads the articles.
import { promisify } from 'node:util';
import { inflateRaw } from 'node:zlib';
import { readResponseBytesCapped } from '../common/http.js';
import { categorizeText, relevanceOf, severityByRules } from './categorize.js';
import { UpstreamError, unescapeHtml, webLink } from './upstream.js';
const EXPORTS_URL = 'https://data.gdeltproject.org/gdeltv2/';
const STEP_MS = 15 * 60_000;
// An events export runs to about 100 KB zipped and 500 KB unzipped; a GKG
// export to about 6 MB and 18 MB. Unzipped off the main thread.
const MAX_ZIP_BYTES = 8 * 1024 * 1024;
const MAX_TABLE_BYTES = 8 * 1024 * 1024;
const MAX_GKG_ZIP_BYTES = 32 * 1024 * 1024;
const MAX_GKG_TABLE_BYTES = 32 * 1024 * 1024;
const inflate = promisify(inflateRaw);
// The most words an address's words make a title of.
const MAX_ADDRESS_WORDS = 24;
const TIMEOUT_MS = 30_000;
const MAX_PLACES = 3;
// Columns of the 2.0 events table (61).
const COL = {
id: 0,
eventCode: 26,
rootCode: 28,
quadClass: 29,
mentions: 31,
geoType: 51,
place: 52,
lat: 56,
lon: 57,
added: 59,
url: 60,
};
// CAMEO event codes as the News layer's categories, and as words for an
// article whose address carries none of its own.
const EVENTS = [
// Aerial weapons, artillery and tanks, bombings.
[/^19[45]|^183/, 'strike', 'Strike reported'],
[/^186|^185/, 'casualties', 'Killing reported'],
[/^181/, 'humanitarian', 'Abduction reported'],
[/^20/, 'casualties', 'Mass violence reported'],
[/^19/, 'ground', 'Fighting reported'],
[/^18/, 'other', 'Violence reported'],
];
const eventKind = (code) =>
EVENTS.find(([pattern]) => pattern.test(code)) || EVENTS.at(-1);
// GDELT's place types: 1 country, 2 US state, 3 US city, 4 world city,
// 5 world state.
const PRECISION = {
1: 'country',
2: 'region',
3: 'city',
4: 'city',
5: 'region',
};
/** The export holding the events GDELT added in the 15 minutes before `ms`. */
export function exportUrl(ms) {
const at = new Date(Math.floor(ms / STEP_MS) * STEP_MS);
const stamp = at.toISOString().replace(/[-:T]/g, '').slice(0, 14);
return `${EXPORTS_URL}${stamp}.export.CSV.zip`;
}
/** An events export's quarter hour (ms), from its name, or NaN. */
export function exportAt(url) {
const match = /(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)00\.export\.CSV\.zip$/.exec(
url,
);
return match
? Date.UTC(match[1], match[2] - 1, match[3], match[4], match[5])
: NaN;
}
const LASTUPDATE_URL = `${EXPORTS_URL}lastupdate.txt`;
/**
* The time of the newest events export GDELT lists as published, in ms, or
* null when its list cannot be read. Asking for one not listed yet would
* be answered 404, and GDELT's CDN keeps that 404 for an hour.
*/
export async function fetchLatestExport({ fetchImpl, signal } = {}) {
try {
const response = await fetchImpl(LASTUPDATE_URL, {
redirect: 'error',
signal: signal
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
: AbortSignal.timeout(TIMEOUT_MS),
});
if (!response.ok) {
await response.body?.cancel?.().catch?.(() => {});
return null;
}
const text = new TextDecoder().decode(
await readResponseBytesCapped(response, 16 * 1024, signal),
);
const at = exportAt(
/\S+\.export\.CSV\.zip/.exec(text)?.[0].replace(/^http:/, 'https:') || '',
);
return Number.isFinite(at) ? at : null;
} catch {
return null;
}
}
/** The GKG export of the same quarter hour as an events export. */
export const gkgUrl = (eventsUrl) =>
eventsUrl.replace(/\.export\.CSV\.zip$/, '.gkg.csv.zip');
/** The day's exports up to `now`, newest first. */
export function exportsOfDay(now, hours = 24) {
const urls = [];
const last = Math.floor(now / STEP_MS) * STEP_MS;
for (let ms = last; ms > now - hours * 3_600_000; ms -= STEP_MS)
urls.push(exportUrl(ms));
return urls;
}
/** The one table inside an export's zip, as text, up to `maxBytes`. */
export async function unzipExport(bytes, maxBytes = MAX_TABLE_BYTES) {
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const fail = () => {
throw new UpstreamError('GDELT export unreadable', { unreadable: true });
};
if (bytes.length < 30 || view.getUint32(0, true) !== 0x04034b50) fail();
// The central directory has the sizes; a local header may leave them 0.
let eocd = -1;
for (let i = bytes.length - 22; i >= Math.max(0, bytes.length - 0x10015); i--)
if (view.getUint32(i, true) === 0x06054b50) {
eocd = i;
break;
}
if (eocd < 0) fail();
const dir = view.getUint32(eocd + 16, true);
if (dir + 46 > bytes.length || view.getUint32(dir, true) !== 0x02014b50)
fail();
const method = view.getUint16(dir + 10, true);
const compressed = view.getUint32(dir + 20, true);
const local = view.getUint32(dir + 42, true);
if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50)
fail();
const start =
local +
30 +
view.getUint16(local + 26, true) +
view.getUint16(local + 28, true);
if (start + compressed > bytes.length) fail();
const data = bytes.subarray(start, start + compressed);
try {
if (method === 0 && data.length > maxBytes) fail();
const table =
method === 0
? data
: method === 8
? await inflate(data, { maxOutputLength: maxBytes })
: fail();
return new TextDecoder().decode(table);
} catch (error) {
if (error instanceof UpstreamError) throw error;
return fail();
}
}
/**
* An article's address as the reports carry it: https, an http one
* upgraded (GDELT keeps the address as it found it), else null.
*/
export const articleLink = (value) =>
webLink(value) || webLink(String(value ?? '').replace(/^http:/, 'https:'));
/**
* The placed events of an export's table, of the stories with physical
* violence in them: those events, and every other placed event of the same
* stories (they help place it).
*/
export function parseEvents(table) {
const all = [];
for (const line of table.split('\n')) {
const f = line.split('\t');
if (f.length < 61) continue;
const violent =
f[COL.quadClass] === '4' && ['18', '19', '20'].includes(f[COL.rootCode]);
const precision = PRECISION[f[COL.geoType]];
const lat = Number(f[COL.lat]);
const lon = Number(f[COL.lon]);
const url = articleLink(f[COL.url]);
const added = /^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/.exec(f[COL.added]);
if (
!precision ||
!f[COL.lat] ||
!f[COL.lon] ||
!Number.isFinite(lat) ||
!Number.isFinite(lon) ||
Math.abs(lat) > 90 ||
Math.abs(lon) > 180 ||
(lat === 0 && lon === 0) ||
!url ||
!added
)
continue;
all.push({
id: f[COL.id],
code: f[COL.eventCode],
violent,
lat,
lon,
place: f[COL.place].trim() || null,
precision,
mentions: Math.max(1, Number(f[COL.mentions]) || 1),
addedAt: `${added[1]}-${added[2]}-${added[3]}T${added[4]}:${added[5]}:${added[6]}.000Z`,
url,
});
}
const stories = new Set(all.filter((e) => e.violent).map(storyKey));
return all.filter((e) => stories.has(storyKey(e)));
}
/**
* The words of an article's address, when it is written in them
* ("three-children-die-in-peshawar-cylinder-exp"), as a sentence; else null.
*/
export function addressWords(url) {
let path;
try {
path = decodeURIComponent(new URL(url).pathname);
} catch {
return null;
}
const segment = path
.split('/')
.map((part) => part.replace(/\.(?:s?html?|php|aspx?)$/i, ''))
.filter((part) => /[-_]/.test(part))
.sort((a, b) => b.length - a.length)[0];
if (!segment) return null;
const words = segment.split(/[-_]+/).filter(Boolean);
const lettered = words.filter((word) => /^\p{L}+$/u.test(word));
if (lettered.length < 4 || lettered.length < 0.7 * words.length) return null;
// Numbers stay ("killed nearly 500"); an id does not: a word of letters
// and digits, five digits or more, or four or more at either end.
const kept = words.filter(
(word, i) =>
/^\p{L}+$/u.test(word) ||
(/^\d{1,4}$/.test(word) &&
!(word.length === 4 && (i === 0 || i === words.length - 1))),
);
const text = kept.slice(0, MAX_ADDRESS_WORDS).join(' ').toLowerCase();
return text.charAt(0).toUpperCase() + text.slice(1);
}
/**
* Address words with the places the story is at written as names
* ("killed in amman" → "killed in Amman"), and "u s" as "US".
*/
export function withNames(words, places) {
let text = words.replace(/(?<![\p{L}])u s(?![\p{L}])/gu, 'US');
const names = new Set(
places.flatMap((place) =>
String(place.place ?? '')
.split(',')
.map((part) => part.replace(/\(.*?\)/g, '').trim())
.filter((part) => part.length > 2 && !/\(general\)/.test(part)),
),
);
for (const name of names) {
const pattern = new RegExp(
`(?<![\\p{L}])${name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\p{L}])`,
'gu',
);
text = text.replace(pattern, name);
}
return text.charAt(0).toUpperCase() + text.slice(1);
}
// A page title's site part: " | The Daily Example", " - Wire Service".
const SITE_PART = /\s+[|–—-]\s+([^|–—-]{2,60})$/u;
/**
* A page title as a headline: entities decoded, the site's name after a
* bar or dash taken off (when the headline before it is still one), or
* null when it is too short to be one.
*/
export function cleanTitle(raw) {
// An agency's tag set on with a bare hyphen ("…attack-Newswire", "…-X").
let title = unescapeHtml(raw).replace(
// After a word in lower case: "Jean-Pierre" keeps its name.
/(?<=(?:^|\s)\p{Ll}\p{L}*|[\p{N}'")])-\p{Lu}\p{L}{0,15}$/u,
'',
);
for (let i = 0; i < 2; i++) {
const match = SITE_PART.exec(title);
if (!match) break;
const rest = title.slice(0, match.index);
if (rest.split(/\s+/).length < 4 || match[1].split(/\s+/).length > 6) break;
title = rest;
}
title = title.trim().slice(0, 300);
return title.length >= 12 && title.split(/\s+/).length >= 3 ? title : null;
}
/**
* GKG page titles by article address (as articleLink writes it), for the
* addresses in `wanted`.
*/
export function parseTitles(table, wanted) {
const titles = new Map();
for (const line of table.split('\n')) {
const f = line.split('\t');
const url = f[4] && articleLink(f[4]);
if (!url || !wanted.has(url)) continue;
const match = /<PAGE_TITLE>(.*?)<\/PAGE_TITLE>/s.exec(f.at(-1) || '');
const title = match && cleanTitle(match[1]);
if (title) titles.set(url, title);
}
return titles;
}
/**
* The story an event belongs to: its article's address words, which the
* copies other outlets run share (an agency story), else its address.
*/
export function storyKey(event) {
return addressWords(event.url)?.toLowerCase() || event.url;
}
const PLACE_WORD = (name) =>
new RegExp(
`(?<![\\p{L}])${name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}s?(?![\\p{L}])`,
'u',
);
/** Whether the words name a place's town or its country ("jordans" too). */
function namedIn(words, place) {
if (!words || !place) return false;
const parts = place
.split(',')
.map((part) => part.replace(/\(.*?\)/g, '').trim())
.filter((part) => part.length > 2);
return [parts[0], parts.at(-1)].some(
(part) => part && PLACE_WORD(part).test(words.toLowerCase()),
);
}
/**
* A story's places, best first, from all its placed events. GDELT places
* an event at a bare country when the text only names a nationality ("a
* Sri Lankan embassy", an Australian outlet's own country), so a town or
* region wins over a country; of those, the ones the article's own words
* name; then the most mentioned.
*/
export function choosePlaces(events, words) {
const byPlace = new Map();
for (const event of events) {
const key = `${event.lat},${event.lon}`;
const place = byPlace.get(key) || { ...event, mentions: 0 };
place.mentions += event.mentions;
byPlace.set(key, place);
}
let places = [...byPlace.values()];
const specific = places.filter((p) => p.precision !== 'country');
if (specific.length) places = specific;
const named = places.filter((p) => namedIn(words, p.place));
if (named.length) places = named;
return places.sort((a, b) => b.mentions - a.mentions).slice(0, MAX_PLACES);
}
// The earlier of two events: by the time GDELT added it, then by its id.
const earlier = (a, b) =>
b.addedAt < a.addedAt ||
(b.addedAt === a.addedAt && Number(b.id) < Number(a.id))
? b
: a;
/**
* One report per story (storyKey), its address and id the first copy's (an
* export adding a more-mentioned event keeps both), placed by choosePlaces,
* titled by a copy's page title (`titleOf`), else its address words, else
* its violent event.
*/
export function eventReports(events, { titleOf = () => null } = {}) {
const stories = new Map();
for (const event of events) {
const key = storyKey(event);
const list = stories.get(key) || [];
list.push(event);
stories.set(key, list);
}
const reports = [];
for (const list of stories.values()) {
const violent = list
.filter((e) => e.violent !== false)
.sort(
(a, b) => b.mentions - a.mentions || a.addedAt.localeCompare(b.addedAt),
);
if (!violent.length) continue;
const [lead] = violent;
const first = list.reduce(earlier);
const { url } = first;
const address = addressWords(url);
const places = choosePlaces(list, address);
const [, eventCategory, words] = eventKind(lead.code);
const page = [url, ...list.map((e) => e.url)]
.map((copy) => titleOf(copy))
.find(Boolean);
const own = page || (address ? withNames(address, places) : null);
const title = own || eventTitle(words, places[0].place);
// The article's words decide whether it is conflict, as any headline's
// would: GDELT codes an election story as "fighting" too. GDELT's code
// names the kind only where the words read as conflict without naming
// one. A story with no words at all has GDELT's code alone to go by,
// which is not enough to list it under Conflicts: it is minor (kept
// for Local) until a page title comes in.
const read = own ? categorizeText({ title: own }) : null;
const tags = read?.tags || [];
const relevance = read
? relevanceOf({ category: read.category, tags, title: own })
: 'minor';
const category = !read
? eventCategory
: read.category !== 'other'
? read.category
: relevance === 'minor'
? 'other'
: eventCategory;
const publisher = new URL(url).hostname.replace(/^www\./, '');
const geo = ({ lat, lon, place, precision }) => ({
lat,
lon,
place,
precision,
confidence: precision === 'city' ? 0.6 : 0.3,
mappable: true,
method: 'source',
});
reports.push({
id: `gdelt-${first.id}`,
title,
url,
publisher,
source: { name: publisher, attribution: 'GDELT Project' },
kind: 'gdelt',
side: 'unknown',
category,
relevance,
severity: Math.max(
1,
severityByRules({ category, tags, title: own || '' }),
),
severityBy: 'rules',
publishedAt: first.addedAt,
geo: geo(places[0]),
locations: places.slice(1).map(geo),
media: [],
mentions: list.reduce((sum, e) => sum + e.mentions, 0),
copies: new Set(list.map((e) => e.url)).size,
// A title of the event's words and its place, when the address had
// none: a renamed place renames it (eventTitle).
...(own ? {} : { eventWords: words }),
titled: page ? 'page' : own ? 'address' : 'event',
});
}
return reports;
}
/** The title of a report without address words, at its (renamed) place. */
export const eventTitle = (words, place) =>
place ? `${words} in ${place}` : words;
/**
* One export's table, or null while GDELT has not published it (GDELT
* lists the newest in lastupdate.txt; see fetchLatestExport).
*/
async function fetchTable(url, { fetchImpl, signal, maxBytes, tableBytes }) {
let response;
try {
response = await fetchImpl(url, {
redirect: 'error',
signal: signal
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
: AbortSignal.timeout(TIMEOUT_MS),
});
} catch {
throw new UpstreamError('GDELT exports unreachable');
}
if (response.status === 404) {
await response.body?.cancel?.().catch?.(() => {});
return null;
}
if (!response.ok) {
await response.body?.cancel?.().catch?.(() => {});
throw new UpstreamError(`GDELT exports answered ${response.status}`, {
status: response.status,
});
}
let bytes;
try {
bytes = await readResponseBytesCapped(response, maxBytes, signal);
} catch (error) {
// Over the cap stays over it; a read cut short may succeed again.
throw new UpstreamError('GDELT export unreadable', {
unreadable: error?.code === 'RESPONSE_TOO_LARGE',
});
}
return unzipExport(bytes, tableBytes);
}
/** One events export's events, or null while it is not published yet. */
export async function fetchExport(url, { fetchImpl, signal } = {}) {
const table = await fetchTable(url, {
fetchImpl,
signal,
maxBytes: MAX_ZIP_BYTES,
tableBytes: MAX_TABLE_BYTES,
});
return table === null ? null : parseEvents(table);
}
/**
* The page titles a GKG export holds for the addresses in `wanted`, or null
* while it is not published yet.
*/
export async function fetchTitles(url, wanted, { fetchImpl, signal } = {}) {
const table = await fetchTable(url, {
fetchImpl,
signal,
maxBytes: MAX_GKG_ZIP_BYTES,
tableBytes: MAX_GKG_TABLE_BYTES,
});
return table === null ? null : parseTitles(table, wanted);
}
+407
View File
@@ -0,0 +1,407 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
addressWords,
cleanTitle,
gkgUrl,
parseTitles,
withNames,
eventReports,
exportsOfDay,
exportUrl,
fetchExport,
parseEvents,
unzipExport,
} from './gdeltEvents.js';
import { eventRow, gkgRow, zipTable } from './gdeltFixtures.mjs';
const ARTICLE =
'https://www.wire.example/rubric-ato/3912-russian-drones-attack-kharkiv-overnight-three-killed.html';
test('the day is 96 exports, one each quarter hour, newest first', () => {
const now = Date.parse('2026-11-05T15:07:30Z');
assert.equal(
exportUrl(now),
'https://data.gdeltproject.org/gdeltv2/20261105150000.export.CSV.zip',
);
const day = exportsOfDay(now);
assert.equal(day.length, 96);
assert.equal(
day[1],
'https://data.gdeltproject.org/gdeltv2/20261105144500.export.CSV.zip',
);
assert.equal(
day.at(-1),
'https://data.gdeltproject.org/gdeltv2/20261104151500.export.CSV.zip',
);
});
test('a story with physical violence is kept with all its placed events', () => {
const rows = [
eventRow({ id: 1, code: '195', url: ARTICLE }),
eventRow({ id: 2, code: '183', url: ARTICLE }),
eventRow({ id: 3, code: '2001', url: ARTICLE }),
// Not physical, but of the same story: kept to place it by.
eventRow({ id: 4, code: '150', url: ARTICLE }),
eventRow({ id: 7, code: '190', quad: '3', url: ARTICLE }),
// A story with no violence in it is left out.
eventRow({
id: 5,
code: '163',
url: 'https://x.example/talks-resume-in-geneva-today-officials',
}),
eventRow({
id: 6,
code: '173',
url: 'https://x.example/talks-resume-in-geneva-today-officials',
}),
// Unplaced, at 0,0, or without a link.
eventRow({ id: 8, geoType: '0', url: ARTICLE }),
eventRow({ id: 9, lat: 0, lon: 0, url: ARTICLE }),
eventRow({ id: 10, url: 'javascript:alert(1)' }),
'short\trow',
];
const events = parseEvents(rows.join('\n'));
assert.deepEqual(
events.map((e) => [e.id, e.violent]),
[
['1', true],
['2', true],
['3', true],
['4', false],
['7', false],
],
);
assert.equal(events[0].precision, 'city');
assert.equal(events[0].addedAt, '2026-11-05T11:45:00.000Z');
});
test("an article is one report, at its most-mentioned event, in its address's words", () => {
const events = parseEvents(
[
eventRow({ id: 11, code: '195', mentions: 2, url: ARTICLE }),
eventRow({
id: 12,
code: '190',
mentions: 9,
url: ARTICLE,
place: "Kupiansk, Kharkivs'ka Oblast', Ukraine",
lat: 49.71,
lon: 37.61,
}),
eventRow({
id: 13,
code: '190',
geoType: '1',
place: 'Ukraine',
lat: 49,
lon: 32,
url: 'https://example.com/a/b/12345',
}),
].join('\n'),
);
const [drones, bare] = eventReports(events);
// The story's first event (by time, then id) names it.
assert.equal(drones.id, 'gdelt-11');
// Its address words, with the place it is at written as a name.
assert.equal(
drones.title,
'Russian drones attack Kharkiv overnight three killed',
);
// Kupiansk has more mentions; the article's own words name Kharkiv.
assert.equal(drones.geo.place, "Kharkiv, Kharkivs'ka Oblast', Ukraine");
assert.equal(drones.geo.precision, 'city');
assert.equal(drones.locations.length, 0, 'Kupiansk is not named');
assert.equal(drones.category, 'strike');
assert.equal(drones.relevance, 'core');
assert.equal(drones.severity, 2);
assert.equal(drones.publisher, 'wire.example');
assert.equal(drones.source.attribution, 'GDELT Project');
assert.equal(drones.side, 'unknown', 'no actor is said');
// No words in the address: GDELT's coding alone, so minor (not listed
// under Conflicts), at least severity 1.
assert.equal(bare.title, 'Fighting reported in Ukraine');
assert.equal(bare.category, 'ground');
assert.equal(bare.relevance, 'minor');
assert.equal(bare.severity, 1);
assert.equal(bare.geo.precision, 'country');
});
test('an article GDELT codes as fighting whose words say otherwise stays out of Conflicts', () => {
const [report] = eventReports(
parseEvents(
eventRow({
url: 'https://news.example.com/2026/11/05/city-council-approves-new-budget-for-schools',
id: 20,
}),
),
);
assert.equal(report.relevance, 'minor');
});
test('address words need four lettered words, most of the address', () => {
assert.equal(addressWords('https://x.example/news/1234567'), null);
assert.equal(addressWords('https://x.example/a/lueop1pzfi'), null);
assert.equal(
addressWords('https://x.example/2026/11/05/a-b-1-2-3-4-5'),
null,
);
assert.equal(
addressWords(
'https://x.example/story/2638789/three-children-die-in-peshawar.html',
),
'Three children die in peshawar',
);
});
test('an export is unzipped and read; one not published yet is null', async () => {
const zip = zipTable(eventRow({ id: 30, code: '193', url: ARTICLE }));
assert.deepEqual(
parseEvents(await unzipExport(zip)).map((e) => e.id),
['30'],
);
const served = await fetchExport('https://data.gdeltproject.org/x.zip', {
fetchImpl: async () => new Response(zip, { status: 200 }),
});
assert.deepEqual(
served.map((e) => e.id),
['30'],
);
const later = await fetchExport('https://data.gdeltproject.org/y.zip', {
fetchImpl: async () => new Response('not found', { status: 404 }),
});
assert.equal(later, null);
await assert.rejects(
fetchExport('https://data.gdeltproject.org/z.zip', {
fetchImpl: async () => new Response('busy', { status: 503 }),
}),
{ name: 'UpstreamError' },
);
await assert.rejects(unzipExport(new Uint8Array([1, 2, 3])), {
name: 'UpstreamError',
});
});
test("a nationality is no place: a town the article's words name wins over a bare country", () => {
const url =
'https://www.sun.example/world/2026/11/05/security-officer-attacked-and-killed-near-sri-lankan-embassy-in-jordans-capital';
const amman = { place: 'Amman, (JO11), Jordan', lat: 31.95, lon: 35.9333 };
const cairo = { place: 'Cairo, Al Qahirah, Egypt', lat: 30.05, lon: 31.25 };
const [report] = eventReports(
parseEvents(
[
// "Sri Lankan embassy": GDELT's most-mentioned place, a bare country.
eventRow({
id: 1,
mentions: 12,
geoType: '1',
place: 'Sri Lanka',
lat: 7,
lon: 81,
url,
}),
eventRow({ id: 2, mentions: 6, ...cairo, url }),
eventRow({ id: 3, mentions: 4, ...amman, url }),
eventRow({ id: 4, mentions: 2, ...amman, url }),
].join('\n'),
),
);
assert.equal(report.geo.place, 'Amman, (JO11), Jordan');
assert.equal(report.geo.precision, 'city');
});
test('an agency story run by several outlets is one report, placed by all their events', () => {
const words = 'jordan-security-officer-killed-near-sri-lanka-embassy';
const times = `https://www.times.example/story/9386807/${words}/`;
const chronicle = `https://www.chronicle.example/story/9386807/${words}/`;
const reports = eventReports(
parseEvents(
[
// The outlet's own country, at country level, on its violent events.
eventRow({
id: 1,
mentions: 6,
geoType: '1',
place: 'Australia',
lat: -25,
lon: 135,
url: times,
added: '20261105141500',
}),
// Its other events are placed in Jordan.
eventRow({
id: 2,
code: '120',
quad: '3',
mentions: 8,
place: 'Karak, Jordan (general), Jordan',
lat: 31.18,
lon: 35.7,
url: times,
added: '20261105141500',
}),
eventRow({
id: 3,
mentions: 4,
geoType: '1',
place: 'Syria',
lat: 35,
lon: 38,
url: chronicle,
added: '20261105151500',
}),
].join('\n'),
),
);
assert.equal(reports.length, 1);
const [story] = reports;
assert.equal(story.copies, 2);
assert.equal(story.url, times, 'the first copy');
assert.equal(story.publishedAt, '2026-11-05T14:15:00.000Z');
assert.equal(story.geo.place, 'Karak, Jordan (general), Jordan');
});
test("a story's title is a copy's page title from the GKG export", () => {
const titles = parseTitles(
[
gkgRow(
ARTICLE,
'Russian drones attack Kharkiv overnight, three killed | Wire',
),
gkgRow('https://elsewhere.example/a', 'Not one of ours, not kept here'),
].join('\n'),
new Set([ARTICLE]),
);
assert.deepEqual(
[...titles],
[[ARTICLE, 'Russian drones attack Kharkiv overnight, three killed']],
);
const [report] = eventReports(
parseEvents(eventRow({ id: 1, url: ARTICLE })),
{
titleOf: (url) => titles.get(url) || null,
},
);
assert.equal(
report.title,
'Russian drones attack Kharkiv overnight, three killed',
);
assert.equal(report.titled, 'page');
assert.equal(
gkgUrl(
'https://data.gdeltproject.org/gdeltv2/20261105141500.export.CSV.zip',
),
'https://data.gdeltproject.org/gdeltv2/20261105141500.gkg.csv.zip',
);
});
test('a page title loses its site name, keeps its words, and must be a headline', () => {
assert.equal(
cleanTitle(
'Yemen&#039;s president sacks ministers | Yemen | The Daily Example',
),
"Yemen's president sacks ministers",
);
assert.equal(
cleanTitle('Strike in Kharkiv - live updates - Wire Service'),
'Strike in Kharkiv - live updates',
);
// Too little before the dash: the dash is part of the headline.
assert.equal(cleanTitle('Iran - Wire Service'), 'Iran - Wire Service');
assert.equal(cleanTitle('Home'), null);
assert.equal(
cleanTitle('Jordanian security officer killed in stabbing attack-Newswire'),
'Jordanian security officer killed in stabbing attack',
);
assert.equal(
cleanTitle('Five killed in Gaza, sources say-X'),
'Five killed in Gaza, sources say',
);
// A hyphenated name is no tag.
assert.equal(
cleanTitle('Talks resume with Jean-Pierre'),
'Talks resume with Jean-Pierre',
);
});
test('address words keep their numbers and name the places the story is at', () => {
const words = addressWords(
'https://x.example/2026/11/05/jordan-says-u-s-envoy-met-officials-in-amman-after-500-killed',
);
assert.equal(
words,
'Jordan says u s envoy met officials in amman after 500 killed',
);
assert.equal(
withNames(words, [{ place: 'Amman, (JO11), Jordan' }]),
'Jordan says US envoy met officials in Amman after 500 killed',
);
});
test("a story's id is its first event's: a later, more-mentioned event keeps it", () => {
const first = eventRow({ id: 40, mentions: 1, url: ARTICLE });
const [before] = eventReports(parseEvents(first));
const [after] = eventReports(
parseEvents(
[
first,
eventRow({
id: 41,
code: '195',
mentions: 30,
url: ARTICLE,
added: '20261105120000',
}),
].join('\n'),
),
);
assert.equal(before.id, 'gdelt-40');
assert.equal(after.id, 'gdelt-40');
assert.equal(after.mentions, 31);
});
test('words that read as no conflict leave GDELT no kind to name', () => {
const [report] = eventReports(
parseEvents(
eventRow({
id: 42,
code: '195',
url: 'https://x.example/2026/11/05/local-bakery-celebrates-fifty-years-of-baking',
}),
),
);
assert.equal(report.relevance, 'minor');
assert.equal(report.category, 'other');
});
test('a GKG address written with http is the https one the events carry', () => {
const plain = ARTICLE.replace('https:', 'http:');
const [event] = parseEvents(eventRow({ id: 43, url: plain }));
assert.equal(event.url, ARTICLE);
const titles = parseTitles(
gkgRow(plain, 'Russian drones attack Kharkiv overnight, three killed'),
new Set([event.url]),
);
assert.equal(
titles.get(ARTICLE),
'Russian drones attack Kharkiv overnight, three killed',
);
});
test('a corrupt or oversized export is an UpstreamError, never a RangeError', async () => {
const zip = zipTable(eventRow({ id: 44, url: ARTICLE }));
const view = new DataView(zip.buffer);
// The central directory's local header offset, pointing past the end.
const dir = view.getUint32(zip.length - 22 + 16, true);
const far = zip.slice();
new DataView(far.buffer).setUint32(dir + 42, zip.length - 10, true);
await assert.rejects(unzipExport(far), { name: 'UpstreamError' });
// The end record cut short.
await assert.rejects(unzipExport(zip.subarray(0, zip.length - 30)), {
name: 'UpstreamError',
});
// Unzipped past the table's cap.
await assert.rejects(unzipExport(zip, 16), { name: 'UpstreamError' });
assert.equal((await unzipExport(zip)).length > 16, true);
});
+72
View File
@@ -0,0 +1,72 @@
// Test fixtures: GDELT 2.0 export rows and their zip, as the export files
// carry them (61 tab-separated columns, one zipped table).
import { deflateRawSync } from 'node:zlib';
/** One events row; only the columns the reader looks at are filled. */
export function eventRow({
id,
code = '190',
quad = '4',
mentions = 1,
geoType = '4',
place = "Kharkiv, Kharkivs'ka Oblast', Ukraine",
lat = 49.9935,
lon = 36.2304,
added = '20261105114500',
url,
}) {
const f = new Array(61).fill('');
f[0] = String(id);
f[26] = code;
f[28] = code.slice(0, 2);
f[29] = quad;
f[31] = String(mentions);
f[51] = geoType;
f[52] = place;
f[56] = String(lat);
f[57] = String(lon);
f[59] = added;
f[60] = url;
return f.join('\t');
}
/** One GKG 2.1 row: an article's address and its page title (27 columns). */
export function gkgRow(url, title) {
const f = new Array(27).fill('');
f[4] = url;
f[26] = `<PAGE_PRECISEPUBTIMESTAMP>20261105114500</PAGE_PRECISEPUBTIMESTAMP><PAGE_TITLE>${title}</PAGE_TITLE>`;
return f.join('\t');
}
/** A single-entry zip of `text`, deflated, sizes in the central directory. */
export function zipTable(text, name = 'table.export.CSV') {
const data = Buffer.from(text, 'utf8');
const packed = deflateRawSync(data);
const nameBytes = Buffer.from(name);
const local = Buffer.alloc(30);
local.writeUInt32LE(0x04034b50, 0);
local.writeUInt16LE(20, 4);
local.writeUInt16LE(0x08, 6); // sizes follow the data (left 0 here)
local.writeUInt16LE(8, 8);
local.writeUInt16LE(nameBytes.length, 26);
const dir = Buffer.alloc(46);
dir.writeUInt32LE(0x02014b50, 0);
dir.writeUInt16LE(20, 4);
dir.writeUInt16LE(20, 6);
dir.writeUInt16LE(0x08, 8);
dir.writeUInt16LE(8, 10);
dir.writeUInt32LE(packed.length, 20);
dir.writeUInt32LE(data.length, 24);
dir.writeUInt16LE(nameBytes.length, 28);
dir.writeUInt32LE(0, 42);
const dirAt = local.length + nameBytes.length + packed.length;
const end = Buffer.alloc(22);
end.writeUInt32LE(0x06054b50, 0);
end.writeUInt16LE(1, 8);
end.writeUInt16LE(1, 10);
end.writeUInt32LE(dir.length + nameBytes.length, 12);
end.writeUInt32LE(dirAt, 16);
return new Uint8Array(
Buffer.concat([local, nameBytes, packed, dir, nameBytes, end]),
);
}
+916
View File
@@ -0,0 +1,916 @@
// The News layer's routes answered by Core itself, when no news server is
// configured (GEV_NEWS_ORIGIN). Conflicts: the day's GDELT events (no key;
// gdeltEvents.js), one report per article at GDELT's place for it, and with
// LIVEUAMAP_API_KEY Liveuamap's mapped items too. Local: World News API
// (WORLDNEWSAPI_KEY) by the country at the view centre; without its key the
// section answers 503 {error: 'no_key', key} and reads nothing, and past
// its day's reads (WORLDNEWSAPI_DAILY_READS) 503 {error: 'key_quota', key}.
// Conflicts' categories, relevance and severity are the shared rules
// (categorize.js); a country's own news keeps World News API's category.
// A news server may replace these readings with its own.
//
// A read that has something to serve is not waited for: what was read is
// served while it is read again. A section that cannot be read answers 502,
// never an empty list.
//
// The answers are the same /api/news contract a news server serves
// (src/layers/news/records.js reads both).
import {
CATEGORY_ORDER,
categorizeText,
relevanceOf,
severityByRules,
} from './categorize.js';
import {
areaAt,
countryAnchor,
countryByCode,
countryPolygons,
} from './areas.js';
import {
eventReports,
fetchLatestExport,
eventTitle,
exportAt,
exportsOfDay,
fetchExport,
fetchTitles,
gkgUrl,
storyKey,
} from './gdeltEvents.js';
import { fetchLiveuamap, fetchWorldNews, UpstreamError } from './upstream.js';
/** The keys each section reads with, by the key registry's id. */
export const NEWS_KEYS = Object.freeze({
liveuamap: 'LIVEUAMAP_API_KEY',
worldnewsapi: 'WORLDNEWSAPI_KEY',
});
/** The one hotspot: the world, its reports drawn as markers at every zoom. */
export const WORLDWIDE_HOTSPOT = 'worldwide';
const HOUR_MS = 3_600_000;
const DAY_MS = 24 * HOUR_MS;
// GDELT publishes an export every 15 minutes and lists the newest in
// lastupdate.txt: the day's missing ones up to it are read every 5 minutes,
// newest first, a dozen a read (the first day's 96 fill in over the next
// reads, a few seconds apart, while each reads something new). One listed
// but missing (404) an hour and a half after its time is taken as never
// published; one that fails GDELT_EXPORT_TRIES times (a try a round), or
// cannot be read at all, is given up on, so one bad file never holds the
// rest back. Only while GDELT itself answers is a file given up on. The first
// answer waits at most FIRST_ANSWER_MS for the first exports.
const GDELT_FRESH_MS = 5 * 60_000;
const GDELT_EXPORTS_PER_READ = 12;
const GDELT_PARALLEL = 4;
const GDELT_NEXT_MS = 2_000;
const GDELT_GONE_MS = 90 * 60_000;
const GDELT_EXPORT_TRIES = 3;
const MAX_GDELT_REPORTS = 5000;
// Events kept of one story, to place it by.
const MAX_STORY_EVENTS = 200;
// Page titles are read from the GKG exports of the newest three hours: one
// runs to 6 MB, so the day's 96 are not fetched. A file that fails is tried
// with the next reads, a few times.
const GDELT_TITLED_EXPORTS = 12;
const GKG_TRIES = 3;
const LIVE_FRESH_MS = 5 * 60_000;
const COUNTRY_FRESH_MS = 30 * 60_000;
// A failed read is not tried again for a minute; what was read before is
// served meanwhile, up to these ages (GDELT's: since an export last came in).
const RETRY_MS = 60_000;
// A read with nothing to serve yet is waited for at most this long, well under
// the browser's own 15 s: it goes on, and the next ask is served from it.
const FIRST_ANSWER_MS = 10_000;
const GDELT_STALE_MS = 2 * HOUR_MS;
const LIVE_STALE_MS = HOUR_MS;
// World News API's terms allow a cache of at most an hour, deleted after:
// a country's news is kept no longer, served or not (dropExpiredCountries).
const COUNTRY_STALE_MS = HOUR_MS;
const MAX_COUNTRIES = 64;
// World News API reads a UTC day, across all countries: each is about 1.5
// of the free plan's 50 daily points.
const DEFAULT_DAILY_READS = 30;
const DEFAULT_LIMIT = 50;
const EMPTY_OUTLINE = Object.freeze({
polygons: [],
borders: [],
disputed: [],
area: null,
});
// A country's own news keeps World News API's category: the rules read
// fighting and crises, and in everyday news their words mislead ("advances
// in the rankings" is no ground fighting). Fighting in the country reaches
// Local as the Conflicts reports placed there.
const WORLD_NEWS_CATEGORY = { politics: 'political', business: 'economy' };
const list = (value) =>
value ? new Set(value.split(',').filter(Boolean)) : null;
/** The rules' reading of a Conflicts headline: category, relevance, severity. */
function readTitle(title) {
const read = categorizeText({ title });
const { category } = read;
return {
category,
relevance: relevanceOf({
category,
tags: read.tags,
title,
dedicated: true,
}),
severity: severityByRules({ category, tags: read.tags, title }),
};
}
/** A one-report event carrying the report's picture, for its card. */
function pictureEvent(id, report, image, video, publisher) {
return {
id,
status: 'single',
size: 1,
reports: 1,
severity: report.severity,
firstAt: report.publishedAt,
lastAt: report.publishedAt,
sides: [],
image: { url: image, publisher, ...(video ? { video } : {}) },
};
}
const media = ({ image, video }) =>
video ? ['video'] : image ? ['photo'] : [];
/** A Liveuamap item as a report of the Conflicts section. */
export function liveuamapReport(item) {
const id = `lum-${item.id}`;
const report = {
id,
hotspotId: WORLDWIDE_HOTSPOT,
hotspotIds: [WORLDWIDE_HOTSPOT],
title: item.title,
url: item.url,
publisher: 'Liveuamap',
source: { name: 'Liveuamap' },
kind: 'liveuamap',
side: 'unknown',
// A conflict map: everything it posts is at least related.
...readTitle(item.title),
severityBy: 'rules',
publishedAt: item.publishedAt,
geo: {
lat: item.lat,
lon: item.lon,
place: item.place,
precision: 'city',
confidence: 1,
mappable: true,
method: 'source',
},
media: media(item),
};
if (item.image)
report.event = pictureEvent(
id,
report,
item.image,
item.video,
'Liveuamap',
);
return report;
}
/**
* A GDELT place named by its town and the area Natural Earth has there (a
* disputed area as itself, else the country): GDELT's own names follow old
* FIPS regions ("Bureij, Yerushalayim, Israel" for a camp in Gaza, "Abuja,
* …, Niger" for Nigeria's capital). Its point stays. A town a disputed
* area is named after ("Jerusalem" in East Jerusalem) lies on both sides
* of its line: GDELT puts the whole city at one point, so it is named as
* the town alone.
*/
export function gdeltPlace(geo) {
const area = areaAt(geo.lat, geo.lon);
const town = String(geo.place ?? '')
.split(',')[0]
.trim();
const straddles =
area?.kind === 'disputed' &&
town &&
area.name.toLowerCase().includes(town.toLowerCase());
const place =
geo.precision === 'country' && area?.kind === 'country'
? area.name
: straddles
? town
: [town, area?.name].filter(Boolean).join(', ');
return { ...geo, place: place || null };
}
/**
* A World News API article as a report of the Local section: placed in its
* publisher's country alone, at a point inside it.
*/
export function worldNewsReport(article, country) {
const id = `wna-${article.id}`;
const anchor = countryAnchor(country);
const report = {
id,
hotspotId: null,
hotspotIds: [],
title: article.title,
url: article.url,
publisher: article.publisher,
// World News API's free plan asks to be credited where its news shows.
source: { name: article.publisher, attribution: 'World News API' },
kind: 'worldnewsapi',
side: 'unknown',
category: WORLD_NEWS_CATEGORY[article.category] || 'other',
// Local news: what the country's own outlets publish, with no severity
// of its own.
relevance: 'minor',
publishedAt: article.publishedAt,
geo: {
lat: anchor.lat,
lon: anchor.lon,
place: country.name,
precision: 'country',
confidence: 1,
mappable: true,
method: 'source',
},
media: media(article),
};
if (article.image)
report.event = pictureEvent(
id,
report,
article.image,
article.video,
article.publisher,
);
return report;
}
const AT = /^(-?\d{1,2}(?:\.\d{1,2})?),(-?\d{1,3}(?:\.\d{1,2})?)$/;
function point(value) {
const match = AT.exec(String(value ?? ''));
if (!match) return null;
const [lat, lon] = [Number(match[1]), Number(match[2])];
return Math.abs(lat) <= 90 && Math.abs(lon) <= 180 ? { lat, lon } : null;
}
const noKey = (key) => ({ status: 503, body: { error: 'no_key', key } });
const rejectedKey = (key) => ({
status: 503,
body: { error: 'key_rejected', key },
});
const spentKey = (key) => ({ status: 503, body: { error: 'key_quota', key } });
/**
* The keyed news routes. `answer(path, params)` resolves to `{status, body}`
* for `/hotspots`, `/feed` and `/outline`, whose queries the proxy has
* already checked against the client's allowlist. `later(run, ms)` runs
* the next GDELT read of a day still filling in (an unref'd timer).
* `firstAnswerMs` bounds the wait for a read with nothing to serve yet.
* @param {{env?: () => object, fetchImpl?: typeof fetch, now?: () => number, log?: (message: string) => void, later?: (run: () => void, ms: number) => void, firstAnswerMs?: number}} [options]
*/
export function createKeyedNews({
env = () => process.env,
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
log = (message) => console.warn(message),
later = (run, ms) => setTimeout(run, ms).unref(),
firstAnswerMs = FIRST_ANSWER_MS,
} = {}) {
const keyOf = (id) => String(env()[NEWS_KEYS[id]] ?? '').trim();
const inFlight = new Map();
// GDELT's stories of the last day (their events and report, by
// storyKey), and the exports read.
const gdelt = {
stories: new Map(),
reports: new Map(),
// Page titles by article address, of the stories kept.
titles: new Map(),
read: new Set(),
// Exports still missing well after their time: not looked for again.
gone: new Set(),
// Failed reads of each export not read yet: `{tries, at}`.
failures: new Map(),
// Exports read among the newest whose GKG titles are not in yet, with
// the tries made: `{tries, triedAt}`.
untitled: new Map(),
titling: null,
// When a read last brought an export in.
readAt: null,
checkedAt: null,
failedAt: null,
};
const live = new Map();
const countries = new Map();
// The keys refused so far, by section: logged once each.
const refused = new Set();
// World News API reads of the UTC day; `spent` once it answered 402.
const budget = { day: null, reads: 0, spent: false, logged: false };
// The areas (disputed area or country) of a placed Conflicts report's
// pins, its first and the others, looked up once.
const reportAreas = new WeakMap();
const areasOf = (report) => {
if (!reportAreas.has(report))
reportAreas.set(
report,
new Set(
[report.geo, ...(report.locations || [])]
.filter(Boolean)
.map((pin) => areaAt(pin.lat, pin.lon)?.key),
),
);
return reportAreas.get(report);
};
/** A failed read in the log: a refused key once, never the key itself. */
function logFailure(section, what, key, error) {
// The day's reads spent: said once, by spendRead.
if (error.quota && error.status === null) return;
if (error.rejected) {
const id = `${section}\0${key}`;
if (refused.has(id)) return;
refused.add(id);
log(
`[news] ${NEWS_KEYS[section]} was refused (${error.status})${section === 'liveuamap' ? "; Conflicts are GDELT's alone" : ''}`,
);
return;
}
log(`[news] ${what} read failed: ${error.message}`);
}
/**
* A cached read (`{key, value, readAt, failedAt, error}`, kept in `store`
* under `id`): served as it is within `freshMs`; past that read again,
* and served meanwhile up to `staleMs`, so only a read with nothing to
* serve is waited for. One read at a time per key; a failed read is tried
* again after RETRY_MS. A new key drops what the old one read.
*/
async function cached(
store,
id,
{ section, key, freshMs, staleMs, read, what },
) {
const at = now();
const stored = store.get(id);
const prior = stored?.key === key ? stored : { key };
if (prior.value && at - prior.readAt < freshMs) return prior;
if (prior.failedAt && at - prior.failedAt < RETRY_MS) return prior;
const flight = `${id}\0${key}`;
if (!inFlight.has(flight))
inFlight.set(
flight,
read(key)
.then(
(value) => ({ key, value, readAt: now() }),
(error) => {
logFailure(section, what, key, error);
// What is held now: news deleted meanwhile stays deleted.
const held = store.get(id);
return {
...(held?.key === key ? held : { key }),
failedAt: now(),
error,
};
},
)
.then((entry) => {
// A read with a key replaced meanwhile is dropped.
if (keyOf(section) === key) keep(store, id, entry);
return entry;
})
.finally(() => inFlight.delete(flight)),
);
if (prior.value && at - prior.readAt < staleMs) return prior;
// Meanwhile a refused key stays refused; an older failure is not this one.
return firstAnswer(inFlight.get(flight), {
...prior,
error: prior.error?.rejected ? prior.error : null,
});
}
/** An entry stored as the most recently used (the countries are capped). */
function keep(store, id, entry) {
store.delete(id);
store.set(id, entry);
while (store.size > MAX_COUNTRIES) store.delete(store.keys().next().value);
}
/** `read`, waited for at most firstAnswerMs; then `fallback`, as it goes on. */
async function firstAnswer(read, fallback) {
let timer;
try {
return await Promise.race([
read,
new Promise((resolve) => {
timer = setTimeout(() => resolve(fallback), firstAnswerMs);
timer.unref?.();
}),
]);
} finally {
clearTimeout(timer);
}
}
/** An entry's reports while they may be served, else why there are none. */
function served(entry, staleMs, id) {
if (entry.value && now() - entry.readAt < staleMs)
return { reports: entry.value, readAt: entry.readAt };
if (entry.error?.rejected) return { rejected: id };
if (entry.error?.quota) return { quota: id };
return { failed: true };
}
async function liveuamapReports() {
const key = keyOf('liveuamap');
if (!key) return { missing: 'liveuamap' };
const entry = await cached(live, 'liveuamap', {
section: 'liveuamap',
key,
freshMs: LIVE_FRESH_MS,
staleMs: LIVE_STALE_MS,
what: 'liveuamap',
read: async (k) =>
(await fetchLiveuamap({ key: k, fetchImpl, now: now() }))
.map(liveuamapReport)
.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)),
});
return served(entry, LIVE_STALE_MS, 'liveuamap');
}
/**
* A story placed (again) from all its events and the titles in: an
* agency story run by several outlets is one report.
*/
function placeStory(key) {
const [report] = eventReports(gdelt.stories.get(key), {
titleOf: (url) => gdelt.titles.get(url) || null,
});
if (!report) return;
const { eventWords, ...rest } = report;
const geo = gdeltPlace(report.geo);
gdelt.reports.set(key, {
...rest,
...(eventWords ? { title: eventTitle(eventWords, geo.place) } : {}),
hotspotId: WORLDWIDE_HOTSPOT,
hotspotIds: [WORLDWIDE_HOTSPOT],
geo,
locations: report.locations.map(gdeltPlace),
});
}
/** An export's events into their stories, each placed again. */
function addEvents(events) {
const changed = new Set();
for (const event of events) {
const key = storyKey(event);
const list = gdelt.stories.get(key) || [];
if (list.length < MAX_STORY_EVENTS) list.push(event);
gdelt.stories.set(key, list);
changed.add(key);
}
for (const key of changed) placeStory(key);
}
/** The stories a GKG export titled, re-titled. */
function addTitles(titles) {
for (const [key, list] of gdelt.stories) {
const titled = list.filter((event) => titles.has(event.url));
if (!titled.length) continue;
for (const { url } of titled) gdelt.titles.set(url, titles.get(url));
placeStory(key);
}
}
/**
* The page titles of the newest exports read, from their GKG exports one
* at a time (each runs to 18 MB unzipped), in the background: no answer
* waits for them, and a story is re-titled as its title comes in. A file
* that fails or is not up yet only leaves address words: it is tried
* again with a later read while among the newest, up to GKG_TRIES times.
*/
function readTitles() {
gdelt.titling ??= (async () => {
let failed = null;
for (const [url, tried] of [...gdelt.untitled]) {
if (!gdelt.untitled.has(url)) continue;
if (tried.triedAt !== null && now() - tried.triedAt < GDELT_FRESH_MS)
continue;
tried.tries += 1;
tried.triedAt = now();
const wanted = new Set();
for (const list of gdelt.stories.values())
for (const event of list) wanted.add(event.url);
let titles = null;
try {
titles = await fetchTitles(gkgUrl(url), wanted, { fetchImpl });
} catch (error) {
failed ??= error;
}
if (titles || tried.tries >= GKG_TRIES) gdelt.untitled.delete(url);
if (titles?.size) addTitles(titles);
}
if (failed) log(`[news] gdelt titles read failed: ${failed.message}`);
})()
.catch((error) => log(`[news] gdelt titles failed: ${error.message}`))
.finally(() => {
gdelt.titling = null;
});
return gdelt.titling;
}
/**
* Up to a dozen of the day's exports not read yet, newest first; whether
* to carry on at once (the round brought one in, and more are missing).
* One not up yet is looked for again; one still missing GDELT_GONE_MS
* after its time is not.
*/
async function readExports() {
const at = now();
const day = exportsOfDay(at);
// Up to the newest GDELT lists; all of the day when its list fails.
const latest = await fetchLatestExport({ fetchImpl });
const listed = (url) => latest === null || exportAt(url) <= latest;
const newest = new Set(day.slice(0, GDELT_TITLED_EXPORTS));
const wanted = (url) =>
listed(url) && !gdelt.read.has(url) && !gdelt.gone.has(url);
const missing = day.filter(wanted).slice(0, GDELT_EXPORTS_PER_READ);
let failed = null;
let added = 0;
const failing = [];
const unreadable = [];
const unpublished = [];
for (let i = 0; i < missing.length; i += GDELT_PARALLEL)
await Promise.all(
missing.slice(i, i + GDELT_PARALLEL).map(async (url) => {
let events;
try {
events = await fetchExport(url, { fetchImpl });
} catch (error) {
failed = error;
(error.unreadable ? unreadable : failing).push(url);
return;
}
if (!events) {
if (at - exportAt(url) >= GDELT_GONE_MS) unpublished.push(url);
return;
}
// The events count now; the titles follow (readTitles).
gdelt.read.add(url);
gdelt.failures.delete(url);
added += 1;
gdelt.readAt = now();
addEvents(events);
if (newest.has(url))
gdelt.untitled.set(url, { tries: 0, triedAt: null });
}),
);
if (failed) log(`[news] gdelt read failed: ${failed.message}`);
// A file is blamed only while GDELT itself answers (its list read, or
// another export of the round read): then a 404 long after its time is
// never published, an unreadable file stays so, and a failing one gets a
// few tries, one a GDELT_FRESH_MS. With GDELT down, no file is blamed.
const answering = added > 0 || latest !== null;
if (answering)
for (const url of [...unpublished, ...unreadable]) gdelt.gone.add(url);
if (added)
for (const url of failing) {
const prior = gdelt.failures.get(url);
if (prior && now() - prior.at < GDELT_FRESH_MS) continue;
const tries = (prior?.tries || 0) + 1;
gdelt.failures.set(url, { tries, at: now() });
if (tries >= GDELT_EXPORT_TRIES) gdelt.gone.add(url);
}
// Nothing read and a failure: tried again after RETRY_MS, not after
// GDELT_FRESH_MS. Something read: the round counts.
if (failed && !added) gdelt.failedAt = now();
else {
gdelt.checkedAt = now();
gdelt.failedAt = null;
}
// Only the day is kept, and titled while among the newest.
const kept = new Set(day);
for (const set of [gdelt.read, gdelt.gone])
for (const url of set) if (!kept.has(url)) set.delete(url);
for (const url of gdelt.failures.keys())
if (!kept.has(url)) gdelt.failures.delete(url);
for (const url of gdelt.untitled.keys())
if (!newest.has(url)) gdelt.untitled.delete(url);
return (
added > 0 && missing.length === GDELT_EXPORTS_PER_READ && day.some(wanted)
);
}
/** GDELT's reports of the last day, newest first, or `{failed}`. */
async function gdeltReports() {
const at = now();
const fresh = gdelt.checkedAt && at - gdelt.checkedAt < GDELT_FRESH_MS;
const resting = gdelt.failedAt && at - gdelt.failedAt < RETRY_MS;
if (!fresh && !resting && !inFlight.has('gdelt')) {
const round = readExports()
.then(
(more) => {
// More of the day to fill in: carry on shortly, without holding
// the answer.
if (!more) return;
gdelt.checkedAt = null;
later(() => void gdeltReports(), GDELT_NEXT_MS);
},
(error) => {
log(`[news] gdelt read failed: ${error.message}`);
gdelt.failedAt = now();
},
)
.finally(() => {
inFlight.delete('gdelt');
void readTitles();
});
inFlight.set('gdelt', round);
}
// Only a read with nothing (recent) to serve is waited for: the
// newest exports' events, not their titles.
const current = () =>
gdelt.readAt !== null && now() - gdelt.readAt < GDELT_STALE_MS;
// A slow GDELT must not outlast the browser's own timeout: what has come
// in by then is served.
if (!current() && inFlight.has('gdelt'))
await firstAnswer(inFlight.get('gdelt'));
if (!current()) return { failed: true };
const dayAgo = new Date(now() - DAY_MS).toISOString();
// Exports are read in parallel, so reports of one time arrive in any
// order: the most mentioned comes first, then by id.
const reports = [...gdelt.reports.values()]
.filter((r) => r.publishedAt >= dayAgo)
.sort(
(a, b) =>
b.publishedAt.localeCompare(a.publishedAt) ||
b.mentions - a.mentions ||
a.id.localeCompare(b.id),
)
.slice(0, MAX_GDELT_REPORTS);
if (reports.length !== gdelt.reports.size) {
const kept = new Set(reports);
for (const [key, report] of gdelt.reports)
if (!kept.has(report)) {
for (const event of gdelt.stories.get(key) || [])
gdelt.titles.delete(event.url);
gdelt.reports.delete(key);
gdelt.stories.delete(key);
}
}
return { reports, readAt: gdelt.readAt };
}
/**
* The Conflicts section's reports: GDELT's, with Liveuamap's when its key
* is set. Liveuamap failing (or refusing its key) leaves GDELT's.
*/
async function conflictReports() {
const [articles, items] = await Promise.all([
gdeltReports(),
keyOf('liveuamap') ? liveuamapReports() : {},
]);
if (!articles.reports && !items.reports) return { failed: true };
return {
reports: [...(articles.reports || []), ...(items.reports || [])].sort(
(a, b) => b.publishedAt.localeCompare(a.publishedAt),
),
readAt: Math.max(articles.readAt || 0, items.readAt || 0),
};
}
const dailyReads = () => {
const raw = String(env().WORLDNEWSAPI_DAILY_READS ?? '').trim();
const reads = raw ? Number(raw) : NaN;
return Number.isInteger(reads) && reads >= 0 ? reads : DEFAULT_DAILY_READS;
};
/** Whether a World News API read is left today (UTC); counts it if so. */
function spendRead() {
const day = new Date(now()).toISOString().slice(0, 10);
if (budget.day !== day)
Object.assign(budget, { day, reads: 0, spent: false, logged: false });
if (!budget.spent && budget.reads < dailyReads()) {
budget.reads += 1;
return true;
}
if (!budget.logged)
log(
"[news] World News API's reads for today are spent; Local reads again after 00:00 UTC",
);
budget.logged = true;
return false;
}
/** World News API's news an hour old is deleted, as its terms ask. */
function dropExpiredCountries() {
for (const [code, entry] of countries)
if (entry.value && now() - entry.readAt >= COUNTRY_STALE_MS) {
const { value, readAt, ...rest } = entry;
countries.set(code, rest);
}
}
async function countryReports(country) {
const key = keyOf('worldnewsapi');
if (!key) return { missing: 'worldnewsapi' };
const stored = countries.get(country.code);
if (stored) keep(countries, country.code, stored);
const entry = await cached(countries, country.code, {
section: 'worldnewsapi',
key,
freshMs: COUNTRY_FRESH_MS,
staleMs: COUNTRY_STALE_MS,
what: `worldnewsapi:${country.code}`,
read: async (k) => {
// Not asked: no status, unlike an answer.
if (!spendRead())
throw new UpstreamError("World News API's reads for today spent", {
quota: true,
});
const { day } = budget;
try {
const reports = (
await fetchWorldNews({
key: k,
country: country.code,
fetchImpl,
now: now(),
})
).map((article) => worldNewsReport(article, country));
// Deleted an hour on, whether or not anything is asked then.
setTimeout(dropExpiredCountries, COUNTRY_STALE_MS + 1000).unref?.();
return reports;
} catch (error) {
// Out of points: nothing more is asked today.
if (error.status === 402) budget.spent = true;
// Never reached (offline): no read of the day it was counted in.
if (error.unreached && budget.day === day && budget.reads > 0)
budget.reads -= 1;
throw error;
}
},
});
return served(entry, COUNTRY_STALE_MS, 'worldnewsapi');
}
const refusal = (read) =>
read.missing
? noKey(read.missing)
: read.rejected
? rejectedKey(read.rejected)
: read.quota
? spentKey(read.quota)
: read.failed
? { status: 502, body: { error: 'news_upstream_error' } }
: null;
async function hotspots(params) {
const read = await conflictReports();
const refused = refusal(read);
if (refused) return refused;
const hours = Number(params.get('hours')) || 24;
const since = new Date(now() - hours * HOUR_MS).toISOString();
// The reports Conflicts lists: a minor one is in no count.
const shown = read.reports.filter(
(r) => r.publishedAt >= since && r.relevance !== 'minor',
);
return {
status: 200,
body: {
hotspots: [
{
id: WORLDWIDE_HOTSPOT,
name: 'Worldwide',
kind: 'conflict',
catchAll: true,
bbox: [-180, -90, 180, 90],
parties: [],
developing: [],
counts: {
last24h: shown.length,
mapped24h: shown.filter((r) => r.geo).length,
prev24h: null,
},
latestAt: shown[0]?.publishedAt ?? null,
},
],
categories: CATEGORY_ORDER,
ingest: { lastRunAt: new Date(read.readAt).toISOString() },
windowHours: hours,
maxWindowHours: 24,
},
};
}
/** Reports filtered, newest first, and paged by `before`. */
function page(reports, params) {
const since = params.get('since');
const before = params.get('before');
const dayAgo = new Date(now() - DAY_MS).toISOString();
const categories = list(params.get('category'));
const relevance = list(params.get('relevance'));
const hotspot = params.get('hotspot');
const limit = Number(params.get('limit')) || DEFAULT_LIMIT;
const kept = reports.filter(
(r) =>
(!hotspot || hotspot === 'all' || r.hotspotIds.includes(hotspot)) &&
(!since || r.publishedAt >= since) &&
(!before || r.publishedAt < before) &&
(!params.has('today') || r.publishedAt >= dayAgo) &&
(!params.has('geo') || r.geo) &&
(!categories || categories.has(r.category)) &&
(!relevance || relevance.has(r.relevance)),
);
const shown = kept.slice(0, limit);
// A page ends between two times: the next asks for those `before` the
// last, and a GDELT export's reports all share its quarter hour.
while (
shown.length < kept.length &&
kept[shown.length].publishedAt === shown.at(-1).publishedAt
)
shown.push(kept[shown.length]);
const events = params.has('events');
return {
reports: events ? shown : shown.map(({ event, ...report }) => report),
nextBefore: kept.length > shown.length ? shown.at(-1).publishedAt : null,
};
}
async function feed(params) {
const at = params.has('at') ? point(params.get('at')) : null;
const area = at ? areaAt(at.lat, at.lon) : null;
if (!area) {
// Conflicts, or Local at sea (by distance, in the browser).
const read = await conflictReports();
return refusal(read) || { status: 200, body: page(read.reports, params) };
}
// Local in a country: its own outlets, with the Conflicts reports
// placed inside it (those failing leave the outlets'). In a disputed
// area no country's outlets speak for it: the Conflicts reports there
// alone, so their failing is the answer's.
const country = area.kind === 'country';
const [national, conflicts] = await Promise.all([
country ? countryReports(area.country) : { reports: [] },
conflictReports(),
]);
const refused = refusal(national) || (!country && refusal(conflicts));
if (refused) return refused;
const inside = (conflicts.reports || []).filter((r) =>
areasOf(r).has(area.key),
);
const reports = [...national.reports, ...inside].sort((a, b) =>
b.publishedAt.localeCompare(a.publishedAt),
);
return { status: 200, body: page(reports, params) };
}
function outline(params) {
if (params.has('at')) {
const at = point(params.get('at'));
const area = at ? areaAt(at.lat, at.lon) : null;
if (!area) return { status: 200, body: EMPTY_OUTLINE };
return {
status: 200,
body: {
...EMPTY_OUTLINE,
polygons:
area.kind === 'country'
? countryPolygons(area.country)
: area.polygons,
area: {
key: area.key,
name: area.name,
kind: area.kind,
country: area.code,
},
},
};
}
// A developing situation's country; the world has no outline.
const country = countryByCode(
String(params.get('country') ?? '').slice(0, 2),
);
return {
status: 200,
body: country
? { ...EMPTY_OUTLINE, polygons: countryPolygons(country) }
: EMPTY_OUTLINE,
};
}
return {
async answer(path, params) {
dropExpiredCountries();
if (path === '/hotspots') return hotspots(params);
if (path === '/feed') return feed(params);
if (path === '/outline') return outline(params);
return { status: 404, body: { error: 'not_found' } };
},
};
}
File diff suppressed because it is too large. Load diff
+283
View File
@@ -0,0 +1,283 @@
// The keyed news APIs Core reads itself when no news server is configured:
// Liveuamap's worldwide feed (`a=mpts`) beside GDELT's events in the
// Conflicts section (gdeltEvents.js), World News API's search by source
// country for Local. Each key stays on the server: Liveuamap's rides in the query, so
// neither a URL nor an error body is ever logged or relayed.
import { readResponseTextCapped } from '../common/http.js';
const LIVEUAMAP_URL = 'https://a.liveuamap.com/api';
const WORLD_NEWS_URL = 'https://api.worldnewsapi.com/search-news';
// 100 Liveuamap items run to about 500 KB.
const LIVEUAMAP_COUNT = 500;
const LIVEUAMAP_MAX_BYTES = 16 * 1024 * 1024;
// One search per country: a point, and a hundredth of one per article, so
// about 1.5 points a read (the free plan has 50 points a day).
const WORLD_NEWS_ARTICLES = 50;
const WORLD_NEWS_MAX_BYTES = 8 * 1024 * 1024;
// World News API's categories that are not the news of the day. Its
// `categories` filter is not asked for: it classifies few articles outside
// English, so it would drop most of a country's own outlets; an article it
// left unclassified is kept.
export const WORLD_NEWS_LEFT_OUT = Object.freeze(
new Set(['sports', 'entertainment', 'lifestyle', 'travel']),
);
const DAY_MS = 86_400_000;
const TIMEOUT_MS = 20_000;
const ENTITIES = {
amp: '&',
quot: '"',
apos: "'",
lt: '<',
gt: '>',
nbsp: ' ',
};
/** Text with HTML entities decoded and whitespace collapsed, or ''. */
export function unescapeHtml(value) {
return String(value ?? '')
.replace(
/&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]{2,6}));/gi,
(all, dec, hex, name) => {
if (name) return ENTITIES[name.toLowerCase()] ?? all;
const code = dec ? Number(dec) : parseInt(hex, 16);
return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : all;
},
)
.replace(/\s+/g, ' ')
.trim();
}
/** An https URL without credentials, or null. */
export function webLink(value) {
try {
const url = new URL(String(value ?? '').trim());
return url.protocol === 'https:' && !url.username && !url.password
? url.href
: null;
} catch {
return null;
}
}
/** A clip a browser plays itself (an https MP4 or WebM file), or null. */
export function playableVideo(value) {
const url = webLink(value);
return url && /\.(?:mp4|m4v|webm)$/i.test(new URL(url).pathname) ? url : null;
}
const finiteNumber = (value) => {
const number = typeof value === 'string' ? Number(value.trim()) : value;
return typeof number === 'number' && Number.isFinite(number) ? number : null;
};
/** Why an upstream could not be read; never carries its URL or body. */
export class UpstreamError extends Error {
constructor(
message,
{
status = null,
rejected = false,
quota = false,
unreadable = false,
unreached = false,
} = {},
) {
super(message);
this.name = 'UpstreamError';
this.status = status;
// The key was refused (401 or 403): the operator's to fix.
this.rejected = rejected;
// The key's allowance is spent (402, or the day's reads): it answers
// again later. A 429 is a per-second limit, not this.
this.quota = quota;
// The answer itself cannot be read (corrupt, or over a size cap): asking
// again gets the same.
this.unreadable = unreadable;
// The request got no answer at all (offline, timed out).
this.unreached = unreached;
}
}
async function getText(
fetchImpl,
url,
{ headers = {}, maxBytes, signal, what },
) {
let response;
try {
response = await fetchImpl(url, {
headers: { Accept: 'application/json', ...headers },
redirect: 'error',
signal: signal
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
: AbortSignal.timeout(TIMEOUT_MS),
});
} catch (cause) {
// A request that timed out may have been read; one refused was not.
const timedOut = cause?.name === 'TimeoutError';
throw new UpstreamError(
timedOut ? `${what} timed out` : `${what} unreachable`,
{ unreached: !timedOut },
);
}
if (!response.ok) {
await response.body?.cancel?.().catch?.(() => {});
throw new UpstreamError(`${what} answered ${response.status}`, {
status: response.status,
rejected: response.status === 401 || response.status === 403,
quota: response.status === 402,
});
}
try {
return await readResponseTextCapped(response, maxBytes, signal);
} catch {
throw new UpstreamError(`${what} answer unreadable`);
}
}
function parseJson(text, what) {
try {
return JSON.parse(text);
} catch {
// Not the parser's message: an error answer may echo the request.
throw new UpstreamError(`${what} answer unreadable`);
}
}
/**
* Liveuamap's items with a headline, a point and a page:
* `{id, title, lat, lon, place, publishedAt, url, image, video}`.
*/
export function parseLiveuamap(payload) {
if (!Array.isArray(payload?.venues))
throw new UpstreamError('Liveuamap answer unreadable');
const items = [];
for (const venue of payload.venues) {
const id = finiteNumber(venue?.id);
const title = unescapeHtml(venue?.name);
const lat = finiteNumber(venue?.lat);
const lon = finiteNumber(venue?.lng);
const seconds = finiteNumber(venue?.timestamp);
const url = webLink(venue?.link);
if (
id === null ||
!title ||
lat === null ||
lon === null ||
Math.abs(lat) > 90 ||
Math.abs(lon) > 180 ||
(lat === 0 && lon === 0) ||
seconds === null ||
!url
)
continue;
items.push({
id: Math.trunc(id),
title,
lat,
lon,
place: unescapeHtml(venue.location) || null,
publishedAt: new Date(seconds * 1000).toISOString(),
url,
image: webLink(venue.img_share),
video: playableVideo(venue.video),
});
}
return items;
}
/** The worldwide feed of the last day, as of `now`. */
export async function fetchLiveuamap({ key, fetchImpl, now, signal }) {
const url = new URL(LIVEUAMAP_URL);
url.search = new URLSearchParams({
a: 'mpts',
resid: '1',
time: String(Math.floor((now - DAY_MS) / 1000)),
count: String(LIVEUAMAP_COUNT),
key,
}).toString();
const text = await getText(fetchImpl, url, {
maxBytes: LIVEUAMAP_MAX_BYTES,
signal,
what: 'Liveuamap',
});
return parseLiveuamap(parseJson(text, 'Liveuamap'));
}
// World News API writes times as "2026-10-30 01:31:00", in UTC.
const worldNewsTime = (ms) =>
new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
function worldNewsDate(value) {
const match = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)$/.exec(
String(value ?? ''),
);
const ms = match ? Date.parse(`${match[1]}T${match[2]}Z`) : NaN;
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
}
/**
* A search's articles with a title, a link and a time, one per title (the
* same story from several outlets is one), sport and entertainment left
* out, newest first:
* `{id, title, url, publisher, language, category, publishedAt, image, video}`.
*/
export function parseWorldNews(payload) {
if (!Array.isArray(payload?.news))
throw new UpstreamError('World News API answer unreadable');
const seen = new Set();
const articles = [];
for (const article of payload.news) {
const id = finiteNumber(article?.id);
const title = unescapeHtml(article?.title);
const url = webLink(article?.url);
const publishedAt = worldNewsDate(article?.publish_date);
const category =
typeof article?.category === 'string'
? article.category.toLowerCase()
: null;
if (id === null || !title || !url || !publishedAt) continue;
if (WORLD_NEWS_LEFT_OUT.has(category)) continue;
const same = title.toLowerCase();
if (seen.has(same)) continue;
seen.add(same);
articles.push({
id: Math.trunc(id),
title,
url,
publisher: new URL(url).hostname.replace(/^www\./, ''),
language:
typeof article.language === 'string' &&
/^[a-z]{2}$/i.test(article.language)
? article.language.toLowerCase()
: null,
category,
publishedAt,
image: webLink(article.image),
video: playableVideo(article.video),
});
}
return articles.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt));
}
/** A country's everyday news of the last day, as of `now`. */
export async function fetchWorldNews({ key, country, fetchImpl, now, signal }) {
const url = new URL(WORLD_NEWS_URL);
url.search = new URLSearchParams({
'source-country': country.toLowerCase(),
// Both ends: sorted by time with neither, a search is cut to 31 days.
'earliest-publish-date': worldNewsTime(now - DAY_MS),
'latest-publish-date': worldNewsTime(now),
sort: 'publish-time',
'sort-direction': 'DESC',
number: String(WORLD_NEWS_ARTICLES),
}).toString();
const text = await getText(fetchImpl, url, {
headers: { 'x-api-key': key },
maxBytes: WORLD_NEWS_MAX_BYTES,
signal,
what: 'World News API',
});
return parseWorldNews(parseJson(text, 'World News API'));
}
+30 -1
View File
@@ -64,6 +64,30 @@ function validMessages(messages) {
return !pending.length;
}
/**
* A tool schema without the `pattern`s Chat Completions refuses: it answers
* 400 for a whole request whose tool uses Unicode property classes (`\p{L}`),
* as `resolve_area`'s area ids do. Every argument is still checked against
* the action's own contract before the tool runs, so the model loses a hint,
* not a rule. Realtime voice refuses them too now (#1618) and uses this as well.
*/
export function chatSchema(schema) {
if (Array.isArray(schema)) return schema.map(chatSchema);
if (!object(schema)) return schema;
return Object.fromEntries(
Object.entries(schema)
.filter(
([key, value]) =>
!(
key === 'pattern' &&
typeof value === 'string' &&
/\\[pP]\{/.test(value)
),
)
.map(([key, value]) => [key, chatSchema(value)]),
);
}
export function createOpenAiChatHandler({
annotationGuidance,
tools = GEV_REALTIME_TOOLS,
@@ -126,7 +150,12 @@ export function createOpenAiChatHandler({
tools: offeredTools(tools, { overpass }).map(
({ name, description, parameters }) => ({
type: 'function',
function: { name, description, parameters, strict: false },
function: {
name,
description,
parameters: chatSchema(parameters),
strict: false,
},
}),
),
},
+5 -4
View File
@@ -26,7 +26,7 @@ function realtimeDirectives(
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
'WORLD QUERIES: query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and other get_* and find_* tools) look up any place without changing the map or enabling layers. Use them for places out of view or data whose layer is off. For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over layers instead; enable the layer first if asked. Use analyst_query and get_entity_context for loaded data or what is on screen. Answer from the result; name its source if supplied and never claim the map changed. If asked to show it, also use the matching app action (fly_to_location, set_layer_visibility).',
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names, or an areaId you already have; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
'AREAS: resolve_area turns a named boundary (country, state, province, county, city, district, natural region) into an areaId; to outline it pass draw=true; annotate_map is for marks, not boundaries. Count inside it → analyst_query kind=area with the areaId (or kind=region); "the area I drew" → kind=drawn; "the area you outlined" → kind=annotation. Imagery of a place → resolve_area, then find_imagery. Ambiguous name: call resolve_area; on needsClarification ask which, never guess.' +
(overpass
@@ -38,7 +38,7 @@ function realtimeDirectives(
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
'Do not mention disabled layers or stale selections.',
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target, find_imagery area. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, resolve_area, find_imagery, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, resolve_area, find_imagery, a searched fly_to_location, select_nearest_aircraft, show_street_level, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers, approximate outlines and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
@@ -58,13 +58,14 @@ function realtimeDirectives(
'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any other mention of "satellite" or "satellites" means the satellites DATA LAYER via set_layer_visibility, never a basemap — except "recent/latest satellite imagery" (the recent-imagery layer) and "weather satellite"/clouds (weather-satellite). "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
'HUD requests ("hud on/off", "switch to minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
'HUD requests ("hud on/off", "switch to minimal/tactical layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
'Sonar is a visual style over any HUD layout. Sonar requests (including "cyber mode", which was replaced by sonar) use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch layouts, reveal the HUD or enable sonar when changing a slider. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
'Bloom/sharpen requests use set_post_processing. Scene requests ("play orbital watch", "stop the scene", "what scenes are there") use control_scene. CCTV camera requests ("next camera", "nearest camera", "select the Congress camera", "show coverage") use control_cctv — the CCTV layer must be enabled first.',
'Radio playback requests use control_radio. "Turn on/start the radio" means action=play; action=enable only reveals Radio markers and must be reserved for explicit "show/enable the Radio layer/markers" requests. After a prepared playback result, briefly confirm any other completed actions and say "Turning on the radio"—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
'"Track/follow <something specific>" (a callsign, ship name, satellite name) uses track_entity. "Take me to the biggest fire" uses track_entity with query "biggest fire" (the fires layer must be enabled). Bare "orbit" means camera orbit of the current landmark. "Stop following/tracking" uses stop_tracking.',
'"Show me which planes are overhead"/"frame the ships"/"show me the satellites above" use frame_overhead with the matching target.',
'Confirmations echo the RESULTING state, never the request: "HUD minimal layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
'STREET LEVEL: "X on Street View/Mapillary/KartaView/Panoramax", "street view of X" → show_street_level once, not fly_to_location; no place = view centre. Street View is never a basemap.',
'For destination requests such as "take me to Italy", "go to NYC", or "show me the Eiffel Tower", call fly_to_location. Prefer known city IDs when available; otherwise pass the plain place query.',
'Navigation-only requests ("take me to X", "go to X", "fly to X") are NOT descriptions: call fly_to_location alone and do NOT also call annotate_map, unless the user explicitly asks to mark the place or you go on to explain specific places there. Never drop a point pin on a region-scale natural feature (a mountain range, desert, sea, or forest) — a single point in the middle of the Rockies is meaningless. If the user explicitly asks to mark such a region, prefer type=area.',
'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
+22 -2
View File
@@ -14,6 +14,7 @@ import {
} from './constants.js';
import { realtimeInstructions } from './instructions.js';
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
import { chatSchema } from './chat.js';
import { resolveOverpassUpstreams } from '../overpass/constants.js';
import {
preferCodexOAuth,
@@ -128,6 +129,19 @@ function createRealtimeOAuthLoginHandler({
};
}
/**
* The tools one Realtime session is offered, as OpenAI accepts them: it now
* refuses a whole session whose tool schema uses `\p{…}` patterns (#1618), as
* Chat Completions did (#1616), so voice drops the same pattern hints chat does.
*/
function realtimeSessionTools(tools, { overpass = false } = {}) {
return offeredTools(tools, { overpass }).map((tool) =>
tool.parameters
? { ...tool, parameters: chatSchema(tool.parameters) }
: tool,
);
}
function createRealtimeTokenHandler({
annotationGuidance,
endpoint = 'https://api.openai.com/v1/realtime/client_secrets',
@@ -224,7 +238,12 @@ function createRealtimeTokenHandler({
if (!bearer) {
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
res.end(
JSON.stringify({
error: 'OPENAI_API_KEY is not set',
code: 'API_KEY_MISSING',
}),
);
return;
}
}
@@ -303,7 +322,7 @@ function createRealtimeTokenHandler({
output: { voice },
},
instructions: realtimeInstructions(annotationGuidance, { overpass }),
tools: offeredTools(tools, { overpass }),
tools: realtimeSessionTools(tools, { overpass }),
tool_choice: 'auto',
},
};
@@ -370,4 +389,5 @@ export {
createRealtimeOAuthLoginHandler,
createRealtimeOAuthStatusHandler,
createRealtimeTokenHandler,
realtimeSessionTools,
};
+28 -1
View File
@@ -65,6 +65,28 @@ export const ACTION_DESCRIPTIONS = {
},
},
},
show_street_level: {
description:
'Fly to a place and open its nearest street-level photo; Street View switches to Google 3D (billed). No query: view centre.',
$position: 1,
parameters: {
properties: {
query: {
description: 'Place or address; "pointer" = under the cursor.',
$position: 1,
},
provider: {
description:
'google = Street View; others by name; omit: the lit source.',
$position: 1,
},
referent: {
description: 'Numbered item n from the last result list.',
$position: 1,
},
},
},
},
adjust_camera_zoom: {
description:
'Move the current Cesium camera closer to or farther from what it is presently looking at. Use for relative zoom requests without changing location.',
@@ -203,7 +225,7 @@ export const ACTION_DESCRIPTIONS = {
},
set_cyber_sonar: {
description:
'Adjust Cyber-only sonar controls. Requires the Cyber HUD layout; never switches layout or HUD visibility. Omitted fields stay unchanged. Returns actual settings and separate map/contact sweep activity. A saved setting does not mean the effect is active.',
'Adjust the Sonar visual style, which works over any HUD layout. Never switches layout or HUD visibility. Omitted fields stay unchanged. Returns actual settings and separate map/contact sweep activity. A saved setting does not mean the effect is active.',
$position: 1,
parameters: {
properties: {
@@ -287,6 +309,11 @@ export const ACTION_DESCRIPTIONS = {
},
},
},
open_stories: {
description:
'Open the Stories Explore list on request ("open Stories", "show the story list"). Turns off voice.',
$position: 1,
},
control_scene: {
description:
'Cinematic scene playback: list scenes, play one scene by name, stop, advance, or read status. Play starts a single named scene and returns immediately.',
+50 -16
View File
@@ -72,6 +72,14 @@ class UpstreamRateLimitError extends Error {
}
}
/** Thrown when the routing service failed or sent no JSON; answered as 502. */
class UpstreamUnavailableError extends Error {
constructor() {
super('routing service unavailable');
this.name = 'UpstreamUnavailableError';
}
}
/**
* Run one outbound request no sooner than the shared gate allows.
* @template T
@@ -167,18 +175,26 @@ async function fetchRoute({
`${base.replace(/\/$/, '')}/route/v1/${osrmProfile}/${coords}` +
`?overview=full&geometries=geojson&alternatives=false&steps=${withSteps ? 'true' : 'false'}`;
const controller = new AbortController();
// The 12 s budget includes queue wait, so Directions (15 s) hears back.
const timer = setTimeout(() => controller.abort(), 12000);
let osrm;
try {
const upstreamRes = await throughUpstreamGate(() =>
fetchImpl(upstream, {
signal: controller.signal,
// The endpoint is configured above; a redirect is the one way out of
// it, so it is refused rather than followed.
redirect: 'error',
headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
}),
);
const upstreamRes = await throughUpstreamGate(async () => {
// Expiry before dispatch reflects our backlog, not an upstream outage.
if (controller.signal.aborted)
throw new RouteBusyError(_upstreamIntervalMs);
try {
return await fetchImpl(upstream, {
signal: controller.signal,
// The endpoint is configured above; a redirect is the one way out of
// it, so it is refused rather than followed.
redirect: 'error',
headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
});
} catch {
throw new UpstreamUnavailableError();
}
});
if (upstreamRes.status === 429) {
await cancelBody(upstreamRes);
const retryAfter = Number(upstreamRes.headers.get('retry-after'));
@@ -186,20 +202,28 @@ async function fetchRoute({
Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : 5,
);
}
if (!upstreamRes.ok) {
// OSRM answers 400 for a route it cannot find; other failures are outages.
if (upstreamRes.status === 400) {
await cancelBody(upstreamRes);
return { payload: null, error: 'no route found' };
}
if (!upstreamRes.ok) {
await cancelBody(upstreamRes);
throw new UpstreamUnavailableError();
}
const ctype = upstreamRes.headers.get('content-type') || '';
if (!ctype.includes('json')) {
await cancelBody(upstreamRes);
return { payload: null, error: 'no route found' };
throw new UpstreamUnavailableError();
}
// Read and parse failures are outages; later ones stay "route proxy error".
try {
osrm = JSON.parse(
await readResponseTextCapped(upstreamRes, ROUTE_MAX_RESPONSE_BYTES),
);
} catch {
throw new UpstreamUnavailableError();
}
const text = await readResponseTextCapped(
upstreamRes,
ROUTE_MAX_RESPONSE_BYTES,
);
osrm = JSON.parse(text);
} finally {
clearTimeout(timer);
}
@@ -227,6 +251,11 @@ export function installRouteMiddleware(
// only shape that asks the upstream for them. A response WITHOUT steps is
// byte for byte what this endpoint has always returned.
middlewares.use('/api/route', async (req, res) => {
if (req.method !== 'GET') {
res.writeHead(405, { 'Content-Type': 'application/json', Allow: 'GET' });
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
const fail = (msg) => {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ ok: false, error: msg }));
@@ -350,6 +379,11 @@ export function installRouteMiddleware(
upstreamError.retryAfterSec,
);
}
if (upstreamError instanceof UpstreamUnavailableError) {
res.writeHead(502, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ ok: false, error: upstreamError.message }));
return;
}
throw upstreamError;
}
if (error || !payload) return fail(error || 'no route found');
+285
View File
@@ -0,0 +1,285 @@
import { open, stat } from 'node:fs/promises';
import path from 'node:path';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { fileURLToPath } from 'node:url';
import { createRailService } from '../../src/sources/railService.js';
import { RAIL_FEED_ID_PATTERN } from '../../src/data/railFeeds.js';
/**
* Where development and preview look for path and schedule packs when
* RAIL_PATHS_DIR is unset: the default output of scripts/build-rail-paths.mjs.
*/
export const DEFAULT_RAIL_PATHS_DIR = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
'../../.cache/rail-paths',
);
/**
* Pack bytes the development server keeps in memory, across path and
* schedule packs: a few of the largest national packs.
*/
export const RAIL_PACK_CACHE_MAX_BYTES = 64 * 1024 * 1024;
/**
* A bounded cache of pack files' bytes, by path, least recently read first
* out. An entry is used only while the file's modification time and size
* are the ones it was read with, so a rebuilt pack is read again; a file
* larger than the whole byte budget is never kept. Reads of the same file at
* the same time share one read.
* @param {number} [maxBytes]
* @returns {{load: (file: string, info: {mtimeMs: number, size: number},
* read: () => Promise<Buffer>) => Promise<Buffer>, bytes: () => number}}
*/
export function createRailPackCache(maxBytes = RAIL_PACK_CACHE_MAX_BYTES) {
/** @type {Map<string, {mtimeMs: number, size: number, bytes: Buffer}>} */
const entries = new Map();
/** @type {Map<string, Promise<Buffer>>} */
const pending = new Map();
let total = 0;
const drop = (file) => {
const entry = entries.get(file);
if (!entry) return;
entries.delete(file);
total -= entry.bytes.byteLength;
};
return {
async load(file, info, read) {
const entry = entries.get(file);
if (entry && entry.mtimeMs === info.mtimeMs && entry.size === info.size) {
// Most recently read last: the first entry is the next one out.
entries.delete(file);
entries.set(file, entry);
return entry.bytes;
}
const version = `${file}|${info.mtimeMs}|${info.size}`;
const reading = pending.get(version);
if (reading) return reading;
const promise = (async () => {
const bytes = await read();
drop(file);
if (bytes.byteLength <= maxBytes) {
entries.set(file, { mtimeMs: info.mtimeMs, size: info.size, bytes });
total += bytes.byteLength;
for (const [oldest] of entries) {
if (total <= maxBytes) break;
drop(oldest);
}
}
return bytes;
})().finally(() => pending.delete(version));
pending.set(version, promise);
return promise;
},
bytes: () => total,
};
}
/**
* A pack reader for the rail service that reads `<dir>/<feedId><suffix>`.
* Ids are re-checked against the registry's id pattern, so a name can never
* leave the directory; a missing file is "no pack", and a file over the cap is
* refused without being read (an error with `code: 'PACK_TOO_LARGE'`). The
* pack is returned as the file's bytes, never decoded to text; with a
* `cache` (createRailPackCache), an unchanged file is read once.
* @param {string} dir
* @param {string} suffix `.json` (path packs) or `.schedule.json`.
* @param {ReturnType<typeof createRailPackCache>|null} [cache]
* @returns {(feedId: string, options?: {maxBytes?: number}) =>
* Promise<Buffer|null>}
*/
function createFsPackReader(dir, suffix, cache = null) {
const root = path.resolve(dir);
return async (feedId, { maxBytes = Infinity } = {}) => {
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
return null;
const file = path.join(root, `${feedId}${suffix}`);
let handle;
try {
handle = await open(file, 'r');
} catch (error) {
if (error?.code === 'ENOENT') return null;
throw error;
}
try {
const info = await handle.stat();
if (!info.isFile()) return null;
if (info.size > maxBytes) {
const error = new Error(`pack is ${info.size} bytes`);
error.code = 'PACK_TOO_LARGE';
throw error;
}
return cache
? await cache.load(file, info, () => handle.readFile())
: await handle.readFile();
} finally {
await handle.close();
}
};
}
/**
* A `readPathPack` that reads `<dir>/<feedId>.json`.
* @param {string} dir
* @param {{cache?: ReturnType<typeof createRailPackCache>|null}} [options]
*/
export function createFsPathPackReader(dir, { cache = null } = {}) {
return createFsPackReader(dir, '.json', cache);
}
/**
* A `readSchedulePack` that reads `<dir>/<feedId>.schedule.json`.
* @param {string} dir
* @param {{cache?: ReturnType<typeof createRailPackCache>|null}} [options]
*/
export function createFsSchedulePackReader(dir, { cache = null } = {}) {
return createFsPackReader(dir, '.schedule.json', cache);
}
/**
* A `hasSchedulePack` for the rail service: whether
* `<dir>/<feedId>.schedule.json` is a file. Only the directory entry is
* read, never the pack; a missing file is false, any other error is thrown
* (the service then lists the feed, so the read shows the failure).
* @param {string} dir
* @returns {(feedId: string) => Promise<boolean>}
*/
export function createFsSchedulePackProbe(dir) {
const root = path.resolve(dir);
return async (feedId) => {
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
return false;
try {
return (await stat(path.join(root, `${feedId}.schedule.json`))).isFile();
} catch (error) {
if (error?.code === 'ENOENT') return false;
throw error;
}
};
}
/**
* A pack version for the rail service: the modification time and size of
* `<dir>/<feedId><suffix>`, or null when there is no such file. Only the
* directory entry is read; a rebuilt pack has a new version.
* @param {string} dir
* @param {string} suffix
* @returns {(feedId: string) => Promise<string|null>}
*/
function createFsPackVersion(dir, suffix) {
const root = path.resolve(dir);
return async (feedId) => {
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
return null;
try {
const info = await stat(path.join(root, `${feedId}${suffix}`));
return info.isFile() ? `${info.mtimeMs}:${info.size}` : null;
} catch (error) {
if (error?.code === 'ENOENT') return null;
throw error;
}
};
}
/**
* A `schedulePackVersion` for the rail service: the modification time and
* size of `<dir>/<feedId>.schedule.json`, or null when there is no such
* file.
* @param {string} dir
* @returns {(feedId: string) => Promise<string|null>}
*/
export function createFsSchedulePackVersion(dir) {
return createFsPackVersion(dir, '.schedule.json');
}
/**
* A `pathPackVersion` for the rail service: the same for
* `<dir>/<feedId>.json`.
* @param {string} dir
* @returns {(feedId: string) => Promise<string|null>}
*/
export function createFsPathPackVersion(dir) {
return createFsPackVersion(dir, '.json');
}
/**
* Write a service Response to a Node response, streaming its body: a pack's
* bytes go to the socket as they are, with no text decode and no copy.
* @param {Response} response
* @param {import('node:http').ServerResponse} res
* @returns {Promise<void>}
*/
export async function sendRailResponse(response, res) {
res.writeHead(response.status, Object.fromEntries(response.headers));
if (!response.body) {
res.end();
return;
}
await pipeline(Readable.fromWeb(response.body), res);
}
/** Connect the reusable rail request service to development and preview. */
export function railProxy(options = {}) {
const pathsDir =
options.pathsDir || process.env.RAIL_PATHS_DIR || DEFAULT_RAIL_PATHS_DIR;
// One byte budget for both kinds of pack.
const cache = createRailPackCache();
const service = createRailService({
...options,
readPathPack:
options.readPathPack || createFsPathPackReader(pathsDir, { cache }),
readSchedulePack:
options.readSchedulePack ||
createFsSchedulePackReader(pathsDir, { cache }),
// Probe the same directory only when its fs reader is the one in use.
hasSchedulePack:
options.hasSchedulePack ||
(options.readSchedulePack ? null : createFsSchedulePackProbe(pathsDir)),
schedulePackVersion:
options.schedulePackVersion ||
(options.readSchedulePack ? null : createFsSchedulePackVersion(pathsDir)),
pathPackVersion:
options.pathPackVersion ||
(options.readPathPack ? null : createFsPathPackVersion(pathsDir)),
});
// A throw here would escape into the HTTP server's event loop and end the
// whole dev server (Node exits on an unhandled rejection or a throwing
// 'close' listener), so failures stay inside the rail routes.
function closeService() {
try {
service.close();
} catch (error) {
console.warn('[rail] provider close failed', error);
}
}
function install(server) {
server.middlewares.use('/api/rail', async (req, res) => {
try {
const response = await service.handle({
url: `http://localhost/api/rail${req.url || '/'}`,
method: req.method,
// Read for If-None-Match on a pack.
headers: req.headers,
});
await sendRailResponse(response, res);
} catch (error) {
// An answer already under way (a viewer that left mid-pack) has
// nothing to add: the stream is closed and that is all.
if (res.headersSent) {
res.destroy();
return;
}
console.warn('[rail] request failed', error);
res.writeHead(500, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Rail provider error' }));
}
});
server.httpServer?.once('close', closeService);
}
return {
name: 'rail-proxy',
closeBundle: closeService,
configureServer: install,
configurePreviewServer: install,
};
}
+6 -1
View File
@@ -3,6 +3,7 @@ import { fetchRegionalPlace } from './place.js';
import { fetchRegionalWeather } from './weather.js';
import { fetchRegionalNews } from './news.js';
import { validRegionalPoint } from './query.js';
import { REGIONAL_NEWS_WINDOW_HOURS } from '../../../src/data/regionalModel.js';
import { coalesceProxyRequest } from '../common/http.js';
// ---------------------------------------------------------------------------
@@ -50,10 +51,11 @@ function regionalBriefProxy({ placeProvider = fetchRegionalPlace } = {}) {
if (!regionalBriefHasAnySource({ place, weather, news })) {
throw new Error('All regional briefing sources unavailable');
}
const retrievedAt = new Date().toISOString();
const payload = {
status:
place && weather && news.status !== 'unavailable' ? 'ready' : 'partial',
retrievedAt: new Date().toISOString(),
retrievedAt,
coordinates: point,
place,
placeStatus: place ? 'ready' : 'unavailable',
@@ -62,6 +64,9 @@ function regionalBriefProxy({ placeProvider = fetchRegionalPlace } = {}) {
newsStatus: news.status,
newsQuery: news.query,
newsSource: news.source,
newsWindowHours: REGIONAL_NEWS_WINDOW_HOURS,
// A cached or stale answer keeps this, so the client can show its age.
newsFetchedAt: retrievedAt,
articles: news.articles,
};
_regionalBriefCache.set(key, { payload, cachedAt: Date.now() });
+35 -8
View File
@@ -1,5 +1,10 @@
import { fetchRegionalText, fetchRegionalJson } from './http.js';
import { normalizeRegionalArticles } from '../../../src/data/regionalModel.js';
import {
REGIONAL_NEWS_WINDOW_HOURS,
googleNewsWindowOperator,
normalizeRegionalArticles,
recentRegionalArticles,
} from '../../../src/data/regionalModel.js';
function decodeRssText(value) {
return String(value || '')
@@ -22,7 +27,10 @@ function rssTag(block, tag) {
);
}
function normalizeRssArticles(xml, limit = 5) {
// Feed items read before the window filter; Google returns about 100.
const RSS_ITEM_CAP = 100;
function normalizeRssArticles(xml, limit = RSS_ITEM_CAP) {
const seen = new Set();
const articles = [];
for (const match of String(xml || '').matchAll(
@@ -57,12 +65,27 @@ function normalizeRssArticles(xml, limit = 5) {
return articles;
}
async function fetchRegionalNews(place) {
/**
* Headlines for a place from the last REGIONAL_NEWS_WINDOW_HOURS, newest
* first: Google News RSS, then GDELT when RSS fails or has nothing recent.
* `empty` means a source answered with nothing in the window; `unavailable`
* means there was no place or no source answered.
* @param {{locality?: string, region?: string, country?: string}|null} place
* @param {{now?: () => number}} [options]
*/
async function fetchRegionalNews(place, { now = Date.now } = {}) {
const query = place?.locality || place?.region || place?.country;
if (!query)
return { status: 'unavailable', query: null, articles: [], source: null };
const recent = (articles) =>
recentRegionalArticles(articles, {
now: now(),
windowHours: REGIONAL_NEWS_WINDOW_HOURS,
limit: 5,
});
let answered = false;
const rssParams = new URLSearchParams({
q: String(query).replace(/["\\]/g, ' ').trim(),
q: `${String(query).replace(/["\\]/g, ' ').trim()} ${googleNewsWindowOperator()}`,
hl: 'en-US',
gl: 'US',
ceid: 'US:en',
@@ -75,7 +98,8 @@ async function fetchRegionalNews(place) {
timeoutMs: 12_000,
},
);
const articles = normalizeRssArticles(xml, 5);
answered = true;
const articles = recent(normalizeRssArticles(xml));
if (articles.length)
return { status: 'ready', query, articles, source: 'Google News RSS' };
} catch {
@@ -87,7 +111,7 @@ async function fetchRegionalNews(place) {
format: 'json',
maxrecords: '5',
sort: 'datedesc',
timespan: '48h',
timespan: `${REGIONAL_NEWS_WINDOW_HOURS}h`,
});
try {
const payload = await fetchRegionalJson(
@@ -97,7 +121,7 @@ async function fetchRegionalNews(place) {
timeoutMs: 12_000,
},
);
const articles = normalizeRegionalArticles(payload, 5);
const articles = recent(normalizeRegionalArticles(payload, 5));
return {
status: articles.length ? 'ready' : 'empty',
query,
@@ -105,7 +129,10 @@ async function fetchRegionalNews(place) {
source: 'GDELT fallback',
};
} catch {
return { status: 'unavailable', query, articles: [], source: null };
// An RSS answer with nothing recent is still a true empty result.
return answered
? { status: 'empty', query, articles: [], source: 'Google News RSS' }
: { status: 'unavailable', query, articles: [], source: null };
}
}
+218
View File
@@ -0,0 +1,218 @@
import {
EMODNET_WFS_URL,
WIKIDATA_QUERY,
mergeShipwreckRecords,
normalizeEmodnetFeatures,
normalizeWikidataResults,
} from '../../src/layers/shipwrecks/records.js';
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
import { clientKey, makeRateLimiter } from './common/rate-limit.js';
const MIB = 1024 * 1024;
/** EMODnet republishes the wreck layer about once a year; Wikidata daily at most. */
const REFRESH_MS = 24 * 60 * 60 * 1000;
/** A source's last good records keep being served this long, marked stale. */
const STALE_MAX_MS = 7 * REFRESH_MS;
/** After a source fails, it is not asked again for this long. */
const RETRY_MS = 15 * 60 * 1000;
/** A refresh with fewer than this share of the last good records is refused. */
const MIN_KEEP_RATIO = 0.5;
// Only the columns the card uses; ship_char/obj_desc histories can name people
// and are not read beyond a short vessel type.
const EMODNET_PROPERTIES = [
'source_id',
'country',
'name',
'least_depth',
'depth_prec',
'loc_prec',
'sink_context',
'sink_yr',
'dating',
'period',
'statutory',
'ship_char',
'obj_type',
'artefacts',
'source_inf',
'website1',
'point_info',
'the_geom',
].join(',');
const EMODNET_URL = `${EMODNET_WFS_URL}&propertyName=${EMODNET_PROPERTIES}`;
const WIKIDATA_URL =
'https://query.wikidata.org/sparql?format=json&query=' +
encodeURIComponent(WIKIDATA_QUERY);
// Wikimedia asks automated clients to name themselves.
const USER_AGENT =
"God's Eye View shipwrecks layer (https://github.com/bilawalsidhu/gods-eye-view)";
/** Fixed-origin, cached EMODnet + Wikidata shipwreck snapshot for dev and preview. */
export function shipwrecksProxy({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
} = {}) {
// Each source keeps its last good records on its own, so one failing (or
// answering with nothing usable) never removes the other's wrecks or its
// own earlier ones. `retryAt` holds a failed source back for RETRY_MS.
const sources = {
emodnet: { load: fetchEmodnet, good: null, retryAt: 0 },
wikidata: { load: fetchWikidata, good: null, retryAt: 0 },
};
let snapshot = null;
const inFlight = new Map();
const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 });
async function upstreamJson(url, maxBytes) {
const signal = AbortSignal.timeout(90_000);
const response = await fetchImpl(url, {
signal,
redirect: 'error',
headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' },
});
if (!response.ok) {
await response.body?.cancel();
throw new Error('upstream_unavailable');
}
return readResponseJsonCapped(response, maxBytes, signal);
}
async function fetchEmodnet() {
const records = normalizeEmodnetFeatures(
await upstreamJson(EMODNET_URL, 32 * MIB),
);
if (!records.length) throw new Error('invalid_emodnet_layer');
return records;
}
async function fetchWikidata() {
const payload = await upstreamJson(WIKIDATA_URL, 16 * MIB);
if (!Array.isArray(payload?.results?.bindings))
throw new Error('invalid_wikidata_results');
const records = normalizeWikidataResults(payload);
// Wikidata lists over a thousand wrecks; none at all is a broken answer.
if (!records.length) throw new Error('invalid_wikidata_results');
return records;
}
const fresh = (state, at) =>
state.good !== null && at - state.good.fetchedAt < REFRESH_MS;
const usable = (state, at) =>
state.good !== null && at - state.good.fetchedAt <= STALE_MAX_MS;
const due = (state, at) => !fresh(state, at) && at >= state.retryAt;
/** Refresh the sources that are due; failures keep their last good records. */
async function refresh() {
const at = now();
const names = Object.keys(sources).filter((name) => due(sources[name], at));
const results = await Promise.allSettled(
names.map((name) => sources[name].load()),
);
const done = now();
results.forEach((result, i) => {
const state = sources[names[i]];
// A valid answer far smaller than the last good set (a truncated or
// half-built upstream) is a failure too, until the old set expires.
const shrunk =
result.status === 'fulfilled' &&
usable(state, done) &&
result.value.length < state.good.records.length * MIN_KEEP_RATIO;
if (result.status === 'fulfilled' && !shrunk)
state.good = { records: result.value, fetchedAt: done };
else state.retryAt = done + RETRY_MS;
if (shrunk)
console.warn(
`[shipwrecks] ${names[i]} refresh refused: ${result.value.length} ` +
`records against ${state.good.records.length} last good`,
);
});
}
/** The served snapshot: every source with usable records, merged once. */
function compose(at) {
const sourceFlags = {};
// How old each source's records are, on this clock, so the page can
// expire them by its own clock without comparing the two.
const ages = {};
let stale = false;
let fetchedAt = null;
for (const [name, state] of Object.entries(sources)) {
sourceFlags[name] = usable(state, at);
ages[name] = sourceFlags[name] ? at - state.good.fetchedAt : null;
if (!sourceFlags[name]) continue;
if (!fresh(state, at)) stale = true;
fetchedAt = Math.min(fetchedAt ?? Infinity, state.good.fetchedAt);
}
if (!sourceFlags.emodnet && !sourceFlags.wikidata) return null;
const key = `${sourceFlags.emodnet && sources.emodnet.good.fetchedAt}:${
sourceFlags.wikidata && sources.wikidata.good.fetchedAt
}`;
if (snapshot?.key !== key) {
// The merge writes EMODnet facts into Wikidata records: work on copies
// so a source's kept records stay as fetched.
const copy = (name) =>
sourceFlags[name]
? sources[name].good.records.map((record) => ({ ...record }))
: [];
snapshot = {
key,
records: mergeShipwreckRecords(copy('emodnet'), copy('wikidata')),
};
}
return {
value: {
fetchedAt,
version: key,
sources: sourceFlags,
ages,
maxAgeMs: STALE_MAX_MS,
records: snapshot.records,
},
stale,
};
}
async function acquire() {
const at = now();
if (Object.values(sources).some((state) => due(state, at)))
await coalesceProxyRequest(inFlight, 'refresh', refresh).promise;
const composed = compose(now());
if (!composed) throw new Error('shipwrecks_unavailable');
return composed;
}
async function handler(req, res) {
const json = (status, value, stale = false) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...(status === 429 ? { 'Retry-After': '60' } : {}),
...(stale ? { 'X-Data-Stale': 'true' } : {}),
});
res.end(JSON.stringify(value));
};
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
const path = (req.url || '/').split('?')[0];
if (path !== '/' && path !== '')
return json(404, { error: 'unknown_route' });
if (!allow(clientKey(req))) return json(429, { error: 'rate_limited' });
try {
const { value, stale } = await acquire();
json(200, stale ? { ...value, stale: true } : value, stale);
} catch {
json(502, { error: 'shipwrecks_unavailable' });
}
}
return {
name: 'shipwrecks',
configureServer({ middlewares }) {
middlewares.use('/api/shipwrecks', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/shipwrecks', handler);
},
};
}
+130 -74
View File
@@ -1,6 +1,9 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
import {
celestrakSatcatUrl,
celestrakTleUrl,
} from '../../../src/data/spaceProviderRequests.js';
import { providerCooldownMs } from './cooldown.js';
import { admitSameSiteKeyless } from '../common/same-site.js';
import { readResponseTextCapped } from '../common/http.js';
@@ -22,84 +25,131 @@ import { readResponseTextCapped } from '../common/http.js';
* 6 h; on upstream failure the freshest stale copy is served (a stale TLE
* beats an empty satellites layer). Pattern mirrors openSkyProxy's
* cache+serve-stale. Adapted from skylight's TleStore (MIT).
*
* The same plugin serves CelesTrak SATCAT records for the satellites layer's
* groups at `/api/celestrak-satcat/<group>` (object type, operational status
* and owner, which file each object under a class). The catalog changes
* slowly, so SATCAT is cached for a day. CelesTrak throttles the client, not
* the endpoint, so a 403 or 429 holds both routes off; any other failure
* holds off only the route that failed, so a bad SATCAT answer never stops
* TLE refreshes.
*/
export function celestrakProxy() {
const TLE_TTL_MS = 6 * 3600_000;
const SATCAT_TTL_MS = 24 * 3600_000;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const groups = new Set([
/** The satellites layer's core groups (src/layers/satellites/policy.js). */
const coreGroups = [
'stations',
'visual',
'gps-ops',
'glo-ops',
'galileo',
'geo',
'starlink',
// Space Missions (`getActiveTle`) reads the full active catalog.
'active',
]);
let cooldownUntil = 0;
const mem = new Map(); // group -> { at: epochMs, body: string }
const inflight = new Map(); // group -> Promise<{at, body}|null>
const diskPath = (group) => path.join(CACHE_DIR, `celestrak-${group}.json`);
async function readDisk(group) {
try {
const parsed = JSON.parse(await fsp.readFile(diskPath(group), 'utf8'));
if (typeof parsed?.body === 'string' && Number.isFinite(parsed?.at))
return parsed;
} catch {
/* no disk cache yet */
}
return null;
}
async function writeDisk(group, entry) {
try {
await fsp.mkdir(CACHE_DIR, { recursive: true });
await fsp.writeFile(diskPath(group), JSON.stringify(entry), 'utf8');
} catch (err) {
console.warn('[celestrak-proxy] cache write failed');
}
}
'resource',
'weather',
'planet',
'sar',
'science',
];
const feeds = [
{
route: '/api/celestrak',
ttlMs: TLE_TTL_MS,
cacheHeader: 'x-tle-cache',
cachedAtHeader: 'X-TLE-Cached-At',
diskName: (group) => `celestrak-${group}.json`,
upstreamUrl: celestrakTleUrl,
groups: new Set([
...coreGroups,
'starlink',
// Space Missions (`getActiveTle`) reads the full active catalog.
'active',
]),
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
validate: (body) => /^1 /m.test(body),
invalid: 'no TLE lines in response',
},
{
route: '/api/celestrak-satcat',
ttlMs: SATCAT_TTL_MS,
cacheHeader: 'x-satcat-cache',
cachedAtHeader: 'X-SATCAT-Cached-At',
diskName: (group) => `celestrak-satcat-${group}.json`,
upstreamUrl: celestrakSatcatUrl,
groups: new Set(coreGroups),
// A real answer, even for an empty group, starts with the CSV header.
validate: (body) => /^OBJECT_NAME,/.test(body),
invalid: 'no SATCAT header in response',
},
];
// CelesTrak throttling (403/429) is per client, so it holds every feed
// off; any other failure holds off only the feed that failed, so a bad
// SATCAT answer never blocks the TLEs the layer cannot do without.
let hostCooldownUntil = 0;
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
async function fetchUpstream(group) {
const url = celestrakTleUrl(group);
const signal = AbortSignal.timeout(20000);
const res = await fetch(url.toString(), {
signal,
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
// descriptive User-Agent with a contact point.
headers: {
'User-Agent':
'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
},
});
if (!res.ok) {
// Group refreshes run in parallel: a later failure without Retry-After
// must never shorten a longer deadline an earlier one already set.
cooldownUntil = Math.max(
cooldownUntil,
Date.now() + providerCooldownMs(res),
);
void res.body?.cancel().catch(() => {});
throw new Error(`HTTP ${res.status}`);
}
const body = await readResponseTextCapped(
res,
CELESTRAK_MAX_RESPONSE_BYTES,
signal,
);
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
return { at: Date.now(), body };
}
function createFeedHandler(feed) {
let feedCooldownUntil = 0;
const cooldownUntil = () => Math.max(hostCooldownUntil, feedCooldownUntil);
const mem = new Map(); // group -> { at: epochMs, body: string }
const inflight = new Map(); // group -> Promise<{at, body}|null>
const diskPath = (group) => path.join(CACHE_DIR, feed.diskName(group));
const installMiddleware = (server) => {
server.middlewares.use('/api/celestrak', async (req, res) => {
async function readDisk(group) {
try {
const parsed = JSON.parse(await fsp.readFile(diskPath(group), 'utf8'));
if (typeof parsed?.body === 'string' && Number.isFinite(parsed?.at))
return parsed;
} catch {
/* no disk cache yet */
}
return null;
}
async function writeDisk(group, entry) {
try {
await fsp.mkdir(CACHE_DIR, { recursive: true });
await fsp.writeFile(diskPath(group), JSON.stringify(entry), 'utf8');
} catch (err) {
console.warn('[celestrak-proxy] cache write failed');
}
}
async function fetchUpstream(group) {
const url = feed.upstreamUrl(group);
const signal = AbortSignal.timeout(20000);
const res = await fetch(url.toString(), {
signal,
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
// descriptive User-Agent with a contact point.
headers: {
'User-Agent':
'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
},
});
if (!res.ok) {
// Group refreshes run in parallel: a later failure without Retry-After
// must never shorten a longer deadline an earlier one already set.
const until = Date.now() + providerCooldownMs(res);
if (res.status === 403 || res.status === 429)
hostCooldownUntil = Math.max(hostCooldownUntil, until);
else feedCooldownUntil = Math.max(feedCooldownUntil, until);
void res.body?.cancel().catch(() => {});
throw new Error(`HTTP ${res.status}`);
}
const body = await readResponseTextCapped(
res,
CELESTRAK_MAX_RESPONSE_BYTES,
signal,
);
if (!feed.validate(body)) throw new Error(feed.invalid);
return { at: Date.now(), body };
}
return async (req, res) => {
const group = String(req.url || '')
.replace(/^\//, '')
.split('?')[0];
@@ -111,7 +161,7 @@ export function celestrakProxy() {
res.end();
return;
}
if (!groups.has(group)) {
if (!feed.groups.has(group)) {
res.writeHead(400, { 'Content-Type': 'text/plain' });
res.end('invalid group');
return;
@@ -123,15 +173,15 @@ export function celestrakProxy() {
if (res.headersSent) return;
res.writeHead(status, {
'Content-Type': 'text/plain',
'x-tle-cache': cacheStatus,
[feed.cacheHeader]: cacheStatus,
'Cache-Control': 'no-store',
...(entry
? { 'X-TLE-Cached-At': new Date(entry.at).toISOString() }
? { [feed.cachedAtHeader]: new Date(entry.at).toISOString() }
: {}),
...(cooldownUntil > 0 && Date.now() < cooldownUntil
...(cooldownUntil() > 0 && Date.now() < cooldownUntil()
? {
'Retry-After': String(
Math.ceil((cooldownUntil - Date.now()) / 1000),
Math.ceil((cooldownUntil() - Date.now()) / 1000),
),
}
: {}),
@@ -145,11 +195,11 @@ export function celestrakProxy() {
entry = await readDisk(group);
if (entry) mem.set(group, entry);
}
if (entry && now - entry.at < TLE_TTL_MS) {
if (entry && now - entry.at < feed.ttlMs) {
send(200, entry.body, 'HIT', entry);
return;
}
if (Date.now() < cooldownUntil) {
if (Date.now() < cooldownUntil()) {
send(
entry ? 200 : 502,
entry?.body || 'celestrak upstream cooling down',
@@ -169,8 +219,8 @@ export function celestrakProxy() {
return fresh;
})
.catch((err) => {
if (Date.now() >= cooldownUntil)
cooldownUntil = Date.now() + 60000;
if (Date.now() >= feedCooldownUntil)
feedCooldownUntil = Date.now() + 60000;
console.warn(
'[celestrak-proxy] refresh failed — serving cache if any',
);
@@ -191,7 +241,13 @@ export function celestrakProxy() {
console.error('[celestrak-proxy] request failed');
send(500, 'celestrak proxy error', 'ERROR');
}
});
};
}
const handlers = feeds.map((feed) => [feed.route, createFeedHandler(feed)]);
const installMiddleware = (server) => {
for (const [route, handler] of handlers)
server.middlewares.use(route, handler);
};
return {
name: 'celestrak-proxy',
+22
View File
@@ -202,3 +202,25 @@ test('a feed fleet-number mode outranks its route mode in the epoch context', (t
assert.equal(h.get('ttc', '4507', 100000).epochs[0].mode, 'tram');
assert.equal(h.get('ttc', 'x1', 100000).epochs[0].mode, 'tram');
});
test('the epoch mode is the live mode: a route type, else hints, else the default', (t) => {
const h = history(t);
const pid = {
id: 'pid',
historyRetention: true,
defaultMode: 'bus',
routeMode: (route) => (route === 'L22' ? 'tram' : 'unknown'),
};
h.ingest(
pid,
[
row(100000, 'a', { routeId: 'L58', routeType: 11 }),
row(100000, 'b', { routeId: 'L22' }),
row(100000, 'c', { routeId: 'L9999' }),
],
100000,
);
assert.equal(h.get('pid', 'a', 100000).epochs[0].mode, 'trolleybus');
assert.equal(h.get('pid', 'b', 100000).epochs[0].mode, 'tram');
// A hint that knows nothing leaves the default, as the live layer does.
assert.equal(h.get('pid', 'c', 100000).epochs[0].mode, 'bus');
});
+2
View File
@@ -27,6 +27,8 @@ const AISSTREAM_DEFAULT_MESSAGE_TYPES = [
'ExtendedClassBPositionReport',
'ShipStaticData',
'StaticDataReport',
// Buoys, beacons and other aids to navigation (AIS message 21).
'AidsToNavigationReport',
];
// Watchdog budgets (policy lives in src/data/aisWatchdog.js). Silence is
// REPORTED quickly and ACTED ON slowly: a dead feed must read as dead within
+69 -6
View File
@@ -1,4 +1,8 @@
import { isRecognizedAisEnvelope } from '../../../src/data/aisStreamAdapter.js';
import {
aidToNavigationTypeName,
isAidToNavigation,
} from '../../../src/data/vesselTypes.js';
export const AISSTREAM_CACHE_MAX = 50000;
export const AISSTREAM_STALE_MS = 30 * 60 * 1000;
// Per-MMSI recent-path ring buffers (PRD WS-F F3). Float32 lat/lon (~1m
@@ -57,8 +61,13 @@ export function ingestAisStreamEnvelope(envelope) {
message.ReportB?.Valid === false
? undefined
: message.ReportB?.ShipType,
Dimension:
message.ReportB?.Valid === false
? undefined
: message.ReportB?.Dimension,
}
: message;
const dimensions = aisDimensions(fields.Dimension);
// Static reports may be partial (including Class B's separate A/B parts).
// Keep known values in the cache as well as in the current live row.
const staticData = {
@@ -84,6 +93,8 @@ export function ingestAisStreamEnvelope(envelope) {
staticCode(live?.imo) ||
staticCode(previous.imo) ||
'',
length: dimensions.length ?? live?.length ?? previous.length ?? null,
beam: dimensions.beam ?? live?.beam ?? previous.beam ?? null,
};
_aisStreamStatic.set(mmsi, staticData);
mergeAisStaticIntoLiveVessel(mmsi, staticData);
@@ -105,14 +116,24 @@ export function ingestAisStreamEnvelope(envelope) {
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return true;
const staticData = _aisStreamStatic.get(mmsi) || {};
const aidReport = messageType === 'AidsToNavigationReport';
_aisStreamVessels.set(mmsi, {
lat,
lon,
name: vesselNameFromAis(metadata, message, staticData) || `MMSI ${mmsi}`,
name:
(aidReport && aidNameFromAis(message)) ||
vesselNameFromAis(metadata, message, staticData) ||
`MMSI ${mmsi}`,
mmsi,
imo: stringValue(message.ImoNumber ?? message.IMO ?? staticData.imo),
type: vesselTypeFromAis(message, staticData),
// An aid-to-navigation report's type is the kind of aid (0 to 31), not
// a ship type: it is carried as the aid's name ("Cardinal Mark N").
type: aidReport
? aidToNavigationTypeName(message.Type)
: vesselTypeFromAis(message, staticData),
destination: stringValue(message.Destination ?? staticData.destination),
length: staticData.length ?? null,
beam: staticData.beam ?? null,
speed: normalizedSpeedOverGround(message.Sog ?? message.SOG),
course: normalizedCourseOverGround(message.Cog ?? message.COG),
heading: normalizedHeading(message.TrueHeading ?? message.Heading),
@@ -122,6 +143,9 @@ export function ingestAisStreamEnvelope(envelope) {
// Use the AIS message's own report time, not server ingest wall-clock —
// trail spacing and dead reckoning depend on true fix epochs.
last_position_epoch: aisEpochSeconds(metadata.time_utc ?? metadata.TimeUtc),
// A virtual aid is broadcast where nothing physical stands. Only such a
// row carries the field, so ship rows stay their size.
...(aidReport && message.VirtualAtoN === true && { virtual: true }),
_updatedAt: Date.now(),
});
@@ -236,6 +260,25 @@ function mergeAisStaticIntoLiveVessel(mmsi, staticData) {
if (staticData.type) existing.type = staticData.type;
if (staticData.destination) existing.destination = staticData.destination;
if (staticData.imo) existing.imo = staticData.imo;
if (staticData.length) existing.length = staticData.length;
if (staticData.beam) existing.beam = staticData.beam;
}
/**
* A static report's hull size from its AIS dimensions: the antenna's
* distances to bow (A), stern (B), port (C) and starboard (D), in metres.
* Zero is AIS's "not available", so a side missing its two parts has none.
* @param {{A?: number, B?: number, C?: number, D?: number}|undefined} dimension
* @returns {{length: number|null, beam: number|null}}
*/
function aisDimensions(dimension) {
const part = (value) => {
const number = numberValue(value);
return number !== null && number > 0 ? number : 0;
};
const length = part(dimension?.A) + part(dimension?.B);
const beam = part(dimension?.C) + part(dimension?.D);
return { length: length > 0 ? length : null, beam: beam > 0 ? beam : null };
}
function vesselNameFromAis(metadata, message, staticData = {}) {
@@ -248,6 +291,15 @@ function vesselNameFromAis(metadata, message, staticData = {}) {
);
}
/**
* An aid's whole name: an aid-to-navigation report carries the first 20
* characters in `Name` and the rest, up to 14 more, in `NameExtension`.
*/
function aidNameFromAis(message) {
const name = stringValue(message.Name);
return name ? stringValue(name + stringValue(message.NameExtension)) : '';
}
function vesselTypeFromAis(message, staticData = {}) {
return stringValue(
message.Type ??
@@ -257,14 +309,25 @@ function vesselTypeFromAis(message, staticData = {}) {
);
}
/**
* The current rows, newest first, with every vessel ahead of every aid to
* navigation: a capped request never gives up a ship for a buoy.
*/
export function aisStreamRows(maxRows) {
const cutoff = Date.now() - AISSTREAM_STALE_MS;
const rows = [];
const vessels = [];
const aids = [];
for (const row of _aisStreamVessels.values()) {
if (row._updatedAt >= cutoff) rows.push(row);
if (row._updatedAt >= cutoff)
(isAidToNavigation(row) ? aids : vessels).push(row);
}
rows.sort((a, b) => b._updatedAt - a._updatedAt);
return rows.slice(0, maxRows).map(({ _updatedAt, ...row }) => row);
const newestFirst = (a, b) => b._updatedAt - a._updatedAt;
vessels.sort(newestFirst);
if (vessels.length < maxRows) aids.sort(newestFirst);
return vessels
.concat(aids)
.slice(0, maxRows)
.map(({ _updatedAt, ...row }) => row);
}
function pruneAisStreamCache() {