mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-11 05:42:22 +00:00
Serve the News layer without a news server: GDELT, Liveuamap and World News API
With GEV_NEWS_ORIGIN unset, Core answers the News layer's routes itself. Conflicts need no key: the last 24 h of GDELT 2.0 event exports, physical violence only (CAMEO 18-20), one report per story (copies of one agency story merged), placed where GDELT put it, a town the article's own words name winning over a bare country, and titled by the article's page title from the GKG export of the same quarter hour (the newest 3 h), else by its address words. GDELT's actor coding is never shown. Exports are read up to the newest GDELT lists; one that cannot be read is given up on without holding the rest back, and an outage of two hours answers 502 rather than an empty feed. With LIVEUAMAP_API_KEY, Liveuamap's mapped items join them. Local reads World News API by the source country at the view centre (WORLDNEWSAPI_KEY, at most WORLDNEWSAPI_DAILY_READS a day), with the Conflicts reports placed inside the area. Disputed areas (11 from Natural Earth, bundled) and territories without a country code are named as themselves and read from no country's outlets; a country's outline leaves them out. Categories, relevance and severity come from shared rules; keyed answers are cached and served while they refresh, and no key or address is logged.
This commit is contained in:
Denis Groshev committed
1 parent
edb5837ae5
commit
34c75ff5de
13 files changed
+4399
-32
No files matched your search
@@ -496,7 +496,17 @@
|
||||
"exports": ["./server/providers/news"],
|
||||
"modules": [
|
||||
"server/providers/news.js",
|
||||
"server/providers/news/keyed.js",
|
||||
"server/providers/news/categorize.js",
|
||||
"server/providers/news/areas.js",
|
||||
"server/providers/news/gdeltEvents.js",
|
||||
"server/providers/news/upstream.js",
|
||||
"server/providers/common/http.js",
|
||||
"src/data/adminBoundaries.js",
|
||||
"src/data/bundledJson.js",
|
||||
"src/data/retryableLoad.js",
|
||||
"src/data/local_data/natural_earth/countries.json",
|
||||
"src/data/local_data/natural_earth/disputed_areas.json",
|
||||
"src/layers/news/source.js",
|
||||
"src/layers/news/records.js",
|
||||
"src/sources/httpBody.js",
|
||||
|
||||
+51
-14
@@ -1,11 +1,14 @@
|
||||
import { newsQueryAllowed } from '../../src/layers/news/source.js';
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
import { createKeyedNews } from './news/keyed.js';
|
||||
|
||||
// News (/api/news/*): Core serves no news of its own. With GEV_NEWS_ORIGIN
|
||||
// set (a news server such as https://news.example), this forwards exactly
|
||||
// the GET requests the news layer sends there and relays the JSON; without
|
||||
// it, every route answers "not configured" and the layer shows itself
|
||||
// unavailable without retrying.
|
||||
// News (/api/news/*). With GEV_NEWS_ORIGIN set (a news server such as
|
||||
// https://news.example), this forwards exactly the GET requests the news
|
||||
// layer sends there and relays the JSON. Without it (or with a value that
|
||||
// is no plain origin, warned of once), Core answers the same routes itself
|
||||
// (news/keyed.js): Conflicts from GDELT's event exports with no key, and
|
||||
// Liveuamap's items beside them with LIVEUAMAP_API_KEY; Local from World
|
||||
// News API, "key required" while WORLDNEWSAPI_KEY is unset.
|
||||
|
||||
const TIMEOUT_MS = 15_000;
|
||||
const MAX_BYTES = 4 * 1024 * 1024;
|
||||
@@ -34,14 +37,42 @@ export function newsOrigin(raw = process.env.GEV_NEWS_ORIGIN) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Text for the log: no address (a key may ride in one), one short line. */
|
||||
const logText = (error) =>
|
||||
`${error?.name || 'Error'}: ${String(error?.message ?? '')}`
|
||||
.replace(/[a-z][a-z0-9+.-]*:\/\/\S*/gi, '<url>')
|
||||
.replace(/\s+/g, ' ')
|
||||
.slice(0, 200);
|
||||
|
||||
/** GEV_NEWS_ORIGIN as newsOrigin reads it; a value it refuses is warned of once. */
|
||||
function configuredOrigin(env, log) {
|
||||
let warned = null;
|
||||
return () => {
|
||||
const raw = String(env().GEV_NEWS_ORIGIN ?? '').trim();
|
||||
const base = newsOrigin(raw);
|
||||
if (!base && raw && warned !== raw) {
|
||||
warned = raw;
|
||||
// Not the value: it may carry credentials.
|
||||
log(
|
||||
'[news] GEV_NEWS_ORIGIN is not a plain http(s) origin; answering with the built-in news backend',
|
||||
);
|
||||
}
|
||||
return base;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A thin, validated GET proxy for the news layer's three read routes.
|
||||
* @param {{origin?: () => string|null, fetchImpl?: typeof fetch, timeoutMs?: number}} [options]
|
||||
* A thin, validated GET proxy for the news layer's three read routes, or
|
||||
* their keyed answers without a news server.
|
||||
* @param {{env?: () => object, log?: (message: string) => void, origin?: () => string|null, fetchImpl?: typeof fetch, timeoutMs?: number, keyed?: {answer: (path: string, params: URLSearchParams) => Promise<{status: number, body: object}>}}} [options]
|
||||
*/
|
||||
export function newsProxy({
|
||||
origin = () => newsOrigin(),
|
||||
env = () => process.env,
|
||||
log = (message) => console.warn(message),
|
||||
origin = configuredOrigin(env, log),
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
timeoutMs = TIMEOUT_MS,
|
||||
keyed = createKeyedNews({ env, fetchImpl, log }),
|
||||
} = {}) {
|
||||
const json = (res, status, value) => {
|
||||
if (res.destroyed) return;
|
||||
@@ -64,12 +95,18 @@ export function newsProxy({
|
||||
if (!newsQueryAllowed(target))
|
||||
return json(res, 400, { error: 'invalid_news_request' });
|
||||
const base = origin();
|
||||
if (!base)
|
||||
return json(res, 503, {
|
||||
error: 'news_not_configured',
|
||||
unavailable: true,
|
||||
retryable: false,
|
||||
});
|
||||
if (!base) {
|
||||
try {
|
||||
const { status, body } = await keyed.answer(
|
||||
local.pathname,
|
||||
target.searchParams,
|
||||
);
|
||||
return json(res, status, body);
|
||||
} catch (error) {
|
||||
log(`[news] built-in news answer failed: ${logText(error)}`);
|
||||
return json(res, 502, { error: 'news_unavailable' });
|
||||
}
|
||||
}
|
||||
try {
|
||||
const upstream = await fetchImpl(
|
||||
new URL(`${target.pathname}${target.search}`, base),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { newsOrigin, newsProxy } from './news.js';
|
||||
import { createKeyedNews } from './news/keyed.js';
|
||||
|
||||
/** Mount the proxy on a fake middleware stack and call it like Connect does. */
|
||||
function mount(options) {
|
||||
@@ -33,7 +34,10 @@ function mount(options) {
|
||||
}
|
||||
|
||||
test('the origin must be a plain http(s) origin', () => {
|
||||
assert.equal(newsOrigin('https://news.example'), 'https://news.example');
|
||||
assert.equal(
|
||||
newsOrigin('http://news.example:8080'),
|
||||
'http://news.example:8080',
|
||||
);
|
||||
assert.equal(
|
||||
newsOrigin('https://news.example/base/'),
|
||||
'https://news.example',
|
||||
@@ -51,23 +55,32 @@ test('the origin must be a plain http(s) origin', () => {
|
||||
assert.equal(newsOrigin(bad), null, String(bad));
|
||||
});
|
||||
|
||||
test('without GEV_NEWS_ORIGIN every route answers not configured and nothing is fetched', async () => {
|
||||
let fetches = 0;
|
||||
test('without GEV_NEWS_ORIGIN Core answers itself: Conflicts from GDELT, Local saying it needs its key', async () => {
|
||||
const hosts = [];
|
||||
const fetchImpl = async (url) => {
|
||||
hosts.push(new URL(String(url)).host);
|
||||
throw new Error('offline');
|
||||
};
|
||||
const call = mount({
|
||||
origin: () => null,
|
||||
fetchImpl: async () => {
|
||||
fetches++;
|
||||
throw new Error('unexpected');
|
||||
},
|
||||
fetchImpl,
|
||||
keyed: createKeyedNews({ env: () => ({}), fetchImpl, log: () => {} }),
|
||||
});
|
||||
const response = await call('/hotspots');
|
||||
assert.equal(response.status, 503);
|
||||
assert.deepEqual(response.json, {
|
||||
error: 'news_not_configured',
|
||||
unavailable: true,
|
||||
retryable: false,
|
||||
});
|
||||
assert.equal(fetches, 0);
|
||||
// GDELT's exports need no key; offline, the read fails like any upstream.
|
||||
const hotspots = await call('/hotspots');
|
||||
assert.equal(hotspots.status, 502);
|
||||
assert.deepEqual(hotspots.json, { error: 'news_upstream_error' });
|
||||
const local = await call('/feed?hotspot=all&at=51.13,71.43');
|
||||
assert.equal(local.status, 503);
|
||||
assert.deepEqual(local.json, { error: 'no_key', key: 'worldnewsapi' });
|
||||
// The area is read from the bundled countries, with no key.
|
||||
const area = await call('/outline?at=51.13,71.43&v=4');
|
||||
assert.equal(area.status, 200);
|
||||
assert.equal(area.json.area.name, 'Kazakhstan');
|
||||
// The client's allowlist still holds.
|
||||
assert.equal((await call('/feed?side=UA')).status, 400);
|
||||
assert.ok(hosts.length > 0);
|
||||
assert.ok(hosts.every((host) => host === 'data.gdeltproject.org'));
|
||||
});
|
||||
|
||||
test('forwards only the client routes, by GET, and relays the JSON', async () => {
|
||||
@@ -139,3 +152,43 @@ test('upstream failures, bad JSON and timeouts become bounded errors', async ()
|
||||
assert.equal(timedOut.status, 504);
|
||||
assert.deepEqual(timedOut.json, { error: 'news_unavailable' });
|
||||
});
|
||||
|
||||
test('a GEV_NEWS_ORIGIN that is no plain origin is warned of once, and Core answers itself', async () => {
|
||||
const logs = [];
|
||||
const asked = [];
|
||||
const call = mount({
|
||||
env: () => ({ GEV_NEWS_ORIGIN: 'https://user:[email protected]' }),
|
||||
log: (line) => logs.push(line),
|
||||
fetchImpl: async () => assert.fail('nothing is forwarded'),
|
||||
keyed: {
|
||||
answer: async (path) => {
|
||||
asked.push(path);
|
||||
return { status: 200, body: { reports: [], nextBefore: null } };
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal((await call('/feed?hotspot=all')).status, 200);
|
||||
assert.equal((await call('/feed?hotspot=all')).status, 200);
|
||||
assert.deepEqual(asked, ['/feed', '/feed']);
|
||||
assert.equal(logs.length, 1);
|
||||
assert.doesNotMatch(logs[0], /secret|news\.example/);
|
||||
});
|
||||
|
||||
test('a built-in answer that throws is a 502, logged in one line without addresses', async () => {
|
||||
const logs = [];
|
||||
const call = mount({
|
||||
env: () => ({}),
|
||||
log: (line) => logs.push(line),
|
||||
keyed: {
|
||||
answer: async () => {
|
||||
throw new TypeError('bad https://api.example/?key=abc\nmore');
|
||||
},
|
||||
},
|
||||
});
|
||||
const failed = await call('/hotspots');
|
||||
assert.equal(failed.status, 502);
|
||||
assert.deepEqual(failed.json, { error: 'news_unavailable' });
|
||||
assert.equal(logs.length, 1);
|
||||
assert.match(logs[0], /TypeError/);
|
||||
assert.doesNotMatch(logs[0], /key=abc|\n/);
|
||||
});
|
||||
@@ -0,0 +1,216 @@
|
||||
// Areas for the News layer, from the bundled Natural Earth packs (public
|
||||
// domain): the disputed area or else the country at a point, its outline in
|
||||
// the /api/news/outline form, and a point inside a country to pin its
|
||||
// national news at. A disputed area is checked first and named as itself,
|
||||
// so no one country's name stands for it; so is a territory the countries
|
||||
// pack gives no code (Somaliland), rather than read as sea. Read once, on
|
||||
// first use.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
import {
|
||||
containsAnchor,
|
||||
geometryOf,
|
||||
pointInRing,
|
||||
} from '../../../src/data/adminBoundaries.js';
|
||||
|
||||
const PACKS = '../../../src/data/local_data/natural_earth/';
|
||||
const PACK = new URL(`${PACKS}countries.json`, import.meta.url);
|
||||
const DISPUTED = new URL(`${PACKS}disputed_areas.json`, import.meta.url);
|
||||
const ISO2 = /^[A-Z]{2}$/;
|
||||
|
||||
let countries = null;
|
||||
let territories = [];
|
||||
|
||||
/**
|
||||
* One entry per ISO code: its name and every Country part's geometry. The
|
||||
* Country features without a code ("-99": Somaliland, Northern Cyprus,
|
||||
* sovereign base areas, no man's lands) are kept apart as territories.
|
||||
*/
|
||||
function loadCountries(read = () => JSON.parse(readFileSync(PACK, 'utf8'))) {
|
||||
if (countries) return countries;
|
||||
const pack = read();
|
||||
const decimals = pack.meta?.decimals ?? 3;
|
||||
const byCode = new Map();
|
||||
const codeless = [];
|
||||
for (const feature of pack.features || []) {
|
||||
// The constituent countries of the United Kingdom repeat its part.
|
||||
if (feature.type !== 'Country') continue;
|
||||
if (!ISO2.test(feature.iso2 || '')) {
|
||||
if (feature.name)
|
||||
codeless.push({ name: feature.name, feature, decimals });
|
||||
continue;
|
||||
}
|
||||
const entry = byCode.get(feature.iso2) || {
|
||||
code: feature.iso2,
|
||||
name: null,
|
||||
size: 0,
|
||||
parts: [],
|
||||
};
|
||||
entry.parts.push({ feature, decimals });
|
||||
// Named after its largest feature: Kazakhstan, not the Baikonur lease.
|
||||
const size = JSON.stringify(feature.polygons).length;
|
||||
if (size > entry.size) {
|
||||
entry.name = feature.name;
|
||||
entry.size = size;
|
||||
}
|
||||
byCode.set(feature.iso2, entry);
|
||||
}
|
||||
countries = byCode;
|
||||
territories = codeless;
|
||||
return countries;
|
||||
}
|
||||
|
||||
/** Test seam: read another pack (or the bundled one again). */
|
||||
export function resetCountries(read) {
|
||||
countries = null;
|
||||
if (read) loadCountries(read);
|
||||
}
|
||||
|
||||
const geometries = (entry) => entry.parts.map((part) => geometryOf(part));
|
||||
|
||||
let disputed = null;
|
||||
const slug = (name) =>
|
||||
name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
|
||||
/** The disputed areas: `{key, name, kind, heldBy, claimedBy, rings}`. */
|
||||
function loadDisputed() {
|
||||
if (disputed) return disputed;
|
||||
const pack = JSON.parse(readFileSync(DISPUTED, 'utf8'));
|
||||
disputed = (pack.areas || []).map((area) => ({
|
||||
key: `disputed-${slug(area.name)}`,
|
||||
name: area.name,
|
||||
kind: area.kind,
|
||||
heldBy: area.heldBy ?? null,
|
||||
claimedBy: area.claimedBy ?? null,
|
||||
rings: area.rings,
|
||||
}));
|
||||
return disputed;
|
||||
}
|
||||
|
||||
/** The disputed area at a point, or null. */
|
||||
export function disputedAt(lat, lon) {
|
||||
return (
|
||||
loadDisputed().find((area) =>
|
||||
area.rings.some((ring) => pointInRing(ring, lat, lon)),
|
||||
) || null
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The area a point is in, as the News layer names it: a disputed area, or
|
||||
* a territory without a code, as itself (`kind: 'disputed'`, `code` null),
|
||||
* else its country (`kind: 'country'`, with `country` the countryAt
|
||||
* entry); null at sea.
|
||||
*/
|
||||
export function areaAt(lat, lon) {
|
||||
const area = disputedAt(lat, lon);
|
||||
if (area)
|
||||
return {
|
||||
...area,
|
||||
kind: 'disputed',
|
||||
held: area.kind,
|
||||
code: null,
|
||||
polygons: area.rings.map((outer) => ({ outer, holes: [] })),
|
||||
};
|
||||
const country = countryAt(lat, lon);
|
||||
if (country)
|
||||
return {
|
||||
key: country.code,
|
||||
name: country.name,
|
||||
kind: 'country',
|
||||
code: country.code,
|
||||
country,
|
||||
};
|
||||
const territory = territoryAt(lat, lon);
|
||||
return territory
|
||||
? {
|
||||
key: `disputed-${slug(territory.name)}`,
|
||||
name: territory.name,
|
||||
kind: 'disputed',
|
||||
held: null,
|
||||
code: null,
|
||||
polygons: geometryOf(territory).polygons.map(([outer, ...holes]) => ({
|
||||
outer,
|
||||
holes,
|
||||
})),
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Whether a point lies in a geometry (geometryOf), across the antimeridian. */
|
||||
function inGeometry(geometry, lat, lon) {
|
||||
const [west, south, east, north] = geometry.bbox;
|
||||
const x = lon - 360 * Math.round((lon - (west + east) / 2) / 360);
|
||||
if (x < west || x > east || lat < south || lat > north) return false;
|
||||
return containsAnchor(geometry.polygons, lat, lon);
|
||||
}
|
||||
|
||||
/** The country at a point, or null at sea or in a territory without a code. */
|
||||
export function countryAt(lat, lon) {
|
||||
for (const entry of loadCountries().values())
|
||||
for (const geometry of geometries(entry))
|
||||
if (inGeometry(geometry, lat, lon)) return entry;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The territory without a code at a point (Somaliland), or null. */
|
||||
function territoryAt(lat, lon) {
|
||||
loadCountries();
|
||||
return (
|
||||
territories.find((part) => inGeometry(geometryOf(part), lat, lon)) || null
|
||||
);
|
||||
}
|
||||
|
||||
/** A country by its ISO code, or null. */
|
||||
export const countryByCode = (code) => loadCountries().get(code) || null;
|
||||
|
||||
/**
|
||||
* A country's outline as /api/news/outline answers it (`{outer, holes}`),
|
||||
* the disputed areas inside it left out as holes (Russia's without Crimea).
|
||||
*/
|
||||
export function countryPolygons(entry) {
|
||||
const areas = loadDisputed();
|
||||
return geometries(entry).flatMap((geometry) =>
|
||||
geometry.polygons.map(([outer, ...holes]) => ({
|
||||
outer,
|
||||
holes: [
|
||||
...holes,
|
||||
...areas
|
||||
.flatMap((area) => area.rings)
|
||||
.filter((ring) => pointInRing(outer, ring[0][1], ring[0][0])),
|
||||
],
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A point inside the country's largest part, for news placed in the country
|
||||
* alone: its label point when that lies inside, else the middle of the
|
||||
* widest span across the part at its middle latitude.
|
||||
*/
|
||||
export function countryAnchor(entry) {
|
||||
const largest = geometries(entry).sort((a, b) => b.areaKm2 - a.areaKm2)[0];
|
||||
const { label, polygons, bbox } = largest;
|
||||
if (containsAnchor(polygons, label.lat, label.lon)) return label;
|
||||
const lat = (bbox[1] + bbox[3]) / 2;
|
||||
const [outer] = polygons.reduce((a, b) =>
|
||||
b[0].length > a[0].length ? b : a,
|
||||
);
|
||||
const crossings = [];
|
||||
for (let i = 0, j = outer.length - 1; i < outer.length; j = i++) {
|
||||
const [x1, y1] = outer[j];
|
||||
const [x2, y2] = outer[i];
|
||||
if (y1 > lat !== y2 > lat)
|
||||
crossings.push(x1 + ((lat - y1) * (x2 - x1)) / (y2 - y1));
|
||||
}
|
||||
crossings.sort((a, b) => a - b);
|
||||
let best = null;
|
||||
for (let i = 0; i + 1 < crossings.length; i += 2)
|
||||
if (!best || crossings[i + 1] - crossings[i] > best[1] - best[0])
|
||||
best = [crossings[i], crossings[i + 1]];
|
||||
return best ? { lat, lon: (best[0] + best[1]) / 2 } : label;
|
||||
}
|
||||
@@ -0,0 +1,873 @@
|
||||
// Rule-based report categories, relevance and severity: the News layer's
|
||||
// base reading, shared by every news server. Multilingual stems (en, uk, ru,
|
||||
// ar, he and more) matched at a word start on normalised text; a trailing `$`
|
||||
// makes a stem whole-word. A title hit counts double. Ties go to the earlier
|
||||
// category in CATEGORY_ORDER, which runs from most to least specific.
|
||||
//
|
||||
// A news server may replace these readings with its own; the rules stay
|
||||
// as the fallback.
|
||||
|
||||
/** Text as the rules match it: folded case, one apostrophe, no vowel marks. */
|
||||
export function normaliseForMatch(text) {
|
||||
return String(text || '')
|
||||
.normalize('NFKC')
|
||||
.toLowerCase()
|
||||
.replace(/ё/g, 'е')
|
||||
.replace(/[’ʼ`´]/g, "'")
|
||||
.replace(/[ً-ٰٟ֑-ׇ]/g, '')
|
||||
.replace(/[أإآ]/g, 'ا');
|
||||
}
|
||||
|
||||
export const CATEGORY_ORDER = Object.freeze([
|
||||
'strike',
|
||||
'alert',
|
||||
'air-defence',
|
||||
'ground',
|
||||
'casualties',
|
||||
'humanitarian',
|
||||
'infrastructure',
|
||||
'disaster',
|
||||
'protest',
|
||||
'political',
|
||||
// Rules only, for the Local section's national news.
|
||||
'economy',
|
||||
]);
|
||||
|
||||
// [stem, weight]; weight defaults to 1.
|
||||
const RULES = {
|
||||
strike: [
|
||||
'strike',
|
||||
'airstrike',
|
||||
'air strike',
|
||||
'missile',
|
||||
'rocket',
|
||||
'drone',
|
||||
'shelling',
|
||||
'shelled',
|
||||
'bomb',
|
||||
'explosion',
|
||||
'blast',
|
||||
'artillery',
|
||||
'shahed',
|
||||
['attack', 0.5],
|
||||
'удар',
|
||||
'ракет',
|
||||
'дрон',
|
||||
'бпла',
|
||||
'обстріл',
|
||||
'обстрел',
|
||||
'вибух',
|
||||
'взрыв',
|
||||
'шахед',
|
||||
'бомб',
|
||||
['атак', 0.5],
|
||||
'غارة',
|
||||
'غارات',
|
||||
'قصف',
|
||||
'صاروخ',
|
||||
'صواريخ',
|
||||
'مسيرة',
|
||||
'انفجار',
|
||||
['استهداف', 0.7],
|
||||
['هجوم', 0.5],
|
||||
'תקיפה',
|
||||
'טיל',
|
||||
'רקט',
|
||||
'כטב"מ',
|
||||
'פיצוץ',
|
||||
'הפצצ',
|
||||
],
|
||||
'air-defence': [
|
||||
'air defen',
|
||||
'intercept',
|
||||
'shot down',
|
||||
'downed',
|
||||
'air raid',
|
||||
'ппо',
|
||||
'збит',
|
||||
'сбит',
|
||||
'перехоп',
|
||||
'перехват',
|
||||
'повітряна тривога',
|
||||
'воздушная тревога',
|
||||
'الدفاع الجوي',
|
||||
'اعتراض',
|
||||
'יירוט',
|
||||
'יורט',
|
||||
'כיפת ברזל',
|
||||
'אזעק',
|
||||
],
|
||||
ground: [
|
||||
'advance$',
|
||||
'advances$',
|
||||
'advancing',
|
||||
'offensive',
|
||||
'captur',
|
||||
'liberat',
|
||||
'assault',
|
||||
'clash',
|
||||
'fighting',
|
||||
'counteroffensive',
|
||||
'frontline',
|
||||
'front line',
|
||||
'battle',
|
||||
'encircl',
|
||||
['troops', 0.5],
|
||||
'наступ',
|
||||
'штурм',
|
||||
'звільн',
|
||||
'освобо',
|
||||
'бої$',
|
||||
'бои$',
|
||||
'контрнаступ',
|
||||
'захоп',
|
||||
'захват',
|
||||
'оточ',
|
||||
'окруж',
|
||||
'продвиж',
|
||||
'просуван',
|
||||
'اشتباك',
|
||||
'معارك',
|
||||
'تقدم',
|
||||
'سيطر',
|
||||
'اقتحام',
|
||||
'توغل',
|
||||
'לחימה',
|
||||
'קרב',
|
||||
'פשיט',
|
||||
'תמרון',
|
||||
],
|
||||
casualties: [
|
||||
'killed',
|
||||
'dead$',
|
||||
'deaths',
|
||||
'wounded',
|
||||
'injured',
|
||||
'casualt',
|
||||
'death toll',
|
||||
'загин',
|
||||
'поранен',
|
||||
'жертв',
|
||||
'вбит',
|
||||
'погиб',
|
||||
'ранен',
|
||||
'убит',
|
||||
'قتلى',
|
||||
'شهداء',
|
||||
'شهيد',
|
||||
'جرحى',
|
||||
'مقتل',
|
||||
'اصاب',
|
||||
'הרוג',
|
||||
'נהרג',
|
||||
'פצוע',
|
||||
'נפצע',
|
||||
],
|
||||
humanitarian: [
|
||||
'evacuat',
|
||||
'aid$',
|
||||
'refugee',
|
||||
'displac',
|
||||
'humanitarian',
|
||||
'famine',
|
||||
'hunger',
|
||||
'convoy',
|
||||
'shelter',
|
||||
'евакуац',
|
||||
'эвакуац',
|
||||
'гуманітар',
|
||||
'гуманитар',
|
||||
'біженц',
|
||||
'беженц',
|
||||
'انسانية',
|
||||
'نازح',
|
||||
'نزوح',
|
||||
'مساعدات',
|
||||
'اجلاء',
|
||||
'مجاعة',
|
||||
'הומניטרי',
|
||||
'פינוי',
|
||||
'סיוע',
|
||||
],
|
||||
infrastructure: [
|
||||
'power outage',
|
||||
'blackout',
|
||||
'electricity',
|
||||
'energy',
|
||||
'grid$',
|
||||
'substation',
|
||||
'water supply',
|
||||
'bridge',
|
||||
'railway',
|
||||
'refinery',
|
||||
'port$',
|
||||
'dam$',
|
||||
'power plant',
|
||||
'енерг',
|
||||
'энерг',
|
||||
'знеструм',
|
||||
'обесточ',
|
||||
'підстанц',
|
||||
'подстанц',
|
||||
'залізниц',
|
||||
'железн',
|
||||
'нпз',
|
||||
'الكهرباء',
|
||||
'محطة',
|
||||
'مياه',
|
||||
'جسر',
|
||||
'חשמל',
|
||||
'תשתית',
|
||||
],
|
||||
disaster: [
|
||||
'earthquake',
|
||||
'flood',
|
||||
'wildfire',
|
||||
'landslide',
|
||||
'cyclone',
|
||||
'hurricane',
|
||||
'typhoon',
|
||||
'tsunami',
|
||||
'eruption',
|
||||
'glof$',
|
||||
'glacial lake',
|
||||
'storm$',
|
||||
'lightning',
|
||||
'землетрус',
|
||||
'землетрясен',
|
||||
'повін',
|
||||
'паводк',
|
||||
'наводнен',
|
||||
'زلزال',
|
||||
'فيضان',
|
||||
'سيول',
|
||||
'רעידת אדמה',
|
||||
'שיטפון',
|
||||
],
|
||||
protest: [
|
||||
'protest',
|
||||
'demonstrat',
|
||||
'rally$',
|
||||
'rallies',
|
||||
'riot',
|
||||
// Not a bare "протест": "протестировал" is "tested".
|
||||
'протест$',
|
||||
'протеста',
|
||||
'протесту',
|
||||
'протестом',
|
||||
'протесте$',
|
||||
'протесты',
|
||||
'протестов',
|
||||
'протестн',
|
||||
'протестую',
|
||||
'протестова',
|
||||
'мітинг',
|
||||
'митинг',
|
||||
'احتجاج',
|
||||
'مظاهر',
|
||||
'تظاهر',
|
||||
'הפגנ',
|
||||
'מחאה',
|
||||
],
|
||||
political: [
|
||||
'talks$',
|
||||
'negotiat',
|
||||
'sanction',
|
||||
'summit',
|
||||
'ceasefire',
|
||||
'truce',
|
||||
'peace',
|
||||
'election',
|
||||
'parliament',
|
||||
['president', 0.5],
|
||||
['minister', 0.5],
|
||||
['statement', 0.5],
|
||||
// A state visit ("King of Jordan arrives in Astana").
|
||||
['visit', 0.5],
|
||||
['delegation', 0.5],
|
||||
['agreement', 0.5],
|
||||
'memorandum',
|
||||
'diplomat',
|
||||
'bilateral',
|
||||
'ambassador',
|
||||
['визит', 0.5],
|
||||
['делегац', 0.5],
|
||||
'дипломат',
|
||||
'переговор',
|
||||
'санкц',
|
||||
'перемир',
|
||||
'вибори',
|
||||
'выборы',
|
||||
['президент', 0.5],
|
||||
['міністр', 0.5],
|
||||
['министр', 0.5],
|
||||
'مفاوضات',
|
||||
'عقوبات',
|
||||
'هدنة',
|
||||
'وقف اطلاق النار',
|
||||
['وزير', 0.5],
|
||||
['رئيس', 0.5],
|
||||
'משא ומתן',
|
||||
'הפסקת אש',
|
||||
'סנקצי',
|
||||
['ראש הממשלה', 0.5],
|
||||
],
|
||||
// Elections and party politics ("How Gina Hinojosa Captured Texas" is a
|
||||
// governor's race, not a battle). They count toward political only when
|
||||
// nothing physical happened (categorizeText): a strike in an election week
|
||||
// is still a strike.
|
||||
election: [
|
||||
'midterm',
|
||||
'electoral',
|
||||
'elect$',
|
||||
'ballot',
|
||||
'voter',
|
||||
'vote$',
|
||||
'votes$',
|
||||
'voting',
|
||||
'polls$',
|
||||
'candidate',
|
||||
'senate race',
|
||||
'referendum',
|
||||
'inaugurat',
|
||||
// Plurals only: "Democratic Republic of Congo", "Republican Guard".
|
||||
'democrats$',
|
||||
'republicans$',
|
||||
'gop$',
|
||||
'избира',
|
||||
'голосован',
|
||||
'виборц',
|
||||
'голосуван',
|
||||
'انتخاب',
|
||||
'בחירות',
|
||||
],
|
||||
};
|
||||
|
||||
// Business, trade, industry and energy, counted only when nothing physical
|
||||
// happened (categorizeText): an oil depot struck is a strike. English,
|
||||
// Russian, Ukrainian, Kazakh-press Russian, Turkish and the European stems.
|
||||
RULES.economy = [
|
||||
'econom',
|
||||
'gdp$',
|
||||
'inflation',
|
||||
'investment',
|
||||
'investor',
|
||||
'export',
|
||||
'imports$',
|
||||
'import$',
|
||||
'tariff',
|
||||
'trade deal',
|
||||
'trade turnover',
|
||||
'central bank',
|
||||
'stock market',
|
||||
'budget',
|
||||
'oil and gas',
|
||||
'oilfield',
|
||||
'oil field',
|
||||
'gas field',
|
||||
'exploration',
|
||||
'drilling',
|
||||
'mining',
|
||||
'industr',
|
||||
'manufactur',
|
||||
'business',
|
||||
'company',
|
||||
'companies',
|
||||
'revenue',
|
||||
'profit',
|
||||
'merger',
|
||||
'tourism',
|
||||
'agricultur',
|
||||
'harvest',
|
||||
'экономи',
|
||||
'инвест',
|
||||
'экспорт',
|
||||
'импорт',
|
||||
'инфляц',
|
||||
'ввп$',
|
||||
'бюджет',
|
||||
'промышлен',
|
||||
'бизнес',
|
||||
'компани',
|
||||
'товарооборот',
|
||||
'економі',
|
||||
'інвест',
|
||||
'експорт',
|
||||
'промисло',
|
||||
'ekonomi',
|
||||
'yatırım',
|
||||
'ihracat',
|
||||
'ithalat',
|
||||
'enflasyon',
|
||||
'sanayi',
|
||||
'şirket',
|
||||
'wirtschaft',
|
||||
'konjunktur',
|
||||
'unternehm',
|
||||
'inversi',
|
||||
'investiss',
|
||||
'investiment',
|
||||
'empresa',
|
||||
'entreprise',
|
||||
'impresa',
|
||||
];
|
||||
|
||||
// Attack words beyond PHYSICAL's harm, which keep a report off economy.
|
||||
const ATTACK = new RegExp(
|
||||
String.raw`(?<![\p{L}])(?:attack|assault|target|hit(?![\p{L}])|fire|plot|war(?![\p{L}])|wartime|атак|удар|пораж|пожар|обстр|війн|войн|هجوم|استهداف|حرب|מתקפ|ataque|attaque|attacco|angriff|saldır)`,
|
||||
'u',
|
||||
);
|
||||
|
||||
function escapeRegex(text) {
|
||||
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
// Spanish, French, Portuguese, Italian and German stems, for outlets that
|
||||
// report in their own language (a train crash in Spain, in Spanish).
|
||||
const EUROPEAN = {
|
||||
strike: [
|
||||
'bombarde',
|
||||
'misil',
|
||||
'míssil',
|
||||
'dron$',
|
||||
'explosión',
|
||||
'explosão',
|
||||
'esplosi',
|
||||
'ataque',
|
||||
'attaque',
|
||||
'attacco',
|
||||
'angriff',
|
||||
'rakete',
|
||||
'luftangriff',
|
||||
],
|
||||
'air-defence': ['antiaére', 'défense aérienne', 'flugabwehr', 'abatid'],
|
||||
ground: [
|
||||
'combate',
|
||||
'combats',
|
||||
'enfrentamiento',
|
||||
'affrontement',
|
||||
'scontri',
|
||||
'gefecht',
|
||||
],
|
||||
casualties: [
|
||||
'muert',
|
||||
'fallecid',
|
||||
'herid',
|
||||
'morts$',
|
||||
'tué',
|
||||
'blessé',
|
||||
'morti',
|
||||
'ferit',
|
||||
'mortos',
|
||||
'ferido',
|
||||
'muere',
|
||||
'mueren',
|
||||
'fallece',
|
||||
'meurt',
|
||||
'morre',
|
||||
'tote$',
|
||||
'verletzt',
|
||||
'víctimas',
|
||||
'victimes',
|
||||
'vittime',
|
||||
],
|
||||
humanitarian: [
|
||||
'evacua',
|
||||
'évacu',
|
||||
'refugiad',
|
||||
'réfugi',
|
||||
'flüchtling',
|
||||
'desplazad',
|
||||
],
|
||||
infrastructure: [
|
||||
'apagón',
|
||||
'panne',
|
||||
'blackout',
|
||||
'descarril',
|
||||
'déraill',
|
||||
'deragliament',
|
||||
'entgleis',
|
||||
'accidente',
|
||||
'accident',
|
||||
'incidente',
|
||||
'choque',
|
||||
'collision',
|
||||
'collisione',
|
||||
'zusammenstoß',
|
||||
'incendio',
|
||||
'incendie',
|
||||
],
|
||||
disaster: [
|
||||
'terremoto',
|
||||
'séisme',
|
||||
'tremblement',
|
||||
'erdbeben',
|
||||
'inundaci',
|
||||
'inondation',
|
||||
'alluvion',
|
||||
'überschwemm',
|
||||
'huracán',
|
||||
'ouragan',
|
||||
'uragano',
|
||||
'tormenta',
|
||||
'tempête',
|
||||
'tempesta',
|
||||
'sturm',
|
||||
'cicl',
|
||||
'erupci',
|
||||
'éruption',
|
||||
'eruzione',
|
||||
],
|
||||
protest: [
|
||||
'protesta',
|
||||
'manifesta',
|
||||
'disturbio',
|
||||
'émeute',
|
||||
'huelga',
|
||||
'grève',
|
||||
'sciopero',
|
||||
'streik',
|
||||
'demonstration',
|
||||
],
|
||||
};
|
||||
for (const [category, stems] of Object.entries(EUROPEAN))
|
||||
RULES[category] = [...(RULES[category] || []), ...stems];
|
||||
|
||||
const COMPILED = Object.entries(RULES).map(([category, stems]) => ({
|
||||
category,
|
||||
rules: stems.map((entry) => {
|
||||
const [stem, weight = 1] = Array.isArray(entry) ? entry : [entry];
|
||||
const whole = stem.endsWith('$');
|
||||
const body = escapeRegex(
|
||||
normaliseForMatch(whole ? stem.slice(0, -1) : stem),
|
||||
).replace(/\s+/g, '\\s+');
|
||||
const arabic = /\p{Script=Arabic}/u.test(stem) ? '(?:[وفبلك]?(?:ال)?)' : '';
|
||||
const hebrew = /\p{Script=Hebrew}/u.test(stem) ? '(?:[ובלמהשכ]{0,2})' : '';
|
||||
return {
|
||||
weight,
|
||||
regex: new RegExp(
|
||||
`(?<![\\p{L}\\p{N}])${arabic}${hebrew}${body}${whole ? '(?![\\p{L}\\p{N}])' : ''}`,
|
||||
'u',
|
||||
),
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
/**
|
||||
* Categorise a report from its title and summary.
|
||||
* @returns {{category: string, tags: string[]}}
|
||||
*/
|
||||
export const RELEVANCE_LEVELS = Object.freeze(['core', 'related', 'minor']);
|
||||
const EVENT_CATEGORIES = new Set([
|
||||
'strike',
|
||||
'air-defence',
|
||||
'ground',
|
||||
'casualties',
|
||||
'disaster',
|
||||
]);
|
||||
// Words that tie a report to armed conflict or crisis, whatever its category
|
||||
// (en, fr, ru, uk, ar, he, es).
|
||||
const CONFLICT_TERMS = new RegExp(
|
||||
[
|
||||
'war',
|
||||
'army',
|
||||
'military',
|
||||
'militar',
|
||||
'troops',
|
||||
'soldier',
|
||||
'militant',
|
||||
'jihad',
|
||||
'terror',
|
||||
'insurg',
|
||||
'rebel',
|
||||
'ceasefire',
|
||||
'truce',
|
||||
'hostage',
|
||||
'sanction',
|
||||
'shell',
|
||||
'drone',
|
||||
'missile',
|
||||
'airstrike',
|
||||
'frappe',
|
||||
'armée',
|
||||
'soldat',
|
||||
'djihad',
|
||||
'fama',
|
||||
'jnim',
|
||||
'houthi',
|
||||
'hezbollah',
|
||||
'hamas',
|
||||
'irgc',
|
||||
'rsf',
|
||||
'wagner',
|
||||
'africa corps',
|
||||
'войн',
|
||||
'війн',
|
||||
'армі',
|
||||
'арми',
|
||||
'всу',
|
||||
'зсу',
|
||||
'бпла',
|
||||
'обстр',
|
||||
'фронт',
|
||||
'حرب',
|
||||
'الجيش',
|
||||
'هجوم',
|
||||
'قصف',
|
||||
'الحوثي',
|
||||
'مسلح',
|
||||
'מלחמה',
|
||||
'צה"ל',
|
||||
'guerra',
|
||||
'ejército',
|
||||
// Violence, coercion and their aftermath.
|
||||
'massacre',
|
||||
'atrocit',
|
||||
'genocide',
|
||||
'arms',
|
||||
'weapon',
|
||||
'detain',
|
||||
'abduct',
|
||||
'kidnap',
|
||||
'air raid',
|
||||
'alert',
|
||||
'jamming',
|
||||
'тривог',
|
||||
'тревог',
|
||||
'відбій',
|
||||
'отбой',
|
||||
'occupied',
|
||||
'occupation',
|
||||
'territor',
|
||||
'convoy',
|
||||
'demining',
|
||||
'forces',
|
||||
'raid',
|
||||
'conflict',
|
||||
'putin',
|
||||
'zelensk',
|
||||
'окуп',
|
||||
'оккуп',
|
||||
// Diplomacy and strategic stakes of the conflict.
|
||||
'ambassador',
|
||||
'mediat',
|
||||
'election',
|
||||
'talks',
|
||||
'negotiat',
|
||||
'envoy',
|
||||
'diplomat',
|
||||
'foreign minister',
|
||||
'condemn',
|
||||
'nuclear',
|
||||
'hormuz',
|
||||
'blockade',
|
||||
'négociation',
|
||||
'переговор',
|
||||
'перемир',
|
||||
'مفاوضات',
|
||||
'هدنة',
|
||||
]
|
||||
// Short words are whole words ("war" is not "warning" or "Warsaw").
|
||||
.map((w) => `(?<![\\p{L}])${w}${w.length < 5 ? '(?![\\p{L}])' : ''}`)
|
||||
.join('|'),
|
||||
'iu',
|
||||
);
|
||||
|
||||
/**
|
||||
* How much a report matters to its hotspot, by rule: `core` for a physical conflict or crisis event, `related` for
|
||||
* news tied to the conflict (by wording, or because the source covers
|
||||
* nothing else), `minor` for the rest of what a broad outlet publishes
|
||||
* (prices, sport, health, culture).
|
||||
*/
|
||||
const termPatterns = new Map();
|
||||
/** A word-start pattern for a hotspot term ("pow" must not match "power"). */
|
||||
function termPattern(term) {
|
||||
if (!termPatterns.has(term))
|
||||
termPatterns.set(
|
||||
term,
|
||||
new RegExp(
|
||||
// Short terms are whole words ("pow" is not "power"); longer ones are
|
||||
// stems ("mobiliz" → mobilized, mobilization).
|
||||
`(?<![\\p{L}])${normaliseForMatch(term).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}${term.length < 5 ? '(?![\\p{L}])' : ''}`,
|
||||
'u',
|
||||
),
|
||||
);
|
||||
return termPatterns.get(term);
|
||||
}
|
||||
|
||||
export function relevanceOf({
|
||||
category,
|
||||
tags = [],
|
||||
title = '',
|
||||
summary = '',
|
||||
dedicated = false,
|
||||
relatedTerms = [],
|
||||
floor = 'minor',
|
||||
}) {
|
||||
const text = normaliseForMatch(`${title} ${summary}`);
|
||||
// A hotspot's own vocabulary counts as conflict wording there (Iran:
|
||||
// "Qalibaf", "IAEA"; Israel: "settler"; Ukraine: "prisoner exchange").
|
||||
const conflict =
|
||||
CONFLICT_TERMS.test(text) ||
|
||||
relatedTerms.some((term) => termPattern(term).test(text));
|
||||
// Any event reading counts, even when another category wins the title
|
||||
// ("the accident toll rises to 74": infrastructure first, casualties too).
|
||||
if (
|
||||
EVENT_CATEGORIES.has(category) ||
|
||||
tags.some((tag) => EVENT_CATEGORIES.has(tag))
|
||||
)
|
||||
return 'core';
|
||||
if (conflict && ['humanitarian', 'infrastructure'].includes(category))
|
||||
return 'core';
|
||||
if (conflict || dedicated || category === 'humanitarian') return 'related';
|
||||
return floor;
|
||||
}
|
||||
|
||||
// Alerts: air-raid alerts and reports of drones or missiles on their way
|
||||
// ("Реактивний БпЛА курсом на Полтаву", "КАБи на Донеччину", "drone heading
|
||||
// for Kyiv", "➡️"). Something is in the air; nothing has hit yet. Most of an
|
||||
// air force's channel is these, and as "strikes" they would merge into real
|
||||
// strike events and inflate them.
|
||||
// A direction counts only after a weapon ("БпЛА … з півдня"), so a convoy
|
||||
// leaving "in the direction of Oleshky" is not an alert.
|
||||
const WEAPON =
|
||||
'(?:бпла|дрон|шахед|герань|ракет|каб|балістик|крилат|боєприпас|боеприпас|uav|drone|missile|shahed|ballistic|cruise|munition)';
|
||||
const DIRECTION =
|
||||
'(?:курсом на|у напрямку|в напрямку|в направлении|у бік|в сторону|повз |з (?:північ|півден|схід|сход|захід|заход)|с (?:север|юг|восток|запад)|на |над |heading |on course|toward)';
|
||||
const ALERT = new RegExp(
|
||||
`${WEAPON}[^.!?]{0,40}${DIRECTION}|^[^\\p{L}]*курсом на|тривог|тревог|загроза застосування|угроза применения|air[- ]raid|threat of (?:missile|drone|ballistic|uav)|[➡⬅⬆⬇↗↘↙↖]`,
|
||||
'u',
|
||||
);
|
||||
// Wording that something did hit: then it is a strike (or air defence), not an alert.
|
||||
const IMPACT =
|
||||
/влуч|вибух|уражен|пошкодж|загин|поранен|зруйн|пожеж|атак|удар по|збит|знищ|попадан|взрыв|поражен|поврежд|погиб|ранен|разруш|сбит|уничтож|\bhit\b|\bhits\b|struck|explosion|blast|killed|injur|wounded|damage|destroy|fire\b|shot down|intercept|attack/u;
|
||||
|
||||
/** Whether a report's title is an alert: a threat on its way, nothing hit yet. */
|
||||
function isAlertText(title = '', summary = '') {
|
||||
const t = normaliseForMatch(`${title} ${summary}`.slice(0, 400));
|
||||
return ALERT.test(t) && !IMPACT.test(t);
|
||||
}
|
||||
|
||||
// "Hit" or "struck" says nothing on its own: what hit decides (a missile is a
|
||||
// strike, a flood or lightning a disaster), and those words score by
|
||||
// themselves. A few phrases use a category's word for another thing.
|
||||
const PHRASES = [
|
||||
[/storm shadow/gu, 'missile'],
|
||||
[/lightning strikes?/gu, 'lightning'],
|
||||
[
|
||||
/(?:general|hunger|labou?r|workers?|nationwide|national|transport|rail|teachers?|doctors?|nurses?) strikes?|on strike|strike action/gu,
|
||||
'protest',
|
||||
],
|
||||
];
|
||||
const rephrase = (text) =>
|
||||
PHRASES.reduce((out, [re, word]) => out.replace(re, word), text);
|
||||
|
||||
// An election story borrows the words of war and disaster ("‘An Earthquake’:
|
||||
// A Democratic Rout", "captured Texas", "a bloodbath for Republicans"). With
|
||||
// election wording and nothing physical (no weapon, no one hurt, no quake's
|
||||
// magnitude), the event categories give way to political.
|
||||
const ELECTION =
|
||||
/(?<![\p{L}])(?:election|midterm|electoral|ballot|voters?(?![\p{L}])|votes?(?![\p{L}])|voting|polls(?![\p{L}])|senate race|referendum|democrats|republicans|gop(?![\p{L}])|[\p{L}]+-elect(?![\p{L}])|выбор|избира|вибор|голосув|голосов|انتخاب|בחירות)/u;
|
||||
const PHYSICAL =
|
||||
/(?<![\p{L}])(?:kill|dead(?![\p{L}])|fighting|clash|air ?strike|strikes?(?![\p{L}])|struck|destroy|downed|ablaze|on fire|catch(?:es)? fire|caught fire|naval mine|landmine|shot down|sunk|sank|deaths?|injur|wounded|casualt|troops|soldier|missile|drone|rocket|shell(?:ing|ed)|artiller|explosi|blast|bomb|gunfire|shot(?![\p{L}])|shooting|magnitude|richter|tremor|evacuat|flood(?:ed|ing|waters)|убит|погиб|ранен|ракет|дрон|обстрел|загин|поранен|обстріл|قتل|جرح|صاروخ|הרג|פצוע|טיל)/u;
|
||||
/** The categories of a physical event (an alert is a threat, not yet one). */
|
||||
export const EVENT_KINDS = new Set([
|
||||
'strike',
|
||||
'air-defence',
|
||||
'ground',
|
||||
'casualties',
|
||||
'disaster',
|
||||
'infrastructure',
|
||||
]);
|
||||
|
||||
export function categorizeText({ title = '', summary = '' } = {}) {
|
||||
if (isAlertText(title)) return { category: 'alert', tags: ['alert'] };
|
||||
const t = rephrase(normaliseForMatch(title));
|
||||
const s = rephrase(normaliseForMatch(summary));
|
||||
const scores = new Map();
|
||||
for (const { category, rules } of COMPILED) {
|
||||
let score = 0;
|
||||
for (const { regex, weight } of rules) {
|
||||
if (regex.test(t)) score += 2 * weight;
|
||||
else if (regex.test(s)) score += weight;
|
||||
}
|
||||
if (score > 0) scores.set(category, score);
|
||||
}
|
||||
const all = `${t} ${s}`;
|
||||
const election = scores.get('election') || 0;
|
||||
scores.delete('election');
|
||||
const physical = PHYSICAL.test(all);
|
||||
// Business words and no harm or attack: "advance exploration" is no
|
||||
// front-line advance; "attack an industrial facility" stays a strike.
|
||||
if (physical || ATTACK.test(all)) scores.delete('economy');
|
||||
else if (scores.has('economy'))
|
||||
for (const kind of EVENT_KINDS) scores.delete(kind);
|
||||
if (!physical) {
|
||||
if (election)
|
||||
scores.set('political', (scores.get('political') || 0) + election);
|
||||
if (ELECTION.test(all)) {
|
||||
for (const kind of EVENT_KINDS) scores.delete(kind);
|
||||
scores.set('political', Math.max(scores.get('political') || 0, 1));
|
||||
}
|
||||
}
|
||||
if (!scores.size) return { category: 'other', tags: [] };
|
||||
const ranked = CATEGORY_ORDER.filter((c) => scores.has(c)).sort(
|
||||
(a, b) => scores.get(b) - scores.get(a),
|
||||
);
|
||||
return { category: ranked[0], tags: ranked };
|
||||
}
|
||||
|
||||
// Severity by rule, for reports without a severity of their own (0 none,
|
||||
// 1 minor incident, 2 significant: deaths, injuries or real damage, 3 major:
|
||||
// mass casualties). Wording in en, uk, ru, ar, he, es, fr, de.
|
||||
const SEVERITY_EVENTS = new Set([...EVENT_CATEGORIES, 'infrastructure']);
|
||||
const MAJOR_HARM =
|
||||
/mass casualt|massacre|dozens (?:of \p{L}+ )?(?:killed|dead)|hundreds (?:of \p{L}+ )?(?:killed|dead)|масов\p{L}* (?:жертв|загибел|гибел)|десятки (?:загибл|погибш|убит|вбит|людей загин|человек погиб)|مجزرة|عشرات (?:القتلى|الشهداء)|טבח/u;
|
||||
const HARM_WORDS =
|
||||
'kill|dead|died|death|injur|wound|casualt|загин|загибл|погиб|убит|вбит|поранен|ранен|постражд|пострадал|травм|قتل|قتيل|شهيد|شهداء|جرح|جريح|إصاب|اصاب|נהרג|הרוג|פצוע|muert|herid|tué|mort|bless|getötet|tote|verletz';
|
||||
const HARM = new RegExp(String.raw`(?<![\p{L}])(?:${HARM_WORDS})`, 'u');
|
||||
// "No casualties", "без жертв": the harm word is there, the harm is not.
|
||||
const NO_HARM =
|
||||
/(?:no|without|zero) (?:casualt|injur|deaths|victims|one (?:was )?(?:hurt|injured))|без (?:жертв|постраждал|пострадавш|загибл|погибш)|не постраждал|не пострадал|لا (?:إصابات|اصابات)|אין נפגעים|sin (?:víctimas|heridos)|sans (?:victime|blessé)/u;
|
||||
const DAMAGE =
|
||||
/destroy|damag|ablaze|on fire|fire broke|explosion|collapse|зруйн|разруш|пошкодж|поврежд|пожеж|пожар|вибух|взрыв|دمار|تدمير|انفجار|حريق|הרס|פיצוץ|שריפה|destru|incendi|explos|zerstör/u;
|
||||
// An age ("a 71-year-old woman", "71-річна", "71-летний") is not a count.
|
||||
const NUMBER = String.raw`(\d{1,3}(?:[,.\s]\d{3})+|\d{1,5})(?![\d,.]*[-\s]*(?:year|yr|річн|рок|летн|год))`;
|
||||
const COUNTS = [
|
||||
// "14 killed", "three people dead" is left to the words; "14 civilians dead".
|
||||
new RegExp(String.raw`${NUMBER}\s+(?:\p{L}+\s+){0,2}(?:${HARM_WORDS})`, 'gu'),
|
||||
// "killing 14", "death toll rises to 74", "kills at least 20".
|
||||
new RegExp(
|
||||
String.raw`(?:kill\p{L}*|toll\p{L}*(?:\s+\p{L}+){0,3}|загинул\p{L}*|погибл\p{L}*|жертв\p{L}*)\s+(?:at least\s+|не менее\s+|щонайменше\s+|до\s+|to\s+)?${NUMBER}`,
|
||||
'gu',
|
||||
),
|
||||
];
|
||||
|
||||
function largestCount(text) {
|
||||
let most = 0;
|
||||
for (const pattern of COUNTS)
|
||||
for (const match of text.matchAll(pattern)) {
|
||||
const n = Number(match[1].replace(/[,.\s]/g, ''));
|
||||
// A year ("since 2022 killed…") is not a toll.
|
||||
if (n < 1900 || n > 2100) most = Math.max(most, n);
|
||||
}
|
||||
return most;
|
||||
}
|
||||
|
||||
/** A report's severity by rule, 0–3. */
|
||||
export function severityByRules({
|
||||
category,
|
||||
tags = [],
|
||||
title = '',
|
||||
summary = '',
|
||||
}) {
|
||||
const text = normaliseForMatch(`${title} ${summary}`);
|
||||
const harmed = HARM.test(text) && !NO_HARM.test(text);
|
||||
if (MAJOR_HARM.test(text) || (harmed && largestCount(text) >= 10)) return 3;
|
||||
const event =
|
||||
SEVERITY_EVENTS.has(category) ||
|
||||
tags.some((tag) => SEVERITY_EVENTS.has(tag));
|
||||
if (harmed && (event || category === 'humanitarian')) return 2;
|
||||
if (event && DAMAGE.test(text)) return 2;
|
||||
return event ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,543 @@
|
||||
// GDELT 2.0 events for the Conflicts section, from the 15-minute export
|
||||
// files (data.gdeltproject.org, plain downloads; no key, and not the
|
||||
// rate-limited search API). Each export is one zipped tab-separated table of
|
||||
// machine-coded events, each placed by GDELT with a point and a place type.
|
||||
//
|
||||
// A story counts when GDELT coded physical violence in it (CAMEO roots 18
|
||||
// assault, 19 fight, 20 mass violence); its place is read from all its
|
||||
// placed events (choosePlace). GDELT's actor fields are machine-coded and
|
||||
// often wrong, so no report says who did what: a report is the story (its
|
||||
// article, and the copies other outlets ran under the same address words).
|
||||
// The events carry no headline: a story's title is its page title from the
|
||||
// GKG export of the same quarter hour (read for the newest hours only, it
|
||||
// runs to 6 MB), else the words of its address. All of it is rules over
|
||||
// GDELT's own fields; nothing reads the articles.
|
||||
|
||||
import { promisify } from 'node:util';
|
||||
import { inflateRaw } from 'node:zlib';
|
||||
|
||||
import { readResponseBytesCapped } from '../common/http.js';
|
||||
import { categorizeText, relevanceOf, severityByRules } from './categorize.js';
|
||||
import { UpstreamError, unescapeHtml, webLink } from './upstream.js';
|
||||
|
||||
const EXPORTS_URL = 'https://data.gdeltproject.org/gdeltv2/';
|
||||
const STEP_MS = 15 * 60_000;
|
||||
// An events export runs to about 100 KB zipped and 500 KB unzipped; a GKG
|
||||
// export to about 6 MB and 18 MB. Unzipped off the main thread.
|
||||
const MAX_ZIP_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_TABLE_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_GKG_ZIP_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_GKG_TABLE_BYTES = 32 * 1024 * 1024;
|
||||
const inflate = promisify(inflateRaw);
|
||||
// The most words an address's words make a title of.
|
||||
const MAX_ADDRESS_WORDS = 24;
|
||||
const TIMEOUT_MS = 30_000;
|
||||
const MAX_PLACES = 3;
|
||||
|
||||
// Columns of the 2.0 events table (61).
|
||||
const COL = {
|
||||
id: 0,
|
||||
eventCode: 26,
|
||||
rootCode: 28,
|
||||
quadClass: 29,
|
||||
mentions: 31,
|
||||
geoType: 51,
|
||||
place: 52,
|
||||
lat: 56,
|
||||
lon: 57,
|
||||
added: 59,
|
||||
url: 60,
|
||||
};
|
||||
|
||||
// CAMEO event codes as the News layer's categories, and as words for an
|
||||
// article whose address carries none of its own.
|
||||
const EVENTS = [
|
||||
// Aerial weapons, artillery and tanks, bombings.
|
||||
[/^19[45]|^183/, 'strike', 'Strike reported'],
|
||||
[/^186|^185/, 'casualties', 'Killing reported'],
|
||||
[/^181/, 'humanitarian', 'Abduction reported'],
|
||||
[/^20/, 'casualties', 'Mass violence reported'],
|
||||
[/^19/, 'ground', 'Fighting reported'],
|
||||
[/^18/, 'other', 'Violence reported'],
|
||||
];
|
||||
const eventKind = (code) =>
|
||||
EVENTS.find(([pattern]) => pattern.test(code)) || EVENTS.at(-1);
|
||||
|
||||
// GDELT's place types: 1 country, 2 US state, 3 US city, 4 world city,
|
||||
// 5 world state.
|
||||
const PRECISION = {
|
||||
1: 'country',
|
||||
2: 'region',
|
||||
3: 'city',
|
||||
4: 'city',
|
||||
5: 'region',
|
||||
};
|
||||
|
||||
/** The export holding the events GDELT added in the 15 minutes before `ms`. */
|
||||
export function exportUrl(ms) {
|
||||
const at = new Date(Math.floor(ms / STEP_MS) * STEP_MS);
|
||||
const stamp = at.toISOString().replace(/[-:T]/g, '').slice(0, 14);
|
||||
return `${EXPORTS_URL}${stamp}.export.CSV.zip`;
|
||||
}
|
||||
|
||||
/** An events export's quarter hour (ms), from its name, or NaN. */
|
||||
export function exportAt(url) {
|
||||
const match = /(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)00\.export\.CSV\.zip$/.exec(
|
||||
url,
|
||||
);
|
||||
return match
|
||||
? Date.UTC(match[1], match[2] - 1, match[3], match[4], match[5])
|
||||
: NaN;
|
||||
}
|
||||
|
||||
const LASTUPDATE_URL = `${EXPORTS_URL}lastupdate.txt`;
|
||||
|
||||
/**
|
||||
* The time of the newest events export GDELT lists as published, in ms, or
|
||||
* null when its list cannot be read. Asking for one not listed yet would
|
||||
* be answered 404, and GDELT's CDN keeps that 404 for an hour.
|
||||
*/
|
||||
export async function fetchLatestExport({ fetchImpl, signal } = {}) {
|
||||
try {
|
||||
const response = await fetchImpl(LASTUPDATE_URL, {
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
return null;
|
||||
}
|
||||
const text = new TextDecoder().decode(
|
||||
await readResponseBytesCapped(response, 16 * 1024, signal),
|
||||
);
|
||||
const at = exportAt(
|
||||
/\S+\.export\.CSV\.zip/.exec(text)?.[0].replace(/^http:/, 'https:') || '',
|
||||
);
|
||||
return Number.isFinite(at) ? at : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The GKG export of the same quarter hour as an events export. */
|
||||
export const gkgUrl = (eventsUrl) =>
|
||||
eventsUrl.replace(/\.export\.CSV\.zip$/, '.gkg.csv.zip');
|
||||
|
||||
/** The day's exports up to `now`, newest first. */
|
||||
export function exportsOfDay(now, hours = 24) {
|
||||
const urls = [];
|
||||
const last = Math.floor(now / STEP_MS) * STEP_MS;
|
||||
for (let ms = last; ms > now - hours * 3_600_000; ms -= STEP_MS)
|
||||
urls.push(exportUrl(ms));
|
||||
return urls;
|
||||
}
|
||||
|
||||
/** The one table inside an export's zip, as text, up to `maxBytes`. */
|
||||
export async function unzipExport(bytes, maxBytes = MAX_TABLE_BYTES) {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const fail = () => {
|
||||
throw new UpstreamError('GDELT export unreadable', { unreadable: true });
|
||||
};
|
||||
if (bytes.length < 30 || view.getUint32(0, true) !== 0x04034b50) fail();
|
||||
// The central directory has the sizes; a local header may leave them 0.
|
||||
let eocd = -1;
|
||||
for (let i = bytes.length - 22; i >= Math.max(0, bytes.length - 0x10015); i--)
|
||||
if (view.getUint32(i, true) === 0x06054b50) {
|
||||
eocd = i;
|
||||
break;
|
||||
}
|
||||
if (eocd < 0) fail();
|
||||
const dir = view.getUint32(eocd + 16, true);
|
||||
if (dir + 46 > bytes.length || view.getUint32(dir, true) !== 0x02014b50)
|
||||
fail();
|
||||
const method = view.getUint16(dir + 10, true);
|
||||
const compressed = view.getUint32(dir + 20, true);
|
||||
const local = view.getUint32(dir + 42, true);
|
||||
if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50)
|
||||
fail();
|
||||
const start =
|
||||
local +
|
||||
30 +
|
||||
view.getUint16(local + 26, true) +
|
||||
view.getUint16(local + 28, true);
|
||||
if (start + compressed > bytes.length) fail();
|
||||
const data = bytes.subarray(start, start + compressed);
|
||||
try {
|
||||
if (method === 0 && data.length > maxBytes) fail();
|
||||
const table =
|
||||
method === 0
|
||||
? data
|
||||
: method === 8
|
||||
? await inflate(data, { maxOutputLength: maxBytes })
|
||||
: fail();
|
||||
return new TextDecoder().decode(table);
|
||||
} catch (error) {
|
||||
if (error instanceof UpstreamError) throw error;
|
||||
return fail();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An article's address as the reports carry it: https, an http one
|
||||
* upgraded (GDELT keeps the address as it found it), else null.
|
||||
*/
|
||||
export const articleLink = (value) =>
|
||||
webLink(value) || webLink(String(value ?? '').replace(/^http:/, 'https:'));
|
||||
|
||||
/**
|
||||
* The placed events of an export's table, of the stories with physical
|
||||
* violence in them: those events, and every other placed event of the same
|
||||
* stories (they help place it).
|
||||
*/
|
||||
export function parseEvents(table) {
|
||||
const all = [];
|
||||
for (const line of table.split('\n')) {
|
||||
const f = line.split('\t');
|
||||
if (f.length < 61) continue;
|
||||
const violent =
|
||||
f[COL.quadClass] === '4' && ['18', '19', '20'].includes(f[COL.rootCode]);
|
||||
const precision = PRECISION[f[COL.geoType]];
|
||||
const lat = Number(f[COL.lat]);
|
||||
const lon = Number(f[COL.lon]);
|
||||
const url = articleLink(f[COL.url]);
|
||||
const added = /^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/.exec(f[COL.added]);
|
||||
if (
|
||||
!precision ||
|
||||
!f[COL.lat] ||
|
||||
!f[COL.lon] ||
|
||||
!Number.isFinite(lat) ||
|
||||
!Number.isFinite(lon) ||
|
||||
Math.abs(lat) > 90 ||
|
||||
Math.abs(lon) > 180 ||
|
||||
(lat === 0 && lon === 0) ||
|
||||
!url ||
|
||||
!added
|
||||
)
|
||||
continue;
|
||||
all.push({
|
||||
id: f[COL.id],
|
||||
code: f[COL.eventCode],
|
||||
violent,
|
||||
lat,
|
||||
lon,
|
||||
place: f[COL.place].trim() || null,
|
||||
precision,
|
||||
mentions: Math.max(1, Number(f[COL.mentions]) || 1),
|
||||
addedAt: `${added[1]}-${added[2]}-${added[3]}T${added[4]}:${added[5]}:${added[6]}.000Z`,
|
||||
url,
|
||||
});
|
||||
}
|
||||
const stories = new Set(all.filter((e) => e.violent).map(storyKey));
|
||||
return all.filter((e) => stories.has(storyKey(e)));
|
||||
}
|
||||
|
||||
/**
|
||||
* The words of an article's address, when it is written in them
|
||||
* ("three-children-die-in-peshawar-cylinder-exp"), as a sentence; else null.
|
||||
*/
|
||||
export function addressWords(url) {
|
||||
let path;
|
||||
try {
|
||||
path = decodeURIComponent(new URL(url).pathname);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const segment = path
|
||||
.split('/')
|
||||
.map((part) => part.replace(/\.(?:s?html?|php|aspx?)$/i, ''))
|
||||
.filter((part) => /[-_]/.test(part))
|
||||
.sort((a, b) => b.length - a.length)[0];
|
||||
if (!segment) return null;
|
||||
const words = segment.split(/[-_]+/).filter(Boolean);
|
||||
const lettered = words.filter((word) => /^\p{L}+$/u.test(word));
|
||||
if (lettered.length < 4 || lettered.length < 0.7 * words.length) return null;
|
||||
// Numbers stay ("killed nearly 500"); an id does not: a word of letters
|
||||
// and digits, five digits or more, or four or more at either end.
|
||||
const kept = words.filter(
|
||||
(word, i) =>
|
||||
/^\p{L}+$/u.test(word) ||
|
||||
(/^\d{1,4}$/.test(word) &&
|
||||
!(word.length === 4 && (i === 0 || i === words.length - 1))),
|
||||
);
|
||||
const text = kept.slice(0, MAX_ADDRESS_WORDS).join(' ').toLowerCase();
|
||||
return text.charAt(0).toUpperCase() + text.slice(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Address words with the places the story is at written as names
|
||||
* ("killed in amman" → "killed in Amman"), and "u s" as "US".
|
||||
*/
|
||||
export function withNames(words, places) {
|
||||
let text = words.replace(/(?<![\p{L}])u s(?![\p{L}])/gu, 'US');
|
||||
const names = new Set(
|
||||
places.flatMap((place) =>
|
||||
String(place.place ?? '')
|
||||
.split(',')
|
||||
.map((part) => part.replace(/\(.*?\)/g, '').trim())
|
||||
.filter((part) => part.length > 2 && !/\(general\)/.test(part)),
|
||||
),
|
||||
);
|
||||
for (const name of names) {
|
||||
const pattern = new RegExp(
|
||||
`(?<![\\p{L}])${name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\p{L}])`,
|
||||
'gu',
|
||||
);
|
||||
text = text.replace(pattern, name);
|
||||
}
|
||||
return text.charAt(0).toUpperCase() + text.slice(1);
|
||||
}
|
||||
|
||||
// A page title's site part: " | The Daily Example", " - Wire Service".
|
||||
const SITE_PART = /\s+[|–—-]\s+([^|–—-]{2,60})$/u;
|
||||
|
||||
/**
|
||||
* A page title as a headline: entities decoded, the site's name after a
|
||||
* bar or dash taken off (when the headline before it is still one), or
|
||||
* null when it is too short to be one.
|
||||
*/
|
||||
export function cleanTitle(raw) {
|
||||
// An agency's tag set on with a bare hyphen ("…attack-Newswire", "…-X").
|
||||
let title = unescapeHtml(raw).replace(
|
||||
// After a word in lower case: "Jean-Pierre" keeps its name.
|
||||
/(?<=(?:^|\s)\p{Ll}\p{L}*|[\p{N}'")])-\p{Lu}\p{L}{0,15}$/u,
|
||||
'',
|
||||
);
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const match = SITE_PART.exec(title);
|
||||
if (!match) break;
|
||||
const rest = title.slice(0, match.index);
|
||||
if (rest.split(/\s+/).length < 4 || match[1].split(/\s+/).length > 6) break;
|
||||
title = rest;
|
||||
}
|
||||
title = title.trim().slice(0, 300);
|
||||
return title.length >= 12 && title.split(/\s+/).length >= 3 ? title : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* GKG page titles by article address (as articleLink writes it), for the
|
||||
* addresses in `wanted`.
|
||||
*/
|
||||
export function parseTitles(table, wanted) {
|
||||
const titles = new Map();
|
||||
for (const line of table.split('\n')) {
|
||||
const f = line.split('\t');
|
||||
const url = f[4] && articleLink(f[4]);
|
||||
if (!url || !wanted.has(url)) continue;
|
||||
const match = /<PAGE_TITLE>(.*?)<\/PAGE_TITLE>/s.exec(f.at(-1) || '');
|
||||
const title = match && cleanTitle(match[1]);
|
||||
if (title) titles.set(url, title);
|
||||
}
|
||||
return titles;
|
||||
}
|
||||
|
||||
/**
|
||||
* The story an event belongs to: its article's address words, which the
|
||||
* copies other outlets run share (an agency story), else its address.
|
||||
*/
|
||||
export function storyKey(event) {
|
||||
return addressWords(event.url)?.toLowerCase() || event.url;
|
||||
}
|
||||
|
||||
const PLACE_WORD = (name) =>
|
||||
new RegExp(
|
||||
`(?<![\\p{L}])${name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}s?(?![\\p{L}])`,
|
||||
'u',
|
||||
);
|
||||
/** Whether the words name a place's town or its country ("jordans" too). */
|
||||
function namedIn(words, place) {
|
||||
if (!words || !place) return false;
|
||||
const parts = place
|
||||
.split(',')
|
||||
.map((part) => part.replace(/\(.*?\)/g, '').trim())
|
||||
.filter((part) => part.length > 2);
|
||||
return [parts[0], parts.at(-1)].some(
|
||||
(part) => part && PLACE_WORD(part).test(words.toLowerCase()),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A story's places, best first, from all its placed events. GDELT places
|
||||
* an event at a bare country when the text only names a nationality ("a
|
||||
* Sri Lankan embassy", an Australian outlet's own country), so a town or
|
||||
* region wins over a country; of those, the ones the article's own words
|
||||
* name; then the most mentioned.
|
||||
*/
|
||||
export function choosePlaces(events, words) {
|
||||
const byPlace = new Map();
|
||||
for (const event of events) {
|
||||
const key = `${event.lat},${event.lon}`;
|
||||
const place = byPlace.get(key) || { ...event, mentions: 0 };
|
||||
place.mentions += event.mentions;
|
||||
byPlace.set(key, place);
|
||||
}
|
||||
let places = [...byPlace.values()];
|
||||
const specific = places.filter((p) => p.precision !== 'country');
|
||||
if (specific.length) places = specific;
|
||||
const named = places.filter((p) => namedIn(words, p.place));
|
||||
if (named.length) places = named;
|
||||
return places.sort((a, b) => b.mentions - a.mentions).slice(0, MAX_PLACES);
|
||||
}
|
||||
|
||||
// The earlier of two events: by the time GDELT added it, then by its id.
|
||||
const earlier = (a, b) =>
|
||||
b.addedAt < a.addedAt ||
|
||||
(b.addedAt === a.addedAt && Number(b.id) < Number(a.id))
|
||||
? b
|
||||
: a;
|
||||
|
||||
/**
|
||||
* One report per story (storyKey), its address and id the first copy's (an
|
||||
* export adding a more-mentioned event keeps both), placed by choosePlaces,
|
||||
* titled by a copy's page title (`titleOf`), else its address words, else
|
||||
* its violent event.
|
||||
*/
|
||||
export function eventReports(events, { titleOf = () => null } = {}) {
|
||||
const stories = new Map();
|
||||
for (const event of events) {
|
||||
const key = storyKey(event);
|
||||
const list = stories.get(key) || [];
|
||||
list.push(event);
|
||||
stories.set(key, list);
|
||||
}
|
||||
const reports = [];
|
||||
for (const list of stories.values()) {
|
||||
const violent = list
|
||||
.filter((e) => e.violent !== false)
|
||||
.sort(
|
||||
(a, b) => b.mentions - a.mentions || a.addedAt.localeCompare(b.addedAt),
|
||||
);
|
||||
if (!violent.length) continue;
|
||||
const [lead] = violent;
|
||||
const first = list.reduce(earlier);
|
||||
const { url } = first;
|
||||
const address = addressWords(url);
|
||||
const places = choosePlaces(list, address);
|
||||
const [, eventCategory, words] = eventKind(lead.code);
|
||||
const page = [url, ...list.map((e) => e.url)]
|
||||
.map((copy) => titleOf(copy))
|
||||
.find(Boolean);
|
||||
const own = page || (address ? withNames(address, places) : null);
|
||||
const title = own || eventTitle(words, places[0].place);
|
||||
// The article's words decide whether it is conflict, as any headline's
|
||||
// would: GDELT codes an election story as "fighting" too. GDELT's code
|
||||
// names the kind only where the words read as conflict without naming
|
||||
// one, or there are no words.
|
||||
const read = own ? categorizeText({ title: own }) : null;
|
||||
const tags = read?.tags || [];
|
||||
const relevance = read
|
||||
? relevanceOf({ category: read.category, tags, title: own })
|
||||
: 'related';
|
||||
const category =
|
||||
read && read.category !== 'other'
|
||||
? read.category
|
||||
: relevance === 'minor'
|
||||
? 'other'
|
||||
: eventCategory;
|
||||
const publisher = new URL(url).hostname.replace(/^www\./, '');
|
||||
const geo = ({ lat, lon, place, precision }) => ({
|
||||
lat,
|
||||
lon,
|
||||
place,
|
||||
precision,
|
||||
confidence: precision === 'city' ? 0.6 : 0.3,
|
||||
mappable: true,
|
||||
method: 'source',
|
||||
});
|
||||
reports.push({
|
||||
id: `gdelt-${first.id}`,
|
||||
title,
|
||||
url,
|
||||
publisher,
|
||||
source: { name: publisher, attribution: 'GDELT Project' },
|
||||
kind: 'gdelt',
|
||||
side: 'unknown',
|
||||
category,
|
||||
relevance,
|
||||
severity: Math.max(
|
||||
1,
|
||||
severityByRules({ category, tags, title: own || '' }),
|
||||
),
|
||||
severityBy: 'rules',
|
||||
publishedAt: first.addedAt,
|
||||
geo: geo(places[0]),
|
||||
locations: places.slice(1).map(geo),
|
||||
media: [],
|
||||
mentions: list.reduce((sum, e) => sum + e.mentions, 0),
|
||||
copies: new Set(list.map((e) => e.url)).size,
|
||||
// A title of the event's words and its place, when the address had
|
||||
// none: a renamed place renames it (eventTitle).
|
||||
...(own ? {} : { eventWords: words }),
|
||||
titled: page ? 'page' : own ? 'address' : 'event',
|
||||
});
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
|
||||
/** The title of a report without address words, at its (renamed) place. */
|
||||
export const eventTitle = (words, place) =>
|
||||
place ? `${words} in ${place}` : words;
|
||||
|
||||
/**
|
||||
* One export's table, or null while GDELT has not published it (GDELT
|
||||
* lists the newest in lastupdate.txt; see fetchLatestExport).
|
||||
*/
|
||||
async function fetchTable(url, { fetchImpl, signal, maxBytes, tableBytes }) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
} catch {
|
||||
throw new UpstreamError('GDELT exports unreachable');
|
||||
}
|
||||
if (response.status === 404) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw new UpstreamError(`GDELT exports answered ${response.status}`, {
|
||||
status: response.status,
|
||||
});
|
||||
}
|
||||
let bytes;
|
||||
try {
|
||||
bytes = await readResponseBytesCapped(response, maxBytes, signal);
|
||||
} catch (error) {
|
||||
// Over the cap stays over it; a read cut short may succeed again.
|
||||
throw new UpstreamError('GDELT export unreadable', {
|
||||
unreadable: error?.code === 'RESPONSE_TOO_LARGE',
|
||||
});
|
||||
}
|
||||
return unzipExport(bytes, tableBytes);
|
||||
}
|
||||
|
||||
/** One events export's events, or null while it is not published yet. */
|
||||
export async function fetchExport(url, { fetchImpl, signal } = {}) {
|
||||
const table = await fetchTable(url, {
|
||||
fetchImpl,
|
||||
signal,
|
||||
maxBytes: MAX_ZIP_BYTES,
|
||||
tableBytes: MAX_TABLE_BYTES,
|
||||
});
|
||||
return table === null ? null : parseEvents(table);
|
||||
}
|
||||
|
||||
/**
|
||||
* The page titles a GKG export holds for the addresses in `wanted`, or null
|
||||
* while it is not published yet.
|
||||
*/
|
||||
export async function fetchTitles(url, wanted, { fetchImpl, signal } = {}) {
|
||||
const table = await fetchTable(url, {
|
||||
fetchImpl,
|
||||
signal,
|
||||
maxBytes: MAX_GKG_ZIP_BYTES,
|
||||
tableBytes: MAX_GKG_TABLE_BYTES,
|
||||
});
|
||||
return table === null ? null : parseTitles(table, wanted);
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
addressWords,
|
||||
cleanTitle,
|
||||
gkgUrl,
|
||||
parseTitles,
|
||||
withNames,
|
||||
eventReports,
|
||||
exportsOfDay,
|
||||
exportUrl,
|
||||
fetchExport,
|
||||
parseEvents,
|
||||
unzipExport,
|
||||
} from './gdeltEvents.js';
|
||||
import { eventRow, gkgRow, zipTable } from './gdeltFixtures.mjs';
|
||||
|
||||
const ARTICLE =
|
||||
'https://www.wire.example/rubric-ato/3912-russian-drones-attack-kharkiv-overnight-three-killed.html';
|
||||
|
||||
test('the day is 96 exports, one each quarter hour, newest first', () => {
|
||||
const now = Date.parse('2026-11-05T15:07:30Z');
|
||||
assert.equal(
|
||||
exportUrl(now),
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105150000.export.CSV.zip',
|
||||
);
|
||||
const day = exportsOfDay(now);
|
||||
assert.equal(day.length, 96);
|
||||
assert.equal(
|
||||
day[1],
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105144500.export.CSV.zip',
|
||||
);
|
||||
assert.equal(
|
||||
day.at(-1),
|
||||
'https://data.gdeltproject.org/gdeltv2/20261104151500.export.CSV.zip',
|
||||
);
|
||||
});
|
||||
|
||||
test('a story with physical violence is kept with all its placed events', () => {
|
||||
const rows = [
|
||||
eventRow({ id: 1, code: '195', url: ARTICLE }),
|
||||
eventRow({ id: 2, code: '183', url: ARTICLE }),
|
||||
eventRow({ id: 3, code: '2001', url: ARTICLE }),
|
||||
// Not physical, but of the same story: kept to place it by.
|
||||
eventRow({ id: 4, code: '150', url: ARTICLE }),
|
||||
eventRow({ id: 7, code: '190', quad: '3', url: ARTICLE }),
|
||||
// A story with no violence in it is left out.
|
||||
eventRow({
|
||||
id: 5,
|
||||
code: '163',
|
||||
url: 'https://x.example/talks-resume-in-geneva-today-officials',
|
||||
}),
|
||||
eventRow({
|
||||
id: 6,
|
||||
code: '173',
|
||||
url: 'https://x.example/talks-resume-in-geneva-today-officials',
|
||||
}),
|
||||
// Unplaced, at 0,0, or without a link.
|
||||
eventRow({ id: 8, geoType: '0', url: ARTICLE }),
|
||||
eventRow({ id: 9, lat: 0, lon: 0, url: ARTICLE }),
|
||||
eventRow({ id: 10, url: 'javascript:alert(1)' }),
|
||||
'short\trow',
|
||||
];
|
||||
const events = parseEvents(rows.join('\n'));
|
||||
assert.deepEqual(
|
||||
events.map((e) => [e.id, e.violent]),
|
||||
[
|
||||
['1', true],
|
||||
['2', true],
|
||||
['3', true],
|
||||
['4', false],
|
||||
['7', false],
|
||||
],
|
||||
);
|
||||
assert.equal(events[0].precision, 'city');
|
||||
assert.equal(events[0].addedAt, '2026-11-05T11:45:00.000Z');
|
||||
});
|
||||
|
||||
test("an article is one report, at its most-mentioned event, in its address's words", () => {
|
||||
const events = parseEvents(
|
||||
[
|
||||
eventRow({ id: 11, code: '195', mentions: 2, url: ARTICLE }),
|
||||
eventRow({
|
||||
id: 12,
|
||||
code: '190',
|
||||
mentions: 9,
|
||||
url: ARTICLE,
|
||||
place: "Kupiansk, Kharkivs'ka Oblast', Ukraine",
|
||||
lat: 49.71,
|
||||
lon: 37.61,
|
||||
}),
|
||||
eventRow({
|
||||
id: 13,
|
||||
code: '190',
|
||||
geoType: '1',
|
||||
place: 'Ukraine',
|
||||
lat: 49,
|
||||
lon: 32,
|
||||
url: 'https://example.com/a/b/12345',
|
||||
}),
|
||||
].join('\n'),
|
||||
);
|
||||
const [drones, bare] = eventReports(events);
|
||||
// The story's first event (by time, then id) names it.
|
||||
assert.equal(drones.id, 'gdelt-11');
|
||||
// Its address words, with the place it is at written as a name.
|
||||
assert.equal(
|
||||
drones.title,
|
||||
'Russian drones attack Kharkiv overnight three killed',
|
||||
);
|
||||
// Kupiansk has more mentions; the article's own words name Kharkiv.
|
||||
assert.equal(drones.geo.place, "Kharkiv, Kharkivs'ka Oblast', Ukraine");
|
||||
assert.equal(drones.geo.precision, 'city');
|
||||
assert.equal(drones.locations.length, 0, 'Kupiansk is not named');
|
||||
assert.equal(drones.category, 'strike');
|
||||
assert.equal(drones.relevance, 'core');
|
||||
assert.equal(drones.severity, 2);
|
||||
assert.equal(drones.publisher, 'wire.example');
|
||||
assert.equal(drones.source.attribution, 'GDELT Project');
|
||||
assert.equal(drones.side, 'unknown', 'no actor is said');
|
||||
// No words in the address: GDELT's coding, related, at least severity 1.
|
||||
assert.equal(bare.title, 'Fighting reported in Ukraine');
|
||||
assert.equal(bare.category, 'ground');
|
||||
assert.equal(bare.relevance, 'related');
|
||||
assert.equal(bare.severity, 1);
|
||||
assert.equal(bare.geo.precision, 'country');
|
||||
});
|
||||
|
||||
test('an article GDELT codes as fighting whose words say otherwise stays out of Conflicts', () => {
|
||||
const [report] = eventReports(
|
||||
parseEvents(
|
||||
eventRow({
|
||||
url: 'https://news.example.com/2026/11/05/city-council-approves-new-budget-for-schools',
|
||||
id: 20,
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert.equal(report.relevance, 'minor');
|
||||
});
|
||||
|
||||
test('address words need four lettered words, most of the address', () => {
|
||||
assert.equal(addressWords('https://x.example/news/1234567'), null);
|
||||
assert.equal(addressWords('https://x.example/a/lueop1pzfi'), null);
|
||||
assert.equal(
|
||||
addressWords('https://x.example/2026/11/05/a-b-1-2-3-4-5'),
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
addressWords(
|
||||
'https://x.example/story/2638789/three-children-die-in-peshawar.html',
|
||||
),
|
||||
'Three children die in peshawar',
|
||||
);
|
||||
});
|
||||
|
||||
test('an export is unzipped and read; one not published yet is null', async () => {
|
||||
const zip = zipTable(eventRow({ id: 30, code: '193', url: ARTICLE }));
|
||||
assert.deepEqual(
|
||||
parseEvents(await unzipExport(zip)).map((e) => e.id),
|
||||
['30'],
|
||||
);
|
||||
const served = await fetchExport('https://data.gdeltproject.org/x.zip', {
|
||||
fetchImpl: async () => new Response(zip, { status: 200 }),
|
||||
});
|
||||
assert.deepEqual(
|
||||
served.map((e) => e.id),
|
||||
['30'],
|
||||
);
|
||||
const later = await fetchExport('https://data.gdeltproject.org/y.zip', {
|
||||
fetchImpl: async () => new Response('not found', { status: 404 }),
|
||||
});
|
||||
assert.equal(later, null);
|
||||
await assert.rejects(
|
||||
fetchExport('https://data.gdeltproject.org/z.zip', {
|
||||
fetchImpl: async () => new Response('busy', { status: 503 }),
|
||||
}),
|
||||
{ name: 'UpstreamError' },
|
||||
);
|
||||
await assert.rejects(unzipExport(new Uint8Array([1, 2, 3])), {
|
||||
name: 'UpstreamError',
|
||||
});
|
||||
});
|
||||
|
||||
test("a nationality is no place: a town the article's words name wins over a bare country", () => {
|
||||
const url =
|
||||
'https://www.sun.example/world/2026/11/05/security-officer-attacked-and-killed-near-sri-lankan-embassy-in-jordans-capital';
|
||||
const amman = { place: 'Amman, (JO11), Jordan', lat: 31.95, lon: 35.9333 };
|
||||
const cairo = { place: 'Cairo, Al Qahirah, Egypt', lat: 30.05, lon: 31.25 };
|
||||
const [report] = eventReports(
|
||||
parseEvents(
|
||||
[
|
||||
// "Sri Lankan embassy": GDELT's most-mentioned place, a bare country.
|
||||
eventRow({
|
||||
id: 1,
|
||||
mentions: 12,
|
||||
geoType: '1',
|
||||
place: 'Sri Lanka',
|
||||
lat: 7,
|
||||
lon: 81,
|
||||
url,
|
||||
}),
|
||||
eventRow({ id: 2, mentions: 6, ...cairo, url }),
|
||||
eventRow({ id: 3, mentions: 4, ...amman, url }),
|
||||
eventRow({ id: 4, mentions: 2, ...amman, url }),
|
||||
].join('\n'),
|
||||
),
|
||||
);
|
||||
assert.equal(report.geo.place, 'Amman, (JO11), Jordan');
|
||||
assert.equal(report.geo.precision, 'city');
|
||||
});
|
||||
|
||||
test('an agency story run by several outlets is one report, placed by all their events', () => {
|
||||
const words = 'jordan-security-officer-killed-near-sri-lanka-embassy';
|
||||
const times = `https://www.times.example/story/9386807/${words}/`;
|
||||
const chronicle = `https://www.chronicle.example/story/9386807/${words}/`;
|
||||
const reports = eventReports(
|
||||
parseEvents(
|
||||
[
|
||||
// The outlet's own country, at country level, on its violent events.
|
||||
eventRow({
|
||||
id: 1,
|
||||
mentions: 6,
|
||||
geoType: '1',
|
||||
place: 'Australia',
|
||||
lat: -25,
|
||||
lon: 135,
|
||||
url: times,
|
||||
added: '20261105141500',
|
||||
}),
|
||||
// Its other events are placed in Jordan.
|
||||
eventRow({
|
||||
id: 2,
|
||||
code: '120',
|
||||
quad: '3',
|
||||
mentions: 8,
|
||||
place: 'Karak, Jordan (general), Jordan',
|
||||
lat: 31.18,
|
||||
lon: 35.7,
|
||||
url: times,
|
||||
added: '20261105141500',
|
||||
}),
|
||||
eventRow({
|
||||
id: 3,
|
||||
mentions: 4,
|
||||
geoType: '1',
|
||||
place: 'Syria',
|
||||
lat: 35,
|
||||
lon: 38,
|
||||
url: chronicle,
|
||||
added: '20261105151500',
|
||||
}),
|
||||
].join('\n'),
|
||||
),
|
||||
);
|
||||
assert.equal(reports.length, 1);
|
||||
const [story] = reports;
|
||||
assert.equal(story.copies, 2);
|
||||
assert.equal(story.url, times, 'the first copy');
|
||||
assert.equal(story.publishedAt, '2026-11-05T14:15:00.000Z');
|
||||
assert.equal(story.geo.place, 'Karak, Jordan (general), Jordan');
|
||||
});
|
||||
|
||||
test("a story's title is a copy's page title from the GKG export", () => {
|
||||
const titles = parseTitles(
|
||||
[
|
||||
gkgRow(
|
||||
ARTICLE,
|
||||
'Russian drones attack Kharkiv overnight, three killed | Wire',
|
||||
),
|
||||
gkgRow('https://elsewhere.example/a', 'Not one of ours, not kept here'),
|
||||
].join('\n'),
|
||||
new Set([ARTICLE]),
|
||||
);
|
||||
assert.deepEqual(
|
||||
[...titles],
|
||||
[[ARTICLE, 'Russian drones attack Kharkiv overnight, three killed']],
|
||||
);
|
||||
const [report] = eventReports(
|
||||
parseEvents(eventRow({ id: 1, url: ARTICLE })),
|
||||
{
|
||||
titleOf: (url) => titles.get(url) || null,
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
report.title,
|
||||
'Russian drones attack Kharkiv overnight, three killed',
|
||||
);
|
||||
assert.equal(report.titled, 'page');
|
||||
assert.equal(
|
||||
gkgUrl(
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105141500.export.CSV.zip',
|
||||
),
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105141500.gkg.csv.zip',
|
||||
);
|
||||
});
|
||||
|
||||
test('a page title loses its site name, keeps its words, and must be a headline', () => {
|
||||
assert.equal(
|
||||
cleanTitle(
|
||||
'Yemen's president sacks ministers | Yemen | The Daily Example',
|
||||
),
|
||||
"Yemen's president sacks ministers",
|
||||
);
|
||||
assert.equal(
|
||||
cleanTitle('Strike in Kharkiv - live updates - Wire Service'),
|
||||
'Strike in Kharkiv - live updates',
|
||||
);
|
||||
// Too little before the dash: the dash is part of the headline.
|
||||
assert.equal(cleanTitle('Iran - Wire Service'), 'Iran - Wire Service');
|
||||
assert.equal(cleanTitle('Home'), null);
|
||||
assert.equal(
|
||||
cleanTitle('Jordanian security officer killed in stabbing attack-Newswire'),
|
||||
'Jordanian security officer killed in stabbing attack',
|
||||
);
|
||||
assert.equal(
|
||||
cleanTitle('Five killed in Gaza, sources say-X'),
|
||||
'Five killed in Gaza, sources say',
|
||||
);
|
||||
// A hyphenated name is no tag.
|
||||
assert.equal(
|
||||
cleanTitle('Talks resume with Jean-Pierre'),
|
||||
'Talks resume with Jean-Pierre',
|
||||
);
|
||||
});
|
||||
|
||||
test('address words keep their numbers and name the places the story is at', () => {
|
||||
const words = addressWords(
|
||||
'https://x.example/2026/11/05/jordan-says-u-s-envoy-met-officials-in-amman-after-500-killed',
|
||||
);
|
||||
assert.equal(
|
||||
words,
|
||||
'Jordan says u s envoy met officials in amman after 500 killed',
|
||||
);
|
||||
assert.equal(
|
||||
withNames(words, [{ place: 'Amman, (JO11), Jordan' }]),
|
||||
'Jordan says US envoy met officials in Amman after 500 killed',
|
||||
);
|
||||
});
|
||||
|
||||
test("a story's id is its first event's: a later, more-mentioned event keeps it", () => {
|
||||
const first = eventRow({ id: 40, mentions: 1, url: ARTICLE });
|
||||
const [before] = eventReports(parseEvents(first));
|
||||
const [after] = eventReports(
|
||||
parseEvents(
|
||||
[
|
||||
first,
|
||||
eventRow({
|
||||
id: 41,
|
||||
code: '195',
|
||||
mentions: 30,
|
||||
url: ARTICLE,
|
||||
added: '20261105120000',
|
||||
}),
|
||||
].join('\n'),
|
||||
),
|
||||
);
|
||||
assert.equal(before.id, 'gdelt-40');
|
||||
assert.equal(after.id, 'gdelt-40');
|
||||
assert.equal(after.mentions, 31);
|
||||
});
|
||||
|
||||
test('words that read as no conflict leave GDELT no kind to name', () => {
|
||||
const [report] = eventReports(
|
||||
parseEvents(
|
||||
eventRow({
|
||||
id: 42,
|
||||
code: '195',
|
||||
url: 'https://x.example/2026/11/05/local-bakery-celebrates-fifty-years-of-baking',
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert.equal(report.relevance, 'minor');
|
||||
assert.equal(report.category, 'other');
|
||||
});
|
||||
|
||||
test('a GKG address written with http is the https one the events carry', () => {
|
||||
const plain = ARTICLE.replace('https:', 'http:');
|
||||
const [event] = parseEvents(eventRow({ id: 43, url: plain }));
|
||||
assert.equal(event.url, ARTICLE);
|
||||
const titles = parseTitles(
|
||||
gkgRow(plain, 'Russian drones attack Kharkiv overnight, three killed'),
|
||||
new Set([event.url]),
|
||||
);
|
||||
assert.equal(
|
||||
titles.get(ARTICLE),
|
||||
'Russian drones attack Kharkiv overnight, three killed',
|
||||
);
|
||||
});
|
||||
|
||||
test('a corrupt or oversized export is an UpstreamError, never a RangeError', async () => {
|
||||
const zip = zipTable(eventRow({ id: 44, url: ARTICLE }));
|
||||
const view = new DataView(zip.buffer);
|
||||
// The central directory's local header offset, pointing past the end.
|
||||
const dir = view.getUint32(zip.length - 22 + 16, true);
|
||||
const far = zip.slice();
|
||||
new DataView(far.buffer).setUint32(dir + 42, zip.length - 10, true);
|
||||
await assert.rejects(unzipExport(far), { name: 'UpstreamError' });
|
||||
// The end record cut short.
|
||||
await assert.rejects(unzipExport(zip.subarray(0, zip.length - 30)), {
|
||||
name: 'UpstreamError',
|
||||
});
|
||||
// Unzipped past the table's cap.
|
||||
await assert.rejects(unzipExport(zip, 16), { name: 'UpstreamError' });
|
||||
assert.equal((await unzipExport(zip)).length > 16, true);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
// Test fixtures: GDELT 2.0 export rows and their zip, as the export files
|
||||
// carry them (61 tab-separated columns, one zipped table).
|
||||
import { deflateRawSync } from 'node:zlib';
|
||||
|
||||
/** One events row; only the columns the reader looks at are filled. */
|
||||
export function eventRow({
|
||||
id,
|
||||
code = '190',
|
||||
quad = '4',
|
||||
mentions = 1,
|
||||
geoType = '4',
|
||||
place = "Kharkiv, Kharkivs'ka Oblast', Ukraine",
|
||||
lat = 49.9935,
|
||||
lon = 36.2304,
|
||||
added = '20261105114500',
|
||||
url,
|
||||
}) {
|
||||
const f = new Array(61).fill('');
|
||||
f[0] = String(id);
|
||||
f[26] = code;
|
||||
f[28] = code.slice(0, 2);
|
||||
f[29] = quad;
|
||||
f[31] = String(mentions);
|
||||
f[51] = geoType;
|
||||
f[52] = place;
|
||||
f[56] = String(lat);
|
||||
f[57] = String(lon);
|
||||
f[59] = added;
|
||||
f[60] = url;
|
||||
return f.join('\t');
|
||||
}
|
||||
|
||||
/** One GKG 2.1 row: an article's address and its page title (27 columns). */
|
||||
export function gkgRow(url, title) {
|
||||
const f = new Array(27).fill('');
|
||||
f[4] = url;
|
||||
f[26] = `<PAGE_PRECISEPUBTIMESTAMP>20261105114500</PAGE_PRECISEPUBTIMESTAMP><PAGE_TITLE>${title}</PAGE_TITLE>`;
|
||||
return f.join('\t');
|
||||
}
|
||||
|
||||
/** A single-entry zip of `text`, deflated, sizes in the central directory. */
|
||||
export function zipTable(text, name = 'table.export.CSV') {
|
||||
const data = Buffer.from(text, 'utf8');
|
||||
const packed = deflateRawSync(data);
|
||||
const nameBytes = Buffer.from(name);
|
||||
const local = Buffer.alloc(30);
|
||||
local.writeUInt32LE(0x04034b50, 0);
|
||||
local.writeUInt16LE(20, 4);
|
||||
local.writeUInt16LE(0x08, 6); // sizes follow the data (left 0 here)
|
||||
local.writeUInt16LE(8, 8);
|
||||
local.writeUInt16LE(nameBytes.length, 26);
|
||||
const dir = Buffer.alloc(46);
|
||||
dir.writeUInt32LE(0x02014b50, 0);
|
||||
dir.writeUInt16LE(20, 4);
|
||||
dir.writeUInt16LE(20, 6);
|
||||
dir.writeUInt16LE(0x08, 8);
|
||||
dir.writeUInt16LE(8, 10);
|
||||
dir.writeUInt32LE(packed.length, 20);
|
||||
dir.writeUInt32LE(data.length, 24);
|
||||
dir.writeUInt16LE(nameBytes.length, 28);
|
||||
dir.writeUInt32LE(0, 42);
|
||||
const dirAt = local.length + nameBytes.length + packed.length;
|
||||
const end = Buffer.alloc(22);
|
||||
end.writeUInt32LE(0x06054b50, 0);
|
||||
end.writeUInt16LE(1, 8);
|
||||
end.writeUInt16LE(1, 10);
|
||||
end.writeUInt32LE(dir.length + nameBytes.length, 12);
|
||||
end.writeUInt32LE(dirAt, 16);
|
||||
return new Uint8Array(
|
||||
Buffer.concat([local, nameBytes, packed, dir, nameBytes, end]),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,851 @@
|
||||
// The News layer's routes answered by Core itself, when no news server is
|
||||
// configured (GEV_NEWS_ORIGIN). Conflicts: the day's GDELT events (no key;
|
||||
// gdeltEvents.js), one report per article at GDELT's place for it, and with
|
||||
// LIVEUAMAP_API_KEY Liveuamap's mapped items too. Local: World News API
|
||||
// (WORLDNEWSAPI_KEY) by the country at the view centre; without its key the
|
||||
// section answers 503 {error: 'no_key', key} and reads nothing, and past
|
||||
// its day's reads (WORLDNEWSAPI_DAILY_READS) 503 {error: 'key_quota', key}.
|
||||
// Categories, relevance and severity are the shared rules (categorize.js);
|
||||
// a news server may replace these readings with its own.
|
||||
//
|
||||
// A read that has something to serve is not waited for: what was read is
|
||||
// served while it is read again. A section that cannot be read answers 502,
|
||||
// never an empty list.
|
||||
//
|
||||
// The answers are the same /api/news contract a news server serves
|
||||
// (src/layers/news/records.js reads both).
|
||||
|
||||
import {
|
||||
CATEGORY_ORDER,
|
||||
categorizeText,
|
||||
relevanceOf,
|
||||
severityByRules,
|
||||
} from './categorize.js';
|
||||
import {
|
||||
areaAt,
|
||||
countryAnchor,
|
||||
countryByCode,
|
||||
countryPolygons,
|
||||
} from './areas.js';
|
||||
import {
|
||||
eventReports,
|
||||
fetchLatestExport,
|
||||
eventTitle,
|
||||
exportAt,
|
||||
exportsOfDay,
|
||||
fetchExport,
|
||||
fetchTitles,
|
||||
gkgUrl,
|
||||
storyKey,
|
||||
} from './gdeltEvents.js';
|
||||
import { fetchLiveuamap, fetchWorldNews, UpstreamError } from './upstream.js';
|
||||
|
||||
/** The keys each section reads with, by the key registry's id. */
|
||||
export const NEWS_KEYS = Object.freeze({
|
||||
liveuamap: 'LIVEUAMAP_API_KEY',
|
||||
worldnewsapi: 'WORLDNEWSAPI_KEY',
|
||||
});
|
||||
|
||||
/** The one hotspot: the world, its reports drawn as markers at every zoom. */
|
||||
export const WORLDWIDE_HOTSPOT = 'worldwide';
|
||||
const HOUR_MS = 3_600_000;
|
||||
const DAY_MS = 24 * HOUR_MS;
|
||||
// GDELT publishes an export every 15 minutes and lists the newest in
|
||||
// lastupdate.txt: the day's missing ones up to it are read every 5 minutes,
|
||||
// newest first, a dozen a read (the first day's 96 fill in over the next
|
||||
// reads, a few seconds apart, while each reads something new). One listed
|
||||
// but missing (404) an hour and a half after its time is taken as never
|
||||
// published; one that fails GDELT_EXPORT_TRIES times, or cannot be read at
|
||||
// all, is given up on, so one bad file never holds the rest back. The first
|
||||
// answer waits at most GDELT_FIRST_MS for the first exports.
|
||||
const GDELT_FRESH_MS = 5 * 60_000;
|
||||
const GDELT_EXPORTS_PER_READ = 12;
|
||||
const GDELT_PARALLEL = 4;
|
||||
const GDELT_NEXT_MS = 2_000;
|
||||
const GDELT_GONE_MS = 90 * 60_000;
|
||||
const GDELT_EXPORT_TRIES = 3;
|
||||
const GDELT_FIRST_MS = 12_000;
|
||||
const MAX_GDELT_REPORTS = 5000;
|
||||
// Events kept of one story, to place it by.
|
||||
const MAX_STORY_EVENTS = 200;
|
||||
// Page titles are read from the GKG exports of the newest three hours: one
|
||||
// runs to 6 MB, so the day's 96 are not fetched. A file that fails is tried
|
||||
// with the next reads, a few times.
|
||||
const GDELT_TITLED_EXPORTS = 12;
|
||||
const GKG_TRIES = 3;
|
||||
const LIVE_FRESH_MS = 5 * 60_000;
|
||||
const COUNTRY_FRESH_MS = 30 * 60_000;
|
||||
// A failed read is not tried again for a minute; what was read before is
|
||||
// served meanwhile, up to these ages (GDELT's: since an export last came in).
|
||||
const RETRY_MS = 60_000;
|
||||
const GDELT_STALE_MS = 2 * HOUR_MS;
|
||||
const LIVE_STALE_MS = HOUR_MS;
|
||||
const COUNTRY_STALE_MS = 6 * HOUR_MS;
|
||||
const MAX_COUNTRIES = 64;
|
||||
// World News API reads a UTC day, across all countries: each is about 1.5
|
||||
// of the free plan's 50 daily points.
|
||||
const DEFAULT_DAILY_READS = 30;
|
||||
const DEFAULT_LIMIT = 50;
|
||||
const EMPTY_OUTLINE = Object.freeze({
|
||||
polygons: [],
|
||||
borders: [],
|
||||
disputed: [],
|
||||
area: null,
|
||||
});
|
||||
// World News API's categories, where the rules find none of their own.
|
||||
const WORLD_NEWS_CATEGORY = { politics: 'political', business: 'economy' };
|
||||
|
||||
const list = (value) =>
|
||||
value ? new Set(value.split(',').filter(Boolean)) : null;
|
||||
|
||||
/** The rules' reading of a headline: category, relevance and severity. */
|
||||
function readTitle(title, { dedicated, fallback = null }) {
|
||||
const read = categorizeText({ title });
|
||||
const category =
|
||||
read.category === 'other' && fallback ? fallback : read.category;
|
||||
return {
|
||||
category,
|
||||
relevance: relevanceOf({ category, tags: read.tags, title, dedicated }),
|
||||
severity: severityByRules({ category, tags: read.tags, title }),
|
||||
};
|
||||
}
|
||||
|
||||
/** A one-report event carrying the report's picture, for its card. */
|
||||
function pictureEvent(id, report, image, video, publisher) {
|
||||
return {
|
||||
id,
|
||||
status: 'single',
|
||||
size: 1,
|
||||
reports: 1,
|
||||
severity: report.severity,
|
||||
firstAt: report.publishedAt,
|
||||
lastAt: report.publishedAt,
|
||||
sides: [],
|
||||
image: { url: image, publisher, ...(video ? { video } : {}) },
|
||||
};
|
||||
}
|
||||
|
||||
const media = ({ image, video }) =>
|
||||
video ? ['video'] : image ? ['photo'] : [];
|
||||
|
||||
/** A Liveuamap item as a report of the Conflicts section. */
|
||||
export function liveuamapReport(item) {
|
||||
const id = `lum-${item.id}`;
|
||||
const report = {
|
||||
id,
|
||||
hotspotId: WORLDWIDE_HOTSPOT,
|
||||
hotspotIds: [WORLDWIDE_HOTSPOT],
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
publisher: 'Liveuamap',
|
||||
source: { name: 'Liveuamap' },
|
||||
kind: 'liveuamap',
|
||||
side: 'unknown',
|
||||
// A conflict map: everything it posts is at least related.
|
||||
...readTitle(item.title, { dedicated: true }),
|
||||
severityBy: 'rules',
|
||||
publishedAt: item.publishedAt,
|
||||
geo: {
|
||||
lat: item.lat,
|
||||
lon: item.lon,
|
||||
place: item.place,
|
||||
precision: 'city',
|
||||
confidence: 1,
|
||||
mappable: true,
|
||||
method: 'source',
|
||||
},
|
||||
media: media(item),
|
||||
};
|
||||
if (item.image)
|
||||
report.event = pictureEvent(
|
||||
id,
|
||||
report,
|
||||
item.image,
|
||||
item.video,
|
||||
'Liveuamap',
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* A GDELT place named by its town and the area Natural Earth has there (a
|
||||
* disputed area as itself, else the country): GDELT's own names follow old
|
||||
* FIPS regions ("Bureij, Yerushalayim, Israel" for a camp in Gaza, "Abuja,
|
||||
* …, Niger" for Nigeria's capital). Its point stays. A town a disputed
|
||||
* area is named after ("Jerusalem" in East Jerusalem) lies on both sides
|
||||
* of its line: GDELT puts the whole city at one point, so it is named as
|
||||
* the town alone.
|
||||
*/
|
||||
export function gdeltPlace(geo) {
|
||||
const area = areaAt(geo.lat, geo.lon);
|
||||
const town = String(geo.place ?? '')
|
||||
.split(',')[0]
|
||||
.trim();
|
||||
const straddles =
|
||||
area?.kind === 'disputed' &&
|
||||
town &&
|
||||
area.name.toLowerCase().includes(town.toLowerCase());
|
||||
const place =
|
||||
geo.precision === 'country' && area?.kind === 'country'
|
||||
? area.name
|
||||
: straddles
|
||||
? town
|
||||
: [town, area?.name].filter(Boolean).join(', ');
|
||||
return { ...geo, place: place || null };
|
||||
}
|
||||
|
||||
/**
|
||||
* A World News API article as a report of the Local section: placed in its
|
||||
* publisher's country alone, at a point inside it.
|
||||
*/
|
||||
export function worldNewsReport(article, country) {
|
||||
const id = `wna-${article.id}`;
|
||||
const anchor = countryAnchor(country);
|
||||
const report = {
|
||||
id,
|
||||
hotspotId: null,
|
||||
hotspotIds: [],
|
||||
title: article.title,
|
||||
url: article.url,
|
||||
publisher: article.publisher,
|
||||
// World News API's free plan asks to be credited where its news shows.
|
||||
source: { name: article.publisher, attribution: 'World News API' },
|
||||
kind: 'worldnewsapi',
|
||||
side: 'unknown',
|
||||
...readTitle(article.title, {
|
||||
dedicated: false,
|
||||
fallback: WORLD_NEWS_CATEGORY[article.category] || null,
|
||||
}),
|
||||
// Local news: what the country's own outlets publish.
|
||||
relevance: 'minor',
|
||||
severityBy: 'rules',
|
||||
publishedAt: article.publishedAt,
|
||||
geo: {
|
||||
lat: anchor.lat,
|
||||
lon: anchor.lon,
|
||||
place: country.name,
|
||||
precision: 'country',
|
||||
confidence: 1,
|
||||
mappable: true,
|
||||
method: 'source',
|
||||
},
|
||||
media: media(article),
|
||||
};
|
||||
if (article.image)
|
||||
report.event = pictureEvent(
|
||||
id,
|
||||
report,
|
||||
article.image,
|
||||
article.video,
|
||||
article.publisher,
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
const AT = /^(-?\d{1,2}(?:\.\d{1,2})?),(-?\d{1,3}(?:\.\d{1,2})?)$/;
|
||||
function point(value) {
|
||||
const match = AT.exec(String(value ?? ''));
|
||||
if (!match) return null;
|
||||
const [lat, lon] = [Number(match[1]), Number(match[2])];
|
||||
return Math.abs(lat) <= 90 && Math.abs(lon) <= 180 ? { lat, lon } : null;
|
||||
}
|
||||
|
||||
const noKey = (key) => ({ status: 503, body: { error: 'no_key', key } });
|
||||
const rejectedKey = (key) => ({
|
||||
status: 503,
|
||||
body: { error: 'key_rejected', key },
|
||||
});
|
||||
const spentKey = (key) => ({ status: 503, body: { error: 'key_quota', key } });
|
||||
|
||||
/**
|
||||
* The keyed news routes. `answer(path, params)` resolves to `{status, body}`
|
||||
* for `/hotspots`, `/feed` and `/outline`, whose queries the proxy has
|
||||
* already checked against the client's allowlist. `later(run, ms)` runs
|
||||
* the next GDELT read of a day still filling in (an unref'd timer).
|
||||
* @param {{env?: () => object, fetchImpl?: typeof fetch, now?: () => number, log?: (message: string) => void, later?: (run: () => void, ms: number) => void}} [options]
|
||||
*/
|
||||
export function createKeyedNews({
|
||||
env = () => process.env,
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
log = (message) => console.warn(message),
|
||||
later = (run, ms) => setTimeout(run, ms).unref(),
|
||||
} = {}) {
|
||||
const keyOf = (id) => String(env()[NEWS_KEYS[id]] ?? '').trim();
|
||||
const inFlight = new Map();
|
||||
// GDELT's stories of the last day (their events and report, by
|
||||
// storyKey), and the exports read.
|
||||
const gdelt = {
|
||||
stories: new Map(),
|
||||
reports: new Map(),
|
||||
// Page titles by article address, of the stories kept.
|
||||
titles: new Map(),
|
||||
read: new Set(),
|
||||
// Exports still missing well after their time: not looked for again.
|
||||
gone: new Set(),
|
||||
// Failed reads of each export not read yet.
|
||||
failures: new Map(),
|
||||
// Exports read among the newest whose GKG titles are not in yet, with
|
||||
// the tries made: `{tries, triedAt}`.
|
||||
untitled: new Map(),
|
||||
titling: null,
|
||||
// When a read last brought an export in, with nothing failing.
|
||||
readAt: null,
|
||||
checkedAt: null,
|
||||
failedAt: null,
|
||||
};
|
||||
const live = new Map();
|
||||
const countries = new Map();
|
||||
// The keys refused so far, by section: logged once each.
|
||||
const refused = new Set();
|
||||
// World News API reads of the UTC day; `spent` once it answered 402.
|
||||
const budget = { day: null, reads: 0, spent: false, logged: false };
|
||||
// The area (disputed area or country) each placed Conflicts report lies
|
||||
// in, looked up once.
|
||||
const reportAreas = new WeakMap();
|
||||
const reportArea = (report) => {
|
||||
if (!reportAreas.has(report))
|
||||
reportAreas.set(
|
||||
report,
|
||||
report.geo
|
||||
? (areaAt(report.geo.lat, report.geo.lon)?.key ?? null)
|
||||
: null,
|
||||
);
|
||||
return reportAreas.get(report);
|
||||
};
|
||||
|
||||
/** A failed read in the log: a refused key once, never the key itself. */
|
||||
function logFailure(section, what, key, error) {
|
||||
// The day's reads spent: said once, by spendRead.
|
||||
if (error.quota && error.status === null) return;
|
||||
if (error.rejected) {
|
||||
const id = `${section}\0${key}`;
|
||||
if (refused.has(id)) return;
|
||||
refused.add(id);
|
||||
log(
|
||||
`[news] ${NEWS_KEYS[section]} was refused (${error.status})${section === 'liveuamap' ? "; Conflicts are GDELT's alone" : ''}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
log(`[news] ${what} read failed: ${error.message}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* A cached read (`{key, value, readAt, failedAt, error}`, kept in `store`
|
||||
* under `id`): served as it is within `freshMs`; past that read again,
|
||||
* and served meanwhile up to `staleMs`, so only a read with nothing to
|
||||
* serve is waited for. One read at a time per key; a failed read is tried
|
||||
* again after RETRY_MS. A new key drops what the old one read.
|
||||
*/
|
||||
async function cached(
|
||||
store,
|
||||
id,
|
||||
{ section, key, freshMs, staleMs, read, what },
|
||||
) {
|
||||
const at = now();
|
||||
const stored = store.get(id);
|
||||
const prior = stored?.key === key ? stored : { key };
|
||||
if (prior.value && at - prior.readAt < freshMs) return prior;
|
||||
if (prior.failedAt && at - prior.failedAt < RETRY_MS) return prior;
|
||||
const flight = `${id}\0${key}`;
|
||||
if (!inFlight.has(flight))
|
||||
inFlight.set(
|
||||
flight,
|
||||
read(key)
|
||||
.then(
|
||||
(value) => ({ key, value, readAt: now() }),
|
||||
(error) => {
|
||||
logFailure(section, what, key, error);
|
||||
return { ...prior, failedAt: now(), error };
|
||||
},
|
||||
)
|
||||
.then((entry) => {
|
||||
// A read with a key replaced meanwhile is dropped.
|
||||
if (keyOf(section) === key) keep(store, id, entry);
|
||||
return entry;
|
||||
})
|
||||
.finally(() => inFlight.delete(flight)),
|
||||
);
|
||||
if (prior.value && at - prior.readAt < staleMs) return prior;
|
||||
return inFlight.get(flight);
|
||||
}
|
||||
|
||||
/** An entry stored as the most recently used (the countries are capped). */
|
||||
function keep(store, id, entry) {
|
||||
store.delete(id);
|
||||
store.set(id, entry);
|
||||
while (store.size > MAX_COUNTRIES) store.delete(store.keys().next().value);
|
||||
}
|
||||
|
||||
/** An entry's reports while they may be served, else why there are none. */
|
||||
function served(entry, staleMs, id) {
|
||||
if (entry.value && now() - entry.readAt < staleMs)
|
||||
return { reports: entry.value, readAt: entry.readAt };
|
||||
if (entry.error?.rejected) return { rejected: id };
|
||||
if (entry.error?.quota) return { quota: id };
|
||||
return { failed: true };
|
||||
}
|
||||
|
||||
async function liveuamapReports() {
|
||||
const key = keyOf('liveuamap');
|
||||
if (!key) return { missing: 'liveuamap' };
|
||||
const entry = await cached(live, 'liveuamap', {
|
||||
section: 'liveuamap',
|
||||
key,
|
||||
freshMs: LIVE_FRESH_MS,
|
||||
staleMs: LIVE_STALE_MS,
|
||||
what: 'liveuamap',
|
||||
read: async (k) =>
|
||||
(await fetchLiveuamap({ key: k, fetchImpl, now: now() }))
|
||||
.map(liveuamapReport)
|
||||
.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)),
|
||||
});
|
||||
return served(entry, LIVE_STALE_MS, 'liveuamap');
|
||||
}
|
||||
|
||||
/**
|
||||
* A story placed (again) from all its events and the titles in: an
|
||||
* agency story run by several outlets is one report.
|
||||
*/
|
||||
function placeStory(key) {
|
||||
const [report] = eventReports(gdelt.stories.get(key), {
|
||||
titleOf: (url) => gdelt.titles.get(url) || null,
|
||||
});
|
||||
if (!report) return;
|
||||
const { eventWords, ...rest } = report;
|
||||
const geo = gdeltPlace(report.geo);
|
||||
gdelt.reports.set(key, {
|
||||
...rest,
|
||||
...(eventWords ? { title: eventTitle(eventWords, geo.place) } : {}),
|
||||
hotspotId: WORLDWIDE_HOTSPOT,
|
||||
hotspotIds: [WORLDWIDE_HOTSPOT],
|
||||
geo,
|
||||
locations: report.locations.map(gdeltPlace),
|
||||
});
|
||||
}
|
||||
|
||||
/** An export's events into their stories, each placed again. */
|
||||
function addEvents(events) {
|
||||
const changed = new Set();
|
||||
for (const event of events) {
|
||||
const key = storyKey(event);
|
||||
const list = gdelt.stories.get(key) || [];
|
||||
if (list.length < MAX_STORY_EVENTS) list.push(event);
|
||||
gdelt.stories.set(key, list);
|
||||
changed.add(key);
|
||||
}
|
||||
for (const key of changed) placeStory(key);
|
||||
}
|
||||
|
||||
/** The stories a GKG export titled, re-titled. */
|
||||
function addTitles(titles) {
|
||||
for (const [key, list] of gdelt.stories) {
|
||||
const titled = list.filter((event) => titles.has(event.url));
|
||||
if (!titled.length) continue;
|
||||
for (const { url } of titled) gdelt.titles.set(url, titles.get(url));
|
||||
placeStory(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The page titles of the newest exports read, from their GKG exports one
|
||||
* at a time (each runs to 18 MB unzipped), in the background: no answer
|
||||
* waits for them, and a story is re-titled as its title comes in. A file
|
||||
* that fails or is not up yet only leaves address words: it is tried
|
||||
* again with a later read while among the newest, up to GKG_TRIES times.
|
||||
*/
|
||||
function readTitles() {
|
||||
gdelt.titling ??= (async () => {
|
||||
let failed = null;
|
||||
for (const [url, tried] of [...gdelt.untitled]) {
|
||||
if (!gdelt.untitled.has(url)) continue;
|
||||
if (tried.triedAt !== null && now() - tried.triedAt < GDELT_FRESH_MS)
|
||||
continue;
|
||||
tried.tries += 1;
|
||||
tried.triedAt = now();
|
||||
const wanted = new Set();
|
||||
for (const list of gdelt.stories.values())
|
||||
for (const event of list) wanted.add(event.url);
|
||||
let titles = null;
|
||||
try {
|
||||
titles = await fetchTitles(gkgUrl(url), wanted, { fetchImpl });
|
||||
} catch (error) {
|
||||
failed ??= error;
|
||||
}
|
||||
if (titles || tried.tries >= GKG_TRIES) gdelt.untitled.delete(url);
|
||||
if (titles?.size) addTitles(titles);
|
||||
}
|
||||
if (failed) log(`[news] gdelt titles read failed: ${failed.message}`);
|
||||
})()
|
||||
.catch((error) => log(`[news] gdelt titles failed: ${error.message}`))
|
||||
.finally(() => {
|
||||
gdelt.titling = null;
|
||||
});
|
||||
return gdelt.titling;
|
||||
}
|
||||
|
||||
/**
|
||||
* Up to a dozen of the day's exports not read yet, newest first; whether
|
||||
* to carry on at once (the round brought one in, and more are missing).
|
||||
* One not up yet is looked for again; one still missing GDELT_GONE_MS
|
||||
* after its time is not.
|
||||
*/
|
||||
async function readExports() {
|
||||
const at = now();
|
||||
const day = exportsOfDay(at);
|
||||
// Up to the newest GDELT lists; all of the day when its list fails.
|
||||
const latest = await fetchLatestExport({ fetchImpl });
|
||||
const listed = (url) => latest === null || exportAt(url) <= latest;
|
||||
const newest = new Set(day.slice(0, GDELT_TITLED_EXPORTS));
|
||||
const wanted = (url) =>
|
||||
listed(url) && !gdelt.read.has(url) && !gdelt.gone.has(url);
|
||||
const missing = day.filter(wanted).slice(0, GDELT_EXPORTS_PER_READ);
|
||||
let failed = null;
|
||||
let added = 0;
|
||||
const failing = [];
|
||||
for (let i = 0; i < missing.length; i += GDELT_PARALLEL)
|
||||
await Promise.all(
|
||||
missing.slice(i, i + GDELT_PARALLEL).map(async (url) => {
|
||||
let events;
|
||||
try {
|
||||
events = await fetchExport(url, { fetchImpl });
|
||||
} catch (error) {
|
||||
failed = error;
|
||||
// An unreadable file stays so.
|
||||
if (error.unreadable) gdelt.gone.add(url);
|
||||
else failing.push(url);
|
||||
return;
|
||||
}
|
||||
if (!events) {
|
||||
if (at - exportAt(url) >= GDELT_GONE_MS) gdelt.gone.add(url);
|
||||
return;
|
||||
}
|
||||
// The events count now; the titles follow (readTitles).
|
||||
gdelt.read.add(url);
|
||||
gdelt.failures.delete(url);
|
||||
added += 1;
|
||||
gdelt.readAt = now();
|
||||
addEvents(events);
|
||||
if (newest.has(url))
|
||||
gdelt.untitled.set(url, { tries: 0, triedAt: null });
|
||||
}),
|
||||
);
|
||||
if (failed) log(`[news] gdelt read failed: ${failed.message}`);
|
||||
// A file that fails while others read is that file's fault, and gets a
|
||||
// few tries; when none read, GDELT itself is down and no file is blamed.
|
||||
if (added)
|
||||
for (const url of failing) {
|
||||
const tries = (gdelt.failures.get(url) || 0) + 1;
|
||||
gdelt.failures.set(url, tries);
|
||||
if (tries >= GDELT_EXPORT_TRIES) gdelt.gone.add(url);
|
||||
}
|
||||
// Nothing read and a failure: tried again after RETRY_MS, not after
|
||||
// GDELT_FRESH_MS. Something read: the round counts.
|
||||
if (failed && !added) gdelt.failedAt = now();
|
||||
else {
|
||||
gdelt.checkedAt = now();
|
||||
gdelt.failedAt = null;
|
||||
}
|
||||
// Only the day is kept, and titled while among the newest.
|
||||
const kept = new Set(day);
|
||||
for (const set of [gdelt.read, gdelt.gone])
|
||||
for (const url of set) if (!kept.has(url)) set.delete(url);
|
||||
for (const url of gdelt.failures.keys())
|
||||
if (!kept.has(url)) gdelt.failures.delete(url);
|
||||
for (const url of gdelt.untitled.keys())
|
||||
if (!newest.has(url)) gdelt.untitled.delete(url);
|
||||
return (
|
||||
added > 0 && missing.length === GDELT_EXPORTS_PER_READ && day.some(wanted)
|
||||
);
|
||||
}
|
||||
|
||||
/** GDELT's reports of the last day, newest first, or `{failed}`. */
|
||||
async function gdeltReports() {
|
||||
const at = now();
|
||||
const fresh = gdelt.checkedAt && at - gdelt.checkedAt < GDELT_FRESH_MS;
|
||||
const resting = gdelt.failedAt && at - gdelt.failedAt < RETRY_MS;
|
||||
if (!fresh && !resting && !inFlight.has('gdelt')) {
|
||||
const round = readExports()
|
||||
.then(
|
||||
(more) => {
|
||||
// More of the day to fill in: carry on shortly, without holding
|
||||
// the answer.
|
||||
if (!more) return;
|
||||
gdelt.checkedAt = null;
|
||||
later(() => void gdeltReports(), GDELT_NEXT_MS);
|
||||
},
|
||||
(error) => {
|
||||
log(`[news] gdelt read failed: ${error.message}`);
|
||||
gdelt.failedAt = now();
|
||||
},
|
||||
)
|
||||
.finally(() => {
|
||||
inFlight.delete('gdelt');
|
||||
void readTitles();
|
||||
});
|
||||
inFlight.set('gdelt', round);
|
||||
}
|
||||
// Only a read with nothing (recent) to serve is waited for: the
|
||||
// newest exports' events, not their titles.
|
||||
const current = () =>
|
||||
gdelt.readAt !== null && now() - gdelt.readAt < GDELT_STALE_MS;
|
||||
if (!current() && inFlight.has('gdelt')) {
|
||||
// At most GDELT_FIRST_MS: a slow GDELT must not outlast the browser's
|
||||
// own timeout; what has come in by then is served.
|
||||
let timer;
|
||||
await Promise.race([
|
||||
inFlight.get('gdelt'),
|
||||
new Promise((resolve) => {
|
||||
timer = setTimeout(resolve, GDELT_FIRST_MS);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
clearTimeout(timer);
|
||||
}
|
||||
if (!current()) return { failed: true };
|
||||
const dayAgo = new Date(now() - DAY_MS).toISOString();
|
||||
const reports = [...gdelt.reports.values()]
|
||||
.filter((r) => r.publishedAt >= dayAgo)
|
||||
.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt))
|
||||
.slice(0, MAX_GDELT_REPORTS);
|
||||
if (reports.length !== gdelt.reports.size) {
|
||||
const kept = new Set(reports);
|
||||
for (const [key, report] of gdelt.reports)
|
||||
if (!kept.has(report)) {
|
||||
for (const event of gdelt.stories.get(key) || [])
|
||||
gdelt.titles.delete(event.url);
|
||||
gdelt.reports.delete(key);
|
||||
gdelt.stories.delete(key);
|
||||
}
|
||||
}
|
||||
return { reports, readAt: gdelt.readAt };
|
||||
}
|
||||
|
||||
/**
|
||||
* The Conflicts section's reports: GDELT's, with Liveuamap's when its key
|
||||
* is set. Liveuamap failing (or refusing its key) leaves GDELT's.
|
||||
*/
|
||||
async function conflictReports() {
|
||||
const [articles, items] = await Promise.all([
|
||||
gdeltReports(),
|
||||
keyOf('liveuamap') ? liveuamapReports() : {},
|
||||
]);
|
||||
if (!articles.reports && !items.reports) return { failed: true };
|
||||
return {
|
||||
reports: [...(articles.reports || []), ...(items.reports || [])].sort(
|
||||
(a, b) => b.publishedAt.localeCompare(a.publishedAt),
|
||||
),
|
||||
readAt: Math.max(articles.readAt || 0, items.readAt || 0),
|
||||
};
|
||||
}
|
||||
|
||||
const dailyReads = () => {
|
||||
const raw = String(env().WORLDNEWSAPI_DAILY_READS ?? '').trim();
|
||||
const reads = raw ? Number(raw) : NaN;
|
||||
return Number.isInteger(reads) && reads >= 0 ? reads : DEFAULT_DAILY_READS;
|
||||
};
|
||||
|
||||
/** Whether a World News API read is left today (UTC); counts it if so. */
|
||||
function spendRead() {
|
||||
const day = new Date(now()).toISOString().slice(0, 10);
|
||||
if (budget.day !== day)
|
||||
Object.assign(budget, { day, reads: 0, spent: false, logged: false });
|
||||
if (!budget.spent && budget.reads < dailyReads()) {
|
||||
budget.reads += 1;
|
||||
return true;
|
||||
}
|
||||
if (!budget.logged)
|
||||
log(
|
||||
"[news] World News API's reads for today are spent; Local reads again after 00:00 UTC",
|
||||
);
|
||||
budget.logged = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
async function countryReports(country) {
|
||||
const key = keyOf('worldnewsapi');
|
||||
if (!key) return { missing: 'worldnewsapi' };
|
||||
const stored = countries.get(country.code);
|
||||
if (stored) keep(countries, country.code, stored);
|
||||
const entry = await cached(countries, country.code, {
|
||||
section: 'worldnewsapi',
|
||||
key,
|
||||
freshMs: COUNTRY_FRESH_MS,
|
||||
staleMs: COUNTRY_STALE_MS,
|
||||
what: `worldnewsapi:${country.code}`,
|
||||
read: async (k) => {
|
||||
// Not asked: no status, unlike an answer.
|
||||
if (!spendRead())
|
||||
throw new UpstreamError("World News API's reads for today spent", {
|
||||
quota: true,
|
||||
});
|
||||
try {
|
||||
return (
|
||||
await fetchWorldNews({
|
||||
key: k,
|
||||
country: country.code,
|
||||
fetchImpl,
|
||||
now: now(),
|
||||
})
|
||||
).map((article) => worldNewsReport(article, country));
|
||||
} catch (error) {
|
||||
// Out of points: nothing more is asked today.
|
||||
if (error.status === 402) budget.spent = true;
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
return served(entry, COUNTRY_STALE_MS, 'worldnewsapi');
|
||||
}
|
||||
|
||||
const refusal = (read) =>
|
||||
read.missing
|
||||
? noKey(read.missing)
|
||||
: read.rejected
|
||||
? rejectedKey(read.rejected)
|
||||
: read.quota
|
||||
? spentKey(read.quota)
|
||||
: read.failed
|
||||
? { status: 502, body: { error: 'news_upstream_error' } }
|
||||
: null;
|
||||
|
||||
async function hotspots(params) {
|
||||
const read = await conflictReports();
|
||||
const refused = refusal(read);
|
||||
if (refused) return refused;
|
||||
const hours = Number(params.get('hours')) || 24;
|
||||
const since = new Date(now() - hours * HOUR_MS).toISOString();
|
||||
const shown = read.reports.filter((r) => r.publishedAt >= since);
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
hotspots: [
|
||||
{
|
||||
id: WORLDWIDE_HOTSPOT,
|
||||
name: 'Worldwide',
|
||||
kind: 'conflict',
|
||||
catchAll: true,
|
||||
bbox: [-180, -90, 180, 90],
|
||||
parties: [],
|
||||
developing: [],
|
||||
counts: {
|
||||
last24h: shown.length,
|
||||
mapped24h: shown.filter((r) => r.geo).length,
|
||||
prev24h: null,
|
||||
},
|
||||
latestAt: shown[0]?.publishedAt ?? null,
|
||||
},
|
||||
],
|
||||
categories: CATEGORY_ORDER,
|
||||
ingest: { lastRunAt: new Date(read.readAt).toISOString() },
|
||||
windowHours: hours,
|
||||
maxWindowHours: 24,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Reports filtered, newest first, and paged by `before`. */
|
||||
function page(reports, params) {
|
||||
const since = params.get('since');
|
||||
const before = params.get('before');
|
||||
const dayAgo = new Date(now() - DAY_MS).toISOString();
|
||||
const categories = list(params.get('category'));
|
||||
const relevance = list(params.get('relevance'));
|
||||
const hotspot = params.get('hotspot');
|
||||
const limit = Number(params.get('limit')) || DEFAULT_LIMIT;
|
||||
const kept = reports.filter(
|
||||
(r) =>
|
||||
(!hotspot || hotspot === 'all' || r.hotspotIds.includes(hotspot)) &&
|
||||
(!since || r.publishedAt >= since) &&
|
||||
(!before || r.publishedAt < before) &&
|
||||
(!params.has('today') || r.publishedAt >= dayAgo) &&
|
||||
(!params.has('geo') || r.geo) &&
|
||||
(!categories || categories.has(r.category)) &&
|
||||
(!relevance || relevance.has(r.relevance)),
|
||||
);
|
||||
const shown = kept.slice(0, limit);
|
||||
// A page ends between two times: the next asks for those `before` the
|
||||
// last, and a GDELT export's reports all share its quarter hour.
|
||||
while (
|
||||
shown.length < kept.length &&
|
||||
kept[shown.length].publishedAt === shown.at(-1).publishedAt
|
||||
)
|
||||
shown.push(kept[shown.length]);
|
||||
const events = params.has('events');
|
||||
return {
|
||||
reports: events ? shown : shown.map(({ event, ...report }) => report),
|
||||
nextBefore: kept.length > shown.length ? shown.at(-1).publishedAt : null,
|
||||
};
|
||||
}
|
||||
|
||||
async function feed(params) {
|
||||
const at = params.has('at') ? point(params.get('at')) : null;
|
||||
const area = at ? areaAt(at.lat, at.lon) : null;
|
||||
if (!area) {
|
||||
// Conflicts, or Local at sea (by distance, in the browser).
|
||||
const read = await conflictReports();
|
||||
return refusal(read) || { status: 200, body: page(read.reports, params) };
|
||||
}
|
||||
// Local in a country: its own outlets, with the Conflicts reports
|
||||
// placed inside it (those failing leave the outlets'). In a disputed
|
||||
// area no country's outlets speak for it: the Conflicts reports there
|
||||
// alone, so their failing is the answer's.
|
||||
const country = area.kind === 'country';
|
||||
const [national, conflicts] = await Promise.all([
|
||||
country ? countryReports(area.country) : { reports: [] },
|
||||
conflictReports(),
|
||||
]);
|
||||
const refused = refusal(national) || (!country && refusal(conflicts));
|
||||
if (refused) return refused;
|
||||
const inside = (conflicts.reports || []).filter(
|
||||
(r) => reportArea(r) === area.key,
|
||||
);
|
||||
const reports = [...national.reports, ...inside].sort((a, b) =>
|
||||
b.publishedAt.localeCompare(a.publishedAt),
|
||||
);
|
||||
return { status: 200, body: page(reports, params) };
|
||||
}
|
||||
|
||||
function outline(params) {
|
||||
if (params.has('at')) {
|
||||
const at = point(params.get('at'));
|
||||
const area = at ? areaAt(at.lat, at.lon) : null;
|
||||
if (!area) return { status: 200, body: EMPTY_OUTLINE };
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
...EMPTY_OUTLINE,
|
||||
polygons:
|
||||
area.kind === 'country'
|
||||
? countryPolygons(area.country)
|
||||
: area.polygons,
|
||||
area: {
|
||||
key: area.key,
|
||||
name: area.name,
|
||||
kind: area.kind,
|
||||
country: area.code,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
// A developing situation's country; the world has no outline.
|
||||
const country = countryByCode(
|
||||
String(params.get('country') ?? '').slice(0, 2),
|
||||
);
|
||||
return {
|
||||
status: 200,
|
||||
body: country
|
||||
? { ...EMPTY_OUTLINE, polygons: countryPolygons(country) }
|
||||
: EMPTY_OUTLINE,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
async answer(path, params) {
|
||||
if (path === '/hotspots') return hotspots(params);
|
||||
if (path === '/feed') return feed(params);
|
||||
if (path === '/outline') return outline(params);
|
||||
return { status: 404, body: { error: 'not_found' } };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,988 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
createKeyedNews,
|
||||
gdeltPlace,
|
||||
NEWS_KEYS,
|
||||
WORLDWIDE_HOTSPOT,
|
||||
} from './keyed.js';
|
||||
import { KEY_SETUP_KEYS } from '../../../src/keySetupCore.mjs';
|
||||
import {
|
||||
normalizeFeedPayload,
|
||||
normalizeHotspotsPayload,
|
||||
normalizeOutlinePayload,
|
||||
} from '../../../src/layers/news/records.js';
|
||||
import { exportsOfDay } from './gdeltEvents.js';
|
||||
import { eventRow, gkgRow, zipTable } from './gdeltFixtures.mjs';
|
||||
|
||||
const NOW = Date.parse('2026-11-05T12:00:00Z');
|
||||
const seconds = (iso) => Date.parse(iso) / 1000;
|
||||
|
||||
// Liveuamap's `a=mpts` answer, cut down: strings where it writes strings.
|
||||
const LIVEUAMAP = {
|
||||
success: true,
|
||||
key: 'redacted',
|
||||
venues: [
|
||||
{
|
||||
id: 101,
|
||||
name: 'Russian drone strike on Kharkiv kills 3, injures 12',
|
||||
lat: '49.99350',
|
||||
lng: '36.23040',
|
||||
location: 'Kharkiv',
|
||||
timestamp: seconds('2026-11-05T11:00:00Z'),
|
||||
link: 'https://liveuamap.com/en/2026/5-november-kharkiv',
|
||||
img_share: 'https://a.liveuamap.com/pics/101.jpg',
|
||||
video: 'https://a.liveuamap.com/clips/101.mp4',
|
||||
},
|
||||
{
|
||||
id: 102,
|
||||
name: 'Floods kill dozens in Astana',
|
||||
lat: '51.16940',
|
||||
lng: '71.44910',
|
||||
location: 'Astana',
|
||||
timestamp: seconds('2026-11-05T10:00:00Z'),
|
||||
link: 'https://asia.liveuamap.com/en/2026/5-november-astana',
|
||||
img_share: '',
|
||||
video: '',
|
||||
},
|
||||
{
|
||||
id: 103,
|
||||
name: 'Officials in "Kherson" report shelling',
|
||||
lat: '46.63540',
|
||||
lng: '32.61690',
|
||||
timestamp: seconds('2026-11-05T09:00:00Z'),
|
||||
link: 'https://liveuamap.com/en/2026/5-november-kherson',
|
||||
},
|
||||
// Without a point, a page or a headline: left out.
|
||||
{ id: 104, name: 'No point', timestamp: seconds('2026-11-05T09:00:00Z') },
|
||||
{
|
||||
id: 105,
|
||||
name: 'No page',
|
||||
lat: '1',
|
||||
lng: '1',
|
||||
timestamp: seconds('2026-11-05T09:00:00Z'),
|
||||
link: 'javascript:alert(1)',
|
||||
},
|
||||
{
|
||||
id: 106,
|
||||
name: '',
|
||||
lat: '1',
|
||||
lng: '1',
|
||||
timestamp: seconds('2026-11-05T09:00:00Z'),
|
||||
link: 'https://liveuamap.com/x',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// World News API's search-news answer, cut down.
|
||||
const WORLD_NEWS = {
|
||||
offset: 0,
|
||||
number: 100,
|
||||
available: 3,
|
||||
news: [
|
||||
{
|
||||
id: 9001,
|
||||
title: 'Visiting minister arrives in Astana for talks',
|
||||
url: 'https://www.daily.example/en/minister-astana-talks',
|
||||
image: 'https://www.daily.example/pics/visit.jpg',
|
||||
publish_date: '2026-11-05 08:30:00',
|
||||
language: 'en',
|
||||
category: 'politics',
|
||||
text: 'The whole article, never kept.',
|
||||
},
|
||||
{
|
||||
id: 9002,
|
||||
title: 'Two oil firms advance subsalt exploration',
|
||||
url: 'https://markets.example/economic/oil-firms-exploration',
|
||||
publish_date: '2026-11-05 07:00:00',
|
||||
language: 'ru',
|
||||
category: 'business',
|
||||
},
|
||||
// Sport is left out.
|
||||
{
|
||||
id: 9004,
|
||||
title: 'Astana Arlan win the cup final',
|
||||
url: 'https://sport.example/cup',
|
||||
publish_date: '2026-11-05 09:00:00',
|
||||
category: 'sports',
|
||||
},
|
||||
// The same story from another outlet is one.
|
||||
{
|
||||
id: 9003,
|
||||
title: 'TWO OIL FIRMS ADVANCE SUBSALT EXPLORATION',
|
||||
url: 'https://portal.test/oil-firms',
|
||||
publish_date: '2026-11-05 06:00:00',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// GDELT's exports around NOW: the newest not published yet, two with
|
||||
// events, the rest of the day empty.
|
||||
const KHARKIV =
|
||||
'https://www.wire.example/rubric-ato/3912-russian-drones-attack-kharkiv-overnight-three-killed.html';
|
||||
const RED_SEA =
|
||||
'https://news.example.com/2026/11/05/houthis-claim-missile-attack-on-tanker-in-red-sea';
|
||||
const COUNCIL =
|
||||
'https://news.example.com/2026/11/05/city-council-approves-new-budget-for-schools';
|
||||
const BARE = 'https://news.example.com/a/b/123456';
|
||||
const exportName = (stamp) =>
|
||||
`https://data.gdeltproject.org/gdeltv2/${stamp}.export.CSV.zip`;
|
||||
const EXPORTS = {
|
||||
[exportName('20261105120000')]: 404,
|
||||
[exportName('20261105114500')]: [
|
||||
eventRow({
|
||||
id: 1,
|
||||
code: '195',
|
||||
mentions: 3,
|
||||
url: KHARKIV,
|
||||
added: '20261105114500',
|
||||
}),
|
||||
eventRow({
|
||||
id: 2,
|
||||
code: '195',
|
||||
mentions: 8,
|
||||
geoType: '1',
|
||||
place: 'Yemen',
|
||||
lat: 15.5,
|
||||
lon: 47.5,
|
||||
url: RED_SEA,
|
||||
added: '20261105114500',
|
||||
}),
|
||||
eventRow({ id: 3, url: COUNCIL, added: '20261105114500' }),
|
||||
],
|
||||
[exportName('20261105113000')]: [
|
||||
eventRow({
|
||||
id: 4,
|
||||
geoType: '1',
|
||||
place: 'Ukraine',
|
||||
lat: 49,
|
||||
lon: 32,
|
||||
url: BARE,
|
||||
added: '20261105113000',
|
||||
}),
|
||||
// An article already read is not a second report.
|
||||
eventRow({ id: 5, code: '190', url: KHARKIV, added: '20261105113000' }),
|
||||
],
|
||||
};
|
||||
const EMPTY = zipTable('');
|
||||
// The page titles of the newest export's quarter hour.
|
||||
const GKG = {
|
||||
[exportName('20261105114500').replace('.export.CSV.zip', '.gkg.csv.zip')]: [
|
||||
gkgRow(
|
||||
KHARKIV,
|
||||
'Russian drones attack Kharkiv overnight, three killed - Wire',
|
||||
),
|
||||
],
|
||||
};
|
||||
|
||||
const host = (call) => call.url.host;
|
||||
|
||||
/**
|
||||
* A fake upstream: answers by host, counts calls, records URLs and headers.
|
||||
* An answer is a payload, a status, or a function giving a Response (one
|
||||
* held open). `unlisted` is the status of an export not in `exports` (an
|
||||
* empty table by default); `gkg` a status for every GKG export.
|
||||
*/
|
||||
function upstream(answers = {}) {
|
||||
const calls = [];
|
||||
const fetchImpl = async (url, init) => {
|
||||
const href = String(url);
|
||||
calls.push({ url: new URL(href), headers: init?.headers || {} });
|
||||
if (href.includes('gdeltproject.org')) {
|
||||
if (typeof answers.gdelt === 'number')
|
||||
return new Response('busy', { status: answers.gdelt });
|
||||
if (href.endsWith('/lastupdate.txt') && answers.lastupdate)
|
||||
return new Response(
|
||||
`95141 9197 http://data.gdeltproject.org/gdeltv2/${answers.lastupdate}.export.CSV.zip
|
||||
`,
|
||||
{ status: 200 },
|
||||
);
|
||||
if (answers.corrupt?.includes(href))
|
||||
return new Response('not a zip at all', { status: 200 });
|
||||
const gkg = href.endsWith('.gkg.csv.zip');
|
||||
const listed = answers.exports ?? EXPORTS;
|
||||
const answer = gkg
|
||||
? (answers.gkg ?? GKG[href])
|
||||
: Object.hasOwn(listed, href)
|
||||
? listed[href]
|
||||
: answers.unlisted;
|
||||
if (typeof answer === 'number')
|
||||
return new Response('', { status: answer });
|
||||
return new Response(answer ? zipTable(answer.join('\n')) : EMPTY, {
|
||||
status: 200,
|
||||
});
|
||||
}
|
||||
const answer = href.includes('liveuamap.com')
|
||||
? (answers.liveuamap ?? LIVEUAMAP)
|
||||
: (answers.worldnews ?? WORLD_NEWS);
|
||||
if (typeof answer === 'function') return answer();
|
||||
if (typeof answer === 'number')
|
||||
return new Response('{"message":"bad key ABC"}', { status: answer });
|
||||
return new Response(JSON.stringify(answer), { status: 200 });
|
||||
};
|
||||
return { calls, fetchImpl };
|
||||
}
|
||||
|
||||
function keyed({ env, answers, clock = { now: NOW }, logs = [] } = {}) {
|
||||
const fake = upstream(answers);
|
||||
// The next GDELT reads, run by settle rather than on a timer.
|
||||
const timers = [];
|
||||
const news = createKeyedNews({
|
||||
env: () =>
|
||||
env ?? { LIVEUAMAP_API_KEY: 'live-key', WORLDNEWSAPI_KEY: 'wna-key' },
|
||||
fetchImpl: fake.fetchImpl,
|
||||
now: () => clock.now,
|
||||
log: (line) => logs.push(line),
|
||||
later: (run) => timers.push(run),
|
||||
});
|
||||
const ask = (path, query = '') =>
|
||||
news.answer(path, new URLSearchParams(query));
|
||||
/**
|
||||
* Let the background reads finish (the rest of the day, the titles):
|
||||
* until nothing is asked for a while, or a bounded number of turns.
|
||||
*/
|
||||
async function settle() {
|
||||
let quiet = 0;
|
||||
for (let turn = 0; turn < 200 && quiet < 5; turn++) {
|
||||
const before = fake.calls.length;
|
||||
for (let i = 0; i < 20; i++) await new Promise((r) => setImmediate(r));
|
||||
await new Promise((r) => setTimeout(r, 1));
|
||||
for (const run of timers.splice(0)) run();
|
||||
quiet = fake.calls.length === before ? quiet + 1 : 0;
|
||||
}
|
||||
}
|
||||
return { ...fake, ask, clock, logs, settle, timers };
|
||||
}
|
||||
|
||||
const CONFLICTS =
|
||||
'hotspot=all&geo=1&today=1&events=1&relevance=core,related&limit=50';
|
||||
const gdeltCalls = (calls) =>
|
||||
calls.filter(
|
||||
(c) =>
|
||||
host(c) === 'data.gdeltproject.org' &&
|
||||
c.url.pathname.endsWith('.export.CSV.zip'),
|
||||
);
|
||||
const gkgCalls = (calls) =>
|
||||
calls.filter((c) => c.url.pathname.endsWith('.gkg.csv.zip'));
|
||||
|
||||
test('without any key Conflicts are the day of GDELT events, one worldwide hotspot', async () => {
|
||||
const { ask, calls, settle } = keyed({ env: {} });
|
||||
const answer = await ask('/hotspots');
|
||||
assert.equal(answer.status, 200);
|
||||
const payload = normalizeHotspotsPayload(answer.body);
|
||||
assert.deepEqual(
|
||||
payload.hotspots.map((h) => [h.id, h.catchAll]),
|
||||
[[WORLDWIDE_HOTSPOT, true]],
|
||||
);
|
||||
assert.equal(payload.hotspots[0].last24h, 4);
|
||||
assert.equal(payload.hotspots[0].mapped24h, 4);
|
||||
assert.equal(payload.maxWindowHours, 24);
|
||||
// The newest dozen first; the rest of the day after, in the background.
|
||||
assert.deepEqual(
|
||||
gdeltCalls(calls)
|
||||
.slice(0, 12)
|
||||
.map((c) => String(c.url)),
|
||||
exportsOfDay(NOW).slice(0, 12),
|
||||
);
|
||||
assert.equal(
|
||||
String(gdeltCalls(calls)[0].url),
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105120000.export.CSV.zip',
|
||||
);
|
||||
await settle();
|
||||
assert.ok(gdeltCalls(calls).length >= 96);
|
||||
assert.ok(calls.every((c) => host(c) === 'data.gdeltproject.org'));
|
||||
// Page titles only for the newest three hours (a GKG export is 6 MB).
|
||||
assert.ok(gkgCalls(calls).length > 0 && gkgCalls(calls).length <= 12);
|
||||
});
|
||||
|
||||
test('a GDELT article is a report at its place, in its words, credited to GDELT', async () => {
|
||||
const { ask, settle } = keyed({ env: {} });
|
||||
// The first answer has the events in, titled by their address words; the
|
||||
// page titles follow in the background.
|
||||
const first = normalizeFeedPayload((await ask('/feed', CONFLICTS)).body);
|
||||
assert.equal(
|
||||
first.reports.find((r) => r.id === 'gdelt-5').title,
|
||||
'Russian drones attack Kharkiv overnight three killed',
|
||||
);
|
||||
await settle();
|
||||
const { reports } = normalizeFeedPayload(
|
||||
(await ask('/feed', CONFLICTS)).body,
|
||||
);
|
||||
// Kharkiv's story was first seen in the earlier export: its first event
|
||||
// names it.
|
||||
assert.deepEqual(
|
||||
reports.map((r) => r.id),
|
||||
['gdelt-2', 'gdelt-5', 'gdelt-4'],
|
||||
);
|
||||
const [redSea, kharkiv, bare] = reports;
|
||||
// Its page title, from the GKG export.
|
||||
assert.equal(
|
||||
kharkiv.title,
|
||||
'Russian drones attack Kharkiv overnight, three killed',
|
||||
);
|
||||
assert.equal(kharkiv.category, 'strike');
|
||||
assert.equal(kharkiv.relevance, 'core');
|
||||
assert.equal(kharkiv.severity, 2);
|
||||
assert.equal(kharkiv.publisher, 'wire.example');
|
||||
assert.equal(kharkiv.attribution, 'GDELT Project');
|
||||
assert.equal(kharkiv.kind, 'gdelt');
|
||||
assert.equal(kharkiv.side, 'unknown');
|
||||
assert.equal(kharkiv.geo.precision, 'city');
|
||||
assert.deepEqual(
|
||||
kharkiv.pins.map(({ lat, lon }) => [lat, lon]),
|
||||
[[49.9935, 36.2304]],
|
||||
);
|
||||
assert.equal(redSea.geo.place, 'Yemen');
|
||||
assert.equal(redSea.geo.precision, 'country');
|
||||
assert.equal(bare.title, 'Fighting reported in Ukraine');
|
||||
assert.equal(bare.relevance, 'related');
|
||||
// The council story GDELT coded as fighting is minor: not in Conflicts.
|
||||
assert.ok(!reports.some((r) => r.url.includes('city-council')));
|
||||
});
|
||||
|
||||
test('with a Liveuamap key its mapped items join GDELT in the one hotspot', async () => {
|
||||
const { ask, calls, settle } = keyed();
|
||||
const answer = await ask('/hotspots');
|
||||
assert.equal(answer.body.hotspots[0].counts.last24h, 7);
|
||||
const live = calls.find((c) => host(c) === 'a.liveuamap.com').url;
|
||||
assert.equal(live.searchParams.get('a'), 'mpts');
|
||||
assert.equal(live.searchParams.get('key'), 'live-key');
|
||||
assert.equal(
|
||||
Number(live.searchParams.get('time')),
|
||||
(NOW - 86_400_000) / 1000,
|
||||
);
|
||||
const { reports } = normalizeFeedPayload(
|
||||
(await ask('/feed', CONFLICTS)).body,
|
||||
);
|
||||
const kharkiv = reports.find((r) => r.id === 'lum-101');
|
||||
assert.equal(kharkiv.category, 'strike');
|
||||
assert.equal(kharkiv.severity, 2);
|
||||
assert.equal(kharkiv.publisher, 'Liveuamap');
|
||||
assert.deepEqual(
|
||||
kharkiv.pins.map(({ lat, lon }) => [lat, lon]),
|
||||
[[49.9935, 36.2304]],
|
||||
);
|
||||
assert.equal(
|
||||
kharkiv.event.image.video,
|
||||
'https://a.liveuamap.com/clips/101.mp4',
|
||||
);
|
||||
assert.deepEqual(kharkiv.media, ['video']);
|
||||
assert.equal(
|
||||
reports.find((r) => r.id === 'lum-103').title,
|
||||
'Officials in "Kherson" report shelling',
|
||||
);
|
||||
// Counts over a shorter window.
|
||||
const short = await ask('/hotspots', 'hours=2');
|
||||
assert.equal(short.body.hotspots[0].counts.last24h, 6);
|
||||
await settle();
|
||||
});
|
||||
|
||||
test('the Conflicts feed pages by time and filters by category and hotspot', async () => {
|
||||
const { ask, settle } = keyed({ env: {} });
|
||||
const all = 'hotspot=all&today=1&relevance=core,related&events=1';
|
||||
const first = await ask('/feed', `${all}&limit=1`);
|
||||
assert.deepEqual(
|
||||
first.body.reports.map((r) => r.id),
|
||||
['gdelt-2'],
|
||||
);
|
||||
assert.equal(first.body.nextBefore, '2026-11-05T11:45:00.000Z');
|
||||
// A page ends between two times: both of 11:30 come together, though
|
||||
// the page asked for one.
|
||||
const next = await ask(
|
||||
'/feed',
|
||||
`${all}&limit=1&before=${first.body.nextBefore}`,
|
||||
);
|
||||
assert.deepEqual(
|
||||
next.body.reports.map((r) => r.id),
|
||||
['gdelt-5', 'gdelt-4'],
|
||||
);
|
||||
assert.equal(next.body.nextBefore, null);
|
||||
const strikes = await ask('/feed', 'hotspot=all&category=strike');
|
||||
assert.equal(strikes.body.reports.length, 2);
|
||||
const elsewhere = await ask('/feed', 'hotspot=ua-ru-war');
|
||||
assert.deepEqual(elsewhere.body.reports, []);
|
||||
await settle();
|
||||
});
|
||||
|
||||
test('an export not published yet is looked for again, and the day is kept a day', async () => {
|
||||
const exports = { ...EXPORTS };
|
||||
const { ask, calls, clock, settle } = keyed({
|
||||
env: {},
|
||||
answers: { exports },
|
||||
});
|
||||
await ask('/hotspots');
|
||||
await settle();
|
||||
const read = gdeltCalls(calls).length;
|
||||
// Within five minutes nothing is asked again.
|
||||
clock.now += 2 * 60_000;
|
||||
await ask('/hotspots');
|
||||
assert.equal(gdeltCalls(calls).length, read);
|
||||
// The late export is published now, with a new article.
|
||||
exports[exportName('20261105120000')] = [
|
||||
eventRow({
|
||||
id: 6,
|
||||
code: '183',
|
||||
url: 'https://news.example.com/2026/11/05/car-bomb-kills-five-in-mogadishu-market',
|
||||
place: 'Mogadishu, Banaadir, Somalia',
|
||||
lat: 2.04,
|
||||
lon: 45.34,
|
||||
added: '20261105120000',
|
||||
}),
|
||||
];
|
||||
clock.now += 4 * 60_000;
|
||||
await ask('/hotspots');
|
||||
await settle();
|
||||
const later = await ask('/hotspots');
|
||||
assert.equal(later.body.hotspots[0].counts.last24h, 5, 'added, not replaced');
|
||||
assert.equal(
|
||||
gdeltCalls(calls)
|
||||
.slice(read)
|
||||
.filter((c) => String(c.url).includes('20261105120000')).length,
|
||||
1,
|
||||
);
|
||||
// A day on, the old ones have aged out.
|
||||
clock.now += 24 * 3_600_000;
|
||||
const dayOn = await ask('/hotspots');
|
||||
await settle();
|
||||
assert.equal(dayOn.body.hotspots[0].counts.last24h, 0);
|
||||
});
|
||||
|
||||
test('Local names the country at the point and outlines it, with no key', async () => {
|
||||
const { ask, calls } = keyed({ env: {} });
|
||||
const answer = await ask('/outline', 'at=51.13,71.43&v=4');
|
||||
const outline = normalizeOutlinePayload(answer.body);
|
||||
assert.deepEqual(outline.area, {
|
||||
key: 'KZ',
|
||||
name: 'Kazakhstan',
|
||||
kind: 'country',
|
||||
country: 'KZ',
|
||||
});
|
||||
assert.ok(outline.polygons.length > 0);
|
||||
const sea = await ask('/outline', 'at=0,-30&v=4');
|
||||
assert.equal(normalizeOutlinePayload(sea.body).area, null);
|
||||
const developing = await ask('/outline', 'country=UA&v=4');
|
||||
assert.ok(developing.body.polygons.length > 0);
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
|
||||
test("Local in a country is its outlets' everyday news, pinned inside it, plus the Conflicts reports there", async () => {
|
||||
const { ask, calls } = keyed();
|
||||
const answer = await ask(
|
||||
'/feed',
|
||||
'hotspot=all&geo=1&today=1&events=1&limit=40&at=51.13,71.43',
|
||||
);
|
||||
assert.equal(answer.status, 200);
|
||||
const search = calls.find((c) => host(c) === 'api.worldnewsapi.com');
|
||||
assert.equal(search.url.pathname, '/search-news');
|
||||
assert.equal(search.url.searchParams.get('source-country'), 'kz');
|
||||
assert.equal(search.url.searchParams.has('categories'), false);
|
||||
assert.equal(
|
||||
search.url.searchParams.get('earliest-publish-date'),
|
||||
'2026-11-04 12:00:00',
|
||||
);
|
||||
assert.equal(search.url.searchParams.has('api-key'), false);
|
||||
assert.equal(search.headers['x-api-key'], 'wna-key');
|
||||
|
||||
const { reports } = normalizeFeedPayload(answer.body);
|
||||
assert.deepEqual(
|
||||
reports.map((r) => r.id),
|
||||
['lum-102', 'wna-9001', 'wna-9002'],
|
||||
);
|
||||
const [, visit, exploration] = reports;
|
||||
assert.equal(visit.category, 'political');
|
||||
assert.equal(visit.relevance, 'minor');
|
||||
assert.equal(visit.publisher, 'daily.example');
|
||||
assert.equal(visit.snippet, null, 'titles and links only');
|
||||
assert.equal(visit.geo.precision, 'country');
|
||||
assert.equal(visit.geo.place, 'Kazakhstan');
|
||||
assert.equal(visit.event.image.publisher, 'daily.example');
|
||||
assert.equal(exploration.category, 'economy');
|
||||
|
||||
// The point each is pinned at lies in the area Local outlines.
|
||||
const outline = normalizeOutlinePayload(
|
||||
(await ask('/outline', 'at=51.13,71.43&v=4')).body,
|
||||
);
|
||||
const { inPolygons } = await import('../../../src/layers/news/model.js');
|
||||
for (const report of reports)
|
||||
assert.ok(inPolygons(outline.polygons, report.pins[0]), report.id);
|
||||
});
|
||||
|
||||
test('a disputed area is named as itself: outlined alone, left out of the country around it, read from no country', async () => {
|
||||
const { ask, calls, settle } = keyed();
|
||||
const crimea = normalizeOutlinePayload(
|
||||
(await ask('/outline', 'at=44.95,34.1&v=4')).body,
|
||||
);
|
||||
assert.deepEqual(crimea.area, {
|
||||
key: 'disputed-crimea',
|
||||
name: 'Crimea',
|
||||
kind: 'disputed',
|
||||
country: null,
|
||||
});
|
||||
const { inPolygons } = await import('../../../src/layers/news/model.js');
|
||||
const russia = normalizeOutlinePayload(
|
||||
(await ask('/outline', 'at=55.75,37.6&v=4')).body,
|
||||
);
|
||||
assert.equal(russia.area.name, 'Russia');
|
||||
assert.equal(inPolygons(russia.polygons, { lat: 44.95, lon: 34.1 }), false);
|
||||
assert.equal(inPolygons(russia.polygons, { lat: 55.75, lon: 37.6 }), true);
|
||||
// Local in Crimea: no national outlets asked, the Conflicts reports there.
|
||||
const local = await ask('/feed', 'hotspot=all&at=44.95,34.1');
|
||||
assert.equal(local.status, 200);
|
||||
assert.ok(calls.every((c) => host(c) !== 'api.worldnewsapi.com'));
|
||||
await settle();
|
||||
});
|
||||
|
||||
test("GDELT's places are named by their town and the area there, not its old regions", () => {
|
||||
const named = (lat, lon, place, precision = 'city') =>
|
||||
gdeltPlace({ lat, lon, place, precision }).place;
|
||||
assert.equal(
|
||||
named(31.43, 34.39, 'Bureij, Yerushalayim, Israel'),
|
||||
'Bureij, Palestine',
|
||||
);
|
||||
assert.equal(
|
||||
named(9.07, 7.48, 'Abuja, Abuja Federal Capital Territory, Niger'),
|
||||
'Abuja, Nigeria',
|
||||
);
|
||||
assert.equal(
|
||||
named(44.95, 34.1, 'Simferopol, Crimea, Ukraine'),
|
||||
'Simferopol, Crimea',
|
||||
);
|
||||
assert.equal(named(48.4, 31.2, 'Ukraine', 'country'), 'Ukraine');
|
||||
});
|
||||
|
||||
test("a country's news is read once per half hour and Liveuamap's feed once per five minutes", async () => {
|
||||
const { ask, calls, clock } = keyed();
|
||||
const local = 'hotspot=all&at=51.13,71.43';
|
||||
await ask('/feed', local);
|
||||
await ask('/feed', local);
|
||||
await ask('/hotspots');
|
||||
const count = (name) => calls.filter((c) => host(c) === name).length;
|
||||
assert.equal(count('api.worldnewsapi.com'), 1);
|
||||
assert.equal(count('a.liveuamap.com'), 1);
|
||||
clock.now += 6 * 60_000;
|
||||
await ask('/feed', local);
|
||||
assert.equal(count('api.worldnewsapi.com'), 1);
|
||||
assert.equal(count('a.liveuamap.com'), 2);
|
||||
clock.now += 25 * 60_000;
|
||||
await ask('/feed', local);
|
||||
assert.equal(count('api.worldnewsapi.com'), 2);
|
||||
});
|
||||
|
||||
test('Local without its key says which key it needs and reads nothing', async () => {
|
||||
const noLocal = keyed({ env: { LIVEUAMAP_API_KEY: 'live-key' } });
|
||||
assert.deepEqual(await noLocal.ask('/feed', 'at=51.13,71.43'), {
|
||||
status: 503,
|
||||
body: { error: 'no_key', key: 'worldnewsapi' },
|
||||
});
|
||||
assert.ok(noLocal.calls.every((c) => host(c) !== 'api.worldnewsapi.com'));
|
||||
// At sea Local is the Conflicts reports by distance, keyless.
|
||||
const sea = await keyed({ env: {} }).ask('/feed', 'hotspot=all&at=0,-30');
|
||||
assert.equal(sea.status, 200);
|
||||
});
|
||||
|
||||
test('a refused Liveuamap key leaves GDELT; World News API refusing names its key', async () => {
|
||||
const logs = [];
|
||||
const refused = keyed({ answers: { liveuamap: 401, worldnews: 403 }, logs });
|
||||
const conflicts = await refused.ask('/hotspots');
|
||||
assert.equal(conflicts.status, 200);
|
||||
assert.equal(conflicts.body.hotspots[0].counts.last24h, 4);
|
||||
assert.deepEqual(await refused.ask('/feed', 'at=51.13,71.43'), {
|
||||
status: 503,
|
||||
body: { error: 'key_rejected', key: 'worldnewsapi' },
|
||||
});
|
||||
// A refused key is logged once, however often it is tried.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
refused.clock.now += 61_000;
|
||||
assert.equal((await refused.ask('/hotspots')).status, 200);
|
||||
}
|
||||
assert.equal(
|
||||
logs.filter((line) => line.includes('LIVEUAMAP_API_KEY was refused'))
|
||||
.length,
|
||||
1,
|
||||
);
|
||||
assert.ok(gdeltCalls(refused.calls).length > 0);
|
||||
assert.ok(
|
||||
refused.calls.filter((c) => host(c) === 'a.liveuamap.com').length > 1,
|
||||
);
|
||||
// Neither key nor answer body reaches the log.
|
||||
assert.ok(logs.length > 0);
|
||||
for (const line of logs) assert.doesNotMatch(line, /live-key|wna-key|ABC/);
|
||||
await refused.settle();
|
||||
});
|
||||
|
||||
test('a failing GDELT is tried again after a minute, and what it read is kept', async () => {
|
||||
const answers = { gdelt: 503 };
|
||||
const flaky = keyed({ env: {}, answers });
|
||||
assert.deepEqual(await flaky.ask('/hotspots'), {
|
||||
status: 502,
|
||||
body: { error: 'news_upstream_error' },
|
||||
});
|
||||
const tried = gdeltCalls(flaky.calls).length;
|
||||
await flaky.ask('/hotspots');
|
||||
assert.equal(gdeltCalls(flaky.calls).length, tried, 'not within a minute');
|
||||
flaky.clock.now += 61_000;
|
||||
delete answers.gdelt;
|
||||
assert.equal((await flaky.ask('/hotspots')).status, 200);
|
||||
await flaky.settle();
|
||||
answers.gdelt = 500;
|
||||
flaky.clock.now += 6 * 60_000;
|
||||
const kept = await flaky.ask('/hotspots');
|
||||
await flaky.settle();
|
||||
assert.equal(kept.status, 200);
|
||||
assert.equal(kept.body.hotspots[0].counts.last24h, 4);
|
||||
});
|
||||
|
||||
test('a new key drops what the old one read', async () => {
|
||||
const env = { LIVEUAMAP_API_KEY: 'old', WORLDNEWSAPI_KEY: 'wna-key' };
|
||||
const { ask, calls } = keyed({ env });
|
||||
await ask('/hotspots');
|
||||
env.LIVEUAMAP_API_KEY = 'new';
|
||||
await ask('/hotspots');
|
||||
assert.deepEqual(
|
||||
calls
|
||||
.filter((c) => host(c) === 'a.liveuamap.com')
|
||||
.map((c) => c.url.searchParams.get('key')),
|
||||
['old', 'new'],
|
||||
);
|
||||
});
|
||||
|
||||
test('each key is one the key registry names', () => {
|
||||
for (const [id, envVar] of Object.entries(NEWS_KEYS))
|
||||
assert.deepEqual(
|
||||
KEY_SETUP_KEYS.find((entry) => entry.id === id)?.envVars,
|
||||
[envVar],
|
||||
id,
|
||||
);
|
||||
});
|
||||
|
||||
const MINUTE = 60_000;
|
||||
const count = (calls, name) => calls.filter((c) => host(c) === name).length;
|
||||
const UPSTREAM_ERROR = { status: 502, body: { error: 'news_upstream_error' } };
|
||||
const KEY_QUOTA = {
|
||||
status: 503,
|
||||
body: { error: 'key_quota', key: 'worldnewsapi' },
|
||||
};
|
||||
|
||||
test('exports never published are given up on, not looked for back to back', async () => {
|
||||
// One export of the day has events; GDELT answers 404 for every other.
|
||||
const quarter = exportName('20261105114500');
|
||||
const { ask, calls, clock, settle } = keyed({
|
||||
env: {},
|
||||
answers: { exports: { [quarter]: EXPORTS[quarter] }, unlisted: 404 },
|
||||
});
|
||||
assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 3);
|
||||
await settle();
|
||||
// A round that brings nothing in is the last until the next five minutes.
|
||||
let seen = gdeltCalls(calls).length;
|
||||
assert.ok(seen <= 2 * 12, `${seen} at first`);
|
||||
const steps = [];
|
||||
for (let i = 0; i < 20; i++) {
|
||||
clock.now += 5 * MINUTE;
|
||||
assert.equal((await ask('/hotspots')).status, 200);
|
||||
await settle();
|
||||
steps.push(gdeltCalls(calls).length - seen);
|
||||
seen = gdeltCalls(calls).length;
|
||||
}
|
||||
// One round each five minutes, until the day has been looked through
|
||||
// once; then only the exports of the last hour and a half (GDELT's CDN
|
||||
// keeps a 404 for an hour, so a late one may still come).
|
||||
assert.ok(
|
||||
steps.every((n) => n <= 12),
|
||||
steps.join(),
|
||||
);
|
||||
assert.ok(
|
||||
steps.slice(-3).every((n) => n <= 7),
|
||||
steps.join(),
|
||||
);
|
||||
assert.ok(seen <= 2 * 12 + 20 * 12, String(seen));
|
||||
});
|
||||
|
||||
test('GDELT with no export read at all is a 502, not an empty Conflicts', async () => {
|
||||
const { ask } = keyed({ env: {}, answers: { exports: {}, unlisted: 404 } });
|
||||
assert.deepEqual(await ask('/hotspots'), UPSTREAM_ERROR);
|
||||
assert.deepEqual(await ask('/feed', CONFLICTS), UPSTREAM_ERROR);
|
||||
});
|
||||
|
||||
test('GDELT failing for two hours answers 502; until then what it read is served', async () => {
|
||||
const answers = {};
|
||||
const { ask, clock, settle } = keyed({ env: {}, answers });
|
||||
await ask('/hotspots');
|
||||
await settle();
|
||||
answers.gdelt = 503;
|
||||
clock.now += 60 * MINUTE;
|
||||
const hour = await ask('/hotspots');
|
||||
assert.equal(hour.status, 200);
|
||||
assert.equal(hour.body.hotspots[0].counts.last24h, 4);
|
||||
clock.now += 61 * MINUTE;
|
||||
assert.deepEqual(await ask('/hotspots'), UPSTREAM_ERROR);
|
||||
assert.deepEqual(await ask('/feed', CONFLICTS), UPSTREAM_ERROR);
|
||||
await settle();
|
||||
});
|
||||
|
||||
test('a failed GKG read keeps the events, titled by their words, and is tried again', async () => {
|
||||
const answers = { gkg: 500 };
|
||||
const { ask, calls, clock, logs, settle } = keyed({ env: {}, answers });
|
||||
const kharkiv = (answer) =>
|
||||
normalizeFeedPayload(answer.body).reports.find((r) => r.id === 'gdelt-5');
|
||||
await ask('/hotspots');
|
||||
await settle();
|
||||
const read = await ask('/feed', CONFLICTS);
|
||||
assert.equal(read.status, 200);
|
||||
assert.equal(read.body.reports.length, 3);
|
||||
assert.equal(
|
||||
kharkiv(read).title,
|
||||
'Russian drones attack Kharkiv overnight three killed',
|
||||
);
|
||||
assert.ok(logs.some((line) => line.includes('titles read failed')));
|
||||
// Each GKG export once, not with every read of the day.
|
||||
assert.equal(gkgCalls(calls).length, 11);
|
||||
delete answers.gkg;
|
||||
clock.now += 5 * MINUTE;
|
||||
await ask('/hotspots');
|
||||
await settle();
|
||||
assert.equal(
|
||||
kharkiv(await ask('/feed', CONFLICTS)).title,
|
||||
'Russian drones attack Kharkiv overnight, three killed',
|
||||
);
|
||||
});
|
||||
|
||||
test('Local where only Conflicts reports speak answers 502 when they fail; in a country the outlets stay', async () => {
|
||||
const { ask, calls } = keyed({ answers: { gdelt: 503, liveuamap: 500 } });
|
||||
// Crimea, and Somaliland (a territory without a code).
|
||||
assert.deepEqual(
|
||||
await ask('/feed', 'hotspot=all&at=44.95,34.1'),
|
||||
UPSTREAM_ERROR,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await ask('/feed', 'hotspot=all&at=9.56,44.06'),
|
||||
UPSTREAM_ERROR,
|
||||
);
|
||||
assert.equal(count(calls, 'api.worldnewsapi.com'), 0);
|
||||
const kazakhstan = await ask('/feed', 'hotspot=all&at=51.13,71.43');
|
||||
assert.equal(kazakhstan.status, 200);
|
||||
assert.deepEqual(
|
||||
kazakhstan.body.reports.map((r) => r.id),
|
||||
['wna-9001', 'wna-9002'],
|
||||
);
|
||||
});
|
||||
|
||||
test('a territory without a code is named as itself, not read as sea', async () => {
|
||||
const { ask, calls, settle } = keyed();
|
||||
for (const [at, key, name] of [
|
||||
['9.56,44.06', 'disputed-somaliland', 'Somaliland'],
|
||||
['35.12,33.94', 'disputed-northern-cyprus', 'Northern Cyprus'],
|
||||
]) {
|
||||
const outline = normalizeOutlinePayload(
|
||||
(await ask('/outline', `at=${at}&v=4`)).body,
|
||||
);
|
||||
assert.deepEqual(outline.area, {
|
||||
key,
|
||||
name,
|
||||
kind: 'disputed',
|
||||
country: null,
|
||||
});
|
||||
assert.ok(outline.polygons.length > 0);
|
||||
}
|
||||
assert.equal(
|
||||
gdeltPlace({
|
||||
lat: 9.56,
|
||||
lon: 44.06,
|
||||
place: 'Hargeysa, Somalia',
|
||||
precision: 'city',
|
||||
}).place,
|
||||
'Hargeysa, Somaliland',
|
||||
);
|
||||
const local = await ask('/feed', 'hotspot=all&at=9.56,44.06');
|
||||
assert.equal(local.status, 200);
|
||||
assert.equal(count(calls, 'api.worldnewsapi.com'), 0);
|
||||
await settle();
|
||||
});
|
||||
|
||||
test('a stale Liveuamap or World News API read is served while it is read again', async () => {
|
||||
const answers = {};
|
||||
const { ask, calls, clock, settle } = keyed({ answers });
|
||||
const local = 'hotspot=all&at=51.13,71.43';
|
||||
const ids = (answer) => answer.body.reports.map((r) => r.id);
|
||||
assert.deepEqual(ids(await ask('/feed', local)), [
|
||||
'lum-102',
|
||||
'wna-9001',
|
||||
'wna-9002',
|
||||
]);
|
||||
// Both read again, and held open: the answer does not wait for them.
|
||||
let release;
|
||||
const held = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const later = (payload) => () =>
|
||||
held.then(() => new Response(JSON.stringify(payload), { status: 200 }));
|
||||
answers.liveuamap = later({ venues: [] });
|
||||
answers.worldnews = later({ news: [] });
|
||||
clock.now += 31 * MINUTE;
|
||||
const served = await ask('/feed', local);
|
||||
assert.equal(served.status, 200);
|
||||
assert.deepEqual(ids(served), ['lum-102', 'wna-9001', 'wna-9002']);
|
||||
assert.equal(count(calls, 'a.liveuamap.com'), 2);
|
||||
assert.equal(count(calls, 'api.worldnewsapi.com'), 2);
|
||||
// Once read, the new answers are served.
|
||||
release();
|
||||
await settle();
|
||||
assert.deepEqual(ids(await ask('/feed', local)), []);
|
||||
// A country not read before is waited for.
|
||||
answers.worldnews = WORLD_NEWS;
|
||||
assert.equal(
|
||||
(await ask('/feed', 'hotspot=all&at=48.85,2.35')).body.reports.length,
|
||||
2,
|
||||
);
|
||||
});
|
||||
|
||||
test('a read in flight with a replaced key is not the answer for the new one', async () => {
|
||||
let release;
|
||||
const held = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const env = { LIVEUAMAP_API_KEY: 'old' };
|
||||
const answers = {
|
||||
liveuamap: () =>
|
||||
held.then(() => new Response(JSON.stringify({ venues: [] }))),
|
||||
};
|
||||
const { ask, calls, settle } = keyed({ env, answers });
|
||||
const before = ask('/hotspots');
|
||||
env.LIVEUAMAP_API_KEY = 'new';
|
||||
delete answers.liveuamap;
|
||||
const after = await ask('/hotspots');
|
||||
assert.equal(after.body.hotspots[0].counts.last24h, 7);
|
||||
assert.deepEqual(
|
||||
calls
|
||||
.filter((c) => host(c) === 'a.liveuamap.com')
|
||||
.map((c) => c.url.searchParams.get('key')),
|
||||
['old', 'new'],
|
||||
);
|
||||
release();
|
||||
await before;
|
||||
// The old key's late answer does not replace the new one's.
|
||||
assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 7);
|
||||
await settle();
|
||||
});
|
||||
|
||||
test("World News API's reads are a daily budget: spent, Local says key_quota and serves what it read", async () => {
|
||||
const { ask, calls, clock, logs, settle } = keyed({
|
||||
env: { WORLDNEWSAPI_KEY: 'wna-key', WORLDNEWSAPI_DAILY_READS: '2' },
|
||||
});
|
||||
const at = (point) => ask('/feed', `hotspot=all&at=${point}`);
|
||||
assert.equal((await at('51.13,71.43')).status, 200);
|
||||
assert.equal((await at('48.85,2.35')).status, 200);
|
||||
// Germany would be the third read today.
|
||||
assert.deepEqual(await at('52.52,13.4'), KEY_QUOTA);
|
||||
assert.equal(count(calls, 'api.worldnewsapi.com'), 2);
|
||||
// Kazakhstan, read earlier, is served from what was read.
|
||||
clock.now += 31 * MINUTE;
|
||||
const kept = await at('51.13,71.43');
|
||||
assert.equal(kept.status, 200);
|
||||
assert.equal(kept.body.reports.length, 2);
|
||||
assert.equal(count(calls, 'api.worldnewsapi.com'), 2);
|
||||
assert.equal(
|
||||
logs.filter((line) => line.includes('reads for today are spent')).length,
|
||||
1,
|
||||
);
|
||||
// A new UTC day has its own reads.
|
||||
clock.now = Date.parse('2026-11-06T00:30:00Z');
|
||||
assert.equal((await at('52.52,13.4')).status, 200);
|
||||
assert.equal(count(calls, 'api.worldnewsapi.com'), 3);
|
||||
await settle();
|
||||
});
|
||||
|
||||
test('World News API answering 402 is key_quota, and nothing more is asked today; a 429 is no quota', async () => {
|
||||
const paid = keyed({ answers: { worldnews: 402 } });
|
||||
assert.deepEqual(await paid.ask('/feed', 'at=51.13,71.43'), KEY_QUOTA);
|
||||
paid.clock.now += 2 * MINUTE;
|
||||
assert.deepEqual(await paid.ask('/feed', 'at=48.85,2.35'), KEY_QUOTA);
|
||||
assert.equal(count(paid.calls, 'api.worldnewsapi.com'), 1);
|
||||
// A 429 is its per-second limit: an ordinary failure, tried again soon.
|
||||
const busy = keyed({ answers: { worldnews: 429 } });
|
||||
assert.deepEqual(await busy.ask('/feed', 'at=51.13,71.43'), UPSTREAM_ERROR);
|
||||
await paid.settle();
|
||||
await busy.settle();
|
||||
});
|
||||
|
||||
test('one unreadable export is given up on and never holds the rest back', async () => {
|
||||
const bad = exportName('20261105114500');
|
||||
const { ask, calls, clock, settle } = keyed({
|
||||
env: {},
|
||||
answers: { corrupt: [bad] },
|
||||
});
|
||||
// The other export's stories are served, from the first answer on (its
|
||||
// copy of the Kharkiv story among them).
|
||||
const first = await ask('/hotspots');
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(first.body.hotspots[0].counts.last24h, 2);
|
||||
await settle();
|
||||
clock.now += 6 * MINUTE;
|
||||
assert.equal((await ask('/hotspots')).status, 200);
|
||||
await settle();
|
||||
const asked = gdeltCalls(calls).filter((c) => String(c.url) === bad).length;
|
||||
assert.equal(asked, 1, 'an unreadable file is not fetched again');
|
||||
});
|
||||
|
||||
test('no export later than the newest GDELT lists is asked for (its CDN keeps a 404 an hour)', async () => {
|
||||
const { ask, calls, settle } = keyed({
|
||||
env: {},
|
||||
answers: { lastupdate: '20261105114500' },
|
||||
});
|
||||
assert.equal((await ask('/hotspots')).status, 200);
|
||||
await settle();
|
||||
assert.ok(
|
||||
!gdeltCalls(calls).some(
|
||||
(c) => String(c.url) === exportName('20261105120000'),
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
gdeltCalls(calls).some(
|
||||
(c) => String(c.url) === exportName('20261105114500'),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('a town a disputed area is named after is named alone, on both sides of its line', () => {
|
||||
assert.equal(
|
||||
gdeltPlace({
|
||||
lat: 31.7667,
|
||||
lon: 35.2333,
|
||||
place: 'Jerusalem, Israel (general), Israel',
|
||||
precision: 'city',
|
||||
}).place,
|
||||
'Jerusalem',
|
||||
);
|
||||
});
|
||||
|
||||
test('World News API is credited on each of its reports', async () => {
|
||||
const { ask, settle } = keyed();
|
||||
const { reports } = normalizeFeedPayload(
|
||||
(await ask('/feed', 'hotspot=all&at=51.13,71.43')).body,
|
||||
);
|
||||
const national = reports.filter((r) => r.kind === 'worldnewsapi');
|
||||
assert.ok(national.length > 0);
|
||||
for (const r of national) assert.equal(r.attribution, 'World News API');
|
||||
await settle();
|
||||
});
|
||||
|
||||
test('GDELT down for a few minutes gives up on no export: they are all read once it is back', async () => {
|
||||
const answers = { gdelt: 503 };
|
||||
const { ask, calls, clock, settle } = keyed({ env: {}, answers });
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await ask('/hotspots');
|
||||
await settle();
|
||||
clock.now += 61_000;
|
||||
}
|
||||
delete answers.gdelt;
|
||||
clock.now += 5 * MINUTE;
|
||||
const back = await ask('/hotspots');
|
||||
await settle();
|
||||
assert.equal(back.status, 200);
|
||||
assert.equal(back.body.hotspots[0].counts.last24h, 4);
|
||||
assert.ok(
|
||||
gdeltCalls(calls).some(
|
||||
(c) => String(c.url) === exportName('20261105114500'),
|
||||
),
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,274 @@
|
||||
// The keyed news APIs Core reads itself when no news server is configured:
|
||||
// Liveuamap's worldwide feed (`a=mpts`) beside GDELT's events in the
|
||||
// Conflicts section (gdeltEvents.js), World News API's search by source
|
||||
// country for Local. Each key stays on the server: Liveuamap's rides in the query, so
|
||||
// neither a URL nor an error body is ever logged or relayed.
|
||||
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
const LIVEUAMAP_URL = 'https://a.liveuamap.com/api';
|
||||
const WORLD_NEWS_URL = 'https://api.worldnewsapi.com/search-news';
|
||||
// 100 Liveuamap items run to about 500 KB.
|
||||
const LIVEUAMAP_COUNT = 500;
|
||||
const LIVEUAMAP_MAX_BYTES = 16 * 1024 * 1024;
|
||||
// One search per country: a point, and a hundredth of one per article, so
|
||||
// about 1.5 points a read (the free plan has 50 points a day).
|
||||
const WORLD_NEWS_ARTICLES = 50;
|
||||
const WORLD_NEWS_MAX_BYTES = 8 * 1024 * 1024;
|
||||
// World News API's categories that are not the news of the day. Its
|
||||
// `categories` filter is not asked for: it classifies few articles outside
|
||||
// English, so it would drop most of a country's own outlets; an article it
|
||||
// left unclassified is kept.
|
||||
export const WORLD_NEWS_LEFT_OUT = Object.freeze(
|
||||
new Set(['sports', 'entertainment', 'lifestyle', 'travel']),
|
||||
);
|
||||
const DAY_MS = 86_400_000;
|
||||
const TIMEOUT_MS = 20_000;
|
||||
|
||||
const ENTITIES = {
|
||||
amp: '&',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
nbsp: ' ',
|
||||
};
|
||||
|
||||
/** Text with HTML entities decoded and whitespace collapsed, or ''. */
|
||||
export function unescapeHtml(value) {
|
||||
return String(value ?? '')
|
||||
.replace(
|
||||
/&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]{2,6}));/gi,
|
||||
(all, dec, hex, name) => {
|
||||
if (name) return ENTITIES[name.toLowerCase()] ?? all;
|
||||
const code = dec ? Number(dec) : parseInt(hex, 16);
|
||||
return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : all;
|
||||
},
|
||||
)
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** An https URL without credentials, or null. */
|
||||
export function webLink(value) {
|
||||
try {
|
||||
const url = new URL(String(value ?? '').trim());
|
||||
return url.protocol === 'https:' && !url.username && !url.password
|
||||
? url.href
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** A clip a browser plays itself (an https MP4 or WebM file), or null. */
|
||||
export function playableVideo(value) {
|
||||
const url = webLink(value);
|
||||
return url && /\.(?:mp4|m4v|webm)$/i.test(new URL(url).pathname) ? url : null;
|
||||
}
|
||||
|
||||
const finiteNumber = (value) => {
|
||||
const number = typeof value === 'string' ? Number(value.trim()) : value;
|
||||
return typeof number === 'number' && Number.isFinite(number) ? number : null;
|
||||
};
|
||||
|
||||
/** Why an upstream could not be read; never carries its URL or body. */
|
||||
export class UpstreamError extends Error {
|
||||
constructor(
|
||||
message,
|
||||
{ status = null, rejected = false, quota = false, unreadable = false } = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'UpstreamError';
|
||||
this.status = status;
|
||||
// The key was refused (401 or 403): the operator's to fix.
|
||||
this.rejected = rejected;
|
||||
// The key's allowance is spent (402, or the day's reads): it answers
|
||||
// again later. A 429 is a per-second limit, not this.
|
||||
this.quota = quota;
|
||||
// The answer itself cannot be read (corrupt, or over a size cap): asking
|
||||
// again gets the same.
|
||||
this.unreadable = unreadable;
|
||||
}
|
||||
}
|
||||
|
||||
async function getText(
|
||||
fetchImpl,
|
||||
url,
|
||||
{ headers = {}, maxBytes, signal, what },
|
||||
) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
headers: { Accept: 'application/json', ...headers },
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
} catch (cause) {
|
||||
throw new UpstreamError(
|
||||
cause?.name === 'TimeoutError'
|
||||
? `${what} timed out`
|
||||
: `${what} unreachable`,
|
||||
);
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw new UpstreamError(`${what} answered ${response.status}`, {
|
||||
status: response.status,
|
||||
rejected: response.status === 401 || response.status === 403,
|
||||
quota: response.status === 402,
|
||||
});
|
||||
}
|
||||
try {
|
||||
return await readResponseTextCapped(response, maxBytes, signal);
|
||||
} catch {
|
||||
throw new UpstreamError(`${what} answer unreadable`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseJson(text, what) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
// Not the parser's message: an error answer may echo the request.
|
||||
throw new UpstreamError(`${what} answer unreadable`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Liveuamap's items with a headline, a point and a page:
|
||||
* `{id, title, lat, lon, place, publishedAt, url, image, video}`.
|
||||
*/
|
||||
export function parseLiveuamap(payload) {
|
||||
if (!Array.isArray(payload?.venues))
|
||||
throw new UpstreamError('Liveuamap answer unreadable');
|
||||
const items = [];
|
||||
for (const venue of payload.venues) {
|
||||
const id = finiteNumber(venue?.id);
|
||||
const title = unescapeHtml(venue?.name);
|
||||
const lat = finiteNumber(venue?.lat);
|
||||
const lon = finiteNumber(venue?.lng);
|
||||
const seconds = finiteNumber(venue?.timestamp);
|
||||
const url = webLink(venue?.link);
|
||||
if (
|
||||
id === null ||
|
||||
!title ||
|
||||
lat === null ||
|
||||
lon === null ||
|
||||
Math.abs(lat) > 90 ||
|
||||
Math.abs(lon) > 180 ||
|
||||
(lat === 0 && lon === 0) ||
|
||||
seconds === null ||
|
||||
!url
|
||||
)
|
||||
continue;
|
||||
items.push({
|
||||
id: Math.trunc(id),
|
||||
title,
|
||||
lat,
|
||||
lon,
|
||||
place: unescapeHtml(venue.location) || null,
|
||||
publishedAt: new Date(seconds * 1000).toISOString(),
|
||||
url,
|
||||
image: webLink(venue.img_share),
|
||||
video: playableVideo(venue.video),
|
||||
});
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
/** The worldwide feed of the last day, as of `now`. */
|
||||
export async function fetchLiveuamap({ key, fetchImpl, now, signal }) {
|
||||
const url = new URL(LIVEUAMAP_URL);
|
||||
url.search = new URLSearchParams({
|
||||
a: 'mpts',
|
||||
resid: '1',
|
||||
time: String(Math.floor((now - DAY_MS) / 1000)),
|
||||
count: String(LIVEUAMAP_COUNT),
|
||||
key,
|
||||
}).toString();
|
||||
const text = await getText(fetchImpl, url, {
|
||||
maxBytes: LIVEUAMAP_MAX_BYTES,
|
||||
signal,
|
||||
what: 'Liveuamap',
|
||||
});
|
||||
return parseLiveuamap(parseJson(text, 'Liveuamap'));
|
||||
}
|
||||
|
||||
// World News API writes times as "2026-10-30 01:31:00", in UTC.
|
||||
const worldNewsTime = (ms) =>
|
||||
new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
|
||||
function worldNewsDate(value) {
|
||||
const match = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)$/.exec(
|
||||
String(value ?? ''),
|
||||
);
|
||||
const ms = match ? Date.parse(`${match[1]}T${match[2]}Z`) : NaN;
|
||||
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A search's articles with a title, a link and a time, one per title (the
|
||||
* same story from several outlets is one), sport and entertainment left
|
||||
* out, newest first:
|
||||
* `{id, title, url, publisher, language, category, publishedAt, image, video}`.
|
||||
*/
|
||||
export function parseWorldNews(payload) {
|
||||
if (!Array.isArray(payload?.news))
|
||||
throw new UpstreamError('World News API answer unreadable');
|
||||
const seen = new Set();
|
||||
const articles = [];
|
||||
for (const article of payload.news) {
|
||||
const id = finiteNumber(article?.id);
|
||||
const title = unescapeHtml(article?.title);
|
||||
const url = webLink(article?.url);
|
||||
const publishedAt = worldNewsDate(article?.publish_date);
|
||||
const category =
|
||||
typeof article?.category === 'string'
|
||||
? article.category.toLowerCase()
|
||||
: null;
|
||||
if (id === null || !title || !url || !publishedAt) continue;
|
||||
if (WORLD_NEWS_LEFT_OUT.has(category)) continue;
|
||||
const same = title.toLowerCase();
|
||||
if (seen.has(same)) continue;
|
||||
seen.add(same);
|
||||
articles.push({
|
||||
id: Math.trunc(id),
|
||||
title,
|
||||
url,
|
||||
publisher: new URL(url).hostname.replace(/^www\./, ''),
|
||||
language:
|
||||
typeof article.language === 'string' &&
|
||||
/^[a-z]{2}$/i.test(article.language)
|
||||
? article.language.toLowerCase()
|
||||
: null,
|
||||
category,
|
||||
publishedAt,
|
||||
image: webLink(article.image),
|
||||
video: playableVideo(article.video),
|
||||
});
|
||||
}
|
||||
return articles.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt));
|
||||
}
|
||||
|
||||
/** A country's everyday news of the last day, as of `now`. */
|
||||
export async function fetchWorldNews({ key, country, fetchImpl, now, signal }) {
|
||||
const url = new URL(WORLD_NEWS_URL);
|
||||
url.search = new URLSearchParams({
|
||||
'source-country': country.toLowerCase(),
|
||||
// Both ends: sorted by time with neither, a search is cut to 31 days.
|
||||
'earliest-publish-date': worldNewsTime(now - DAY_MS),
|
||||
'latest-publish-date': worldNewsTime(now),
|
||||
sort: 'publish-time',
|
||||
'sort-direction': 'DESC',
|
||||
number: String(WORLD_NEWS_ARTICLES),
|
||||
}).toString();
|
||||
const text = await getText(fetchImpl, url, {
|
||||
headers: { 'x-api-key': key },
|
||||
maxBytes: WORLD_NEWS_MAX_BYTES,
|
||||
signal,
|
||||
what: 'World News API',
|
||||
});
|
||||
return parseWorldNews(parseJson(text, 'World News API'));
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
# Natural Earth packs
|
||||
|
||||
Offline named-region polygons for the voice-annotation resolver
|
||||
(`src/data/naturalEarthRegions.js`) — "outline the Alps" resolves to the real
|
||||
range geometry with no network dependency.
|
||||
Offline polygons with no network dependency. The voice-annotation resolver
|
||||
(`src/data/naturalEarthRegions.js`) reads the named regions, so "outline the
|
||||
Alps" resolves to the real range geometry; area annotations read the
|
||||
countries, states and provinces (`src/data/adminBoundaries.js`); and the News
|
||||
layer reads the countries and the disputed areas
|
||||
(`server/providers/news/areas.js`).
|
||||
|
||||
| File | Source dataset | Features |
|
||||
|------|----------------|----------|
|
||||
@@ -10,6 +13,7 @@ range geometry with no network dependency.
|
||||
| `marine.json` | `ne_10m_geography_marine_polys` (seas, gulfs, straits, bays, …) | 292 named |
|
||||
| `countries.json` | `ne_10m_admin_0_countries` (countries) | 256 |
|
||||
| `states_provinces.json` | `ne_10m_admin_1_states_provinces` (states, provinces and other first-level units) | 4,587 named |
|
||||
| `disputed_areas.json` | `ne_10m_admin_0_disputed_areas` (disputed, breakaway and UN-patrolled areas; a hand-picked set, see below) | 11 of 99 |
|
||||
|
||||
**Source:** Natural Earth 10m physical vectors, via the canonical
|
||||
[nvkelso/natural-earth-vector](https://github.com/nvkelso/natural-earth-vector)
|
||||
@@ -103,3 +107,42 @@ Includes England, Scotland, Wales and Northern Ireland from
|
||||
`ne_10m_admin_0_map_units.geojson` at the same commit (SHA-256
|
||||
`57da82be755f4afccd8f3b14251bb2752f5df1395f47d2d86f817470c4a48862`).
|
||||
Result: 945,843 bytes; 260 features, 2,088 parts, 122,788 vertices.
|
||||
|
||||
## Disputed areas (`disputed_areas.json`)
|
||||
|
||||
The News layer's areas that no one country's name may stand for
|
||||
(`server/providers/news/areas.js`): a place inside one is named with the
|
||||
area ("Crimea"), and Local there reads no country's outlets.
|
||||
|
||||
**Source:** `geojson/ne_10m_admin_0_disputed_areas.geojson` at the same pinned
|
||||
commit (SHA-256
|
||||
`9cafef8b7dfb6b164dc58f218f981f4ace9f716f6c03795d4c62d1ac9f3d50f5`), fetched
|
||||
2026-11-05. Public domain, as above.
|
||||
|
||||
**Selection:** a hand-picked 11 of the source's 99 features. No rule picks
|
||||
them; this is the list:
|
||||
|
||||
| Area | Natural Earth's note (`NOTE_BRK`) |
|
||||
|------|-----------------------------------|
|
||||
| Golan Heights | Admin. By Israel; Claimed by Syria |
|
||||
| UNDOF Zone | UN Patrolled Zone in Syria |
|
||||
| Transnistria | Self admin.; Claimed by Moldova |
|
||||
| Abyei | Admin. by Sudan; Claimed by South Sudan |
|
||||
| Crimea | Admin. by Russia; Claimed by Ukraine |
|
||||
| Shebaa Farms | Admin. By Israel; Claimed by Lebanon |
|
||||
| Bir Tawil | Between Egypt and Sudan |
|
||||
| Halayib Triangle | Admin. by Egypt; Claimed by Sudan |
|
||||
| East Jerusalem | Admin. By Israel; Claimed by Palestine |
|
||||
| Abu Musa I. | Admin. by Iran; Claimed by UAE |
|
||||
| Doumera Island | Admin. By Eritrea; Claimed by Djibouti |
|
||||
|
||||
The source names three of these by a country ("Moldova", "Sudan", "Iran");
|
||||
the pack names them as the area. The 2014 Donetsk and Luhansk lines are left
|
||||
out as out of date. Every other feature is left out too, so a place there is
|
||||
named by its country in `countries.json`.
|
||||
|
||||
**Curation:** `kind` (`held`, `self`, `un`, `unclaimed`), `heldBy` and
|
||||
`claimedBy` are read from `NOTE_BRK`, and `note` keeps it as published. Every
|
||||
part is kept; each ring is Douglas-Peucker simplified at 0.003°, coordinates
|
||||
are rounded to 3 decimals and rings are stored open. Provenance is in the
|
||||
pack's `meta`. Result: 14,504 bytes.
|
||||
@@ -0,0 +1 @@
|
||||
{"meta":{"title":"Natural Earth admin-0 disputed areas, for the News layer","source":"ne_10m_admin_0_disputed_areas","url":"https://raw.githubusercontent.com/nvkelso/natural-earth-vector/ca96624a56bd078437bca8184e78163e5039ad19/geojson/ne_10m_admin_0_disputed_areas.geojson","sha256":"9cafef8b7dfb6b164dc58f218f981f4ace9f716f6c03795d4c62d1ac9f3d50f5","fetched":"2026-11-05","license":"Public domain (https://www.naturalearthdata.com/about/terms-of-use/)","curation":"A hand-picked 11 of the source's 99 features: Golan Heights, UNDOF Zone, Transnistria, Abyei, Crimea, Shebaa Farms, Bir Tawil, Halayib Triangle, East Jerusalem, Abu Musa I. and Doumera Island. The source names three of them by a country (Moldova, Sudan, Iran); they are named here as the area. kind, heldBy and claimedBy are read from NOTE_BRK; note is NOTE_BRK as published. The 2014 Donetsk and Luhansk lines are left out as out of date. Every part kept; each ring Douglas-Peucker simplified at 0.003 degrees, coordinates rounded to 3 decimals, rings stored open."},"areas":[{"name":"Golan Heights","kind":"held","heldBy":"IL","claimedBy":"SY","note":"Admin. By Israel; Claimed by Syria","rings":[[[35.821,33.407],[35.81,33.36],[35.786,33.343],[35.764,33.334],[35.802,33.312],[35.769,33.273],[35.804,33.248],[35.808,33.202],[35.83,33.19],[35.833,33.161],[35.811,33.127],[35.811,33.112],[35.849,33.099],[35.846,33.085],[35.859,32.99],[35.888,32.945],[35.85,32.896],[35.838,32.866],[35.842,32.854],[35.834,32.828],[35.784,32.778],[35.758,32.744],[35.74,32.741],[35.685,32.711],[35.636,32.679],[35.612,32.682],[35.595,32.707],[35.6,32.717],[35.634,32.752],[35.635,32.769],[35.641,32.772],[35.635,32.794],[35.641,32.82],[35.638,32.832],[35.641,32.865],[35.638,32.875],[35.602,32.909],[35.615,32.921],[35.62,32.984],[35.588,33.018],[35.621,33.05],[35.634,33.052],[35.637,33.077],[35.656,33.125],[35.672,33.15],[35.661,33.213],[35.66,33.236],[35.666,33.247],[35.635,33.249],[35.608,33.239],[35.605,33.244],[35.618,33.256],[35.62,33.27],[35.631,33.275],[35.654,33.262],[35.712,33.307],[35.718,33.317],[35.716,33.327],[35.744,33.331],[35.769,33.343],[35.786,33.358],[35.806,33.391]]]},{"name":"UNDOF Zone","kind":"un","note":"UN Patrolled Zone in Syria","rings":[[[35.758,32.744],[35.784,32.778],[35.834,32.828],[35.842,32.854],[35.838,32.866],[35.85,32.896],[35.888,32.945],[35.859,32.99],[35.846,33.085],[35.849,33.099],[35.811,33.112],[35.811,33.127],[35.833,33.161],[35.83,33.19],[35.808,33.202],[35.804,33.248],[35.769,33.273],[35.802,33.312],[35.764,33.334],[35.786,33.343],[35.81,33.36],[35.821,33.407],[35.87,33.431],[35.873,33.416],[35.87,33.407],[35.845,33.389],[35.83,33.365],[35.83,33.351],[35.843,33.322],[35.845,33.304],[35.84,33.295],[35.819,33.284],[35.82,33.281],[35.851,33.276],[35.866,33.226],[35.889,33.199],[35.882,33.189],[35.885,33.179],[35.9,33.18],[35.911,33.166],[35.911,33.135],[35.892,33.124],[35.897,33.117],[35.903,33.117],[35.903,33.11],[35.868,33.107],[35.865,33.082],[35.877,32.993],[35.898,32.986],[35.924,32.968],[35.912,32.947],[35.89,32.929],[35.861,32.88],[35.843,32.822],[35.815,32.787],[35.791,32.774],[35.77,32.748],[35.764,32.747]]]},{"name":"Transnistria","kind":"self","claimedBy":"MD","note":"Self admin.; Claimed by Moldova","rings":[[[29.899,46.553],[29.868,46.58],[29.84,46.582],[29.831,46.588],[29.824,46.601],[29.776,46.58],[29.728,46.594],[29.748,46.617],[29.776,46.629],[29.759,46.658],[29.749,46.665],[29.728,46.67],[29.703,46.665],[29.696,46.676],[29.667,46.676],[29.69,46.707],[29.698,46.728],[29.665,46.801],[29.652,46.813],[29.62,46.815],[29.611,46.827],[29.619,46.832],[29.615,46.841],[29.59,46.824],[29.563,46.787],[29.549,46.793],[29.543,46.793],[29.531,46.783],[29.512,46.787],[29.498,46.801],[29.502,46.821],[29.495,46.828],[29.494,46.862],[29.477,46.889],[29.471,46.941],[29.46,46.953],[29.436,46.957],[29.376,46.935],[29.35,46.937],[29.316,46.958],[29.296,46.992],[29.325,46.988],[29.371,47.003],[29.378,47.012],[29.374,47.011],[29.364,47.033],[29.381,47.029],[29.383,47.042],[29.378,47.071],[29.372,47.078],[29.33,47.108],[29.296,47.122],[29.291,47.14],[29.237,47.162],[29.227,47.149],[29.221,47.149],[29.213,47.158],[29.188,47.162],[29.179,47.156],[29.18,47.145],[29.194,47.124],[29.19,47.111],[29.157,47.113],[29.148,47.096],[29.125,47.074],[29.099,47.092],[29.07,47.101],[29.095,47.135],[29.112,47.15],[29.145,47.163],[29.164,47.192],[29.179,47.197],[29.164,47.231],[29.094,47.31],[29.08,47.332],[29.079,47.348],[29.089,47.351],[29.123,47.329],[29.14,47.329],[29.154,47.336],[29.165,47.348],[29.139,47.376],[29.083,47.406],[29.091,47.437],[29.108,47.454],[29.111,47.464],[29.104,47.478],[29.093,47.487],[29.049,47.506],[29.034,47.518],[29.025,47.533],[29.016,47.582],[28.993,47.605],[28.987,47.622],[28.996,47.651],[28.988,47.661],[28.981,47.687],[28.976,47.743],[28.981,47.765],[29.018,47.827],[29.019,47.853],[28.981,47.875],[28.861,47.899],[28.829,47.917],[28.831,47.941],[28.813,47.965],[28.785,47.983],[28.754,47.979],[28.719,47.999],[28.698,48.022],[28.682,48.031]Line truncated
|
||||
Reference in new issue
Block a user