From 34c75ff5de39b486f21a8d8801bf674f8caac012 Mon Sep 17 00:00:00 2001 From: Denis Groshev Date: Thu, 5 Nov 2026 19:45:18 +0100 Subject: [PATCH] Serve the News layer without a news server: GDELT, Liveuamap and World News API With GEV_NEWS_ORIGIN unset, Core answers the News layer's routes itself. Conflicts need no key: the last 24 h of GDELT 2.0 event exports, physical violence only (CAMEO 18-20), one report per story (copies of one agency story merged), placed where GDELT put it, a town the article's own words name winning over a bare country, and titled by the article's page title from the GKG export of the same quarter hour (the newest 3 h), else by its address words. GDELT's actor coding is never shown. Exports are read up to the newest GDELT lists; one that cannot be read is given up on without holding the rest back, and an outage of two hours answers 502 rather than an empty feed. With LIVEUAMAP_API_KEY, Liveuamap's mapped items join them. Local reads World News API by the source country at the view centre (WORLDNEWSAPI_KEY, at most WORLDNEWSAPI_DAILY_READS a day), with the Conflicts reports placed inside the area. Disputed areas (11 from Natural Earth, bundled) and territories without a country code are named as themselves and read from no country's outlets; a country's outline leaves them out. Categories, relevance and severity come from shared rules; keyed answers are cached and served while they refresh, and no key or address is logged. --- scripts/package-boundaries.json | 10 + server/providers/news.js | 65 +- server/providers/news.test.mjs | 83 +- server/providers/news/areas.js | 216 ++++ server/providers/news/categorize.js | 873 ++++++++++++++++ server/providers/news/gdeltEvents.js | 543 ++++++++++ server/providers/news/gdeltEvents.test.mjs | 406 +++++++ server/providers/news/gdeltFixtures.mjs | 72 ++ server/providers/news/keyed.js | 851 +++++++++++++++ server/providers/news/keyed.test.mjs | 988 ++++++++++++++++++ server/providers/news/upstream.js | 274 +++++ src/data/local_data/natural_earth/README.md | 49 +- .../natural_earth/disputed_areas.json | 1 + 13 files changed, 4399 insertions(+), 32 deletions(-) create mode 100644 server/providers/news/areas.js create mode 100644 server/providers/news/categorize.js create mode 100644 server/providers/news/gdeltEvents.js create mode 100644 server/providers/news/gdeltEvents.test.mjs create mode 100644 server/providers/news/gdeltFixtures.mjs create mode 100644 server/providers/news/keyed.js create mode 100644 server/providers/news/keyed.test.mjs create mode 100644 server/providers/news/upstream.js create mode 100644 src/data/local_data/natural_earth/disputed_areas.json diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index ca8491892..a9fe65232 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -496,7 +496,17 @@ "exports": ["./server/providers/news"], "modules": [ "server/providers/news.js", + "server/providers/news/keyed.js", + "server/providers/news/categorize.js", + "server/providers/news/areas.js", + "server/providers/news/gdeltEvents.js", + "server/providers/news/upstream.js", "server/providers/common/http.js", + "src/data/adminBoundaries.js", + "src/data/bundledJson.js", + "src/data/retryableLoad.js", + "src/data/local_data/natural_earth/countries.json", + "src/data/local_data/natural_earth/disputed_areas.json", "src/layers/news/source.js", "src/layers/news/records.js", "src/sources/httpBody.js", diff --git a/server/providers/news.js b/server/providers/news.js index 5d10ab2b1..0ad78665a 100644 --- a/server/providers/news.js +++ b/server/providers/news.js @@ -1,11 +1,14 @@ import { newsQueryAllowed } from '../../src/layers/news/source.js'; import { readResponseTextCapped } from './common/http.js'; +import { createKeyedNews } from './news/keyed.js'; -// News (/api/news/*): Core serves no news of its own. With GEV_NEWS_ORIGIN -// set (a news server such as https://news.example), this forwards exactly -// the GET requests the news layer sends there and relays the JSON; without -// it, every route answers "not configured" and the layer shows itself -// unavailable without retrying. +// News (/api/news/*). With GEV_NEWS_ORIGIN set (a news server such as +// https://news.example), this forwards exactly the GET requests the news +// layer sends there and relays the JSON. Without it (or with a value that +// is no plain origin, warned of once), Core answers the same routes itself +// (news/keyed.js): Conflicts from GDELT's event exports with no key, and +// Liveuamap's items beside them with LIVEUAMAP_API_KEY; Local from World +// News API, "key required" while WORLDNEWSAPI_KEY is unset. const TIMEOUT_MS = 15_000; const MAX_BYTES = 4 * 1024 * 1024; @@ -34,14 +37,42 @@ export function newsOrigin(raw = process.env.GEV_NEWS_ORIGIN) { } } +/** Text for the log: no address (a key may ride in one), one short line. */ +const logText = (error) => + `${error?.name || 'Error'}: ${String(error?.message ?? '')}` + .replace(/[a-z][a-z0-9+.-]*:\/\/\S*/gi, '') + .replace(/\s+/g, ' ') + .slice(0, 200); + +/** GEV_NEWS_ORIGIN as newsOrigin reads it; a value it refuses is warned of once. */ +function configuredOrigin(env, log) { + let warned = null; + return () => { + const raw = String(env().GEV_NEWS_ORIGIN ?? '').trim(); + const base = newsOrigin(raw); + if (!base && raw && warned !== raw) { + warned = raw; + // Not the value: it may carry credentials. + log( + '[news] GEV_NEWS_ORIGIN is not a plain http(s) origin; answering with the built-in news backend', + ); + } + return base; + }; +} + /** - * A thin, validated GET proxy for the news layer's three read routes. - * @param {{origin?: () => string|null, fetchImpl?: typeof fetch, timeoutMs?: number}} [options] + * A thin, validated GET proxy for the news layer's three read routes, or + * their keyed answers without a news server. + * @param {{env?: () => object, log?: (message: string) => void, origin?: () => string|null, fetchImpl?: typeof fetch, timeoutMs?: number, keyed?: {answer: (path: string, params: URLSearchParams) => Promise<{status: number, body: object}>}}} [options] */ export function newsProxy({ - origin = () => newsOrigin(), + env = () => process.env, + log = (message) => console.warn(message), + origin = configuredOrigin(env, log), fetchImpl = (...args) => globalThis.fetch(...args), timeoutMs = TIMEOUT_MS, + keyed = createKeyedNews({ env, fetchImpl, log }), } = {}) { const json = (res, status, value) => { if (res.destroyed) return; @@ -64,12 +95,18 @@ export function newsProxy({ if (!newsQueryAllowed(target)) return json(res, 400, { error: 'invalid_news_request' }); const base = origin(); - if (!base) - return json(res, 503, { - error: 'news_not_configured', - unavailable: true, - retryable: false, - }); + if (!base) { + try { + const { status, body } = await keyed.answer( + local.pathname, + target.searchParams, + ); + return json(res, status, body); + } catch (error) { + log(`[news] built-in news answer failed: ${logText(error)}`); + return json(res, 502, { error: 'news_unavailable' }); + } + } try { const upstream = await fetchImpl( new URL(`${target.pathname}${target.search}`, base), diff --git a/server/providers/news.test.mjs b/server/providers/news.test.mjs index 73defce9f..07c1f0fd1 100644 --- a/server/providers/news.test.mjs +++ b/server/providers/news.test.mjs @@ -1,6 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { newsOrigin, newsProxy } from './news.js'; +import { createKeyedNews } from './news/keyed.js'; /** Mount the proxy on a fake middleware stack and call it like Connect does. */ function mount(options) { @@ -33,7 +34,10 @@ function mount(options) { } test('the origin must be a plain http(s) origin', () => { - assert.equal(newsOrigin('https://news.example'), 'https://news.example'); + assert.equal( + newsOrigin('http://news.example:8080'), + 'http://news.example:8080', + ); assert.equal( newsOrigin('https://news.example/base/'), 'https://news.example', @@ -51,23 +55,32 @@ test('the origin must be a plain http(s) origin', () => { assert.equal(newsOrigin(bad), null, String(bad)); }); -test('without GEV_NEWS_ORIGIN every route answers not configured and nothing is fetched', async () => { - let fetches = 0; +test('without GEV_NEWS_ORIGIN Core answers itself: Conflicts from GDELT, Local saying it needs its key', async () => { + const hosts = []; + const fetchImpl = async (url) => { + hosts.push(new URL(String(url)).host); + throw new Error('offline'); + }; const call = mount({ origin: () => null, - fetchImpl: async () => { - fetches++; - throw new Error('unexpected'); - }, + fetchImpl, + keyed: createKeyedNews({ env: () => ({}), fetchImpl, log: () => {} }), }); - const response = await call('/hotspots'); - assert.equal(response.status, 503); - assert.deepEqual(response.json, { - error: 'news_not_configured', - unavailable: true, - retryable: false, - }); - assert.equal(fetches, 0); + // GDELT's exports need no key; offline, the read fails like any upstream. + const hotspots = await call('/hotspots'); + assert.equal(hotspots.status, 502); + assert.deepEqual(hotspots.json, { error: 'news_upstream_error' }); + const local = await call('/feed?hotspot=all&at=51.13,71.43'); + assert.equal(local.status, 503); + assert.deepEqual(local.json, { error: 'no_key', key: 'worldnewsapi' }); + // The area is read from the bundled countries, with no key. + const area = await call('/outline?at=51.13,71.43&v=4'); + assert.equal(area.status, 200); + assert.equal(area.json.area.name, 'Kazakhstan'); + // The client's allowlist still holds. + assert.equal((await call('/feed?side=UA')).status, 400); + assert.ok(hosts.length > 0); + assert.ok(hosts.every((host) => host === 'data.gdeltproject.org')); }); test('forwards only the client routes, by GET, and relays the JSON', async () => { @@ -139,3 +152,43 @@ test('upstream failures, bad JSON and timeouts become bounded errors', async () assert.equal(timedOut.status, 504); assert.deepEqual(timedOut.json, { error: 'news_unavailable' }); }); + +test('a GEV_NEWS_ORIGIN that is no plain origin is warned of once, and Core answers itself', async () => { + const logs = []; + const asked = []; + const call = mount({ + env: () => ({ GEV_NEWS_ORIGIN: 'https://user:secret@news.example' }), + log: (line) => logs.push(line), + fetchImpl: async () => assert.fail('nothing is forwarded'), + keyed: { + answer: async (path) => { + asked.push(path); + return { status: 200, body: { reports: [], nextBefore: null } }; + }, + }, + }); + assert.equal((await call('/feed?hotspot=all')).status, 200); + assert.equal((await call('/feed?hotspot=all')).status, 200); + assert.deepEqual(asked, ['/feed', '/feed']); + assert.equal(logs.length, 1); + assert.doesNotMatch(logs[0], /secret|news\.example/); +}); + +test('a built-in answer that throws is a 502, logged in one line without addresses', async () => { + const logs = []; + const call = mount({ + env: () => ({}), + log: (line) => logs.push(line), + keyed: { + answer: async () => { + throw new TypeError('bad https://api.example/?key=abc\nmore'); + }, + }, + }); + const failed = await call('/hotspots'); + assert.equal(failed.status, 502); + assert.deepEqual(failed.json, { error: 'news_unavailable' }); + assert.equal(logs.length, 1); + assert.match(logs[0], /TypeError/); + assert.doesNotMatch(logs[0], /key=abc|\n/); +}); diff --git a/server/providers/news/areas.js b/server/providers/news/areas.js new file mode 100644 index 000000000..d283fe876 --- /dev/null +++ b/server/providers/news/areas.js @@ -0,0 +1,216 @@ +// Areas for the News layer, from the bundled Natural Earth packs (public +// domain): the disputed area or else the country at a point, its outline in +// the /api/news/outline form, and a point inside a country to pin its +// national news at. A disputed area is checked first and named as itself, +// so no one country's name stands for it; so is a territory the countries +// pack gives no code (Somaliland), rather than read as sea. Read once, on +// first use. + +import { readFileSync } from 'node:fs'; + +import { + containsAnchor, + geometryOf, + pointInRing, +} from '../../../src/data/adminBoundaries.js'; + +const PACKS = '../../../src/data/local_data/natural_earth/'; +const PACK = new URL(`${PACKS}countries.json`, import.meta.url); +const DISPUTED = new URL(`${PACKS}disputed_areas.json`, import.meta.url); +const ISO2 = /^[A-Z]{2}$/; + +let countries = null; +let territories = []; + +/** + * One entry per ISO code: its name and every Country part's geometry. The + * Country features without a code ("-99": Somaliland, Northern Cyprus, + * sovereign base areas, no man's lands) are kept apart as territories. + */ +function loadCountries(read = () => JSON.parse(readFileSync(PACK, 'utf8'))) { + if (countries) return countries; + const pack = read(); + const decimals = pack.meta?.decimals ?? 3; + const byCode = new Map(); + const codeless = []; + for (const feature of pack.features || []) { + // The constituent countries of the United Kingdom repeat its part. + if (feature.type !== 'Country') continue; + if (!ISO2.test(feature.iso2 || '')) { + if (feature.name) + codeless.push({ name: feature.name, feature, decimals }); + continue; + } + const entry = byCode.get(feature.iso2) || { + code: feature.iso2, + name: null, + size: 0, + parts: [], + }; + entry.parts.push({ feature, decimals }); + // Named after its largest feature: Kazakhstan, not the Baikonur lease. + const size = JSON.stringify(feature.polygons).length; + if (size > entry.size) { + entry.name = feature.name; + entry.size = size; + } + byCode.set(feature.iso2, entry); + } + countries = byCode; + territories = codeless; + return countries; +} + +/** Test seam: read another pack (or the bundled one again). */ +export function resetCountries(read) { + countries = null; + if (read) loadCountries(read); +} + +const geometries = (entry) => entry.parts.map((part) => geometryOf(part)); + +let disputed = null; +const slug = (name) => + name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + +/** The disputed areas: `{key, name, kind, heldBy, claimedBy, rings}`. */ +function loadDisputed() { + if (disputed) return disputed; + const pack = JSON.parse(readFileSync(DISPUTED, 'utf8')); + disputed = (pack.areas || []).map((area) => ({ + key: `disputed-${slug(area.name)}`, + name: area.name, + kind: area.kind, + heldBy: area.heldBy ?? null, + claimedBy: area.claimedBy ?? null, + rings: area.rings, + })); + return disputed; +} + +/** The disputed area at a point, or null. */ +export function disputedAt(lat, lon) { + return ( + loadDisputed().find((area) => + area.rings.some((ring) => pointInRing(ring, lat, lon)), + ) || null + ); +} + +/** + * The area a point is in, as the News layer names it: a disputed area, or + * a territory without a code, as itself (`kind: 'disputed'`, `code` null), + * else its country (`kind: 'country'`, with `country` the countryAt + * entry); null at sea. + */ +export function areaAt(lat, lon) { + const area = disputedAt(lat, lon); + if (area) + return { + ...area, + kind: 'disputed', + held: area.kind, + code: null, + polygons: area.rings.map((outer) => ({ outer, holes: [] })), + }; + const country = countryAt(lat, lon); + if (country) + return { + key: country.code, + name: country.name, + kind: 'country', + code: country.code, + country, + }; + const territory = territoryAt(lat, lon); + return territory + ? { + key: `disputed-${slug(territory.name)}`, + name: territory.name, + kind: 'disputed', + held: null, + code: null, + polygons: geometryOf(territory).polygons.map(([outer, ...holes]) => ({ + outer, + holes, + })), + } + : null; +} + +/** Whether a point lies in a geometry (geometryOf), across the antimeridian. */ +function inGeometry(geometry, lat, lon) { + const [west, south, east, north] = geometry.bbox; + const x = lon - 360 * Math.round((lon - (west + east) / 2) / 360); + if (x < west || x > east || lat < south || lat > north) return false; + return containsAnchor(geometry.polygons, lat, lon); +} + +/** The country at a point, or null at sea or in a territory without a code. */ +export function countryAt(lat, lon) { + for (const entry of loadCountries().values()) + for (const geometry of geometries(entry)) + if (inGeometry(geometry, lat, lon)) return entry; + return null; +} + +/** The territory without a code at a point (Somaliland), or null. */ +function territoryAt(lat, lon) { + loadCountries(); + return ( + territories.find((part) => inGeometry(geometryOf(part), lat, lon)) || null + ); +} + +/** A country by its ISO code, or null. */ +export const countryByCode = (code) => loadCountries().get(code) || null; + +/** + * A country's outline as /api/news/outline answers it (`{outer, holes}`), + * the disputed areas inside it left out as holes (Russia's without Crimea). + */ +export function countryPolygons(entry) { + const areas = loadDisputed(); + return geometries(entry).flatMap((geometry) => + geometry.polygons.map(([outer, ...holes]) => ({ + outer, + holes: [ + ...holes, + ...areas + .flatMap((area) => area.rings) + .filter((ring) => pointInRing(outer, ring[0][1], ring[0][0])), + ], + })), + ); +} + +/** + * A point inside the country's largest part, for news placed in the country + * alone: its label point when that lies inside, else the middle of the + * widest span across the part at its middle latitude. + */ +export function countryAnchor(entry) { + const largest = geometries(entry).sort((a, b) => b.areaKm2 - a.areaKm2)[0]; + const { label, polygons, bbox } = largest; + if (containsAnchor(polygons, label.lat, label.lon)) return label; + const lat = (bbox[1] + bbox[3]) / 2; + const [outer] = polygons.reduce((a, b) => + b[0].length > a[0].length ? b : a, + ); + const crossings = []; + for (let i = 0, j = outer.length - 1; i < outer.length; j = i++) { + const [x1, y1] = outer[j]; + const [x2, y2] = outer[i]; + if (y1 > lat !== y2 > lat) + crossings.push(x1 + ((lat - y1) * (x2 - x1)) / (y2 - y1)); + } + crossings.sort((a, b) => a - b); + let best = null; + for (let i = 0; i + 1 < crossings.length; i += 2) + if (!best || crossings[i + 1] - crossings[i] > best[1] - best[0]) + best = [crossings[i], crossings[i + 1]]; + return best ? { lat, lon: (best[0] + best[1]) / 2 } : label; +} diff --git a/server/providers/news/categorize.js b/server/providers/news/categorize.js new file mode 100644 index 000000000..4574246de --- /dev/null +++ b/server/providers/news/categorize.js @@ -0,0 +1,873 @@ +// Rule-based report categories, relevance and severity: the News layer's +// base reading, shared by every news server. Multilingual stems (en, uk, ru, +// ar, he and more) matched at a word start on normalised text; a trailing `$` +// makes a stem whole-word. A title hit counts double. Ties go to the earlier +// category in CATEGORY_ORDER, which runs from most to least specific. +// +// A news server may replace these readings with its own; the rules stay +// as the fallback. + +/** Text as the rules match it: folded case, one apostrophe, no vowel marks. */ +export function normaliseForMatch(text) { + return String(text || '') + .normalize('NFKC') + .toLowerCase() + .replace(/ё/g, 'е') + .replace(/[’ʼ`´]/g, "'") + .replace(/[ً-ٰٟ֑-ׇ]/g, '') + .replace(/[أإآ]/g, 'ا'); +} + +export const CATEGORY_ORDER = Object.freeze([ + 'strike', + 'alert', + 'air-defence', + 'ground', + 'casualties', + 'humanitarian', + 'infrastructure', + 'disaster', + 'protest', + 'political', + // Rules only, for the Local section's national news. + 'economy', +]); + +// [stem, weight]; weight defaults to 1. +const RULES = { + strike: [ + 'strike', + 'airstrike', + 'air strike', + 'missile', + 'rocket', + 'drone', + 'shelling', + 'shelled', + 'bomb', + 'explosion', + 'blast', + 'artillery', + 'shahed', + ['attack', 0.5], + 'удар', + 'ракет', + 'дрон', + 'бпла', + 'обстріл', + 'обстрел', + 'вибух', + 'взрыв', + 'шахед', + 'бомб', + ['атак', 0.5], + 'غارة', + 'غارات', + 'قصف', + 'صاروخ', + 'صواريخ', + 'مسيرة', + 'انفجار', + ['استهداف', 0.7], + ['هجوم', 0.5], + 'תקיפה', + 'טיל', + 'רקט', + 'כטב"מ', + 'פיצוץ', + 'הפצצ', + ], + 'air-defence': [ + 'air defen', + 'intercept', + 'shot down', + 'downed', + 'air raid', + 'ппо', + 'збит', + 'сбит', + 'перехоп', + 'перехват', + 'повітряна тривога', + 'воздушная тревога', + 'الدفاع الجوي', + 'اعتراض', + 'יירוט', + 'יורט', + 'כיפת ברזל', + 'אזעק', + ], + ground: [ + 'advance$', + 'advances$', + 'advancing', + 'offensive', + 'captur', + 'liberat', + 'assault', + 'clash', + 'fighting', + 'counteroffensive', + 'frontline', + 'front line', + 'battle', + 'encircl', + ['troops', 0.5], + 'наступ', + 'штурм', + 'звільн', + 'освобо', + 'бої$', + 'бои$', + 'контрнаступ', + 'захоп', + 'захват', + 'оточ', + 'окруж', + 'продвиж', + 'просуван', + 'اشتباك', + 'معارك', + 'تقدم', + 'سيطر', + 'اقتحام', + 'توغل', + 'לחימה', + 'קרב', + 'פשיט', + 'תמרון', + ], + casualties: [ + 'killed', + 'dead$', + 'deaths', + 'wounded', + 'injured', + 'casualt', + 'death toll', + 'загин', + 'поранен', + 'жертв', + 'вбит', + 'погиб', + 'ранен', + 'убит', + 'قتلى', + 'شهداء', + 'شهيد', + 'جرحى', + 'مقتل', + 'اصاب', + 'הרוג', + 'נהרג', + 'פצוע', + 'נפצע', + ], + humanitarian: [ + 'evacuat', + 'aid$', + 'refugee', + 'displac', + 'humanitarian', + 'famine', + 'hunger', + 'convoy', + 'shelter', + 'евакуац', + 'эвакуац', + 'гуманітар', + 'гуманитар', + 'біженц', + 'беженц', + 'انسانية', + 'نازح', + 'نزوح', + 'مساعدات', + 'اجلاء', + 'مجاعة', + 'הומניטרי', + 'פינוי', + 'סיוע', + ], + infrastructure: [ + 'power outage', + 'blackout', + 'electricity', + 'energy', + 'grid$', + 'substation', + 'water supply', + 'bridge', + 'railway', + 'refinery', + 'port$', + 'dam$', + 'power plant', + 'енерг', + 'энерг', + 'знеструм', + 'обесточ', + 'підстанц', + 'подстанц', + 'залізниц', + 'железн', + 'нпз', + 'الكهرباء', + 'محطة', + 'مياه', + 'جسر', + 'חשמל', + 'תשתית', + ], + disaster: [ + 'earthquake', + 'flood', + 'wildfire', + 'landslide', + 'cyclone', + 'hurricane', + 'typhoon', + 'tsunami', + 'eruption', + 'glof$', + 'glacial lake', + 'storm$', + 'lightning', + 'землетрус', + 'землетрясен', + 'повін', + 'паводк', + 'наводнен', + 'زلزال', + 'فيضان', + 'سيول', + 'רעידת אדמה', + 'שיטפון', + ], + protest: [ + 'protest', + 'demonstrat', + 'rally$', + 'rallies', + 'riot', + // Not a bare "протест": "протестировал" is "tested". + 'протест$', + 'протеста', + 'протесту', + 'протестом', + 'протесте$', + 'протесты', + 'протестов', + 'протестн', + 'протестую', + 'протестова', + 'мітинг', + 'митинг', + 'احتجاج', + 'مظاهر', + 'تظاهر', + 'הפגנ', + 'מחאה', + ], + political: [ + 'talks$', + 'negotiat', + 'sanction', + 'summit', + 'ceasefire', + 'truce', + 'peace', + 'election', + 'parliament', + ['president', 0.5], + ['minister', 0.5], + ['statement', 0.5], + // A state visit ("King of Jordan arrives in Astana"). + ['visit', 0.5], + ['delegation', 0.5], + ['agreement', 0.5], + 'memorandum', + 'diplomat', + 'bilateral', + 'ambassador', + ['визит', 0.5], + ['делегац', 0.5], + 'дипломат', + 'переговор', + 'санкц', + 'перемир', + 'вибори', + 'выборы', + ['президент', 0.5], + ['міністр', 0.5], + ['министр', 0.5], + 'مفاوضات', + 'عقوبات', + 'هدنة', + 'وقف اطلاق النار', + ['وزير', 0.5], + ['رئيس', 0.5], + 'משא ומתן', + 'הפסקת אש', + 'סנקצי', + ['ראש הממשלה', 0.5], + ], + // Elections and party politics ("How Gina Hinojosa Captured Texas" is a + // governor's race, not a battle). They count toward political only when + // nothing physical happened (categorizeText): a strike in an election week + // is still a strike. + election: [ + 'midterm', + 'electoral', + 'elect$', + 'ballot', + 'voter', + 'vote$', + 'votes$', + 'voting', + 'polls$', + 'candidate', + 'senate race', + 'referendum', + 'inaugurat', + // Plurals only: "Democratic Republic of Congo", "Republican Guard". + 'democrats$', + 'republicans$', + 'gop$', + 'избира', + 'голосован', + 'виборц', + 'голосуван', + 'انتخاب', + 'בחירות', + ], +}; + +// Business, trade, industry and energy, counted only when nothing physical +// happened (categorizeText): an oil depot struck is a strike. English, +// Russian, Ukrainian, Kazakh-press Russian, Turkish and the European stems. +RULES.economy = [ + 'econom', + 'gdp$', + 'inflation', + 'investment', + 'investor', + 'export', + 'imports$', + 'import$', + 'tariff', + 'trade deal', + 'trade turnover', + 'central bank', + 'stock market', + 'budget', + 'oil and gas', + 'oilfield', + 'oil field', + 'gas field', + 'exploration', + 'drilling', + 'mining', + 'industr', + 'manufactur', + 'business', + 'company', + 'companies', + 'revenue', + 'profit', + 'merger', + 'tourism', + 'agricultur', + 'harvest', + 'экономи', + 'инвест', + 'экспорт', + 'импорт', + 'инфляц', + 'ввп$', + 'бюджет', + 'промышлен', + 'бизнес', + 'компани', + 'товарооборот', + 'економі', + 'інвест', + 'експорт', + 'промисло', + 'ekonomi', + 'yatırım', + 'ihracat', + 'ithalat', + 'enflasyon', + 'sanayi', + 'şirket', + 'wirtschaft', + 'konjunktur', + 'unternehm', + 'inversi', + 'investiss', + 'investiment', + 'empresa', + 'entreprise', + 'impresa', +]; + +// Attack words beyond PHYSICAL's harm, which keep a report off economy. +const ATTACK = new RegExp( + String.raw`(? ({ + category, + rules: stems.map((entry) => { + const [stem, weight = 1] = Array.isArray(entry) ? entry : [entry]; + const whole = stem.endsWith('$'); + const body = escapeRegex( + normaliseForMatch(whole ? stem.slice(0, -1) : stem), + ).replace(/\s+/g, '\\s+'); + const arabic = /\p{Script=Arabic}/u.test(stem) ? '(?:[وفبلك]?(?:ال)?)' : ''; + const hebrew = /\p{Script=Hebrew}/u.test(stem) ? '(?:[ובלמהשכ]{0,2})' : ''; + return { + weight, + regex: new RegExp( + `(? `(? termPattern(term).test(text)); + // Any event reading counts, even when another category wins the title + // ("the accident toll rises to 74": infrastructure first, casualties too). + if ( + EVENT_CATEGORIES.has(category) || + tags.some((tag) => EVENT_CATEGORIES.has(tag)) + ) + return 'core'; + if (conflict && ['humanitarian', 'infrastructure'].includes(category)) + return 'core'; + if (conflict || dedicated || category === 'humanitarian') return 'related'; + return floor; +} + +// Alerts: air-raid alerts and reports of drones or missiles on their way +// ("Реактивний БпЛА курсом на Полтаву", "КАБи на Донеччину", "drone heading +// for Kyiv", "➡️"). Something is in the air; nothing has hit yet. Most of an +// air force's channel is these, and as "strikes" they would merge into real +// strike events and inflate them. +// A direction counts only after a weapon ("БпЛА … з півдня"), so a convoy +// leaving "in the direction of Oleshky" is not an alert. +const WEAPON = + '(?:бпла|дрон|шахед|герань|ракет|каб|балістик|крилат|боєприпас|боеприпас|uav|drone|missile|shahed|ballistic|cruise|munition)'; +const DIRECTION = + '(?:курсом на|у напрямку|в напрямку|в направлении|у бік|в сторону|повз |з (?:північ|півден|схід|сход|захід|заход)|с (?:север|юг|восток|запад)|на |над |heading |on course|toward)'; +const ALERT = new RegExp( + `${WEAPON}[^.!?]{0,40}${DIRECTION}|^[^\\p{L}]*курсом на|тривог|тревог|загроза застосування|угроза применения|air[- ]raid|threat of (?:missile|drone|ballistic|uav)|[➡⬅⬆⬇↗↘↙↖]`, + 'u', +); +// Wording that something did hit: then it is a strike (or air defence), not an alert. +const IMPACT = + /влуч|вибух|уражен|пошкодж|загин|поранен|зруйн|пожеж|атак|удар по|збит|знищ|попадан|взрыв|поражен|поврежд|погиб|ранен|разруш|сбит|уничтож|\bhit\b|\bhits\b|struck|explosion|blast|killed|injur|wounded|damage|destroy|fire\b|shot down|intercept|attack/u; + +/** Whether a report's title is an alert: a threat on its way, nothing hit yet. */ +function isAlertText(title = '', summary = '') { + const t = normaliseForMatch(`${title} ${summary}`.slice(0, 400)); + return ALERT.test(t) && !IMPACT.test(t); +} + +// "Hit" or "struck" says nothing on its own: what hit decides (a missile is a +// strike, a flood or lightning a disaster), and those words score by +// themselves. A few phrases use a category's word for another thing. +const PHRASES = [ + [/storm shadow/gu, 'missile'], + [/lightning strikes?/gu, 'lightning'], + [ + /(?:general|hunger|labou?r|workers?|nationwide|national|transport|rail|teachers?|doctors?|nurses?) strikes?|on strike|strike action/gu, + 'protest', + ], +]; +const rephrase = (text) => + PHRASES.reduce((out, [re, word]) => out.replace(re, word), text); + +// An election story borrows the words of war and disaster ("‘An Earthquake’: +// A Democratic Rout", "captured Texas", "a bloodbath for Republicans"). With +// election wording and nothing physical (no weapon, no one hurt, no quake's +// magnitude), the event categories give way to political. +const ELECTION = + /(? 0) scores.set(category, score); + } + const all = `${t} ${s}`; + const election = scores.get('election') || 0; + scores.delete('election'); + const physical = PHYSICAL.test(all); + // Business words and no harm or attack: "advance exploration" is no + // front-line advance; "attack an industrial facility" stays a strike. + if (physical || ATTACK.test(all)) scores.delete('economy'); + else if (scores.has('economy')) + for (const kind of EVENT_KINDS) scores.delete(kind); + if (!physical) { + if (election) + scores.set('political', (scores.get('political') || 0) + election); + if (ELECTION.test(all)) { + for (const kind of EVENT_KINDS) scores.delete(kind); + scores.set('political', Math.max(scores.get('political') || 0, 1)); + } + } + if (!scores.size) return { category: 'other', tags: [] }; + const ranked = CATEGORY_ORDER.filter((c) => scores.has(c)).sort( + (a, b) => scores.get(b) - scores.get(a), + ); + return { category: ranked[0], tags: ranked }; +} + +// Severity by rule, for reports without a severity of their own (0 none, +// 1 minor incident, 2 significant: deaths, injuries or real damage, 3 major: +// mass casualties). Wording in en, uk, ru, ar, he, es, fr, de. +const SEVERITY_EVENTS = new Set([...EVENT_CATEGORIES, 'infrastructure']); +const MAJOR_HARM = + /mass casualt|massacre|dozens (?:of \p{L}+ )?(?:killed|dead)|hundreds (?:of \p{L}+ )?(?:killed|dead)|масов\p{L}* (?:жертв|загибел|гибел)|десятки (?:загибл|погибш|убит|вбит|людей загин|человек погиб)|مجزرة|عشرات (?:القتلى|الشهداء)|טבח/u; +const HARM_WORDS = + 'kill|dead|died|death|injur|wound|casualt|загин|загибл|погиб|убит|вбит|поранен|ранен|постражд|пострадал|травм|قتل|قتيل|شهيد|شهداء|جرح|جريح|إصاب|اصاب|נהרג|הרוג|פצוע|muert|herid|tué|mort|bless|getötet|tote|verletz'; +const HARM = new RegExp(String.raw`(? 2100) most = Math.max(most, n); + } + return most; +} + +/** A report's severity by rule, 0–3. */ +export function severityByRules({ + category, + tags = [], + title = '', + summary = '', +}) { + const text = normaliseForMatch(`${title} ${summary}`); + const harmed = HARM.test(text) && !NO_HARM.test(text); + if (MAJOR_HARM.test(text) || (harmed && largestCount(text) >= 10)) return 3; + const event = + SEVERITY_EVENTS.has(category) || + tags.some((tag) => SEVERITY_EVENTS.has(tag)); + if (harmed && (event || category === 'humanitarian')) return 2; + if (event && DAMAGE.test(text)) return 2; + return event ? 1 : 0; +} diff --git a/server/providers/news/gdeltEvents.js b/server/providers/news/gdeltEvents.js new file mode 100644 index 000000000..e390dc060 --- /dev/null +++ b/server/providers/news/gdeltEvents.js @@ -0,0 +1,543 @@ +// GDELT 2.0 events for the Conflicts section, from the 15-minute export +// files (data.gdeltproject.org, plain downloads; no key, and not the +// rate-limited search API). Each export is one zipped tab-separated table of +// machine-coded events, each placed by GDELT with a point and a place type. +// +// A story counts when GDELT coded physical violence in it (CAMEO roots 18 +// assault, 19 fight, 20 mass violence); its place is read from all its +// placed events (choosePlace). GDELT's actor fields are machine-coded and +// often wrong, so no report says who did what: a report is the story (its +// article, and the copies other outlets ran under the same address words). +// The events carry no headline: a story's title is its page title from the +// GKG export of the same quarter hour (read for the newest hours only, it +// runs to 6 MB), else the words of its address. All of it is rules over +// GDELT's own fields; nothing reads the articles. + +import { promisify } from 'node:util'; +import { inflateRaw } from 'node:zlib'; + +import { readResponseBytesCapped } from '../common/http.js'; +import { categorizeText, relevanceOf, severityByRules } from './categorize.js'; +import { UpstreamError, unescapeHtml, webLink } from './upstream.js'; + +const EXPORTS_URL = 'https://data.gdeltproject.org/gdeltv2/'; +const STEP_MS = 15 * 60_000; +// An events export runs to about 100 KB zipped and 500 KB unzipped; a GKG +// export to about 6 MB and 18 MB. Unzipped off the main thread. +const MAX_ZIP_BYTES = 8 * 1024 * 1024; +const MAX_TABLE_BYTES = 8 * 1024 * 1024; +const MAX_GKG_ZIP_BYTES = 32 * 1024 * 1024; +const MAX_GKG_TABLE_BYTES = 32 * 1024 * 1024; +const inflate = promisify(inflateRaw); +// The most words an address's words make a title of. +const MAX_ADDRESS_WORDS = 24; +const TIMEOUT_MS = 30_000; +const MAX_PLACES = 3; + +// Columns of the 2.0 events table (61). +const COL = { + id: 0, + eventCode: 26, + rootCode: 28, + quadClass: 29, + mentions: 31, + geoType: 51, + place: 52, + lat: 56, + lon: 57, + added: 59, + url: 60, +}; + +// CAMEO event codes as the News layer's categories, and as words for an +// article whose address carries none of its own. +const EVENTS = [ + // Aerial weapons, artillery and tanks, bombings. + [/^19[45]|^183/, 'strike', 'Strike reported'], + [/^186|^185/, 'casualties', 'Killing reported'], + [/^181/, 'humanitarian', 'Abduction reported'], + [/^20/, 'casualties', 'Mass violence reported'], + [/^19/, 'ground', 'Fighting reported'], + [/^18/, 'other', 'Violence reported'], +]; +const eventKind = (code) => + EVENTS.find(([pattern]) => pattern.test(code)) || EVENTS.at(-1); + +// GDELT's place types: 1 country, 2 US state, 3 US city, 4 world city, +// 5 world state. +const PRECISION = { + 1: 'country', + 2: 'region', + 3: 'city', + 4: 'city', + 5: 'region', +}; + +/** The export holding the events GDELT added in the 15 minutes before `ms`. */ +export function exportUrl(ms) { + const at = new Date(Math.floor(ms / STEP_MS) * STEP_MS); + const stamp = at.toISOString().replace(/[-:T]/g, '').slice(0, 14); + return `${EXPORTS_URL}${stamp}.export.CSV.zip`; +} + +/** An events export's quarter hour (ms), from its name, or NaN. */ +export function exportAt(url) { + const match = /(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)00\.export\.CSV\.zip$/.exec( + url, + ); + return match + ? Date.UTC(match[1], match[2] - 1, match[3], match[4], match[5]) + : NaN; +} + +const LASTUPDATE_URL = `${EXPORTS_URL}lastupdate.txt`; + +/** + * The time of the newest events export GDELT lists as published, in ms, or + * null when its list cannot be read. Asking for one not listed yet would + * be answered 404, and GDELT's CDN keeps that 404 for an hour. + */ +export async function fetchLatestExport({ fetchImpl, signal } = {}) { + try { + const response = await fetchImpl(LASTUPDATE_URL, { + redirect: 'error', + signal: signal + ? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)]) + : AbortSignal.timeout(TIMEOUT_MS), + }); + if (!response.ok) { + await response.body?.cancel?.().catch?.(() => {}); + return null; + } + const text = new TextDecoder().decode( + await readResponseBytesCapped(response, 16 * 1024, signal), + ); + const at = exportAt( + /\S+\.export\.CSV\.zip/.exec(text)?.[0].replace(/^http:/, 'https:') || '', + ); + return Number.isFinite(at) ? at : null; + } catch { + return null; + } +} + +/** The GKG export of the same quarter hour as an events export. */ +export const gkgUrl = (eventsUrl) => + eventsUrl.replace(/\.export\.CSV\.zip$/, '.gkg.csv.zip'); + +/** The day's exports up to `now`, newest first. */ +export function exportsOfDay(now, hours = 24) { + const urls = []; + const last = Math.floor(now / STEP_MS) * STEP_MS; + for (let ms = last; ms > now - hours * 3_600_000; ms -= STEP_MS) + urls.push(exportUrl(ms)); + return urls; +} + +/** The one table inside an export's zip, as text, up to `maxBytes`. */ +export async function unzipExport(bytes, maxBytes = MAX_TABLE_BYTES) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const fail = () => { + throw new UpstreamError('GDELT export unreadable', { unreadable: true }); + }; + if (bytes.length < 30 || view.getUint32(0, true) !== 0x04034b50) fail(); + // The central directory has the sizes; a local header may leave them 0. + let eocd = -1; + for (let i = bytes.length - 22; i >= Math.max(0, bytes.length - 0x10015); i--) + if (view.getUint32(i, true) === 0x06054b50) { + eocd = i; + break; + } + if (eocd < 0) fail(); + const dir = view.getUint32(eocd + 16, true); + if (dir + 46 > bytes.length || view.getUint32(dir, true) !== 0x02014b50) + fail(); + const method = view.getUint16(dir + 10, true); + const compressed = view.getUint32(dir + 20, true); + const local = view.getUint32(dir + 42, true); + if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50) + fail(); + const start = + local + + 30 + + view.getUint16(local + 26, true) + + view.getUint16(local + 28, true); + if (start + compressed > bytes.length) fail(); + const data = bytes.subarray(start, start + compressed); + try { + if (method === 0 && data.length > maxBytes) fail(); + const table = + method === 0 + ? data + : method === 8 + ? await inflate(data, { maxOutputLength: maxBytes }) + : fail(); + return new TextDecoder().decode(table); + } catch (error) { + if (error instanceof UpstreamError) throw error; + return fail(); + } +} + +/** + * An article's address as the reports carry it: https, an http one + * upgraded (GDELT keeps the address as it found it), else null. + */ +export const articleLink = (value) => + webLink(value) || webLink(String(value ?? '').replace(/^http:/, 'https:')); + +/** + * The placed events of an export's table, of the stories with physical + * violence in them: those events, and every other placed event of the same + * stories (they help place it). + */ +export function parseEvents(table) { + const all = []; + for (const line of table.split('\n')) { + const f = line.split('\t'); + if (f.length < 61) continue; + const violent = + f[COL.quadClass] === '4' && ['18', '19', '20'].includes(f[COL.rootCode]); + const precision = PRECISION[f[COL.geoType]]; + const lat = Number(f[COL.lat]); + const lon = Number(f[COL.lon]); + const url = articleLink(f[COL.url]); + const added = /^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/.exec(f[COL.added]); + if ( + !precision || + !f[COL.lat] || + !f[COL.lon] || + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 || + (lat === 0 && lon === 0) || + !url || + !added + ) + continue; + all.push({ + id: f[COL.id], + code: f[COL.eventCode], + violent, + lat, + lon, + place: f[COL.place].trim() || null, + precision, + mentions: Math.max(1, Number(f[COL.mentions]) || 1), + addedAt: `${added[1]}-${added[2]}-${added[3]}T${added[4]}:${added[5]}:${added[6]}.000Z`, + url, + }); + } + const stories = new Set(all.filter((e) => e.violent).map(storyKey)); + return all.filter((e) => stories.has(storyKey(e))); +} + +/** + * The words of an article's address, when it is written in them + * ("three-children-die-in-peshawar-cylinder-exp"), as a sentence; else null. + */ +export function addressWords(url) { + let path; + try { + path = decodeURIComponent(new URL(url).pathname); + } catch { + return null; + } + const segment = path + .split('/') + .map((part) => part.replace(/\.(?:s?html?|php|aspx?)$/i, '')) + .filter((part) => /[-_]/.test(part)) + .sort((a, b) => b.length - a.length)[0]; + if (!segment) return null; + const words = segment.split(/[-_]+/).filter(Boolean); + const lettered = words.filter((word) => /^\p{L}+$/u.test(word)); + if (lettered.length < 4 || lettered.length < 0.7 * words.length) return null; + // Numbers stay ("killed nearly 500"); an id does not: a word of letters + // and digits, five digits or more, or four or more at either end. + const kept = words.filter( + (word, i) => + /^\p{L}+$/u.test(word) || + (/^\d{1,4}$/.test(word) && + !(word.length === 4 && (i === 0 || i === words.length - 1))), + ); + const text = kept.slice(0, MAX_ADDRESS_WORDS).join(' ').toLowerCase(); + return text.charAt(0).toUpperCase() + text.slice(1); +} + +/** + * Address words with the places the story is at written as names + * ("killed in amman" → "killed in Amman"), and "u s" as "US". + */ +export function withNames(words, places) { + let text = words.replace(/(? + String(place.place ?? '') + .split(',') + .map((part) => part.replace(/\(.*?\)/g, '').trim()) + .filter((part) => part.length > 2 && !/\(general\)/.test(part)), + ), + ); + for (const name of names) { + const pattern = new RegExp( + `(? 6) break; + title = rest; + } + title = title.trim().slice(0, 300); + return title.length >= 12 && title.split(/\s+/).length >= 3 ? title : null; +} + +/** + * GKG page titles by article address (as articleLink writes it), for the + * addresses in `wanted`. + */ +export function parseTitles(table, wanted) { + const titles = new Map(); + for (const line of table.split('\n')) { + const f = line.split('\t'); + const url = f[4] && articleLink(f[4]); + if (!url || !wanted.has(url)) continue; + const match = /(.*?)<\/PAGE_TITLE>/s.exec(f.at(-1) || ''); + const title = match && cleanTitle(match[1]); + if (title) titles.set(url, title); + } + return titles; +} + +/** + * The story an event belongs to: its article's address words, which the + * copies other outlets run share (an agency story), else its address. + */ +export function storyKey(event) { + return addressWords(event.url)?.toLowerCase() || event.url; +} + +const PLACE_WORD = (name) => + new RegExp( + `(? part.replace(/\(.*?\)/g, '').trim()) + .filter((part) => part.length > 2); + return [parts[0], parts.at(-1)].some( + (part) => part && PLACE_WORD(part).test(words.toLowerCase()), + ); +} + +/** + * A story's places, best first, from all its placed events. GDELT places + * an event at a bare country when the text only names a nationality ("a + * Sri Lankan embassy", an Australian outlet's own country), so a town or + * region wins over a country; of those, the ones the article's own words + * name; then the most mentioned. + */ +export function choosePlaces(events, words) { + const byPlace = new Map(); + for (const event of events) { + const key = `${event.lat},${event.lon}`; + const place = byPlace.get(key) || { ...event, mentions: 0 }; + place.mentions += event.mentions; + byPlace.set(key, place); + } + let places = [...byPlace.values()]; + const specific = places.filter((p) => p.precision !== 'country'); + if (specific.length) places = specific; + const named = places.filter((p) => namedIn(words, p.place)); + if (named.length) places = named; + return places.sort((a, b) => b.mentions - a.mentions).slice(0, MAX_PLACES); +} + +// The earlier of two events: by the time GDELT added it, then by its id. +const earlier = (a, b) => + b.addedAt < a.addedAt || + (b.addedAt === a.addedAt && Number(b.id) < Number(a.id)) + ? b + : a; + +/** + * One report per story (storyKey), its address and id the first copy's (an + * export adding a more-mentioned event keeps both), placed by choosePlaces, + * titled by a copy's page title (`titleOf`), else its address words, else + * its violent event. + */ +export function eventReports(events, { titleOf = () => null } = {}) { + const stories = new Map(); + for (const event of events) { + const key = storyKey(event); + const list = stories.get(key) || []; + list.push(event); + stories.set(key, list); + } + const reports = []; + for (const list of stories.values()) { + const violent = list + .filter((e) => e.violent !== false) + .sort( + (a, b) => b.mentions - a.mentions || a.addedAt.localeCompare(b.addedAt), + ); + if (!violent.length) continue; + const [lead] = violent; + const first = list.reduce(earlier); + const { url } = first; + const address = addressWords(url); + const places = choosePlaces(list, address); + const [, eventCategory, words] = eventKind(lead.code); + const page = [url, ...list.map((e) => e.url)] + .map((copy) => titleOf(copy)) + .find(Boolean); + const own = page || (address ? withNames(address, places) : null); + const title = own || eventTitle(words, places[0].place); + // The article's words decide whether it is conflict, as any headline's + // would: GDELT codes an election story as "fighting" too. GDELT's code + // names the kind only where the words read as conflict without naming + // one, or there are no words. + const read = own ? categorizeText({ title: own }) : null; + const tags = read?.tags || []; + const relevance = read + ? relevanceOf({ category: read.category, tags, title: own }) + : 'related'; + const category = + read && read.category !== 'other' + ? read.category + : relevance === 'minor' + ? 'other' + : eventCategory; + const publisher = new URL(url).hostname.replace(/^www\./, ''); + const geo = ({ lat, lon, place, precision }) => ({ + lat, + lon, + place, + precision, + confidence: precision === 'city' ? 0.6 : 0.3, + mappable: true, + method: 'source', + }); + reports.push({ + id: `gdelt-${first.id}`, + title, + url, + publisher, + source: { name: publisher, attribution: 'GDELT Project' }, + kind: 'gdelt', + side: 'unknown', + category, + relevance, + severity: Math.max( + 1, + severityByRules({ category, tags, title: own || '' }), + ), + severityBy: 'rules', + publishedAt: first.addedAt, + geo: geo(places[0]), + locations: places.slice(1).map(geo), + media: [], + mentions: list.reduce((sum, e) => sum + e.mentions, 0), + copies: new Set(list.map((e) => e.url)).size, + // A title of the event's words and its place, when the address had + // none: a renamed place renames it (eventTitle). + ...(own ? {} : { eventWords: words }), + titled: page ? 'page' : own ? 'address' : 'event', + }); + } + return reports; +} + +/** The title of a report without address words, at its (renamed) place. */ +export const eventTitle = (words, place) => + place ? `${words} in ${place}` : words; + +/** + * One export's table, or null while GDELT has not published it (GDELT + * lists the newest in lastupdate.txt; see fetchLatestExport). + */ +async function fetchTable(url, { fetchImpl, signal, maxBytes, tableBytes }) { + let response; + try { + response = await fetchImpl(url, { + redirect: 'error', + signal: signal + ? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)]) + : AbortSignal.timeout(TIMEOUT_MS), + }); + } catch { + throw new UpstreamError('GDELT exports unreachable'); + } + if (response.status === 404) { + await response.body?.cancel?.().catch?.(() => {}); + return null; + } + if (!response.ok) { + await response.body?.cancel?.().catch?.(() => {}); + throw new UpstreamError(`GDELT exports answered ${response.status}`, { + status: response.status, + }); + } + let bytes; + try { + bytes = await readResponseBytesCapped(response, maxBytes, signal); + } catch (error) { + // Over the cap stays over it; a read cut short may succeed again. + throw new UpstreamError('GDELT export unreadable', { + unreadable: error?.code === 'RESPONSE_TOO_LARGE', + }); + } + return unzipExport(bytes, tableBytes); +} + +/** One events export's events, or null while it is not published yet. */ +export async function fetchExport(url, { fetchImpl, signal } = {}) { + const table = await fetchTable(url, { + fetchImpl, + signal, + maxBytes: MAX_ZIP_BYTES, + tableBytes: MAX_TABLE_BYTES, + }); + return table === null ? null : parseEvents(table); +} + +/** + * The page titles a GKG export holds for the addresses in `wanted`, or null + * while it is not published yet. + */ +export async function fetchTitles(url, wanted, { fetchImpl, signal } = {}) { + const table = await fetchTable(url, { + fetchImpl, + signal, + maxBytes: MAX_GKG_ZIP_BYTES, + tableBytes: MAX_GKG_TABLE_BYTES, + }); + return table === null ? null : parseTitles(table, wanted); +} diff --git a/server/providers/news/gdeltEvents.test.mjs b/server/providers/news/gdeltEvents.test.mjs new file mode 100644 index 000000000..3d0919080 --- /dev/null +++ b/server/providers/news/gdeltEvents.test.mjs @@ -0,0 +1,406 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + addressWords, + cleanTitle, + gkgUrl, + parseTitles, + withNames, + eventReports, + exportsOfDay, + exportUrl, + fetchExport, + parseEvents, + unzipExport, +} from './gdeltEvents.js'; +import { eventRow, gkgRow, zipTable } from './gdeltFixtures.mjs'; + +const ARTICLE = + 'https://www.wire.example/rubric-ato/3912-russian-drones-attack-kharkiv-overnight-three-killed.html'; + +test('the day is 96 exports, one each quarter hour, newest first', () => { + const now = Date.parse('2026-11-05T15:07:30Z'); + assert.equal( + exportUrl(now), + 'https://data.gdeltproject.org/gdeltv2/20261105150000.export.CSV.zip', + ); + const day = exportsOfDay(now); + assert.equal(day.length, 96); + assert.equal( + day[1], + 'https://data.gdeltproject.org/gdeltv2/20261105144500.export.CSV.zip', + ); + assert.equal( + day.at(-1), + 'https://data.gdeltproject.org/gdeltv2/20261104151500.export.CSV.zip', + ); +}); + +test('a story with physical violence is kept with all its placed events', () => { + const rows = [ + eventRow({ id: 1, code: '195', url: ARTICLE }), + eventRow({ id: 2, code: '183', url: ARTICLE }), + eventRow({ id: 3, code: '2001', url: ARTICLE }), + // Not physical, but of the same story: kept to place it by. + eventRow({ id: 4, code: '150', url: ARTICLE }), + eventRow({ id: 7, code: '190', quad: '3', url: ARTICLE }), + // A story with no violence in it is left out. + eventRow({ + id: 5, + code: '163', + url: 'https://x.example/talks-resume-in-geneva-today-officials', + }), + eventRow({ + id: 6, + code: '173', + url: 'https://x.example/talks-resume-in-geneva-today-officials', + }), + // Unplaced, at 0,0, or without a link. + eventRow({ id: 8, geoType: '0', url: ARTICLE }), + eventRow({ id: 9, lat: 0, lon: 0, url: ARTICLE }), + eventRow({ id: 10, url: 'javascript:alert(1)' }), + 'short\trow', + ]; + const events = parseEvents(rows.join('\n')); + assert.deepEqual( + events.map((e) => [e.id, e.violent]), + [ + ['1', true], + ['2', true], + ['3', true], + ['4', false], + ['7', false], + ], + ); + assert.equal(events[0].precision, 'city'); + assert.equal(events[0].addedAt, '2026-11-05T11:45:00.000Z'); +}); + +test("an article is one report, at its most-mentioned event, in its address's words", () => { + const events = parseEvents( + [ + eventRow({ id: 11, code: '195', mentions: 2, url: ARTICLE }), + eventRow({ + id: 12, + code: '190', + mentions: 9, + url: ARTICLE, + place: "Kupiansk, Kharkivs'ka Oblast', Ukraine", + lat: 49.71, + lon: 37.61, + }), + eventRow({ + id: 13, + code: '190', + geoType: '1', + place: 'Ukraine', + lat: 49, + lon: 32, + url: 'https://example.com/a/b/12345', + }), + ].join('\n'), + ); + const [drones, bare] = eventReports(events); + // The story's first event (by time, then id) names it. + assert.equal(drones.id, 'gdelt-11'); + // Its address words, with the place it is at written as a name. + assert.equal( + drones.title, + 'Russian drones attack Kharkiv overnight three killed', + ); + // Kupiansk has more mentions; the article's own words name Kharkiv. + assert.equal(drones.geo.place, "Kharkiv, Kharkivs'ka Oblast', Ukraine"); + assert.equal(drones.geo.precision, 'city'); + assert.equal(drones.locations.length, 0, 'Kupiansk is not named'); + assert.equal(drones.category, 'strike'); + assert.equal(drones.relevance, 'core'); + assert.equal(drones.severity, 2); + assert.equal(drones.publisher, 'wire.example'); + assert.equal(drones.source.attribution, 'GDELT Project'); + assert.equal(drones.side, 'unknown', 'no actor is said'); + // No words in the address: GDELT's coding, related, at least severity 1. + assert.equal(bare.title, 'Fighting reported in Ukraine'); + assert.equal(bare.category, 'ground'); + assert.equal(bare.relevance, 'related'); + assert.equal(bare.severity, 1); + assert.equal(bare.geo.precision, 'country'); +}); + +test('an article GDELT codes as fighting whose words say otherwise stays out of Conflicts', () => { + const [report] = eventReports( + parseEvents( + eventRow({ + url: 'https://news.example.com/2026/11/05/city-council-approves-new-budget-for-schools', + id: 20, + }), + ), + ); + assert.equal(report.relevance, 'minor'); +}); + +test('address words need four lettered words, most of the address', () => { + assert.equal(addressWords('https://x.example/news/1234567'), null); + assert.equal(addressWords('https://x.example/a/lueop1pzfi'), null); + assert.equal( + addressWords('https://x.example/2026/11/05/a-b-1-2-3-4-5'), + null, + ); + assert.equal( + addressWords( + 'https://x.example/story/2638789/three-children-die-in-peshawar.html', + ), + 'Three children die in peshawar', + ); +}); + +test('an export is unzipped and read; one not published yet is null', async () => { + const zip = zipTable(eventRow({ id: 30, code: '193', url: ARTICLE })); + assert.deepEqual( + parseEvents(await unzipExport(zip)).map((e) => e.id), + ['30'], + ); + const served = await fetchExport('https://data.gdeltproject.org/x.zip', { + fetchImpl: async () => new Response(zip, { status: 200 }), + }); + assert.deepEqual( + served.map((e) => e.id), + ['30'], + ); + const later = await fetchExport('https://data.gdeltproject.org/y.zip', { + fetchImpl: async () => new Response('not found', { status: 404 }), + }); + assert.equal(later, null); + await assert.rejects( + fetchExport('https://data.gdeltproject.org/z.zip', { + fetchImpl: async () => new Response('busy', { status: 503 }), + }), + { name: 'UpstreamError' }, + ); + await assert.rejects(unzipExport(new Uint8Array([1, 2, 3])), { + name: 'UpstreamError', + }); +}); + +test("a nationality is no place: a town the article's words name wins over a bare country", () => { + const url = + 'https://www.sun.example/world/2026/11/05/security-officer-attacked-and-killed-near-sri-lankan-embassy-in-jordans-capital'; + const amman = { place: 'Amman, (JO11), Jordan', lat: 31.95, lon: 35.9333 }; + const cairo = { place: 'Cairo, Al Qahirah, Egypt', lat: 30.05, lon: 31.25 }; + const [report] = eventReports( + parseEvents( + [ + // "Sri Lankan embassy": GDELT's most-mentioned place, a bare country. + eventRow({ + id: 1, + mentions: 12, + geoType: '1', + place: 'Sri Lanka', + lat: 7, + lon: 81, + url, + }), + eventRow({ id: 2, mentions: 6, ...cairo, url }), + eventRow({ id: 3, mentions: 4, ...amman, url }), + eventRow({ id: 4, mentions: 2, ...amman, url }), + ].join('\n'), + ), + ); + assert.equal(report.geo.place, 'Amman, (JO11), Jordan'); + assert.equal(report.geo.precision, 'city'); +}); + +test('an agency story run by several outlets is one report, placed by all their events', () => { + const words = 'jordan-security-officer-killed-near-sri-lanka-embassy'; + const times = `https://www.times.example/story/9386807/${words}/`; + const chronicle = `https://www.chronicle.example/story/9386807/${words}/`; + const reports = eventReports( + parseEvents( + [ + // The outlet's own country, at country level, on its violent events. + eventRow({ + id: 1, + mentions: 6, + geoType: '1', + place: 'Australia', + lat: -25, + lon: 135, + url: times, + added: '20261105141500', + }), + // Its other events are placed in Jordan. + eventRow({ + id: 2, + code: '120', + quad: '3', + mentions: 8, + place: 'Karak, Jordan (general), Jordan', + lat: 31.18, + lon: 35.7, + url: times, + added: '20261105141500', + }), + eventRow({ + id: 3, + mentions: 4, + geoType: '1', + place: 'Syria', + lat: 35, + lon: 38, + url: chronicle, + added: '20261105151500', + }), + ].join('\n'), + ), + ); + assert.equal(reports.length, 1); + const [story] = reports; + assert.equal(story.copies, 2); + assert.equal(story.url, times, 'the first copy'); + assert.equal(story.publishedAt, '2026-11-05T14:15:00.000Z'); + assert.equal(story.geo.place, 'Karak, Jordan (general), Jordan'); +}); + +test("a story's title is a copy's page title from the GKG export", () => { + const titles = parseTitles( + [ + gkgRow( + ARTICLE, + 'Russian drones attack Kharkiv overnight, three killed | Wire', + ), + gkgRow('https://elsewhere.example/a', 'Not one of ours, not kept here'), + ].join('\n'), + new Set([ARTICLE]), + ); + assert.deepEqual( + [...titles], + [[ARTICLE, 'Russian drones attack Kharkiv overnight, three killed']], + ); + const [report] = eventReports( + parseEvents(eventRow({ id: 1, url: ARTICLE })), + { + titleOf: (url) => titles.get(url) || null, + }, + ); + assert.equal( + report.title, + 'Russian drones attack Kharkiv overnight, three killed', + ); + assert.equal(report.titled, 'page'); + assert.equal( + gkgUrl( + 'https://data.gdeltproject.org/gdeltv2/20261105141500.export.CSV.zip', + ), + 'https://data.gdeltproject.org/gdeltv2/20261105141500.gkg.csv.zip', + ); +}); + +test('a page title loses its site name, keeps its words, and must be a headline', () => { + assert.equal( + cleanTitle( + 'Yemen's president sacks ministers | Yemen | The Daily Example', + ), + "Yemen's president sacks ministers", + ); + assert.equal( + cleanTitle('Strike in Kharkiv - live updates - Wire Service'), + 'Strike in Kharkiv - live updates', + ); + // Too little before the dash: the dash is part of the headline. + assert.equal(cleanTitle('Iran - Wire Service'), 'Iran - Wire Service'); + assert.equal(cleanTitle('Home'), null); + assert.equal( + cleanTitle('Jordanian security officer killed in stabbing attack-Newswire'), + 'Jordanian security officer killed in stabbing attack', + ); + assert.equal( + cleanTitle('Five killed in Gaza, sources say-X'), + 'Five killed in Gaza, sources say', + ); + // A hyphenated name is no tag. + assert.equal( + cleanTitle('Talks resume with Jean-Pierre'), + 'Talks resume with Jean-Pierre', + ); +}); + +test('address words keep their numbers and name the places the story is at', () => { + const words = addressWords( + 'https://x.example/2026/11/05/jordan-says-u-s-envoy-met-officials-in-amman-after-500-killed', + ); + assert.equal( + words, + 'Jordan says u s envoy met officials in amman after 500 killed', + ); + assert.equal( + withNames(words, [{ place: 'Amman, (JO11), Jordan' }]), + 'Jordan says US envoy met officials in Amman after 500 killed', + ); +}); + +test("a story's id is its first event's: a later, more-mentioned event keeps it", () => { + const first = eventRow({ id: 40, mentions: 1, url: ARTICLE }); + const [before] = eventReports(parseEvents(first)); + const [after] = eventReports( + parseEvents( + [ + first, + eventRow({ + id: 41, + code: '195', + mentions: 30, + url: ARTICLE, + added: '20261105120000', + }), + ].join('\n'), + ), + ); + assert.equal(before.id, 'gdelt-40'); + assert.equal(after.id, 'gdelt-40'); + assert.equal(after.mentions, 31); +}); + +test('words that read as no conflict leave GDELT no kind to name', () => { + const [report] = eventReports( + parseEvents( + eventRow({ + id: 42, + code: '195', + url: 'https://x.example/2026/11/05/local-bakery-celebrates-fifty-years-of-baking', + }), + ), + ); + assert.equal(report.relevance, 'minor'); + assert.equal(report.category, 'other'); +}); + +test('a GKG address written with http is the https one the events carry', () => { + const plain = ARTICLE.replace('https:', 'http:'); + const [event] = parseEvents(eventRow({ id: 43, url: plain })); + assert.equal(event.url, ARTICLE); + const titles = parseTitles( + gkgRow(plain, 'Russian drones attack Kharkiv overnight, three killed'), + new Set([event.url]), + ); + assert.equal( + titles.get(ARTICLE), + 'Russian drones attack Kharkiv overnight, three killed', + ); +}); + +test('a corrupt or oversized export is an UpstreamError, never a RangeError', async () => { + const zip = zipTable(eventRow({ id: 44, url: ARTICLE })); + const view = new DataView(zip.buffer); + // The central directory's local header offset, pointing past the end. + const dir = view.getUint32(zip.length - 22 + 16, true); + const far = zip.slice(); + new DataView(far.buffer).setUint32(dir + 42, zip.length - 10, true); + await assert.rejects(unzipExport(far), { name: 'UpstreamError' }); + // The end record cut short. + await assert.rejects(unzipExport(zip.subarray(0, zip.length - 30)), { + name: 'UpstreamError', + }); + // Unzipped past the table's cap. + await assert.rejects(unzipExport(zip, 16), { name: 'UpstreamError' }); + assert.equal((await unzipExport(zip)).length > 16, true); +}); diff --git a/server/providers/news/gdeltFixtures.mjs b/server/providers/news/gdeltFixtures.mjs new file mode 100644 index 000000000..08b947525 --- /dev/null +++ b/server/providers/news/gdeltFixtures.mjs @@ -0,0 +1,72 @@ +// Test fixtures: GDELT 2.0 export rows and their zip, as the export files +// carry them (61 tab-separated columns, one zipped table). +import { deflateRawSync } from 'node:zlib'; + +/** One events row; only the columns the reader looks at are filled. */ +export function eventRow({ + id, + code = '190', + quad = '4', + mentions = 1, + geoType = '4', + place = "Kharkiv, Kharkivs'ka Oblast', Ukraine", + lat = 49.9935, + lon = 36.2304, + added = '20261105114500', + url, +}) { + const f = new Array(61).fill(''); + f[0] = String(id); + f[26] = code; + f[28] = code.slice(0, 2); + f[29] = quad; + f[31] = String(mentions); + f[51] = geoType; + f[52] = place; + f[56] = String(lat); + f[57] = String(lon); + f[59] = added; + f[60] = url; + return f.join('\t'); +} + +/** One GKG 2.1 row: an article's address and its page title (27 columns). */ +export function gkgRow(url, title) { + const f = new Array(27).fill(''); + f[4] = url; + f[26] = `20261105114500${title}`; + return f.join('\t'); +} + +/** A single-entry zip of `text`, deflated, sizes in the central directory. */ +export function zipTable(text, name = 'table.export.CSV') { + const data = Buffer.from(text, 'utf8'); + const packed = deflateRawSync(data); + const nameBytes = Buffer.from(name); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(0x08, 6); // sizes follow the data (left 0 here) + local.writeUInt16LE(8, 8); + local.writeUInt16LE(nameBytes.length, 26); + const dir = Buffer.alloc(46); + dir.writeUInt32LE(0x02014b50, 0); + dir.writeUInt16LE(20, 4); + dir.writeUInt16LE(20, 6); + dir.writeUInt16LE(0x08, 8); + dir.writeUInt16LE(8, 10); + dir.writeUInt32LE(packed.length, 20); + dir.writeUInt32LE(data.length, 24); + dir.writeUInt16LE(nameBytes.length, 28); + dir.writeUInt32LE(0, 42); + const dirAt = local.length + nameBytes.length + packed.length; + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(1, 8); + end.writeUInt16LE(1, 10); + end.writeUInt32LE(dir.length + nameBytes.length, 12); + end.writeUInt32LE(dirAt, 16); + return new Uint8Array( + Buffer.concat([local, nameBytes, packed, dir, nameBytes, end]), + ); +} diff --git a/server/providers/news/keyed.js b/server/providers/news/keyed.js new file mode 100644 index 000000000..2a89d7a69 --- /dev/null +++ b/server/providers/news/keyed.js @@ -0,0 +1,851 @@ +// The News layer's routes answered by Core itself, when no news server is +// configured (GEV_NEWS_ORIGIN). Conflicts: the day's GDELT events (no key; +// gdeltEvents.js), one report per article at GDELT's place for it, and with +// LIVEUAMAP_API_KEY Liveuamap's mapped items too. Local: World News API +// (WORLDNEWSAPI_KEY) by the country at the view centre; without its key the +// section answers 503 {error: 'no_key', key} and reads nothing, and past +// its day's reads (WORLDNEWSAPI_DAILY_READS) 503 {error: 'key_quota', key}. +// Categories, relevance and severity are the shared rules (categorize.js); +// a news server may replace these readings with its own. +// +// A read that has something to serve is not waited for: what was read is +// served while it is read again. A section that cannot be read answers 502, +// never an empty list. +// +// The answers are the same /api/news contract a news server serves +// (src/layers/news/records.js reads both). + +import { + CATEGORY_ORDER, + categorizeText, + relevanceOf, + severityByRules, +} from './categorize.js'; +import { + areaAt, + countryAnchor, + countryByCode, + countryPolygons, +} from './areas.js'; +import { + eventReports, + fetchLatestExport, + eventTitle, + exportAt, + exportsOfDay, + fetchExport, + fetchTitles, + gkgUrl, + storyKey, +} from './gdeltEvents.js'; +import { fetchLiveuamap, fetchWorldNews, UpstreamError } from './upstream.js'; + +/** The keys each section reads with, by the key registry's id. */ +export const NEWS_KEYS = Object.freeze({ + liveuamap: 'LIVEUAMAP_API_KEY', + worldnewsapi: 'WORLDNEWSAPI_KEY', +}); + +/** The one hotspot: the world, its reports drawn as markers at every zoom. */ +export const WORLDWIDE_HOTSPOT = 'worldwide'; +const HOUR_MS = 3_600_000; +const DAY_MS = 24 * HOUR_MS; +// GDELT publishes an export every 15 minutes and lists the newest in +// lastupdate.txt: the day's missing ones up to it are read every 5 minutes, +// newest first, a dozen a read (the first day's 96 fill in over the next +// reads, a few seconds apart, while each reads something new). One listed +// but missing (404) an hour and a half after its time is taken as never +// published; one that fails GDELT_EXPORT_TRIES times, or cannot be read at +// all, is given up on, so one bad file never holds the rest back. The first +// answer waits at most GDELT_FIRST_MS for the first exports. +const GDELT_FRESH_MS = 5 * 60_000; +const GDELT_EXPORTS_PER_READ = 12; +const GDELT_PARALLEL = 4; +const GDELT_NEXT_MS = 2_000; +const GDELT_GONE_MS = 90 * 60_000; +const GDELT_EXPORT_TRIES = 3; +const GDELT_FIRST_MS = 12_000; +const MAX_GDELT_REPORTS = 5000; +// Events kept of one story, to place it by. +const MAX_STORY_EVENTS = 200; +// Page titles are read from the GKG exports of the newest three hours: one +// runs to 6 MB, so the day's 96 are not fetched. A file that fails is tried +// with the next reads, a few times. +const GDELT_TITLED_EXPORTS = 12; +const GKG_TRIES = 3; +const LIVE_FRESH_MS = 5 * 60_000; +const COUNTRY_FRESH_MS = 30 * 60_000; +// A failed read is not tried again for a minute; what was read before is +// served meanwhile, up to these ages (GDELT's: since an export last came in). +const RETRY_MS = 60_000; +const GDELT_STALE_MS = 2 * HOUR_MS; +const LIVE_STALE_MS = HOUR_MS; +const COUNTRY_STALE_MS = 6 * HOUR_MS; +const MAX_COUNTRIES = 64; +// World News API reads a UTC day, across all countries: each is about 1.5 +// of the free plan's 50 daily points. +const DEFAULT_DAILY_READS = 30; +const DEFAULT_LIMIT = 50; +const EMPTY_OUTLINE = Object.freeze({ + polygons: [], + borders: [], + disputed: [], + area: null, +}); +// World News API's categories, where the rules find none of their own. +const WORLD_NEWS_CATEGORY = { politics: 'political', business: 'economy' }; + +const list = (value) => + value ? new Set(value.split(',').filter(Boolean)) : null; + +/** The rules' reading of a headline: category, relevance and severity. */ +function readTitle(title, { dedicated, fallback = null }) { + const read = categorizeText({ title }); + const category = + read.category === 'other' && fallback ? fallback : read.category; + return { + category, + relevance: relevanceOf({ category, tags: read.tags, title, dedicated }), + severity: severityByRules({ category, tags: read.tags, title }), + }; +} + +/** A one-report event carrying the report's picture, for its card. */ +function pictureEvent(id, report, image, video, publisher) { + return { + id, + status: 'single', + size: 1, + reports: 1, + severity: report.severity, + firstAt: report.publishedAt, + lastAt: report.publishedAt, + sides: [], + image: { url: image, publisher, ...(video ? { video } : {}) }, + }; +} + +const media = ({ image, video }) => + video ? ['video'] : image ? ['photo'] : []; + +/** A Liveuamap item as a report of the Conflicts section. */ +export function liveuamapReport(item) { + const id = `lum-${item.id}`; + const report = { + id, + hotspotId: WORLDWIDE_HOTSPOT, + hotspotIds: [WORLDWIDE_HOTSPOT], + title: item.title, + url: item.url, + publisher: 'Liveuamap', + source: { name: 'Liveuamap' }, + kind: 'liveuamap', + side: 'unknown', + // A conflict map: everything it posts is at least related. + ...readTitle(item.title, { dedicated: true }), + severityBy: 'rules', + publishedAt: item.publishedAt, + geo: { + lat: item.lat, + lon: item.lon, + place: item.place, + precision: 'city', + confidence: 1, + mappable: true, + method: 'source', + }, + media: media(item), + }; + if (item.image) + report.event = pictureEvent( + id, + report, + item.image, + item.video, + 'Liveuamap', + ); + return report; +} + +/** + * A GDELT place named by its town and the area Natural Earth has there (a + * disputed area as itself, else the country): GDELT's own names follow old + * FIPS regions ("Bureij, Yerushalayim, Israel" for a camp in Gaza, "Abuja, + * …, Niger" for Nigeria's capital). Its point stays. A town a disputed + * area is named after ("Jerusalem" in East Jerusalem) lies on both sides + * of its line: GDELT puts the whole city at one point, so it is named as + * the town alone. + */ +export function gdeltPlace(geo) { + const area = areaAt(geo.lat, geo.lon); + const town = String(geo.place ?? '') + .split(',')[0] + .trim(); + const straddles = + area?.kind === 'disputed' && + town && + area.name.toLowerCase().includes(town.toLowerCase()); + const place = + geo.precision === 'country' && area?.kind === 'country' + ? area.name + : straddles + ? town + : [town, area?.name].filter(Boolean).join(', '); + return { ...geo, place: place || null }; +} + +/** + * A World News API article as a report of the Local section: placed in its + * publisher's country alone, at a point inside it. + */ +export function worldNewsReport(article, country) { + const id = `wna-${article.id}`; + const anchor = countryAnchor(country); + const report = { + id, + hotspotId: null, + hotspotIds: [], + title: article.title, + url: article.url, + publisher: article.publisher, + // World News API's free plan asks to be credited where its news shows. + source: { name: article.publisher, attribution: 'World News API' }, + kind: 'worldnewsapi', + side: 'unknown', + ...readTitle(article.title, { + dedicated: false, + fallback: WORLD_NEWS_CATEGORY[article.category] || null, + }), + // Local news: what the country's own outlets publish. + relevance: 'minor', + severityBy: 'rules', + publishedAt: article.publishedAt, + geo: { + lat: anchor.lat, + lon: anchor.lon, + place: country.name, + precision: 'country', + confidence: 1, + mappable: true, + method: 'source', + }, + media: media(article), + }; + if (article.image) + report.event = pictureEvent( + id, + report, + article.image, + article.video, + article.publisher, + ); + return report; +} + +const AT = /^(-?\d{1,2}(?:\.\d{1,2})?),(-?\d{1,3}(?:\.\d{1,2})?)$/; +function point(value) { + const match = AT.exec(String(value ?? '')); + if (!match) return null; + const [lat, lon] = [Number(match[1]), Number(match[2])]; + return Math.abs(lat) <= 90 && Math.abs(lon) <= 180 ? { lat, lon } : null; +} + +const noKey = (key) => ({ status: 503, body: { error: 'no_key', key } }); +const rejectedKey = (key) => ({ + status: 503, + body: { error: 'key_rejected', key }, +}); +const spentKey = (key) => ({ status: 503, body: { error: 'key_quota', key } }); + +/** + * The keyed news routes. `answer(path, params)` resolves to `{status, body}` + * for `/hotspots`, `/feed` and `/outline`, whose queries the proxy has + * already checked against the client's allowlist. `later(run, ms)` runs + * the next GDELT read of a day still filling in (an unref'd timer). + * @param {{env?: () => object, fetchImpl?: typeof fetch, now?: () => number, log?: (message: string) => void, later?: (run: () => void, ms: number) => void}} [options] + */ +export function createKeyedNews({ + env = () => process.env, + fetchImpl = (...args) => globalThis.fetch(...args), + now = () => Date.now(), + log = (message) => console.warn(message), + later = (run, ms) => setTimeout(run, ms).unref(), +} = {}) { + const keyOf = (id) => String(env()[NEWS_KEYS[id]] ?? '').trim(); + const inFlight = new Map(); + // GDELT's stories of the last day (their events and report, by + // storyKey), and the exports read. + const gdelt = { + stories: new Map(), + reports: new Map(), + // Page titles by article address, of the stories kept. + titles: new Map(), + read: new Set(), + // Exports still missing well after their time: not looked for again. + gone: new Set(), + // Failed reads of each export not read yet. + failures: new Map(), + // Exports read among the newest whose GKG titles are not in yet, with + // the tries made: `{tries, triedAt}`. + untitled: new Map(), + titling: null, + // When a read last brought an export in, with nothing failing. + readAt: null, + checkedAt: null, + failedAt: null, + }; + const live = new Map(); + const countries = new Map(); + // The keys refused so far, by section: logged once each. + const refused = new Set(); + // World News API reads of the UTC day; `spent` once it answered 402. + const budget = { day: null, reads: 0, spent: false, logged: false }; + // The area (disputed area or country) each placed Conflicts report lies + // in, looked up once. + const reportAreas = new WeakMap(); + const reportArea = (report) => { + if (!reportAreas.has(report)) + reportAreas.set( + report, + report.geo + ? (areaAt(report.geo.lat, report.geo.lon)?.key ?? null) + : null, + ); + return reportAreas.get(report); + }; + + /** A failed read in the log: a refused key once, never the key itself. */ + function logFailure(section, what, key, error) { + // The day's reads spent: said once, by spendRead. + if (error.quota && error.status === null) return; + if (error.rejected) { + const id = `${section}\0${key}`; + if (refused.has(id)) return; + refused.add(id); + log( + `[news] ${NEWS_KEYS[section]} was refused (${error.status})${section === 'liveuamap' ? "; Conflicts are GDELT's alone" : ''}`, + ); + return; + } + log(`[news] ${what} read failed: ${error.message}`); + } + + /** + * A cached read (`{key, value, readAt, failedAt, error}`, kept in `store` + * under `id`): served as it is within `freshMs`; past that read again, + * and served meanwhile up to `staleMs`, so only a read with nothing to + * serve is waited for. One read at a time per key; a failed read is tried + * again after RETRY_MS. A new key drops what the old one read. + */ + async function cached( + store, + id, + { section, key, freshMs, staleMs, read, what }, + ) { + const at = now(); + const stored = store.get(id); + const prior = stored?.key === key ? stored : { key }; + if (prior.value && at - prior.readAt < freshMs) return prior; + if (prior.failedAt && at - prior.failedAt < RETRY_MS) return prior; + const flight = `${id}\0${key}`; + if (!inFlight.has(flight)) + inFlight.set( + flight, + read(key) + .then( + (value) => ({ key, value, readAt: now() }), + (error) => { + logFailure(section, what, key, error); + return { ...prior, failedAt: now(), error }; + }, + ) + .then((entry) => { + // A read with a key replaced meanwhile is dropped. + if (keyOf(section) === key) keep(store, id, entry); + return entry; + }) + .finally(() => inFlight.delete(flight)), + ); + if (prior.value && at - prior.readAt < staleMs) return prior; + return inFlight.get(flight); + } + + /** An entry stored as the most recently used (the countries are capped). */ + function keep(store, id, entry) { + store.delete(id); + store.set(id, entry); + while (store.size > MAX_COUNTRIES) store.delete(store.keys().next().value); + } + + /** An entry's reports while they may be served, else why there are none. */ + function served(entry, staleMs, id) { + if (entry.value && now() - entry.readAt < staleMs) + return { reports: entry.value, readAt: entry.readAt }; + if (entry.error?.rejected) return { rejected: id }; + if (entry.error?.quota) return { quota: id }; + return { failed: true }; + } + + async function liveuamapReports() { + const key = keyOf('liveuamap'); + if (!key) return { missing: 'liveuamap' }; + const entry = await cached(live, 'liveuamap', { + section: 'liveuamap', + key, + freshMs: LIVE_FRESH_MS, + staleMs: LIVE_STALE_MS, + what: 'liveuamap', + read: async (k) => + (await fetchLiveuamap({ key: k, fetchImpl, now: now() })) + .map(liveuamapReport) + .sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)), + }); + return served(entry, LIVE_STALE_MS, 'liveuamap'); + } + + /** + * A story placed (again) from all its events and the titles in: an + * agency story run by several outlets is one report. + */ + function placeStory(key) { + const [report] = eventReports(gdelt.stories.get(key), { + titleOf: (url) => gdelt.titles.get(url) || null, + }); + if (!report) return; + const { eventWords, ...rest } = report; + const geo = gdeltPlace(report.geo); + gdelt.reports.set(key, { + ...rest, + ...(eventWords ? { title: eventTitle(eventWords, geo.place) } : {}), + hotspotId: WORLDWIDE_HOTSPOT, + hotspotIds: [WORLDWIDE_HOTSPOT], + geo, + locations: report.locations.map(gdeltPlace), + }); + } + + /** An export's events into their stories, each placed again. */ + function addEvents(events) { + const changed = new Set(); + for (const event of events) { + const key = storyKey(event); + const list = gdelt.stories.get(key) || []; + if (list.length < MAX_STORY_EVENTS) list.push(event); + gdelt.stories.set(key, list); + changed.add(key); + } + for (const key of changed) placeStory(key); + } + + /** The stories a GKG export titled, re-titled. */ + function addTitles(titles) { + for (const [key, list] of gdelt.stories) { + const titled = list.filter((event) => titles.has(event.url)); + if (!titled.length) continue; + for (const { url } of titled) gdelt.titles.set(url, titles.get(url)); + placeStory(key); + } + } + + /** + * The page titles of the newest exports read, from their GKG exports one + * at a time (each runs to 18 MB unzipped), in the background: no answer + * waits for them, and a story is re-titled as its title comes in. A file + * that fails or is not up yet only leaves address words: it is tried + * again with a later read while among the newest, up to GKG_TRIES times. + */ + function readTitles() { + gdelt.titling ??= (async () => { + let failed = null; + for (const [url, tried] of [...gdelt.untitled]) { + if (!gdelt.untitled.has(url)) continue; + if (tried.triedAt !== null && now() - tried.triedAt < GDELT_FRESH_MS) + continue; + tried.tries += 1; + tried.triedAt = now(); + const wanted = new Set(); + for (const list of gdelt.stories.values()) + for (const event of list) wanted.add(event.url); + let titles = null; + try { + titles = await fetchTitles(gkgUrl(url), wanted, { fetchImpl }); + } catch (error) { + failed ??= error; + } + if (titles || tried.tries >= GKG_TRIES) gdelt.untitled.delete(url); + if (titles?.size) addTitles(titles); + } + if (failed) log(`[news] gdelt titles read failed: ${failed.message}`); + })() + .catch((error) => log(`[news] gdelt titles failed: ${error.message}`)) + .finally(() => { + gdelt.titling = null; + }); + return gdelt.titling; + } + + /** + * Up to a dozen of the day's exports not read yet, newest first; whether + * to carry on at once (the round brought one in, and more are missing). + * One not up yet is looked for again; one still missing GDELT_GONE_MS + * after its time is not. + */ + async function readExports() { + const at = now(); + const day = exportsOfDay(at); + // Up to the newest GDELT lists; all of the day when its list fails. + const latest = await fetchLatestExport({ fetchImpl }); + const listed = (url) => latest === null || exportAt(url) <= latest; + const newest = new Set(day.slice(0, GDELT_TITLED_EXPORTS)); + const wanted = (url) => + listed(url) && !gdelt.read.has(url) && !gdelt.gone.has(url); + const missing = day.filter(wanted).slice(0, GDELT_EXPORTS_PER_READ); + let failed = null; + let added = 0; + const failing = []; + for (let i = 0; i < missing.length; i += GDELT_PARALLEL) + await Promise.all( + missing.slice(i, i + GDELT_PARALLEL).map(async (url) => { + let events; + try { + events = await fetchExport(url, { fetchImpl }); + } catch (error) { + failed = error; + // An unreadable file stays so. + if (error.unreadable) gdelt.gone.add(url); + else failing.push(url); + return; + } + if (!events) { + if (at - exportAt(url) >= GDELT_GONE_MS) gdelt.gone.add(url); + return; + } + // The events count now; the titles follow (readTitles). + gdelt.read.add(url); + gdelt.failures.delete(url); + added += 1; + gdelt.readAt = now(); + addEvents(events); + if (newest.has(url)) + gdelt.untitled.set(url, { tries: 0, triedAt: null }); + }), + ); + if (failed) log(`[news] gdelt read failed: ${failed.message}`); + // A file that fails while others read is that file's fault, and gets a + // few tries; when none read, GDELT itself is down and no file is blamed. + if (added) + for (const url of failing) { + const tries = (gdelt.failures.get(url) || 0) + 1; + gdelt.failures.set(url, tries); + if (tries >= GDELT_EXPORT_TRIES) gdelt.gone.add(url); + } + // Nothing read and a failure: tried again after RETRY_MS, not after + // GDELT_FRESH_MS. Something read: the round counts. + if (failed && !added) gdelt.failedAt = now(); + else { + gdelt.checkedAt = now(); + gdelt.failedAt = null; + } + // Only the day is kept, and titled while among the newest. + const kept = new Set(day); + for (const set of [gdelt.read, gdelt.gone]) + for (const url of set) if (!kept.has(url)) set.delete(url); + for (const url of gdelt.failures.keys()) + if (!kept.has(url)) gdelt.failures.delete(url); + for (const url of gdelt.untitled.keys()) + if (!newest.has(url)) gdelt.untitled.delete(url); + return ( + added > 0 && missing.length === GDELT_EXPORTS_PER_READ && day.some(wanted) + ); + } + + /** GDELT's reports of the last day, newest first, or `{failed}`. */ + async function gdeltReports() { + const at = now(); + const fresh = gdelt.checkedAt && at - gdelt.checkedAt < GDELT_FRESH_MS; + const resting = gdelt.failedAt && at - gdelt.failedAt < RETRY_MS; + if (!fresh && !resting && !inFlight.has('gdelt')) { + const round = readExports() + .then( + (more) => { + // More of the day to fill in: carry on shortly, without holding + // the answer. + if (!more) return; + gdelt.checkedAt = null; + later(() => void gdeltReports(), GDELT_NEXT_MS); + }, + (error) => { + log(`[news] gdelt read failed: ${error.message}`); + gdelt.failedAt = now(); + }, + ) + .finally(() => { + inFlight.delete('gdelt'); + void readTitles(); + }); + inFlight.set('gdelt', round); + } + // Only a read with nothing (recent) to serve is waited for: the + // newest exports' events, not their titles. + const current = () => + gdelt.readAt !== null && now() - gdelt.readAt < GDELT_STALE_MS; + if (!current() && inFlight.has('gdelt')) { + // At most GDELT_FIRST_MS: a slow GDELT must not outlast the browser's + // own timeout; what has come in by then is served. + let timer; + await Promise.race([ + inFlight.get('gdelt'), + new Promise((resolve) => { + timer = setTimeout(resolve, GDELT_FIRST_MS); + timer.unref?.(); + }), + ]); + clearTimeout(timer); + } + if (!current()) return { failed: true }; + const dayAgo = new Date(now() - DAY_MS).toISOString(); + const reports = [...gdelt.reports.values()] + .filter((r) => r.publishedAt >= dayAgo) + .sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)) + .slice(0, MAX_GDELT_REPORTS); + if (reports.length !== gdelt.reports.size) { + const kept = new Set(reports); + for (const [key, report] of gdelt.reports) + if (!kept.has(report)) { + for (const event of gdelt.stories.get(key) || []) + gdelt.titles.delete(event.url); + gdelt.reports.delete(key); + gdelt.stories.delete(key); + } + } + return { reports, readAt: gdelt.readAt }; + } + + /** + * The Conflicts section's reports: GDELT's, with Liveuamap's when its key + * is set. Liveuamap failing (or refusing its key) leaves GDELT's. + */ + async function conflictReports() { + const [articles, items] = await Promise.all([ + gdeltReports(), + keyOf('liveuamap') ? liveuamapReports() : {}, + ]); + if (!articles.reports && !items.reports) return { failed: true }; + return { + reports: [...(articles.reports || []), ...(items.reports || [])].sort( + (a, b) => b.publishedAt.localeCompare(a.publishedAt), + ), + readAt: Math.max(articles.readAt || 0, items.readAt || 0), + }; + } + + const dailyReads = () => { + const raw = String(env().WORLDNEWSAPI_DAILY_READS ?? '').trim(); + const reads = raw ? Number(raw) : NaN; + return Number.isInteger(reads) && reads >= 0 ? reads : DEFAULT_DAILY_READS; + }; + + /** Whether a World News API read is left today (UTC); counts it if so. */ + function spendRead() { + const day = new Date(now()).toISOString().slice(0, 10); + if (budget.day !== day) + Object.assign(budget, { day, reads: 0, spent: false, logged: false }); + if (!budget.spent && budget.reads < dailyReads()) { + budget.reads += 1; + return true; + } + if (!budget.logged) + log( + "[news] World News API's reads for today are spent; Local reads again after 00:00 UTC", + ); + budget.logged = true; + return false; + } + + async function countryReports(country) { + const key = keyOf('worldnewsapi'); + if (!key) return { missing: 'worldnewsapi' }; + const stored = countries.get(country.code); + if (stored) keep(countries, country.code, stored); + const entry = await cached(countries, country.code, { + section: 'worldnewsapi', + key, + freshMs: COUNTRY_FRESH_MS, + staleMs: COUNTRY_STALE_MS, + what: `worldnewsapi:${country.code}`, + read: async (k) => { + // Not asked: no status, unlike an answer. + if (!spendRead()) + throw new UpstreamError("World News API's reads for today spent", { + quota: true, + }); + try { + return ( + await fetchWorldNews({ + key: k, + country: country.code, + fetchImpl, + now: now(), + }) + ).map((article) => worldNewsReport(article, country)); + } catch (error) { + // Out of points: nothing more is asked today. + if (error.status === 402) budget.spent = true; + throw error; + } + }, + }); + return served(entry, COUNTRY_STALE_MS, 'worldnewsapi'); + } + + const refusal = (read) => + read.missing + ? noKey(read.missing) + : read.rejected + ? rejectedKey(read.rejected) + : read.quota + ? spentKey(read.quota) + : read.failed + ? { status: 502, body: { error: 'news_upstream_error' } } + : null; + + async function hotspots(params) { + const read = await conflictReports(); + const refused = refusal(read); + if (refused) return refused; + const hours = Number(params.get('hours')) || 24; + const since = new Date(now() - hours * HOUR_MS).toISOString(); + const shown = read.reports.filter((r) => r.publishedAt >= since); + return { + status: 200, + body: { + hotspots: [ + { + id: WORLDWIDE_HOTSPOT, + name: 'Worldwide', + kind: 'conflict', + catchAll: true, + bbox: [-180, -90, 180, 90], + parties: [], + developing: [], + counts: { + last24h: shown.length, + mapped24h: shown.filter((r) => r.geo).length, + prev24h: null, + }, + latestAt: shown[0]?.publishedAt ?? null, + }, + ], + categories: CATEGORY_ORDER, + ingest: { lastRunAt: new Date(read.readAt).toISOString() }, + windowHours: hours, + maxWindowHours: 24, + }, + }; + } + + /** Reports filtered, newest first, and paged by `before`. */ + function page(reports, params) { + const since = params.get('since'); + const before = params.get('before'); + const dayAgo = new Date(now() - DAY_MS).toISOString(); + const categories = list(params.get('category')); + const relevance = list(params.get('relevance')); + const hotspot = params.get('hotspot'); + const limit = Number(params.get('limit')) || DEFAULT_LIMIT; + const kept = reports.filter( + (r) => + (!hotspot || hotspot === 'all' || r.hotspotIds.includes(hotspot)) && + (!since || r.publishedAt >= since) && + (!before || r.publishedAt < before) && + (!params.has('today') || r.publishedAt >= dayAgo) && + (!params.has('geo') || r.geo) && + (!categories || categories.has(r.category)) && + (!relevance || relevance.has(r.relevance)), + ); + const shown = kept.slice(0, limit); + // A page ends between two times: the next asks for those `before` the + // last, and a GDELT export's reports all share its quarter hour. + while ( + shown.length < kept.length && + kept[shown.length].publishedAt === shown.at(-1).publishedAt + ) + shown.push(kept[shown.length]); + const events = params.has('events'); + return { + reports: events ? shown : shown.map(({ event, ...report }) => report), + nextBefore: kept.length > shown.length ? shown.at(-1).publishedAt : null, + }; + } + + async function feed(params) { + const at = params.has('at') ? point(params.get('at')) : null; + const area = at ? areaAt(at.lat, at.lon) : null; + if (!area) { + // Conflicts, or Local at sea (by distance, in the browser). + const read = await conflictReports(); + return refusal(read) || { status: 200, body: page(read.reports, params) }; + } + // Local in a country: its own outlets, with the Conflicts reports + // placed inside it (those failing leave the outlets'). In a disputed + // area no country's outlets speak for it: the Conflicts reports there + // alone, so their failing is the answer's. + const country = area.kind === 'country'; + const [national, conflicts] = await Promise.all([ + country ? countryReports(area.country) : { reports: [] }, + conflictReports(), + ]); + const refused = refusal(national) || (!country && refusal(conflicts)); + if (refused) return refused; + const inside = (conflicts.reports || []).filter( + (r) => reportArea(r) === area.key, + ); + const reports = [...national.reports, ...inside].sort((a, b) => + b.publishedAt.localeCompare(a.publishedAt), + ); + return { status: 200, body: page(reports, params) }; + } + + function outline(params) { + if (params.has('at')) { + const at = point(params.get('at')); + const area = at ? areaAt(at.lat, at.lon) : null; + if (!area) return { status: 200, body: EMPTY_OUTLINE }; + return { + status: 200, + body: { + ...EMPTY_OUTLINE, + polygons: + area.kind === 'country' + ? countryPolygons(area.country) + : area.polygons, + area: { + key: area.key, + name: area.name, + kind: area.kind, + country: area.code, + }, + }, + }; + } + // A developing situation's country; the world has no outline. + const country = countryByCode( + String(params.get('country') ?? '').slice(0, 2), + ); + return { + status: 200, + body: country + ? { ...EMPTY_OUTLINE, polygons: countryPolygons(country) } + : EMPTY_OUTLINE, + }; + } + + return { + async answer(path, params) { + if (path === '/hotspots') return hotspots(params); + if (path === '/feed') return feed(params); + if (path === '/outline') return outline(params); + return { status: 404, body: { error: 'not_found' } }; + }, + }; +} diff --git a/server/providers/news/keyed.test.mjs b/server/providers/news/keyed.test.mjs new file mode 100644 index 000000000..c7b6706e9 --- /dev/null +++ b/server/providers/news/keyed.test.mjs @@ -0,0 +1,988 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + createKeyedNews, + gdeltPlace, + NEWS_KEYS, + WORLDWIDE_HOTSPOT, +} from './keyed.js'; +import { KEY_SETUP_KEYS } from '../../../src/keySetupCore.mjs'; +import { + normalizeFeedPayload, + normalizeHotspotsPayload, + normalizeOutlinePayload, +} from '../../../src/layers/news/records.js'; +import { exportsOfDay } from './gdeltEvents.js'; +import { eventRow, gkgRow, zipTable } from './gdeltFixtures.mjs'; + +const NOW = Date.parse('2026-11-05T12:00:00Z'); +const seconds = (iso) => Date.parse(iso) / 1000; + +// Liveuamap's `a=mpts` answer, cut down: strings where it writes strings. +const LIVEUAMAP = { + success: true, + key: 'redacted', + venues: [ + { + id: 101, + name: 'Russian drone strike on Kharkiv kills 3, injures 12', + lat: '49.99350', + lng: '36.23040', + location: 'Kharkiv', + timestamp: seconds('2026-11-05T11:00:00Z'), + link: 'https://liveuamap.com/en/2026/5-november-kharkiv', + img_share: 'https://a.liveuamap.com/pics/101.jpg', + video: 'https://a.liveuamap.com/clips/101.mp4', + }, + { + id: 102, + name: 'Floods kill dozens in Astana', + lat: '51.16940', + lng: '71.44910', + location: 'Astana', + timestamp: seconds('2026-11-05T10:00:00Z'), + link: 'https://asia.liveuamap.com/en/2026/5-november-astana', + img_share: '', + video: '', + }, + { + id: 103, + name: 'Officials in "Kherson" report shelling', + lat: '46.63540', + lng: '32.61690', + timestamp: seconds('2026-11-05T09:00:00Z'), + link: 'https://liveuamap.com/en/2026/5-november-kherson', + }, + // Without a point, a page or a headline: left out. + { id: 104, name: 'No point', timestamp: seconds('2026-11-05T09:00:00Z') }, + { + id: 105, + name: 'No page', + lat: '1', + lng: '1', + timestamp: seconds('2026-11-05T09:00:00Z'), + link: 'javascript:alert(1)', + }, + { + id: 106, + name: '', + lat: '1', + lng: '1', + timestamp: seconds('2026-11-05T09:00:00Z'), + link: 'https://liveuamap.com/x', + }, + ], +}; + +// World News API's search-news answer, cut down. +const WORLD_NEWS = { + offset: 0, + number: 100, + available: 3, + news: [ + { + id: 9001, + title: 'Visiting minister arrives in Astana for talks', + url: 'https://www.daily.example/en/minister-astana-talks', + image: 'https://www.daily.example/pics/visit.jpg', + publish_date: '2026-11-05 08:30:00', + language: 'en', + category: 'politics', + text: 'The whole article, never kept.', + }, + { + id: 9002, + title: 'Two oil firms advance subsalt exploration', + url: 'https://markets.example/economic/oil-firms-exploration', + publish_date: '2026-11-05 07:00:00', + language: 'ru', + category: 'business', + }, + // Sport is left out. + { + id: 9004, + title: 'Astana Arlan win the cup final', + url: 'https://sport.example/cup', + publish_date: '2026-11-05 09:00:00', + category: 'sports', + }, + // The same story from another outlet is one. + { + id: 9003, + title: 'TWO OIL FIRMS ADVANCE SUBSALT EXPLORATION', + url: 'https://portal.test/oil-firms', + publish_date: '2026-11-05 06:00:00', + }, + ], +}; + +// GDELT's exports around NOW: the newest not published yet, two with +// events, the rest of the day empty. +const KHARKIV = + 'https://www.wire.example/rubric-ato/3912-russian-drones-attack-kharkiv-overnight-three-killed.html'; +const RED_SEA = + 'https://news.example.com/2026/11/05/houthis-claim-missile-attack-on-tanker-in-red-sea'; +const COUNCIL = + 'https://news.example.com/2026/11/05/city-council-approves-new-budget-for-schools'; +const BARE = 'https://news.example.com/a/b/123456'; +const exportName = (stamp) => + `https://data.gdeltproject.org/gdeltv2/${stamp}.export.CSV.zip`; +const EXPORTS = { + [exportName('20261105120000')]: 404, + [exportName('20261105114500')]: [ + eventRow({ + id: 1, + code: '195', + mentions: 3, + url: KHARKIV, + added: '20261105114500', + }), + eventRow({ + id: 2, + code: '195', + mentions: 8, + geoType: '1', + place: 'Yemen', + lat: 15.5, + lon: 47.5, + url: RED_SEA, + added: '20261105114500', + }), + eventRow({ id: 3, url: COUNCIL, added: '20261105114500' }), + ], + [exportName('20261105113000')]: [ + eventRow({ + id: 4, + geoType: '1', + place: 'Ukraine', + lat: 49, + lon: 32, + url: BARE, + added: '20261105113000', + }), + // An article already read is not a second report. + eventRow({ id: 5, code: '190', url: KHARKIV, added: '20261105113000' }), + ], +}; +const EMPTY = zipTable(''); +// The page titles of the newest export's quarter hour. +const GKG = { + [exportName('20261105114500').replace('.export.CSV.zip', '.gkg.csv.zip')]: [ + gkgRow( + KHARKIV, + 'Russian drones attack Kharkiv overnight, three killed - Wire', + ), + ], +}; + +const host = (call) => call.url.host; + +/** + * A fake upstream: answers by host, counts calls, records URLs and headers. + * An answer is a payload, a status, or a function giving a Response (one + * held open). `unlisted` is the status of an export not in `exports` (an + * empty table by default); `gkg` a status for every GKG export. + */ +function upstream(answers = {}) { + const calls = []; + const fetchImpl = async (url, init) => { + const href = String(url); + calls.push({ url: new URL(href), headers: init?.headers || {} }); + if (href.includes('gdeltproject.org')) { + if (typeof answers.gdelt === 'number') + return new Response('busy', { status: answers.gdelt }); + if (href.endsWith('/lastupdate.txt') && answers.lastupdate) + return new Response( + `95141 9197 http://data.gdeltproject.org/gdeltv2/${answers.lastupdate}.export.CSV.zip +`, + { status: 200 }, + ); + if (answers.corrupt?.includes(href)) + return new Response('not a zip at all', { status: 200 }); + const gkg = href.endsWith('.gkg.csv.zip'); + const listed = answers.exports ?? EXPORTS; + const answer = gkg + ? (answers.gkg ?? GKG[href]) + : Object.hasOwn(listed, href) + ? listed[href] + : answers.unlisted; + if (typeof answer === 'number') + return new Response('', { status: answer }); + return new Response(answer ? zipTable(answer.join('\n')) : EMPTY, { + status: 200, + }); + } + const answer = href.includes('liveuamap.com') + ? (answers.liveuamap ?? LIVEUAMAP) + : (answers.worldnews ?? WORLD_NEWS); + if (typeof answer === 'function') return answer(); + if (typeof answer === 'number') + return new Response('{"message":"bad key ABC"}', { status: answer }); + return new Response(JSON.stringify(answer), { status: 200 }); + }; + return { calls, fetchImpl }; +} + +function keyed({ env, answers, clock = { now: NOW }, logs = [] } = {}) { + const fake = upstream(answers); + // The next GDELT reads, run by settle rather than on a timer. + const timers = []; + const news = createKeyedNews({ + env: () => + env ?? { LIVEUAMAP_API_KEY: 'live-key', WORLDNEWSAPI_KEY: 'wna-key' }, + fetchImpl: fake.fetchImpl, + now: () => clock.now, + log: (line) => logs.push(line), + later: (run) => timers.push(run), + }); + const ask = (path, query = '') => + news.answer(path, new URLSearchParams(query)); + /** + * Let the background reads finish (the rest of the day, the titles): + * until nothing is asked for a while, or a bounded number of turns. + */ + async function settle() { + let quiet = 0; + for (let turn = 0; turn < 200 && quiet < 5; turn++) { + const before = fake.calls.length; + for (let i = 0; i < 20; i++) await new Promise((r) => setImmediate(r)); + await new Promise((r) => setTimeout(r, 1)); + for (const run of timers.splice(0)) run(); + quiet = fake.calls.length === before ? quiet + 1 : 0; + } + } + return { ...fake, ask, clock, logs, settle, timers }; +} + +const CONFLICTS = + 'hotspot=all&geo=1&today=1&events=1&relevance=core,related&limit=50'; +const gdeltCalls = (calls) => + calls.filter( + (c) => + host(c) === 'data.gdeltproject.org' && + c.url.pathname.endsWith('.export.CSV.zip'), + ); +const gkgCalls = (calls) => + calls.filter((c) => c.url.pathname.endsWith('.gkg.csv.zip')); + +test('without any key Conflicts are the day of GDELT events, one worldwide hotspot', async () => { + const { ask, calls, settle } = keyed({ env: {} }); + const answer = await ask('/hotspots'); + assert.equal(answer.status, 200); + const payload = normalizeHotspotsPayload(answer.body); + assert.deepEqual( + payload.hotspots.map((h) => [h.id, h.catchAll]), + [[WORLDWIDE_HOTSPOT, true]], + ); + assert.equal(payload.hotspots[0].last24h, 4); + assert.equal(payload.hotspots[0].mapped24h, 4); + assert.equal(payload.maxWindowHours, 24); + // The newest dozen first; the rest of the day after, in the background. + assert.deepEqual( + gdeltCalls(calls) + .slice(0, 12) + .map((c) => String(c.url)), + exportsOfDay(NOW).slice(0, 12), + ); + assert.equal( + String(gdeltCalls(calls)[0].url), + 'https://data.gdeltproject.org/gdeltv2/20261105120000.export.CSV.zip', + ); + await settle(); + assert.ok(gdeltCalls(calls).length >= 96); + assert.ok(calls.every((c) => host(c) === 'data.gdeltproject.org')); + // Page titles only for the newest three hours (a GKG export is 6 MB). + assert.ok(gkgCalls(calls).length > 0 && gkgCalls(calls).length <= 12); +}); + +test('a GDELT article is a report at its place, in its words, credited to GDELT', async () => { + const { ask, settle } = keyed({ env: {} }); + // The first answer has the events in, titled by their address words; the + // page titles follow in the background. + const first = normalizeFeedPayload((await ask('/feed', CONFLICTS)).body); + assert.equal( + first.reports.find((r) => r.id === 'gdelt-5').title, + 'Russian drones attack Kharkiv overnight three killed', + ); + await settle(); + const { reports } = normalizeFeedPayload( + (await ask('/feed', CONFLICTS)).body, + ); + // Kharkiv's story was first seen in the earlier export: its first event + // names it. + assert.deepEqual( + reports.map((r) => r.id), + ['gdelt-2', 'gdelt-5', 'gdelt-4'], + ); + const [redSea, kharkiv, bare] = reports; + // Its page title, from the GKG export. + assert.equal( + kharkiv.title, + 'Russian drones attack Kharkiv overnight, three killed', + ); + assert.equal(kharkiv.category, 'strike'); + assert.equal(kharkiv.relevance, 'core'); + assert.equal(kharkiv.severity, 2); + assert.equal(kharkiv.publisher, 'wire.example'); + assert.equal(kharkiv.attribution, 'GDELT Project'); + assert.equal(kharkiv.kind, 'gdelt'); + assert.equal(kharkiv.side, 'unknown'); + assert.equal(kharkiv.geo.precision, 'city'); + assert.deepEqual( + kharkiv.pins.map(({ lat, lon }) => [lat, lon]), + [[49.9935, 36.2304]], + ); + assert.equal(redSea.geo.place, 'Yemen'); + assert.equal(redSea.geo.precision, 'country'); + assert.equal(bare.title, 'Fighting reported in Ukraine'); + assert.equal(bare.relevance, 'related'); + // The council story GDELT coded as fighting is minor: not in Conflicts. + assert.ok(!reports.some((r) => r.url.includes('city-council'))); +}); + +test('with a Liveuamap key its mapped items join GDELT in the one hotspot', async () => { + const { ask, calls, settle } = keyed(); + const answer = await ask('/hotspots'); + assert.equal(answer.body.hotspots[0].counts.last24h, 7); + const live = calls.find((c) => host(c) === 'a.liveuamap.com').url; + assert.equal(live.searchParams.get('a'), 'mpts'); + assert.equal(live.searchParams.get('key'), 'live-key'); + assert.equal( + Number(live.searchParams.get('time')), + (NOW - 86_400_000) / 1000, + ); + const { reports } = normalizeFeedPayload( + (await ask('/feed', CONFLICTS)).body, + ); + const kharkiv = reports.find((r) => r.id === 'lum-101'); + assert.equal(kharkiv.category, 'strike'); + assert.equal(kharkiv.severity, 2); + assert.equal(kharkiv.publisher, 'Liveuamap'); + assert.deepEqual( + kharkiv.pins.map(({ lat, lon }) => [lat, lon]), + [[49.9935, 36.2304]], + ); + assert.equal( + kharkiv.event.image.video, + 'https://a.liveuamap.com/clips/101.mp4', + ); + assert.deepEqual(kharkiv.media, ['video']); + assert.equal( + reports.find((r) => r.id === 'lum-103').title, + 'Officials in "Kherson" report shelling', + ); + // Counts over a shorter window. + const short = await ask('/hotspots', 'hours=2'); + assert.equal(short.body.hotspots[0].counts.last24h, 6); + await settle(); +}); + +test('the Conflicts feed pages by time and filters by category and hotspot', async () => { + const { ask, settle } = keyed({ env: {} }); + const all = 'hotspot=all&today=1&relevance=core,related&events=1'; + const first = await ask('/feed', `${all}&limit=1`); + assert.deepEqual( + first.body.reports.map((r) => r.id), + ['gdelt-2'], + ); + assert.equal(first.body.nextBefore, '2026-11-05T11:45:00.000Z'); + // A page ends between two times: both of 11:30 come together, though + // the page asked for one. + const next = await ask( + '/feed', + `${all}&limit=1&before=${first.body.nextBefore}`, + ); + assert.deepEqual( + next.body.reports.map((r) => r.id), + ['gdelt-5', 'gdelt-4'], + ); + assert.equal(next.body.nextBefore, null); + const strikes = await ask('/feed', 'hotspot=all&category=strike'); + assert.equal(strikes.body.reports.length, 2); + const elsewhere = await ask('/feed', 'hotspot=ua-ru-war'); + assert.deepEqual(elsewhere.body.reports, []); + await settle(); +}); + +test('an export not published yet is looked for again, and the day is kept a day', async () => { + const exports = { ...EXPORTS }; + const { ask, calls, clock, settle } = keyed({ + env: {}, + answers: { exports }, + }); + await ask('/hotspots'); + await settle(); + const read = gdeltCalls(calls).length; + // Within five minutes nothing is asked again. + clock.now += 2 * 60_000; + await ask('/hotspots'); + assert.equal(gdeltCalls(calls).length, read); + // The late export is published now, with a new article. + exports[exportName('20261105120000')] = [ + eventRow({ + id: 6, + code: '183', + url: 'https://news.example.com/2026/11/05/car-bomb-kills-five-in-mogadishu-market', + place: 'Mogadishu, Banaadir, Somalia', + lat: 2.04, + lon: 45.34, + added: '20261105120000', + }), + ]; + clock.now += 4 * 60_000; + await ask('/hotspots'); + await settle(); + const later = await ask('/hotspots'); + assert.equal(later.body.hotspots[0].counts.last24h, 5, 'added, not replaced'); + assert.equal( + gdeltCalls(calls) + .slice(read) + .filter((c) => String(c.url).includes('20261105120000')).length, + 1, + ); + // A day on, the old ones have aged out. + clock.now += 24 * 3_600_000; + const dayOn = await ask('/hotspots'); + await settle(); + assert.equal(dayOn.body.hotspots[0].counts.last24h, 0); +}); + +test('Local names the country at the point and outlines it, with no key', async () => { + const { ask, calls } = keyed({ env: {} }); + const answer = await ask('/outline', 'at=51.13,71.43&v=4'); + const outline = normalizeOutlinePayload(answer.body); + assert.deepEqual(outline.area, { + key: 'KZ', + name: 'Kazakhstan', + kind: 'country', + country: 'KZ', + }); + assert.ok(outline.polygons.length > 0); + const sea = await ask('/outline', 'at=0,-30&v=4'); + assert.equal(normalizeOutlinePayload(sea.body).area, null); + const developing = await ask('/outline', 'country=UA&v=4'); + assert.ok(developing.body.polygons.length > 0); + assert.equal(calls.length, 0); +}); + +test("Local in a country is its outlets' everyday news, pinned inside it, plus the Conflicts reports there", async () => { + const { ask, calls } = keyed(); + const answer = await ask( + '/feed', + 'hotspot=all&geo=1&today=1&events=1&limit=40&at=51.13,71.43', + ); + assert.equal(answer.status, 200); + const search = calls.find((c) => host(c) === 'api.worldnewsapi.com'); + assert.equal(search.url.pathname, '/search-news'); + assert.equal(search.url.searchParams.get('source-country'), 'kz'); + assert.equal(search.url.searchParams.has('categories'), false); + assert.equal( + search.url.searchParams.get('earliest-publish-date'), + '2026-11-04 12:00:00', + ); + assert.equal(search.url.searchParams.has('api-key'), false); + assert.equal(search.headers['x-api-key'], 'wna-key'); + + const { reports } = normalizeFeedPayload(answer.body); + assert.deepEqual( + reports.map((r) => r.id), + ['lum-102', 'wna-9001', 'wna-9002'], + ); + const [, visit, exploration] = reports; + assert.equal(visit.category, 'political'); + assert.equal(visit.relevance, 'minor'); + assert.equal(visit.publisher, 'daily.example'); + assert.equal(visit.snippet, null, 'titles and links only'); + assert.equal(visit.geo.precision, 'country'); + assert.equal(visit.geo.place, 'Kazakhstan'); + assert.equal(visit.event.image.publisher, 'daily.example'); + assert.equal(exploration.category, 'economy'); + + // The point each is pinned at lies in the area Local outlines. + const outline = normalizeOutlinePayload( + (await ask('/outline', 'at=51.13,71.43&v=4')).body, + ); + const { inPolygons } = await import('../../../src/layers/news/model.js'); + for (const report of reports) + assert.ok(inPolygons(outline.polygons, report.pins[0]), report.id); +}); + +test('a disputed area is named as itself: outlined alone, left out of the country around it, read from no country', async () => { + const { ask, calls, settle } = keyed(); + const crimea = normalizeOutlinePayload( + (await ask('/outline', 'at=44.95,34.1&v=4')).body, + ); + assert.deepEqual(crimea.area, { + key: 'disputed-crimea', + name: 'Crimea', + kind: 'disputed', + country: null, + }); + const { inPolygons } = await import('../../../src/layers/news/model.js'); + const russia = normalizeOutlinePayload( + (await ask('/outline', 'at=55.75,37.6&v=4')).body, + ); + assert.equal(russia.area.name, 'Russia'); + assert.equal(inPolygons(russia.polygons, { lat: 44.95, lon: 34.1 }), false); + assert.equal(inPolygons(russia.polygons, { lat: 55.75, lon: 37.6 }), true); + // Local in Crimea: no national outlets asked, the Conflicts reports there. + const local = await ask('/feed', 'hotspot=all&at=44.95,34.1'); + assert.equal(local.status, 200); + assert.ok(calls.every((c) => host(c) !== 'api.worldnewsapi.com')); + await settle(); +}); + +test("GDELT's places are named by their town and the area there, not its old regions", () => { + const named = (lat, lon, place, precision = 'city') => + gdeltPlace({ lat, lon, place, precision }).place; + assert.equal( + named(31.43, 34.39, 'Bureij, Yerushalayim, Israel'), + 'Bureij, Palestine', + ); + assert.equal( + named(9.07, 7.48, 'Abuja, Abuja Federal Capital Territory, Niger'), + 'Abuja, Nigeria', + ); + assert.equal( + named(44.95, 34.1, 'Simferopol, Crimea, Ukraine'), + 'Simferopol, Crimea', + ); + assert.equal(named(48.4, 31.2, 'Ukraine', 'country'), 'Ukraine'); +}); + +test("a country's news is read once per half hour and Liveuamap's feed once per five minutes", async () => { + const { ask, calls, clock } = keyed(); + const local = 'hotspot=all&at=51.13,71.43'; + await ask('/feed', local); + await ask('/feed', local); + await ask('/hotspots'); + const count = (name) => calls.filter((c) => host(c) === name).length; + assert.equal(count('api.worldnewsapi.com'), 1); + assert.equal(count('a.liveuamap.com'), 1); + clock.now += 6 * 60_000; + await ask('/feed', local); + assert.equal(count('api.worldnewsapi.com'), 1); + assert.equal(count('a.liveuamap.com'), 2); + clock.now += 25 * 60_000; + await ask('/feed', local); + assert.equal(count('api.worldnewsapi.com'), 2); +}); + +test('Local without its key says which key it needs and reads nothing', async () => { + const noLocal = keyed({ env: { LIVEUAMAP_API_KEY: 'live-key' } }); + assert.deepEqual(await noLocal.ask('/feed', 'at=51.13,71.43'), { + status: 503, + body: { error: 'no_key', key: 'worldnewsapi' }, + }); + assert.ok(noLocal.calls.every((c) => host(c) !== 'api.worldnewsapi.com')); + // At sea Local is the Conflicts reports by distance, keyless. + const sea = await keyed({ env: {} }).ask('/feed', 'hotspot=all&at=0,-30'); + assert.equal(sea.status, 200); +}); + +test('a refused Liveuamap key leaves GDELT; World News API refusing names its key', async () => { + const logs = []; + const refused = keyed({ answers: { liveuamap: 401, worldnews: 403 }, logs }); + const conflicts = await refused.ask('/hotspots'); + assert.equal(conflicts.status, 200); + assert.equal(conflicts.body.hotspots[0].counts.last24h, 4); + assert.deepEqual(await refused.ask('/feed', 'at=51.13,71.43'), { + status: 503, + body: { error: 'key_rejected', key: 'worldnewsapi' }, + }); + // A refused key is logged once, however often it is tried. + for (let i = 0; i < 3; i++) { + refused.clock.now += 61_000; + assert.equal((await refused.ask('/hotspots')).status, 200); + } + assert.equal( + logs.filter((line) => line.includes('LIVEUAMAP_API_KEY was refused')) + .length, + 1, + ); + assert.ok(gdeltCalls(refused.calls).length > 0); + assert.ok( + refused.calls.filter((c) => host(c) === 'a.liveuamap.com').length > 1, + ); + // Neither key nor answer body reaches the log. + assert.ok(logs.length > 0); + for (const line of logs) assert.doesNotMatch(line, /live-key|wna-key|ABC/); + await refused.settle(); +}); + +test('a failing GDELT is tried again after a minute, and what it read is kept', async () => { + const answers = { gdelt: 503 }; + const flaky = keyed({ env: {}, answers }); + assert.deepEqual(await flaky.ask('/hotspots'), { + status: 502, + body: { error: 'news_upstream_error' }, + }); + const tried = gdeltCalls(flaky.calls).length; + await flaky.ask('/hotspots'); + assert.equal(gdeltCalls(flaky.calls).length, tried, 'not within a minute'); + flaky.clock.now += 61_000; + delete answers.gdelt; + assert.equal((await flaky.ask('/hotspots')).status, 200); + await flaky.settle(); + answers.gdelt = 500; + flaky.clock.now += 6 * 60_000; + const kept = await flaky.ask('/hotspots'); + await flaky.settle(); + assert.equal(kept.status, 200); + assert.equal(kept.body.hotspots[0].counts.last24h, 4); +}); + +test('a new key drops what the old one read', async () => { + const env = { LIVEUAMAP_API_KEY: 'old', WORLDNEWSAPI_KEY: 'wna-key' }; + const { ask, calls } = keyed({ env }); + await ask('/hotspots'); + env.LIVEUAMAP_API_KEY = 'new'; + await ask('/hotspots'); + assert.deepEqual( + calls + .filter((c) => host(c) === 'a.liveuamap.com') + .map((c) => c.url.searchParams.get('key')), + ['old', 'new'], + ); +}); + +test('each key is one the key registry names', () => { + for (const [id, envVar] of Object.entries(NEWS_KEYS)) + assert.deepEqual( + KEY_SETUP_KEYS.find((entry) => entry.id === id)?.envVars, + [envVar], + id, + ); +}); + +const MINUTE = 60_000; +const count = (calls, name) => calls.filter((c) => host(c) === name).length; +const UPSTREAM_ERROR = { status: 502, body: { error: 'news_upstream_error' } }; +const KEY_QUOTA = { + status: 503, + body: { error: 'key_quota', key: 'worldnewsapi' }, +}; + +test('exports never published are given up on, not looked for back to back', async () => { + // One export of the day has events; GDELT answers 404 for every other. + const quarter = exportName('20261105114500'); + const { ask, calls, clock, settle } = keyed({ + env: {}, + answers: { exports: { [quarter]: EXPORTS[quarter] }, unlisted: 404 }, + }); + assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 3); + await settle(); + // A round that brings nothing in is the last until the next five minutes. + let seen = gdeltCalls(calls).length; + assert.ok(seen <= 2 * 12, `${seen} at first`); + const steps = []; + for (let i = 0; i < 20; i++) { + clock.now += 5 * MINUTE; + assert.equal((await ask('/hotspots')).status, 200); + await settle(); + steps.push(gdeltCalls(calls).length - seen); + seen = gdeltCalls(calls).length; + } + // One round each five minutes, until the day has been looked through + // once; then only the exports of the last hour and a half (GDELT's CDN + // keeps a 404 for an hour, so a late one may still come). + assert.ok( + steps.every((n) => n <= 12), + steps.join(), + ); + assert.ok( + steps.slice(-3).every((n) => n <= 7), + steps.join(), + ); + assert.ok(seen <= 2 * 12 + 20 * 12, String(seen)); +}); + +test('GDELT with no export read at all is a 502, not an empty Conflicts', async () => { + const { ask } = keyed({ env: {}, answers: { exports: {}, unlisted: 404 } }); + assert.deepEqual(await ask('/hotspots'), UPSTREAM_ERROR); + assert.deepEqual(await ask('/feed', CONFLICTS), UPSTREAM_ERROR); +}); + +test('GDELT failing for two hours answers 502; until then what it read is served', async () => { + const answers = {}; + const { ask, clock, settle } = keyed({ env: {}, answers }); + await ask('/hotspots'); + await settle(); + answers.gdelt = 503; + clock.now += 60 * MINUTE; + const hour = await ask('/hotspots'); + assert.equal(hour.status, 200); + assert.equal(hour.body.hotspots[0].counts.last24h, 4); + clock.now += 61 * MINUTE; + assert.deepEqual(await ask('/hotspots'), UPSTREAM_ERROR); + assert.deepEqual(await ask('/feed', CONFLICTS), UPSTREAM_ERROR); + await settle(); +}); + +test('a failed GKG read keeps the events, titled by their words, and is tried again', async () => { + const answers = { gkg: 500 }; + const { ask, calls, clock, logs, settle } = keyed({ env: {}, answers }); + const kharkiv = (answer) => + normalizeFeedPayload(answer.body).reports.find((r) => r.id === 'gdelt-5'); + await ask('/hotspots'); + await settle(); + const read = await ask('/feed', CONFLICTS); + assert.equal(read.status, 200); + assert.equal(read.body.reports.length, 3); + assert.equal( + kharkiv(read).title, + 'Russian drones attack Kharkiv overnight three killed', + ); + assert.ok(logs.some((line) => line.includes('titles read failed'))); + // Each GKG export once, not with every read of the day. + assert.equal(gkgCalls(calls).length, 11); + delete answers.gkg; + clock.now += 5 * MINUTE; + await ask('/hotspots'); + await settle(); + assert.equal( + kharkiv(await ask('/feed', CONFLICTS)).title, + 'Russian drones attack Kharkiv overnight, three killed', + ); +}); + +test('Local where only Conflicts reports speak answers 502 when they fail; in a country the outlets stay', async () => { + const { ask, calls } = keyed({ answers: { gdelt: 503, liveuamap: 500 } }); + // Crimea, and Somaliland (a territory without a code). + assert.deepEqual( + await ask('/feed', 'hotspot=all&at=44.95,34.1'), + UPSTREAM_ERROR, + ); + assert.deepEqual( + await ask('/feed', 'hotspot=all&at=9.56,44.06'), + UPSTREAM_ERROR, + ); + assert.equal(count(calls, 'api.worldnewsapi.com'), 0); + const kazakhstan = await ask('/feed', 'hotspot=all&at=51.13,71.43'); + assert.equal(kazakhstan.status, 200); + assert.deepEqual( + kazakhstan.body.reports.map((r) => r.id), + ['wna-9001', 'wna-9002'], + ); +}); + +test('a territory without a code is named as itself, not read as sea', async () => { + const { ask, calls, settle } = keyed(); + for (const [at, key, name] of [ + ['9.56,44.06', 'disputed-somaliland', 'Somaliland'], + ['35.12,33.94', 'disputed-northern-cyprus', 'Northern Cyprus'], + ]) { + const outline = normalizeOutlinePayload( + (await ask('/outline', `at=${at}&v=4`)).body, + ); + assert.deepEqual(outline.area, { + key, + name, + kind: 'disputed', + country: null, + }); + assert.ok(outline.polygons.length > 0); + } + assert.equal( + gdeltPlace({ + lat: 9.56, + lon: 44.06, + place: 'Hargeysa, Somalia', + precision: 'city', + }).place, + 'Hargeysa, Somaliland', + ); + const local = await ask('/feed', 'hotspot=all&at=9.56,44.06'); + assert.equal(local.status, 200); + assert.equal(count(calls, 'api.worldnewsapi.com'), 0); + await settle(); +}); + +test('a stale Liveuamap or World News API read is served while it is read again', async () => { + const answers = {}; + const { ask, calls, clock, settle } = keyed({ answers }); + const local = 'hotspot=all&at=51.13,71.43'; + const ids = (answer) => answer.body.reports.map((r) => r.id); + assert.deepEqual(ids(await ask('/feed', local)), [ + 'lum-102', + 'wna-9001', + 'wna-9002', + ]); + // Both read again, and held open: the answer does not wait for them. + let release; + const held = new Promise((resolve) => { + release = resolve; + }); + const later = (payload) => () => + held.then(() => new Response(JSON.stringify(payload), { status: 200 })); + answers.liveuamap = later({ venues: [] }); + answers.worldnews = later({ news: [] }); + clock.now += 31 * MINUTE; + const served = await ask('/feed', local); + assert.equal(served.status, 200); + assert.deepEqual(ids(served), ['lum-102', 'wna-9001', 'wna-9002']); + assert.equal(count(calls, 'a.liveuamap.com'), 2); + assert.equal(count(calls, 'api.worldnewsapi.com'), 2); + // Once read, the new answers are served. + release(); + await settle(); + assert.deepEqual(ids(await ask('/feed', local)), []); + // A country not read before is waited for. + answers.worldnews = WORLD_NEWS; + assert.equal( + (await ask('/feed', 'hotspot=all&at=48.85,2.35')).body.reports.length, + 2, + ); +}); + +test('a read in flight with a replaced key is not the answer for the new one', async () => { + let release; + const held = new Promise((resolve) => { + release = resolve; + }); + const env = { LIVEUAMAP_API_KEY: 'old' }; + const answers = { + liveuamap: () => + held.then(() => new Response(JSON.stringify({ venues: [] }))), + }; + const { ask, calls, settle } = keyed({ env, answers }); + const before = ask('/hotspots'); + env.LIVEUAMAP_API_KEY = 'new'; + delete answers.liveuamap; + const after = await ask('/hotspots'); + assert.equal(after.body.hotspots[0].counts.last24h, 7); + assert.deepEqual( + calls + .filter((c) => host(c) === 'a.liveuamap.com') + .map((c) => c.url.searchParams.get('key')), + ['old', 'new'], + ); + release(); + await before; + // The old key's late answer does not replace the new one's. + assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 7); + await settle(); +}); + +test("World News API's reads are a daily budget: spent, Local says key_quota and serves what it read", async () => { + const { ask, calls, clock, logs, settle } = keyed({ + env: { WORLDNEWSAPI_KEY: 'wna-key', WORLDNEWSAPI_DAILY_READS: '2' }, + }); + const at = (point) => ask('/feed', `hotspot=all&at=${point}`); + assert.equal((await at('51.13,71.43')).status, 200); + assert.equal((await at('48.85,2.35')).status, 200); + // Germany would be the third read today. + assert.deepEqual(await at('52.52,13.4'), KEY_QUOTA); + assert.equal(count(calls, 'api.worldnewsapi.com'), 2); + // Kazakhstan, read earlier, is served from what was read. + clock.now += 31 * MINUTE; + const kept = await at('51.13,71.43'); + assert.equal(kept.status, 200); + assert.equal(kept.body.reports.length, 2); + assert.equal(count(calls, 'api.worldnewsapi.com'), 2); + assert.equal( + logs.filter((line) => line.includes('reads for today are spent')).length, + 1, + ); + // A new UTC day has its own reads. + clock.now = Date.parse('2026-11-06T00:30:00Z'); + assert.equal((await at('52.52,13.4')).status, 200); + assert.equal(count(calls, 'api.worldnewsapi.com'), 3); + await settle(); +}); + +test('World News API answering 402 is key_quota, and nothing more is asked today; a 429 is no quota', async () => { + const paid = keyed({ answers: { worldnews: 402 } }); + assert.deepEqual(await paid.ask('/feed', 'at=51.13,71.43'), KEY_QUOTA); + paid.clock.now += 2 * MINUTE; + assert.deepEqual(await paid.ask('/feed', 'at=48.85,2.35'), KEY_QUOTA); + assert.equal(count(paid.calls, 'api.worldnewsapi.com'), 1); + // A 429 is its per-second limit: an ordinary failure, tried again soon. + const busy = keyed({ answers: { worldnews: 429 } }); + assert.deepEqual(await busy.ask('/feed', 'at=51.13,71.43'), UPSTREAM_ERROR); + await paid.settle(); + await busy.settle(); +}); + +test('one unreadable export is given up on and never holds the rest back', async () => { + const bad = exportName('20261105114500'); + const { ask, calls, clock, settle } = keyed({ + env: {}, + answers: { corrupt: [bad] }, + }); + // The other export's stories are served, from the first answer on (its + // copy of the Kharkiv story among them). + const first = await ask('/hotspots'); + assert.equal(first.status, 200); + assert.equal(first.body.hotspots[0].counts.last24h, 2); + await settle(); + clock.now += 6 * MINUTE; + assert.equal((await ask('/hotspots')).status, 200); + await settle(); + const asked = gdeltCalls(calls).filter((c) => String(c.url) === bad).length; + assert.equal(asked, 1, 'an unreadable file is not fetched again'); +}); + +test('no export later than the newest GDELT lists is asked for (its CDN keeps a 404 an hour)', async () => { + const { ask, calls, settle } = keyed({ + env: {}, + answers: { lastupdate: '20261105114500' }, + }); + assert.equal((await ask('/hotspots')).status, 200); + await settle(); + assert.ok( + !gdeltCalls(calls).some( + (c) => String(c.url) === exportName('20261105120000'), + ), + ); + assert.ok( + gdeltCalls(calls).some( + (c) => String(c.url) === exportName('20261105114500'), + ), + ); +}); + +test('a town a disputed area is named after is named alone, on both sides of its line', () => { + assert.equal( + gdeltPlace({ + lat: 31.7667, + lon: 35.2333, + place: 'Jerusalem, Israel (general), Israel', + precision: 'city', + }).place, + 'Jerusalem', + ); +}); + +test('World News API is credited on each of its reports', async () => { + const { ask, settle } = keyed(); + const { reports } = normalizeFeedPayload( + (await ask('/feed', 'hotspot=all&at=51.13,71.43')).body, + ); + const national = reports.filter((r) => r.kind === 'worldnewsapi'); + assert.ok(national.length > 0); + for (const r of national) assert.equal(r.attribution, 'World News API'); + await settle(); +}); + +test('GDELT down for a few minutes gives up on no export: they are all read once it is back', async () => { + const answers = { gdelt: 503 }; + const { ask, calls, clock, settle } = keyed({ env: {}, answers }); + for (let i = 0; i < 4; i++) { + await ask('/hotspots'); + await settle(); + clock.now += 61_000; + } + delete answers.gdelt; + clock.now += 5 * MINUTE; + const back = await ask('/hotspots'); + await settle(); + assert.equal(back.status, 200); + assert.equal(back.body.hotspots[0].counts.last24h, 4); + assert.ok( + gdeltCalls(calls).some( + (c) => String(c.url) === exportName('20261105114500'), + ), + ); +}); diff --git a/server/providers/news/upstream.js b/server/providers/news/upstream.js new file mode 100644 index 000000000..0f834ee7a --- /dev/null +++ b/server/providers/news/upstream.js @@ -0,0 +1,274 @@ +// The keyed news APIs Core reads itself when no news server is configured: +// Liveuamap's worldwide feed (`a=mpts`) beside GDELT's events in the +// Conflicts section (gdeltEvents.js), World News API's search by source +// country for Local. Each key stays on the server: Liveuamap's rides in the query, so +// neither a URL nor an error body is ever logged or relayed. + +import { readResponseTextCapped } from '../common/http.js'; + +const LIVEUAMAP_URL = 'https://a.liveuamap.com/api'; +const WORLD_NEWS_URL = 'https://api.worldnewsapi.com/search-news'; +// 100 Liveuamap items run to about 500 KB. +const LIVEUAMAP_COUNT = 500; +const LIVEUAMAP_MAX_BYTES = 16 * 1024 * 1024; +// One search per country: a point, and a hundredth of one per article, so +// about 1.5 points a read (the free plan has 50 points a day). +const WORLD_NEWS_ARTICLES = 50; +const WORLD_NEWS_MAX_BYTES = 8 * 1024 * 1024; +// World News API's categories that are not the news of the day. Its +// `categories` filter is not asked for: it classifies few articles outside +// English, so it would drop most of a country's own outlets; an article it +// left unclassified is kept. +export const WORLD_NEWS_LEFT_OUT = Object.freeze( + new Set(['sports', 'entertainment', 'lifestyle', 'travel']), +); +const DAY_MS = 86_400_000; +const TIMEOUT_MS = 20_000; + +const ENTITIES = { + amp: '&', + quot: '"', + apos: "'", + lt: '<', + gt: '>', + nbsp: ' ', +}; + +/** Text with HTML entities decoded and whitespace collapsed, or ''. */ +export function unescapeHtml(value) { + return String(value ?? '') + .replace( + /&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]{2,6}));/gi, + (all, dec, hex, name) => { + if (name) return ENTITIES[name.toLowerCase()] ?? all; + const code = dec ? Number(dec) : parseInt(hex, 16); + return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : all; + }, + ) + .replace(/\s+/g, ' ') + .trim(); +} + +/** An https URL without credentials, or null. */ +export function webLink(value) { + try { + const url = new URL(String(value ?? '').trim()); + return url.protocol === 'https:' && !url.username && !url.password + ? url.href + : null; + } catch { + return null; + } +} + +/** A clip a browser plays itself (an https MP4 or WebM file), or null. */ +export function playableVideo(value) { + const url = webLink(value); + return url && /\.(?:mp4|m4v|webm)$/i.test(new URL(url).pathname) ? url : null; +} + +const finiteNumber = (value) => { + const number = typeof value === 'string' ? Number(value.trim()) : value; + return typeof number === 'number' && Number.isFinite(number) ? number : null; +}; + +/** Why an upstream could not be read; never carries its URL or body. */ +export class UpstreamError extends Error { + constructor( + message, + { status = null, rejected = false, quota = false, unreadable = false } = {}, + ) { + super(message); + this.name = 'UpstreamError'; + this.status = status; + // The key was refused (401 or 403): the operator's to fix. + this.rejected = rejected; + // The key's allowance is spent (402, or the day's reads): it answers + // again later. A 429 is a per-second limit, not this. + this.quota = quota; + // The answer itself cannot be read (corrupt, or over a size cap): asking + // again gets the same. + this.unreadable = unreadable; + } +} + +async function getText( + fetchImpl, + url, + { headers = {}, maxBytes, signal, what }, +) { + let response; + try { + response = await fetchImpl(url, { + headers: { Accept: 'application/json', ...headers }, + redirect: 'error', + signal: signal + ? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)]) + : AbortSignal.timeout(TIMEOUT_MS), + }); + } catch (cause) { + throw new UpstreamError( + cause?.name === 'TimeoutError' + ? `${what} timed out` + : `${what} unreachable`, + ); + } + if (!response.ok) { + await response.body?.cancel?.().catch?.(() => {}); + throw new UpstreamError(`${what} answered ${response.status}`, { + status: response.status, + rejected: response.status === 401 || response.status === 403, + quota: response.status === 402, + }); + } + try { + return await readResponseTextCapped(response, maxBytes, signal); + } catch { + throw new UpstreamError(`${what} answer unreadable`); + } +} + +function parseJson(text, what) { + try { + return JSON.parse(text); + } catch { + // Not the parser's message: an error answer may echo the request. + throw new UpstreamError(`${what} answer unreadable`); + } +} + +/** + * Liveuamap's items with a headline, a point and a page: + * `{id, title, lat, lon, place, publishedAt, url, image, video}`. + */ +export function parseLiveuamap(payload) { + if (!Array.isArray(payload?.venues)) + throw new UpstreamError('Liveuamap answer unreadable'); + const items = []; + for (const venue of payload.venues) { + const id = finiteNumber(venue?.id); + const title = unescapeHtml(venue?.name); + const lat = finiteNumber(venue?.lat); + const lon = finiteNumber(venue?.lng); + const seconds = finiteNumber(venue?.timestamp); + const url = webLink(venue?.link); + if ( + id === null || + !title || + lat === null || + lon === null || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 || + (lat === 0 && lon === 0) || + seconds === null || + !url + ) + continue; + items.push({ + id: Math.trunc(id), + title, + lat, + lon, + place: unescapeHtml(venue.location) || null, + publishedAt: new Date(seconds * 1000).toISOString(), + url, + image: webLink(venue.img_share), + video: playableVideo(venue.video), + }); + } + return items; +} + +/** The worldwide feed of the last day, as of `now`. */ +export async function fetchLiveuamap({ key, fetchImpl, now, signal }) { + const url = new URL(LIVEUAMAP_URL); + url.search = new URLSearchParams({ + a: 'mpts', + resid: '1', + time: String(Math.floor((now - DAY_MS) / 1000)), + count: String(LIVEUAMAP_COUNT), + key, + }).toString(); + const text = await getText(fetchImpl, url, { + maxBytes: LIVEUAMAP_MAX_BYTES, + signal, + what: 'Liveuamap', + }); + return parseLiveuamap(parseJson(text, 'Liveuamap')); +} + +// World News API writes times as "2026-10-30 01:31:00", in UTC. +const worldNewsTime = (ms) => + new Date(ms).toISOString().slice(0, 19).replace('T', ' '); +function worldNewsDate(value) { + const match = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)$/.exec( + String(value ?? ''), + ); + const ms = match ? Date.parse(`${match[1]}T${match[2]}Z`) : NaN; + return Number.isFinite(ms) ? new Date(ms).toISOString() : null; +} + +/** + * A search's articles with a title, a link and a time, one per title (the + * same story from several outlets is one), sport and entertainment left + * out, newest first: + * `{id, title, url, publisher, language, category, publishedAt, image, video}`. + */ +export function parseWorldNews(payload) { + if (!Array.isArray(payload?.news)) + throw new UpstreamError('World News API answer unreadable'); + const seen = new Set(); + const articles = []; + for (const article of payload.news) { + const id = finiteNumber(article?.id); + const title = unescapeHtml(article?.title); + const url = webLink(article?.url); + const publishedAt = worldNewsDate(article?.publish_date); + const category = + typeof article?.category === 'string' + ? article.category.toLowerCase() + : null; + if (id === null || !title || !url || !publishedAt) continue; + if (WORLD_NEWS_LEFT_OUT.has(category)) continue; + const same = title.toLowerCase(); + if (seen.has(same)) continue; + seen.add(same); + articles.push({ + id: Math.trunc(id), + title, + url, + publisher: new URL(url).hostname.replace(/^www\./, ''), + language: + typeof article.language === 'string' && + /^[a-z]{2}$/i.test(article.language) + ? article.language.toLowerCase() + : null, + category, + publishedAt, + image: webLink(article.image), + video: playableVideo(article.video), + }); + } + return articles.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)); +} + +/** A country's everyday news of the last day, as of `now`. */ +export async function fetchWorldNews({ key, country, fetchImpl, now, signal }) { + const url = new URL(WORLD_NEWS_URL); + url.search = new URLSearchParams({ + 'source-country': country.toLowerCase(), + // Both ends: sorted by time with neither, a search is cut to 31 days. + 'earliest-publish-date': worldNewsTime(now - DAY_MS), + 'latest-publish-date': worldNewsTime(now), + sort: 'publish-time', + 'sort-direction': 'DESC', + number: String(WORLD_NEWS_ARTICLES), + }).toString(); + const text = await getText(fetchImpl, url, { + headers: { 'x-api-key': key }, + maxBytes: WORLD_NEWS_MAX_BYTES, + signal, + what: 'World News API', + }); + return parseWorldNews(parseJson(text, 'World News API')); +} diff --git a/src/data/local_data/natural_earth/README.md b/src/data/local_data/natural_earth/README.md index d3a325585..c486383c1 100644 --- a/src/data/local_data/natural_earth/README.md +++ b/src/data/local_data/natural_earth/README.md @@ -1,8 +1,11 @@ # Natural Earth packs -Offline named-region polygons for the voice-annotation resolver -(`src/data/naturalEarthRegions.js`) — "outline the Alps" resolves to the real -range geometry with no network dependency. +Offline polygons with no network dependency. The voice-annotation resolver +(`src/data/naturalEarthRegions.js`) reads the named regions, so "outline the +Alps" resolves to the real range geometry; area annotations read the +countries, states and provinces (`src/data/adminBoundaries.js`); and the News +layer reads the countries and the disputed areas +(`server/providers/news/areas.js`). | File | Source dataset | Features | |------|----------------|----------| @@ -10,6 +13,7 @@ range geometry with no network dependency. | `marine.json` | `ne_10m_geography_marine_polys` (seas, gulfs, straits, bays, …) | 292 named | | `countries.json` | `ne_10m_admin_0_countries` (countries) | 256 | | `states_provinces.json` | `ne_10m_admin_1_states_provinces` (states, provinces and other first-level units) | 4,587 named | +| `disputed_areas.json` | `ne_10m_admin_0_disputed_areas` (disputed, breakaway and UN-patrolled areas; a hand-picked set, see below) | 11 of 99 | **Source:** Natural Earth 10m physical vectors, via the canonical [nvkelso/natural-earth-vector](https://github.com/nvkelso/natural-earth-vector) @@ -103,3 +107,42 @@ Includes England, Scotland, Wales and Northern Ireland from `ne_10m_admin_0_map_units.geojson` at the same commit (SHA-256 `57da82be755f4afccd8f3b14251bb2752f5df1395f47d2d86f817470c4a48862`). Result: 945,843 bytes; 260 features, 2,088 parts, 122,788 vertices. + +## Disputed areas (`disputed_areas.json`) + +The News layer's areas that no one country's name may stand for +(`server/providers/news/areas.js`): a place inside one is named with the +area ("Crimea"), and Local there reads no country's outlets. + +**Source:** `geojson/ne_10m_admin_0_disputed_areas.geojson` at the same pinned +commit (SHA-256 +`9cafef8b7dfb6b164dc58f218f981f4ace9f716f6c03795d4c62d1ac9f3d50f5`), fetched +2026-11-05. Public domain, as above. + +**Selection:** a hand-picked 11 of the source's 99 features. No rule picks +them; this is the list: + +| Area | Natural Earth's note (`NOTE_BRK`) | +|------|-----------------------------------| +| Golan Heights | Admin. By Israel; Claimed by Syria | +| UNDOF Zone | UN Patrolled Zone in Syria | +| Transnistria | Self admin.; Claimed by Moldova | +| Abyei | Admin. by Sudan; Claimed by South Sudan | +| Crimea | Admin. by Russia; Claimed by Ukraine | +| Shebaa Farms | Admin. By Israel; Claimed by Lebanon | +| Bir Tawil | Between Egypt and Sudan | +| Halayib Triangle | Admin. by Egypt; Claimed by Sudan | +| East Jerusalem | Admin. By Israel; Claimed by Palestine | +| Abu Musa I. | Admin. by Iran; Claimed by UAE | +| Doumera Island | Admin. By Eritrea; Claimed by Djibouti | + +The source names three of these by a country ("Moldova", "Sudan", "Iran"); +the pack names them as the area. The 2014 Donetsk and Luhansk lines are left +out as out of date. Every other feature is left out too, so a place there is +named by its country in `countries.json`. + +**Curation:** `kind` (`held`, `self`, `un`, `unclaimed`), `heldBy` and +`claimedBy` are read from `NOTE_BRK`, and `note` keeps it as published. Every +part is kept; each ring is Douglas-Peucker simplified at 0.003°, coordinates +are rounded to 3 decimals and rings are stored open. Provenance is in the +pack's `meta`. Result: 14,504 bytes. diff --git a/src/data/local_data/natural_earth/disputed_areas.json b/src/data/local_data/natural_earth/disputed_areas.json new file mode 100644 index 000000000..078669288 --- /dev/null +++ b/src/data/local_data/natural_earth/disputed_areas.json @@ -0,0 +1 @@ +{"meta":{"title":"Natural Earth admin-0 disputed areas, for the News layer","source":"ne_10m_admin_0_disputed_areas","url":"https://raw.githubusercontent.com/nvkelso/natural-earth-vector/ca96624a56bd078437bca8184e78163e5039ad19/geojson/ne_10m_admin_0_disputed_areas.geojson","sha256":"9cafef8b7dfb6b164dc58f218f981f4ace9f716f6c03795d4c62d1ac9f3d50f5","fetched":"2026-11-05","license":"Public domain (https://www.naturalearthdata.com/about/terms-of-use/)","curation":"A hand-picked 11 of the source's 99 features: Golan Heights, UNDOF Zone, Transnistria, Abyei, Crimea, Shebaa Farms, Bir Tawil, Halayib Triangle, East Jerusalem, Abu Musa I. and Doumera Island. The source names three of them by a country (Moldova, Sudan, Iran); they are named here as the area. kind, heldBy and claimedBy are read from NOTE_BRK; note is NOTE_BRK as published. The 2014 Donetsk and Luhansk lines are left out as out of date. Every part kept; each ring Douglas-Peucker simplified at 0.003 degrees, coordinates rounded to 3 decimals, rings stored open."},"areas":[{"name":"Golan Heights","kind":"held","heldBy":"IL","claimedBy":"SY","note":"Admin. By Israel; Claimed by Syria","rings":[[[35.821,33.407],[35.81,33.36],[35.786,33.343],[35.764,33.334],[35.802,33.312],[35.769,33.273],[35.804,33.248],[35.808,33.202],[35.83,33.19],[35.833,33.161],[35.811,33.127],[35.811,33.112],[35.849,33.099],[35.846,33.085],[35.859,32.99],[35.888,32.945],[35.85,32.896],[35.838,32.866],[35.842,32.854],[35.834,32.828],[35.784,32.778],[35.758,32.744],[35.74,32.741],[35.685,32.711],[35.636,32.679],[35.612,32.682],[35.595,32.707],[35.6,32.717],[35.634,32.752],[35.635,32.769],[35.641,32.772],[35.635,32.794],[35.641,32.82],[35.638,32.832],[35.641,32.865],[35.638,32.875],[35.602,32.909],[35.615,32.921],[35.62,32.984],[35.588,33.018],[35.621,33.05],[35.634,33.052],[35.637,33.077],[35.656,33.125],[35.672,33.15],[35.661,33.213],[35.66,33.236],[35.666,33.247],[35.635,33.249],[35.608,33.239],[35.605,33.244],[35.618,33.256],[35.62,33.27],[35.631,33.275],[35.654,33.262],[35.712,33.307],[35.718,33.317],[35.716,33.327],[35.744,33.331],[35.769,33.343],[35.786,33.358],[35.806,33.391]]]},{"name":"UNDOF Zone","kind":"un","note":"UN Patrolled Zone in Syria","rings":[[[35.758,32.744],[35.784,32.778],[35.834,32.828],[35.842,32.854],[35.838,32.866],[35.85,32.896],[35.888,32.945],[35.859,32.99],[35.846,33.085],[35.849,33.099],[35.811,33.112],[35.811,33.127],[35.833,33.161],[35.83,33.19],[35.808,33.202],[35.804,33.248],[35.769,33.273],[35.802,33.312],[35.764,33.334],[35.786,33.343],[35.81,33.36],[35.821,33.407],[35.87,33.431],[35.873,33.416],[35.87,33.407],[35.845,33.389],[35.83,33.365],[35.83,33.351],[35.843,33.322],[35.845,33.304],[35.84,33.295],[35.819,33.284],[35.82,33.281],[35.851,33.276],[35.866,33.226],[35.889,33.199],[35.882,33.189],[35.885,33.179],[35.9,33.18],[35.911,33.166],[35.911,33.135],[35.892,33.124],[35.897,33.117],[35.903,33.117],[35.903,33.11],[35.868,33.107],[35.865,33.082],[35.877,32.993],[35.898,32.986],[35.924,32.968],[35.912,32.947],[35.89,32.929],[35.861,32.88],[35.843,32.822],[35.815,32.787],[35.791,32.774],[35.77,32.748],[35.764,32.747]]]},{"name":"Transnistria","kind":"self","claimedBy":"MD","note":"Self admin.; Claimed by Moldova","rings":[[[29.899,46.553],[29.868,46.58],[29.84,46.582],[29.831,46.588],[29.824,46.601],[29.776,46.58],[29.728,46.594],[29.748,46.617],[29.776,46.629],[29.759,46.658],[29.749,46.665],[29.728,46.67],[29.703,46.665],[29.696,46.676],[29.667,46.676],[29.69,46.707],[29.698,46.728],[29.665,46.801],[29.652,46.813],[29.62,46.815],[29.611,46.827],[29.619,46.832],[29.615,46.841],[29.59,46.824],[29.563,46.787],[29.549,46.793],[29.543,46.793],[29.531,46.783],[29.512,46.787],[29.498,46.801],[29.502,46.821],[29.495,46.828],[29.494,46.862],[29.477,46.889],[29.471,46.941],[29.46,46.953],[29.436,46.957],[29.376,46.935],[29.35,46.937],[29.316,46.958],[29.296,46.992],[29.325,46.988],[29.371,47.003],[29.378,47.012],[29.374,47.011],[29.364,47.033],[29.381,47.029],[29.383,47.042],[29.378,47.071],[29.372,47.078],[29.33,47.108],[29.296,47.122],[29.291,47.14],[29.237,47.162],[29.227,47.149],[29.221,47.149],[29.213,47.158],[29.188,47.162],[29.179,47.156],[29.18,47.145],[29.194,47.124],[29.19,47.111],[29.157,47.113],[29.148,47.096],[29.125,47.074],[29.099,47.092],[29.07,47.101],[29.095,47.135],[29.112,47.15],[29.145,47.163],[29.164,47.192],[29.179,47.197],[29.164,47.231],[29.094,47.31],[29.08,47.332],[29.079,47.348],[29.089,47.351],[29.123,47.329],[29.14,47.329],[29.154,47.336],[29.165,47.348],[29.139,47.376],[29.083,47.406],[29.091,47.437],[29.108,47.454],[29.111,47.464],[29.104,47.478],[29.093,47.487],[29.049,47.506],[29.034,47.518],[29.025,47.533],[29.016,47.582],[28.993,47.605],[28.987,47.622],[28.996,47.651],[28.988,47.661],[28.981,47.687],[28.976,47.743],[28.981,47.765],[29.018,47.827],[29.019,47.853],[28.981,47.875],[28.861,47.899],[28.829,47.917],[28.831,47.941],[28.813,47.965],[28.785,47.983],[28.754,47.979],[28.719,47.999],[28.698,48.022],[28.682,48.031],[28.652,48.019],[28.631,48.034],[28.619,48.033],[28.56,47.998],[28.538,47.994],[28.528,48.009],[28.533,48.02],[28.597,48.067],[28.628,48.063],[28.633,48.084],[28.63,48.092],[28.584,48.102],[28.538,48.12],[28.518,48.121],[28.501,48.113],[28.519,48.149],[28.542,48.156],[28.574,48.155],[28.666,48.129],[28.771,48.124],[28.799,48.112],[28.806,48.104],[28.812,48.083],[28.828,48.057],[28.833,48.025],[28.856,48.008],[28.883,47.977],[28.915,47.953],[28.95,47.935],[28.981,47.926],[29.017,47.931],[29.061,47.97],[29.093,47.98],[29.11,47.98],[29.136,47.968],[29.148,47.955],[29.173,47.891],[29.236,47.871],[29.233,47.857],[29.187,47.818],[29.178,47.801],[29.178,47.79],[29.187,47.783],[29.222,47.775],[29.235,47.767],[29.239,47.756],[29.197,47.718],[29.191,47.686],[29.192,47.651],[29.182,47.613],[29.131,47.56],[29.117,47.533],[29.13,47.493],[29.164,47.439],[29.182,47.43],[29.202,47.447],[29.218,47.452],[29.233,47.446],[29.251,47.425],[29.281,47.411],[29.301,47.378],[29.346,47.366],[29.359,47.353],[29.368,47.308],[29.394,47.285],[29.426,47.279],[29.442,47.282],[29.459,47.293],[29.481,47.338],[29.521,47.339],[29.557,47.324],[29.57,47.307],[29.579,47.284],[29.58,47.26],[29.569,47.243],[29.544,47.234],[29.54,47.213],[29.551,47.16],[29.545,47.136],[29.53,47.124],[29.481,47.117],[29.479,47.108],[29.52,47.06],[29.531,47.066],[29.551,47.09],[29.57,47.091],[29.595,47.075],[29.602,47.061],[29.603,47.046],[29.595,47.032],[29.572,47.012],[29.559,46.946],[29.567,46.935],[29.598,46.935],[29.632,46.914],[29.696,46.905],[29.712,46.893],[29.736,46.867],[29.754,46.858],[29.844,46.854],[29.872,46.838],[29.89,46.807],[29.917,46.814],[29.928,46.81],[29.932,46.772],[29.951,46.743],[29.953,46.725],[29.944,46.662],[29.932,46.632],[29.931,46.619],[29.95,46.579],[29.938,46.557],[29.916,46.554]]]},{"name":"Abyei","kind":"held","heldBy":"SD","claimedBy":"SS","note":"Admin. by Sudan; Claimed by South Sudan","rings":[[[27.895,9.595],[27.825,9.744],[27.825,10.16],[28.989,10.16],[28.989,9.587],[28.914,9.529],[28.849,9.466],[28.829,9.427],[28.831,9.349],[28.844,9.325],[28.045,9.331],[27.997,9.385]]]},{"name":"Crimea","kind":"held","heldBy":"RU","claimedBy":"UA","note":"Admin. by Russia; Claimed by Ukraine","rings":[[[33.629,46.125],[33.66,46.148],[33.671,46.173],[33.662,46.21],[33.67,46.218],[33.716,46.196],[33.755,46.193],[33.802,46.203],[33.846,46.196],[34.026,46.116],[34.105,46.095],[34.161,46.091],[34.201,46.1],[34.236,46.098],[34.301,46.07],[34.334,46.07],[34.362,46.056],[34.411,45.988],[34.465,45.955],[34.504,45.951],[34.53,45.981],[34.57,45.976],[34.642,45.981],[34.689,45.974],[34.735,45.953],[34.763,45.932],[34.784,45.899],[34.784,45.829],[34.789,45.804],[34.81,45.785],[34.871,45.773],[34.917,45.748],[34.952,45.722],[34.976,45.717],[35.002,45.729],[35.044,45.67],[35.332,45.371],[35.481,45.298],[35.518,45.295],[35.695,45.33],[35.716,45.341],[35.743,45.361],[35.753,45.386],[35.767,45.389],[35.788,45.407],[35.804,45.449],[35.817,45.464],[35.833,45.471],[35.858,45.469],[35.86,45.46],[35.847,45.426],[35.854,45.414],[35.947,45.372],[35.986,45.363],[36.022,45.368],[36.052,45.395],[36.069,45.43],[36.118,45.457],[36.137,45.462],[36.252,45.461],[36.284,45.477],[36.3,45.473],[36.307,45.461],[36.299,45.443],[36.312,45.444],[36.341,45.457],[36.412,45.436],[36.471,45.446],[36.491,45.443],[36.533,45.409],[36.549,45.412],[36.567,45.425],[36.583,45.427],[36.614,45.409],[36.638,45.351],[36.631,45.341],[36.586,45.334],[36.545,45.347],[36.502,45.347],[36.489,45.336],[36.484,45.316],[36.497,45.301],[36.491,45.292],[36.45,45.285],[36.437,45.272],[36.418,45.23],[36.409,45.149],[36.415,45.128],[36.455,45.092],[36.453,45.076],[36.44,45.067],[36.368,45.049],[36.268,45.044],[36.258,45.039],[36.244,45.011],[36.225,45.008],[36.136,45.021],[36.079,45.046],[36.079,45.039],[36.048,45.049],[36.025,45.033],[36.008,45.01],[35.993,44.998],[35.917,45.001],[35.848,44.992],[35.828,44.999],[35.813,45.032],[35.783,45.058],[35.706,45.095],[35.634,45.115],[35.591,45.12],[35.547,45.12],[35.506,45.114],[35.47,45.101],[35.408,45.062],[35.397,45.035],[35.401,45.025],[35.435,45.011],[35.435,45.005],[35.374,44.998],[35.362,44.987],[35.361,44.974],[35.367,44.943],[35.319,44.97],[35.298,44.956],[35.273,44.967],[35.256,44.959],[35.233,44.926],[35.215,44.914],[35.154,44.901],[35.13,44.869],[35.108,44.825],[35.081,44.794],[35.038,44.799],[35.006,44.835],[34.986,44.84],[34.942,44.84],[34.908,44.821],[34.89,44.818],[34.857,44.824],[34.827,44.812],[34.757,44.821],[34.721,44.811],[34.651,44.777],[34.613,44.765],[34.533,44.753],[34.5,44.743],[34.475,44.724],[34.473,44.73],[34.469,44.73],[34.414,44.674],[34.38,44.631],[34.331,44.545],[34.271,44.545],[34.243,44.507],[34.23,44.5],[34.182,44.493],[34.147,44.45],[34.103,44.425],[34.066,44.42],[33.958,44.383],[33.938,44.381],[33.852,44.402],[33.808,44.395],[33.721,44.4],[33.645,44.425],[33.632,44.434],[33.62,44.462],[33.578,44.497],[33.518,44.491],[33.487,44.51],[33.38,44.565],[33.372,44.576],[33.371,44.588],[33.375,44.585],[33.405,44.59],[33.436,44.605],[33.477,44.607],[33.5,44.614],[33.551,44.615],[33.572,44.621],[33.525,44.629],[33.516,44.634],[33.515,44.649],[33.544,44.683],[33.548,44.699],[33.525,44.772],[33.516,44.785],[33.528,44.793],[33.546,44.837],[33.56,44.85],[33.598,44.859],[33.609,44.872],[33.62,44.909],[33.62,44.931],[33.606,44.987],[33.585,45.018],[33.578,45.053],[33.563,45.094],[33.537,45.117],[33.469,45.149],[33.413,45.185],[33.396,45.19],[33.355,45.186],[33.26,45.156],[33.24,45.18],[33.174,45.197],[33.123,45.238],[33.098,45.248],[33.079,45.272],[33.043,45.285],[32.941,45.343],[32.899,45.354],[32.75,45.363],[32.729,45.361],[32.663,45.32],[32.591,45.32],[32.535,45.338],[32.513,45.341],[32.482,45.409],[32.495,45.436],[32.525,45.458],[32.644,45.522],[32.705,45.531],[32.735,45.545],[32.758,45.565],[32.773,45.567],[32.814,45.559],[32.828,45.562],[32.827,45.596],[32.868,45.63],[32.937,45.663],[33.176,45.743],[33.181,45.755],[33.159,45.769],[33.161,45.779],[33.169,45.784],[33.195,45.765],[33.23,45.751],[33.265,45.754],[33.338,45.786],[33.374,45.797],[33.41,45.827],[33.482,45.854],[33.508,45.843],[33.53,45.842],[33.554,45.847],[33.578,45.868],[33.62,45.882],[33.642,45.876],[33.659,45.859],[33.692,45.861],[33.694,45.914],[33.709,45.91],[33.777,45.93],[33.74,45.948],[33.682,45.958],[33.657,45.957],[33.632,45.944],[33.62,45.951],[33.647,45.98],[33.632,45.988],[33.631,45.997],[33.644,46.015],[33.628,46.068],[33.648,46.09],[33.644,46.105]]]},{"name":"Shebaa Farms","kind":"held","heldBy":"IL","claimedBy":"LB","note":"Admin. By Israel; Claimed by Lebanon","rings":[[[35.631,33.275],[35.652,33.277],[35.66,33.289],[35.699,33.323],[35.716,33.327],[35.718,33.317],[35.712,33.307],[35.676,33.276],[35.654,33.262],[35.642,33.266]]]},{"name":"Bir Tawil","kind":"unclaimed","note":"Between Egypt and Sudan","rings":[[[33.181,21.995],[34.084,21.995],[33.999,21.767],[33.558,21.711],[33.304,21.903]]]},{"name":"Halayib Triangle","kind":"held","heldBy":"EG","claimedBy":"SD","note":"Admin. by Egypt; Claimed by Sudan","rings":[[[36.884,21.996],[34.084,21.995],[34.147,22.192],[34.682,22.298],[34.944,22.858],[35.212,22.786],[35.621,23.139],[35.655,23.074],[35.668,22.983],[35.695,22.932],[35.737,22.884],[35.764,22.87],[35.723,22.932],[35.743,22.931],[35.756,22.922],[35.771,22.89],[35.785,22.833],[35.801,22.823],[35.848,22.763],[35.886,22.736],[35.94,22.717],[35.969,22.698],[36.121,22.676],[36.198,22.652],[36.226,22.635],[36.275,22.592],[36.291,22.568],[36.296,22.522],[36.313,22.492],[36.375,22.445],[36.405,22.433],[36.402,22.449],[36.407,22.449],[36.428,22.426],[36.423,22.398],[36.44,22.356],[36.508,22.328],[36.563,22.297],[36.623,22.25],[36.714,22.169],[36.733,22.16],[36.748,22.161],[36.757,22.177],[36.782,22.164],[36.796,22.151],[36.818,22.109],[36.872,22.074],[36.895,22.066],[36.898,22.017],[36.884,21.997]],[[36.651,22.272],[36.661,22.262],[36.66,22.244],[36.658,22.255],[36.631,22.252],[36.637,22.277],[36.649,22.281],[36.652,22.278],[36.644,22.273]],[[36.204,22.67],[36.207,22.66],[36.201,22.652],[36.185,22.673],[36.17,22.684],[36.194,22.685]],[[36.063,22.738],[36.067,22.728],[36.062,22.73],[36.056,22.743],[36.043,22.732],[36.047,22.726],[36.042,22.721],[36.038,22.737],[36.056,22.761],[36.066,22.767],[36.062,22.755],[36.064,22.743]],[[36.228,22.793],[36.25,22.789],[36.207,22.784],[36.197,22.791],[36.218,22.793],[36.214,22.814],[36.224,22.803],[36.222,22.793]]]},{"name":"East Jerusalem","kind":"held","heldBy":"IL","claimedBy":"PS","note":"Admin. By Israel; Claimed by Palestine","rings":[[[35.186,31.809],[35.184,31.825],[35.197,31.826],[35.216,31.82],[35.216,31.839],[35.205,31.877],[35.212,31.881],[35.222,31.879],[35.227,31.85],[35.233,31.843],[35.25,31.844],[35.257,31.839],[35.252,31.831],[35.265,31.826],[35.256,31.812],[35.263,31.79],[35.253,31.77],[35.263,31.749],[35.252,31.738],[35.242,31.711],[35.238,31.709],[35.224,31.719],[35.219,31.715],[35.205,31.723],[35.179,31.72],[35.167,31.732],[35.149,31.737],[35.209,31.747],[35.217,31.736],[35.223,31.735],[35.227,31.744],[35.243,31.751],[35.242,31.759],[35.23,31.762],[35.233,31.769],[35.224,31.801],[35.198,31.81]]]},{"name":"Abu Musa I.","kind":"held","heldBy":"IR","claimedBy":"AE","note":"Admin. by Iran; Claimed by UAE","rings":[[[55.049,25.886],[55.054,25.865],[55.033,25.855],[55.023,25.863],[55.018,25.86],[55.012,25.882],[55.044,25.893]]]},{"name":"Doumera Island","kind":"held","heldBy":"ER","claimedBy":"DJ","note":"Admin. By Eritrea; Claimed by Djibouti","rings":[[[43.14,12.72],[43.152,12.721],[43.152,12.713],[43.143,12.712],[43.139,12.718]]]}]}