mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
The bundled rules read fighting and crises; in everyday headlines their
words mislead ("Kazakhstan Advances in Global Digital Rankings" read as
ground fighting). A country's own stories in Local now keep World News
API's category (politics, business, else other) and carry no severity.
Fighting in the area still shows through the Conflicts reports placed
there.
860 lines
30 KiB
JavaScript
860 lines
30 KiB
JavaScript
// The News layer's routes answered by Core itself, when no news server is
|
|
// configured (GEV_NEWS_ORIGIN). Conflicts: the day's GDELT events (no key;
|
|
// gdeltEvents.js), one report per article at GDELT's place for it, and with
|
|
// LIVEUAMAP_API_KEY Liveuamap's mapped items too. Local: World News API
|
|
// (WORLDNEWSAPI_KEY) by the country at the view centre; without its key the
|
|
// section answers 503 {error: 'no_key', key} and reads nothing, and past
|
|
// its day's reads (WORLDNEWSAPI_DAILY_READS) 503 {error: 'key_quota', key}.
|
|
// Conflicts' categories, relevance and severity are the shared rules
|
|
// (categorize.js); a country's own news keeps World News API's category.
|
|
// A news server may replace these readings with its own.
|
|
//
|
|
// A read that has something to serve is not waited for: what was read is
|
|
// served while it is read again. A section that cannot be read answers 502,
|
|
// never an empty list.
|
|
//
|
|
// The answers are the same /api/news contract a news server serves
|
|
// (src/layers/news/records.js reads both).
|
|
|
|
import {
|
|
CATEGORY_ORDER,
|
|
categorizeText,
|
|
relevanceOf,
|
|
severityByRules,
|
|
} from './categorize.js';
|
|
import {
|
|
areaAt,
|
|
countryAnchor,
|
|
countryByCode,
|
|
countryPolygons,
|
|
} from './areas.js';
|
|
import {
|
|
eventReports,
|
|
fetchLatestExport,
|
|
eventTitle,
|
|
exportAt,
|
|
exportsOfDay,
|
|
fetchExport,
|
|
fetchTitles,
|
|
gkgUrl,
|
|
storyKey,
|
|
} from './gdeltEvents.js';
|
|
import { fetchLiveuamap, fetchWorldNews, UpstreamError } from './upstream.js';
|
|
|
|
/** The keys each section reads with, by the key registry's id. */
|
|
export const NEWS_KEYS = Object.freeze({
|
|
liveuamap: 'LIVEUAMAP_API_KEY',
|
|
worldnewsapi: 'WORLDNEWSAPI_KEY',
|
|
});
|
|
|
|
/** The one hotspot: the world, its reports drawn as markers at every zoom. */
|
|
export const WORLDWIDE_HOTSPOT = 'worldwide';
|
|
const HOUR_MS = 3_600_000;
|
|
const DAY_MS = 24 * HOUR_MS;
|
|
// GDELT publishes an export every 15 minutes and lists the newest in
|
|
// lastupdate.txt: the day's missing ones up to it are read every 5 minutes,
|
|
// newest first, a dozen a read (the first day's 96 fill in over the next
|
|
// reads, a few seconds apart, while each reads something new). One listed
|
|
// but missing (404) an hour and a half after its time is taken as never
|
|
// published; one that fails GDELT_EXPORT_TRIES times (a try a round), or
|
|
// cannot be read at all, is given up on, so one bad file never holds the
|
|
// rest back. Only while GDELT itself answers is a file given up on. The first
|
|
// answer waits at most GDELT_FIRST_MS for the first exports.
|
|
const GDELT_FRESH_MS = 5 * 60_000;
|
|
const GDELT_EXPORTS_PER_READ = 12;
|
|
const GDELT_PARALLEL = 4;
|
|
const GDELT_NEXT_MS = 2_000;
|
|
const GDELT_GONE_MS = 90 * 60_000;
|
|
const GDELT_EXPORT_TRIES = 3;
|
|
const GDELT_FIRST_MS = 12_000;
|
|
const MAX_GDELT_REPORTS = 5000;
|
|
// Events kept of one story, to place it by.
|
|
const MAX_STORY_EVENTS = 200;
|
|
// Page titles are read from the GKG exports of the newest three hours: one
|
|
// runs to 6 MB, so the day's 96 are not fetched. A file that fails is tried
|
|
// with the next reads, a few times.
|
|
const GDELT_TITLED_EXPORTS = 12;
|
|
const GKG_TRIES = 3;
|
|
const LIVE_FRESH_MS = 5 * 60_000;
|
|
const COUNTRY_FRESH_MS = 30 * 60_000;
|
|
// A failed read is not tried again for a minute; what was read before is
|
|
// served meanwhile, up to these ages (GDELT's: since an export last came in).
|
|
const RETRY_MS = 60_000;
|
|
const GDELT_STALE_MS = 2 * HOUR_MS;
|
|
const LIVE_STALE_MS = HOUR_MS;
|
|
const COUNTRY_STALE_MS = 6 * HOUR_MS;
|
|
const MAX_COUNTRIES = 64;
|
|
// World News API reads a UTC day, across all countries: each is about 1.5
|
|
// of the free plan's 50 daily points.
|
|
const DEFAULT_DAILY_READS = 30;
|
|
const DEFAULT_LIMIT = 50;
|
|
const EMPTY_OUTLINE = Object.freeze({
|
|
polygons: [],
|
|
borders: [],
|
|
disputed: [],
|
|
area: null,
|
|
});
|
|
// A country's own news keeps World News API's category: the rules read
|
|
// fighting and crises, and in everyday news their words mislead ("advances
|
|
// in the rankings" is no ground fighting). Fighting in the country reaches
|
|
// Local as the Conflicts reports placed there.
|
|
const WORLD_NEWS_CATEGORY = { politics: 'political', business: 'economy' };
|
|
|
|
const list = (value) =>
|
|
value ? new Set(value.split(',').filter(Boolean)) : null;
|
|
|
|
/** The rules' reading of a headline: category, relevance and severity. */
|
|
function readTitle(title, { dedicated }) {
|
|
const read = categorizeText({ title });
|
|
const { category } = read;
|
|
return {
|
|
category,
|
|
relevance: relevanceOf({ category, tags: read.tags, title, dedicated }),
|
|
severity: severityByRules({ category, tags: read.tags, title }),
|
|
};
|
|
}
|
|
|
|
/** A one-report event carrying the report's picture, for its card. */
|
|
function pictureEvent(id, report, image, video, publisher) {
|
|
return {
|
|
id,
|
|
status: 'single',
|
|
size: 1,
|
|
reports: 1,
|
|
severity: report.severity,
|
|
firstAt: report.publishedAt,
|
|
lastAt: report.publishedAt,
|
|
sides: [],
|
|
image: { url: image, publisher, ...(video ? { video } : {}) },
|
|
};
|
|
}
|
|
|
|
const media = ({ image, video }) =>
|
|
video ? ['video'] : image ? ['photo'] : [];
|
|
|
|
/** A Liveuamap item as a report of the Conflicts section. */
|
|
export function liveuamapReport(item) {
|
|
const id = `lum-${item.id}`;
|
|
const report = {
|
|
id,
|
|
hotspotId: WORLDWIDE_HOTSPOT,
|
|
hotspotIds: [WORLDWIDE_HOTSPOT],
|
|
title: item.title,
|
|
url: item.url,
|
|
publisher: 'Liveuamap',
|
|
source: { name: 'Liveuamap' },
|
|
kind: 'liveuamap',
|
|
side: 'unknown',
|
|
// A conflict map: everything it posts is at least related.
|
|
...readTitle(item.title, { dedicated: true }),
|
|
severityBy: 'rules',
|
|
publishedAt: item.publishedAt,
|
|
geo: {
|
|
lat: item.lat,
|
|
lon: item.lon,
|
|
place: item.place,
|
|
precision: 'city',
|
|
confidence: 1,
|
|
mappable: true,
|
|
method: 'source',
|
|
},
|
|
media: media(item),
|
|
};
|
|
if (item.image)
|
|
report.event = pictureEvent(
|
|
id,
|
|
report,
|
|
item.image,
|
|
item.video,
|
|
'Liveuamap',
|
|
);
|
|
return report;
|
|
}
|
|
|
|
/**
|
|
* A GDELT place named by its town and the area Natural Earth has there (a
|
|
* disputed area as itself, else the country): GDELT's own names follow old
|
|
* FIPS regions ("Bureij, Yerushalayim, Israel" for a camp in Gaza, "Abuja,
|
|
* …, Niger" for Nigeria's capital). Its point stays. A town a disputed
|
|
* area is named after ("Jerusalem" in East Jerusalem) lies on both sides
|
|
* of its line: GDELT puts the whole city at one point, so it is named as
|
|
* the town alone.
|
|
*/
|
|
export function gdeltPlace(geo) {
|
|
const area = areaAt(geo.lat, geo.lon);
|
|
const town = String(geo.place ?? '')
|
|
.split(',')[0]
|
|
.trim();
|
|
const straddles =
|
|
area?.kind === 'disputed' &&
|
|
town &&
|
|
area.name.toLowerCase().includes(town.toLowerCase());
|
|
const place =
|
|
geo.precision === 'country' && area?.kind === 'country'
|
|
? area.name
|
|
: straddles
|
|
? town
|
|
: [town, area?.name].filter(Boolean).join(', ');
|
|
return { ...geo, place: place || null };
|
|
}
|
|
|
|
/**
|
|
* A World News API article as a report of the Local section: placed in its
|
|
* publisher's country alone, at a point inside it.
|
|
*/
|
|
export function worldNewsReport(article, country) {
|
|
const id = `wna-${article.id}`;
|
|
const anchor = countryAnchor(country);
|
|
const report = {
|
|
id,
|
|
hotspotId: null,
|
|
hotspotIds: [],
|
|
title: article.title,
|
|
url: article.url,
|
|
publisher: article.publisher,
|
|
// World News API's free plan asks to be credited where its news shows.
|
|
source: { name: article.publisher, attribution: 'World News API' },
|
|
kind: 'worldnewsapi',
|
|
side: 'unknown',
|
|
category: WORLD_NEWS_CATEGORY[article.category] || 'other',
|
|
// Local news: what the country's own outlets publish, with no severity
|
|
// of its own.
|
|
relevance: 'minor',
|
|
publishedAt: article.publishedAt,
|
|
geo: {
|
|
lat: anchor.lat,
|
|
lon: anchor.lon,
|
|
place: country.name,
|
|
precision: 'country',
|
|
confidence: 1,
|
|
mappable: true,
|
|
method: 'source',
|
|
},
|
|
media: media(article),
|
|
};
|
|
if (article.image)
|
|
report.event = pictureEvent(
|
|
id,
|
|
report,
|
|
article.image,
|
|
article.video,
|
|
article.publisher,
|
|
);
|
|
return report;
|
|
}
|
|
|
|
const AT = /^(-?\d{1,2}(?:\.\d{1,2})?),(-?\d{1,3}(?:\.\d{1,2})?)$/;
|
|
function point(value) {
|
|
const match = AT.exec(String(value ?? ''));
|
|
if (!match) return null;
|
|
const [lat, lon] = [Number(match[1]), Number(match[2])];
|
|
return Math.abs(lat) <= 90 && Math.abs(lon) <= 180 ? { lat, lon } : null;
|
|
}
|
|
|
|
const noKey = (key) => ({ status: 503, body: { error: 'no_key', key } });
|
|
const rejectedKey = (key) => ({
|
|
status: 503,
|
|
body: { error: 'key_rejected', key },
|
|
});
|
|
const spentKey = (key) => ({ status: 503, body: { error: 'key_quota', key } });
|
|
|
|
/**
|
|
* The keyed news routes. `answer(path, params)` resolves to `{status, body}`
|
|
* for `/hotspots`, `/feed` and `/outline`, whose queries the proxy has
|
|
* already checked against the client's allowlist. `later(run, ms)` runs
|
|
* the next GDELT read of a day still filling in (an unref'd timer).
|
|
* @param {{env?: () => object, fetchImpl?: typeof fetch, now?: () => number, log?: (message: string) => void, later?: (run: () => void, ms: number) => void}} [options]
|
|
*/
|
|
export function createKeyedNews({
|
|
env = () => process.env,
|
|
fetchImpl = (...args) => globalThis.fetch(...args),
|
|
now = () => Date.now(),
|
|
log = (message) => console.warn(message),
|
|
later = (run, ms) => setTimeout(run, ms).unref(),
|
|
} = {}) {
|
|
const keyOf = (id) => String(env()[NEWS_KEYS[id]] ?? '').trim();
|
|
const inFlight = new Map();
|
|
// GDELT's stories of the last day (their events and report, by
|
|
// storyKey), and the exports read.
|
|
const gdelt = {
|
|
stories: new Map(),
|
|
reports: new Map(),
|
|
// Page titles by article address, of the stories kept.
|
|
titles: new Map(),
|
|
read: new Set(),
|
|
// Exports still missing well after their time: not looked for again.
|
|
gone: new Set(),
|
|
// Failed reads of each export not read yet: `{tries, at}`.
|
|
failures: new Map(),
|
|
// Exports read among the newest whose GKG titles are not in yet, with
|
|
// the tries made: `{tries, triedAt}`.
|
|
untitled: new Map(),
|
|
titling: null,
|
|
// When a read last brought an export in.
|
|
readAt: null,
|
|
checkedAt: null,
|
|
failedAt: null,
|
|
};
|
|
const live = new Map();
|
|
const countries = new Map();
|
|
// The keys refused so far, by section: logged once each.
|
|
const refused = new Set();
|
|
// World News API reads of the UTC day; `spent` once it answered 402.
|
|
const budget = { day: null, reads: 0, spent: false, logged: false };
|
|
// The area (disputed area or country) each placed Conflicts report lies
|
|
// in, looked up once.
|
|
const reportAreas = new WeakMap();
|
|
const reportArea = (report) => {
|
|
if (!reportAreas.has(report))
|
|
reportAreas.set(
|
|
report,
|
|
report.geo
|
|
? (areaAt(report.geo.lat, report.geo.lon)?.key ?? null)
|
|
: null,
|
|
);
|
|
return reportAreas.get(report);
|
|
};
|
|
|
|
/** A failed read in the log: a refused key once, never the key itself. */
|
|
function logFailure(section, what, key, error) {
|
|
// The day's reads spent: said once, by spendRead.
|
|
if (error.quota && error.status === null) return;
|
|
if (error.rejected) {
|
|
const id = `${section}\0${key}`;
|
|
if (refused.has(id)) return;
|
|
refused.add(id);
|
|
log(
|
|
`[news] ${NEWS_KEYS[section]} was refused (${error.status})${section === 'liveuamap' ? "; Conflicts are GDELT's alone" : ''}`,
|
|
);
|
|
return;
|
|
}
|
|
log(`[news] ${what} read failed: ${error.message}`);
|
|
}
|
|
|
|
/**
|
|
* A cached read (`{key, value, readAt, failedAt, error}`, kept in `store`
|
|
* under `id`): served as it is within `freshMs`; past that read again,
|
|
* and served meanwhile up to `staleMs`, so only a read with nothing to
|
|
* serve is waited for. One read at a time per key; a failed read is tried
|
|
* again after RETRY_MS. A new key drops what the old one read.
|
|
*/
|
|
async function cached(
|
|
store,
|
|
id,
|
|
{ section, key, freshMs, staleMs, read, what },
|
|
) {
|
|
const at = now();
|
|
const stored = store.get(id);
|
|
const prior = stored?.key === key ? stored : { key };
|
|
if (prior.value && at - prior.readAt < freshMs) return prior;
|
|
if (prior.failedAt && at - prior.failedAt < RETRY_MS) return prior;
|
|
const flight = `${id}\0${key}`;
|
|
if (!inFlight.has(flight))
|
|
inFlight.set(
|
|
flight,
|
|
read(key)
|
|
.then(
|
|
(value) => ({ key, value, readAt: now() }),
|
|
(error) => {
|
|
logFailure(section, what, key, error);
|
|
return { ...prior, failedAt: now(), error };
|
|
},
|
|
)
|
|
.then((entry) => {
|
|
// A read with a key replaced meanwhile is dropped.
|
|
if (keyOf(section) === key) keep(store, id, entry);
|
|
return entry;
|
|
})
|
|
.finally(() => inFlight.delete(flight)),
|
|
);
|
|
if (prior.value && at - prior.readAt < staleMs) return prior;
|
|
return inFlight.get(flight);
|
|
}
|
|
|
|
/** An entry stored as the most recently used (the countries are capped). */
|
|
function keep(store, id, entry) {
|
|
store.delete(id);
|
|
store.set(id, entry);
|
|
while (store.size > MAX_COUNTRIES) store.delete(store.keys().next().value);
|
|
}
|
|
|
|
/** An entry's reports while they may be served, else why there are none. */
|
|
function served(entry, staleMs, id) {
|
|
if (entry.value && now() - entry.readAt < staleMs)
|
|
return { reports: entry.value, readAt: entry.readAt };
|
|
if (entry.error?.rejected) return { rejected: id };
|
|
if (entry.error?.quota) return { quota: id };
|
|
return { failed: true };
|
|
}
|
|
|
|
async function liveuamapReports() {
|
|
const key = keyOf('liveuamap');
|
|
if (!key) return { missing: 'liveuamap' };
|
|
const entry = await cached(live, 'liveuamap', {
|
|
section: 'liveuamap',
|
|
key,
|
|
freshMs: LIVE_FRESH_MS,
|
|
staleMs: LIVE_STALE_MS,
|
|
what: 'liveuamap',
|
|
read: async (k) =>
|
|
(await fetchLiveuamap({ key: k, fetchImpl, now: now() }))
|
|
.map(liveuamapReport)
|
|
.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)),
|
|
});
|
|
return served(entry, LIVE_STALE_MS, 'liveuamap');
|
|
}
|
|
|
|
/**
|
|
* A story placed (again) from all its events and the titles in: an
|
|
* agency story run by several outlets is one report.
|
|
*/
|
|
function placeStory(key) {
|
|
const [report] = eventReports(gdelt.stories.get(key), {
|
|
titleOf: (url) => gdelt.titles.get(url) || null,
|
|
});
|
|
if (!report) return;
|
|
const { eventWords, ...rest } = report;
|
|
const geo = gdeltPlace(report.geo);
|
|
gdelt.reports.set(key, {
|
|
...rest,
|
|
...(eventWords ? { title: eventTitle(eventWords, geo.place) } : {}),
|
|
hotspotId: WORLDWIDE_HOTSPOT,
|
|
hotspotIds: [WORLDWIDE_HOTSPOT],
|
|
geo,
|
|
locations: report.locations.map(gdeltPlace),
|
|
});
|
|
}
|
|
|
|
/** An export's events into their stories, each placed again. */
|
|
function addEvents(events) {
|
|
const changed = new Set();
|
|
for (const event of events) {
|
|
const key = storyKey(event);
|
|
const list = gdelt.stories.get(key) || [];
|
|
if (list.length < MAX_STORY_EVENTS) list.push(event);
|
|
gdelt.stories.set(key, list);
|
|
changed.add(key);
|
|
}
|
|
for (const key of changed) placeStory(key);
|
|
}
|
|
|
|
/** The stories a GKG export titled, re-titled. */
|
|
function addTitles(titles) {
|
|
for (const [key, list] of gdelt.stories) {
|
|
const titled = list.filter((event) => titles.has(event.url));
|
|
if (!titled.length) continue;
|
|
for (const { url } of titled) gdelt.titles.set(url, titles.get(url));
|
|
placeStory(key);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The page titles of the newest exports read, from their GKG exports one
|
|
* at a time (each runs to 18 MB unzipped), in the background: no answer
|
|
* waits for them, and a story is re-titled as its title comes in. A file
|
|
* that fails or is not up yet only leaves address words: it is tried
|
|
* again with a later read while among the newest, up to GKG_TRIES times.
|
|
*/
|
|
function readTitles() {
|
|
gdelt.titling ??= (async () => {
|
|
let failed = null;
|
|
for (const [url, tried] of [...gdelt.untitled]) {
|
|
if (!gdelt.untitled.has(url)) continue;
|
|
if (tried.triedAt !== null && now() - tried.triedAt < GDELT_FRESH_MS)
|
|
continue;
|
|
tried.tries += 1;
|
|
tried.triedAt = now();
|
|
const wanted = new Set();
|
|
for (const list of gdelt.stories.values())
|
|
for (const event of list) wanted.add(event.url);
|
|
let titles = null;
|
|
try {
|
|
titles = await fetchTitles(gkgUrl(url), wanted, { fetchImpl });
|
|
} catch (error) {
|
|
failed ??= error;
|
|
}
|
|
if (titles || tried.tries >= GKG_TRIES) gdelt.untitled.delete(url);
|
|
if (titles?.size) addTitles(titles);
|
|
}
|
|
if (failed) log(`[news] gdelt titles read failed: ${failed.message}`);
|
|
})()
|
|
.catch((error) => log(`[news] gdelt titles failed: ${error.message}`))
|
|
.finally(() => {
|
|
gdelt.titling = null;
|
|
});
|
|
return gdelt.titling;
|
|
}
|
|
|
|
/**
|
|
* Up to a dozen of the day's exports not read yet, newest first; whether
|
|
* to carry on at once (the round brought one in, and more are missing).
|
|
* One not up yet is looked for again; one still missing GDELT_GONE_MS
|
|
* after its time is not.
|
|
*/
|
|
async function readExports() {
|
|
const at = now();
|
|
const day = exportsOfDay(at);
|
|
// Up to the newest GDELT lists; all of the day when its list fails.
|
|
const latest = await fetchLatestExport({ fetchImpl });
|
|
const listed = (url) => latest === null || exportAt(url) <= latest;
|
|
const newest = new Set(day.slice(0, GDELT_TITLED_EXPORTS));
|
|
const wanted = (url) =>
|
|
listed(url) && !gdelt.read.has(url) && !gdelt.gone.has(url);
|
|
const missing = day.filter(wanted).slice(0, GDELT_EXPORTS_PER_READ);
|
|
let failed = null;
|
|
let added = 0;
|
|
const failing = [];
|
|
const unreadable = [];
|
|
const unpublished = [];
|
|
for (let i = 0; i < missing.length; i += GDELT_PARALLEL)
|
|
await Promise.all(
|
|
missing.slice(i, i + GDELT_PARALLEL).map(async (url) => {
|
|
let events;
|
|
try {
|
|
events = await fetchExport(url, { fetchImpl });
|
|
} catch (error) {
|
|
failed = error;
|
|
(error.unreadable ? unreadable : failing).push(url);
|
|
return;
|
|
}
|
|
if (!events) {
|
|
if (at - exportAt(url) >= GDELT_GONE_MS) unpublished.push(url);
|
|
return;
|
|
}
|
|
// The events count now; the titles follow (readTitles).
|
|
gdelt.read.add(url);
|
|
gdelt.failures.delete(url);
|
|
added += 1;
|
|
gdelt.readAt = now();
|
|
addEvents(events);
|
|
if (newest.has(url))
|
|
gdelt.untitled.set(url, { tries: 0, triedAt: null });
|
|
}),
|
|
);
|
|
if (failed) log(`[news] gdelt read failed: ${failed.message}`);
|
|
// A file is blamed only while GDELT itself answers (its list read, or
|
|
// another export of the round read): then a 404 long after its time is
|
|
// never published, an unreadable file stays so, and a failing one gets a
|
|
// few tries, one a GDELT_FRESH_MS. With GDELT down, no file is blamed.
|
|
const answering = added > 0 || latest !== null;
|
|
if (answering)
|
|
for (const url of [...unpublished, ...unreadable]) gdelt.gone.add(url);
|
|
if (added)
|
|
for (const url of failing) {
|
|
const prior = gdelt.failures.get(url);
|
|
if (prior && now() - prior.at < GDELT_FRESH_MS) continue;
|
|
const tries = (prior?.tries || 0) + 1;
|
|
gdelt.failures.set(url, { tries, at: now() });
|
|
if (tries >= GDELT_EXPORT_TRIES) gdelt.gone.add(url);
|
|
}
|
|
// Nothing read and a failure: tried again after RETRY_MS, not after
|
|
// GDELT_FRESH_MS. Something read: the round counts.
|
|
if (failed && !added) gdelt.failedAt = now();
|
|
else {
|
|
gdelt.checkedAt = now();
|
|
gdelt.failedAt = null;
|
|
}
|
|
// Only the day is kept, and titled while among the newest.
|
|
const kept = new Set(day);
|
|
for (const set of [gdelt.read, gdelt.gone])
|
|
for (const url of set) if (!kept.has(url)) set.delete(url);
|
|
for (const url of gdelt.failures.keys())
|
|
if (!kept.has(url)) gdelt.failures.delete(url);
|
|
for (const url of gdelt.untitled.keys())
|
|
if (!newest.has(url)) gdelt.untitled.delete(url);
|
|
return (
|
|
added > 0 && missing.length === GDELT_EXPORTS_PER_READ && day.some(wanted)
|
|
);
|
|
}
|
|
|
|
/** GDELT's reports of the last day, newest first, or `{failed}`. */
|
|
async function gdeltReports() {
|
|
const at = now();
|
|
const fresh = gdelt.checkedAt && at - gdelt.checkedAt < GDELT_FRESH_MS;
|
|
const resting = gdelt.failedAt && at - gdelt.failedAt < RETRY_MS;
|
|
if (!fresh && !resting && !inFlight.has('gdelt')) {
|
|
const round = readExports()
|
|
.then(
|
|
(more) => {
|
|
// More of the day to fill in: carry on shortly, without holding
|
|
// the answer.
|
|
if (!more) return;
|
|
gdelt.checkedAt = null;
|
|
later(() => void gdeltReports(), GDELT_NEXT_MS);
|
|
},
|
|
(error) => {
|
|
log(`[news] gdelt read failed: ${error.message}`);
|
|
gdelt.failedAt = now();
|
|
},
|
|
)
|
|
.finally(() => {
|
|
inFlight.delete('gdelt');
|
|
void readTitles();
|
|
});
|
|
inFlight.set('gdelt', round);
|
|
}
|
|
// Only a read with nothing (recent) to serve is waited for: the
|
|
// newest exports' events, not their titles.
|
|
const current = () =>
|
|
gdelt.readAt !== null && now() - gdelt.readAt < GDELT_STALE_MS;
|
|
if (!current() && inFlight.has('gdelt')) {
|
|
// At most GDELT_FIRST_MS: a slow GDELT must not outlast the browser's
|
|
// own timeout; what has come in by then is served.
|
|
let timer;
|
|
await Promise.race([
|
|
inFlight.get('gdelt'),
|
|
new Promise((resolve) => {
|
|
timer = setTimeout(resolve, GDELT_FIRST_MS);
|
|
timer.unref?.();
|
|
}),
|
|
]);
|
|
clearTimeout(timer);
|
|
}
|
|
if (!current()) return { failed: true };
|
|
const dayAgo = new Date(now() - DAY_MS).toISOString();
|
|
const reports = [...gdelt.reports.values()]
|
|
.filter((r) => r.publishedAt >= dayAgo)
|
|
.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt))
|
|
.slice(0, MAX_GDELT_REPORTS);
|
|
if (reports.length !== gdelt.reports.size) {
|
|
const kept = new Set(reports);
|
|
for (const [key, report] of gdelt.reports)
|
|
if (!kept.has(report)) {
|
|
for (const event of gdelt.stories.get(key) || [])
|
|
gdelt.titles.delete(event.url);
|
|
gdelt.reports.delete(key);
|
|
gdelt.stories.delete(key);
|
|
}
|
|
}
|
|
return { reports, readAt: gdelt.readAt };
|
|
}
|
|
|
|
/**
|
|
* The Conflicts section's reports: GDELT's, with Liveuamap's when its key
|
|
* is set. Liveuamap failing (or refusing its key) leaves GDELT's.
|
|
*/
|
|
async function conflictReports() {
|
|
const [articles, items] = await Promise.all([
|
|
gdeltReports(),
|
|
keyOf('liveuamap') ? liveuamapReports() : {},
|
|
]);
|
|
if (!articles.reports && !items.reports) return { failed: true };
|
|
return {
|
|
reports: [...(articles.reports || []), ...(items.reports || [])].sort(
|
|
(a, b) => b.publishedAt.localeCompare(a.publishedAt),
|
|
),
|
|
readAt: Math.max(articles.readAt || 0, items.readAt || 0),
|
|
};
|
|
}
|
|
|
|
const dailyReads = () => {
|
|
const raw = String(env().WORLDNEWSAPI_DAILY_READS ?? '').trim();
|
|
const reads = raw ? Number(raw) : NaN;
|
|
return Number.isInteger(reads) && reads >= 0 ? reads : DEFAULT_DAILY_READS;
|
|
};
|
|
|
|
/** Whether a World News API read is left today (UTC); counts it if so. */
|
|
function spendRead() {
|
|
const day = new Date(now()).toISOString().slice(0, 10);
|
|
if (budget.day !== day)
|
|
Object.assign(budget, { day, reads: 0, spent: false, logged: false });
|
|
if (!budget.spent && budget.reads < dailyReads()) {
|
|
budget.reads += 1;
|
|
return true;
|
|
}
|
|
if (!budget.logged)
|
|
log(
|
|
"[news] World News API's reads for today are spent; Local reads again after 00:00 UTC",
|
|
);
|
|
budget.logged = true;
|
|
return false;
|
|
}
|
|
|
|
async function countryReports(country) {
|
|
const key = keyOf('worldnewsapi');
|
|
if (!key) return { missing: 'worldnewsapi' };
|
|
const stored = countries.get(country.code);
|
|
if (stored) keep(countries, country.code, stored);
|
|
const entry = await cached(countries, country.code, {
|
|
section: 'worldnewsapi',
|
|
key,
|
|
freshMs: COUNTRY_FRESH_MS,
|
|
staleMs: COUNTRY_STALE_MS,
|
|
what: `worldnewsapi:${country.code}`,
|
|
read: async (k) => {
|
|
// Not asked: no status, unlike an answer.
|
|
if (!spendRead())
|
|
throw new UpstreamError("World News API's reads for today spent", {
|
|
quota: true,
|
|
});
|
|
try {
|
|
return (
|
|
await fetchWorldNews({
|
|
key: k,
|
|
country: country.code,
|
|
fetchImpl,
|
|
now: now(),
|
|
})
|
|
).map((article) => worldNewsReport(article, country));
|
|
} catch (error) {
|
|
// Out of points: nothing more is asked today.
|
|
if (error.status === 402) budget.spent = true;
|
|
throw error;
|
|
}
|
|
},
|
|
});
|
|
return served(entry, COUNTRY_STALE_MS, 'worldnewsapi');
|
|
}
|
|
|
|
const refusal = (read) =>
|
|
read.missing
|
|
? noKey(read.missing)
|
|
: read.rejected
|
|
? rejectedKey(read.rejected)
|
|
: read.quota
|
|
? spentKey(read.quota)
|
|
: read.failed
|
|
? { status: 502, body: { error: 'news_upstream_error' } }
|
|
: null;
|
|
|
|
async function hotspots(params) {
|
|
const read = await conflictReports();
|
|
const refused = refusal(read);
|
|
if (refused) return refused;
|
|
const hours = Number(params.get('hours')) || 24;
|
|
const since = new Date(now() - hours * HOUR_MS).toISOString();
|
|
const shown = read.reports.filter((r) => r.publishedAt >= since);
|
|
return {
|
|
status: 200,
|
|
body: {
|
|
hotspots: [
|
|
{
|
|
id: WORLDWIDE_HOTSPOT,
|
|
name: 'Worldwide',
|
|
kind: 'conflict',
|
|
catchAll: true,
|
|
bbox: [-180, -90, 180, 90],
|
|
parties: [],
|
|
developing: [],
|
|
counts: {
|
|
last24h: shown.length,
|
|
mapped24h: shown.filter((r) => r.geo).length,
|
|
prev24h: null,
|
|
},
|
|
latestAt: shown[0]?.publishedAt ?? null,
|
|
},
|
|
],
|
|
categories: CATEGORY_ORDER,
|
|
ingest: { lastRunAt: new Date(read.readAt).toISOString() },
|
|
windowHours: hours,
|
|
maxWindowHours: 24,
|
|
},
|
|
};
|
|
}
|
|
|
|
/** Reports filtered, newest first, and paged by `before`. */
|
|
function page(reports, params) {
|
|
const since = params.get('since');
|
|
const before = params.get('before');
|
|
const dayAgo = new Date(now() - DAY_MS).toISOString();
|
|
const categories = list(params.get('category'));
|
|
const relevance = list(params.get('relevance'));
|
|
const hotspot = params.get('hotspot');
|
|
const limit = Number(params.get('limit')) || DEFAULT_LIMIT;
|
|
const kept = reports.filter(
|
|
(r) =>
|
|
(!hotspot || hotspot === 'all' || r.hotspotIds.includes(hotspot)) &&
|
|
(!since || r.publishedAt >= since) &&
|
|
(!before || r.publishedAt < before) &&
|
|
(!params.has('today') || r.publishedAt >= dayAgo) &&
|
|
(!params.has('geo') || r.geo) &&
|
|
(!categories || categories.has(r.category)) &&
|
|
(!relevance || relevance.has(r.relevance)),
|
|
);
|
|
const shown = kept.slice(0, limit);
|
|
// A page ends between two times: the next asks for those `before` the
|
|
// last, and a GDELT export's reports all share its quarter hour.
|
|
while (
|
|
shown.length < kept.length &&
|
|
kept[shown.length].publishedAt === shown.at(-1).publishedAt
|
|
)
|
|
shown.push(kept[shown.length]);
|
|
const events = params.has('events');
|
|
return {
|
|
reports: events ? shown : shown.map(({ event, ...report }) => report),
|
|
nextBefore: kept.length > shown.length ? shown.at(-1).publishedAt : null,
|
|
};
|
|
}
|
|
|
|
async function feed(params) {
|
|
const at = params.has('at') ? point(params.get('at')) : null;
|
|
const area = at ? areaAt(at.lat, at.lon) : null;
|
|
if (!area) {
|
|
// Conflicts, or Local at sea (by distance, in the browser).
|
|
const read = await conflictReports();
|
|
return refusal(read) || { status: 200, body: page(read.reports, params) };
|
|
}
|
|
// Local in a country: its own outlets, with the Conflicts reports
|
|
// placed inside it (those failing leave the outlets'). In a disputed
|
|
// area no country's outlets speak for it: the Conflicts reports there
|
|
// alone, so their failing is the answer's.
|
|
const country = area.kind === 'country';
|
|
const [national, conflicts] = await Promise.all([
|
|
country ? countryReports(area.country) : { reports: [] },
|
|
conflictReports(),
|
|
]);
|
|
const refused = refusal(national) || (!country && refusal(conflicts));
|
|
if (refused) return refused;
|
|
const inside = (conflicts.reports || []).filter(
|
|
(r) => reportArea(r) === area.key,
|
|
);
|
|
const reports = [...national.reports, ...inside].sort((a, b) =>
|
|
b.publishedAt.localeCompare(a.publishedAt),
|
|
);
|
|
return { status: 200, body: page(reports, params) };
|
|
}
|
|
|
|
function outline(params) {
|
|
if (params.has('at')) {
|
|
const at = point(params.get('at'));
|
|
const area = at ? areaAt(at.lat, at.lon) : null;
|
|
if (!area) return { status: 200, body: EMPTY_OUTLINE };
|
|
return {
|
|
status: 200,
|
|
body: {
|
|
...EMPTY_OUTLINE,
|
|
polygons:
|
|
area.kind === 'country'
|
|
? countryPolygons(area.country)
|
|
: area.polygons,
|
|
area: {
|
|
key: area.key,
|
|
name: area.name,
|
|
kind: area.kind,
|
|
country: area.code,
|
|
},
|
|
},
|
|
};
|
|
}
|
|
// A developing situation's country; the world has no outline.
|
|
const country = countryByCode(
|
|
String(params.get('country') ?? '').slice(0, 2),
|
|
);
|
|
return {
|
|
status: 200,
|
|
body: country
|
|
? { ...EMPTY_OUTLINE, polygons: countryPolygons(country) }
|
|
: EMPTY_OUTLINE,
|
|
};
|
|
}
|
|
|
|
return {
|
|
async answer(path, params) {
|
|
if (path === '/hotspots') return hotspots(params);
|
|
if (path === '/feed') return feed(params);
|
|
if (path === '/outline') return outline(params);
|
|
return { status: 404, body: { error: 'not_found' } };
|
|
},
|
|
};
|
|
}
|