mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
Serve the News layer without a news server: GDELT, Liveuamap and World News API
With GEV_NEWS_ORIGIN unset, Core answers the News layer's routes itself. Conflicts need no key: the last 24 h of GDELT 2.0 event exports, physical violence only (CAMEO 18-20), one report per story (copies of one agency story merged), placed where GDELT put it, a town the article's own words name winning over a bare country, and titled by the article's page title from the GKG export of the same quarter hour (the newest 3 h), else by its address words. GDELT's actor coding is never shown. Exports are read up to the newest GDELT lists; one that cannot be read is given up on without holding the rest back, and an outage of two hours answers 502 rather than an empty feed. With LIVEUAMAP_API_KEY, Liveuamap's mapped items join them. Local reads World News API by the source country at the view centre (WORLDNEWSAPI_KEY, at most WORLDNEWSAPI_DAILY_READS a day), with the Conflicts reports placed inside the area. Disputed areas (11 from Natural Earth, bundled) and territories without a country code are named as themselves and read from no country's outlets; a country's outline leaves them out. Categories, relevance and severity come from shared rules; keyed answers are cached and served while they refresh, and no key or address is logged.
This commit is contained in:
1 parent
edb5837ae5
commit
34c75ff5de
13 files changed
+4399
-32
No files matched your search
@@ -0,0 +1,274 @@
|
||||
// The keyed news APIs Core reads itself when no news server is configured:
|
||||
// Liveuamap's worldwide feed (`a=mpts`) beside GDELT's events in the
|
||||
// Conflicts section (gdeltEvents.js), World News API's search by source
|
||||
// country for Local. Each key stays on the server: Liveuamap's rides in the query, so
|
||||
// neither a URL nor an error body is ever logged or relayed.
|
||||
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
const LIVEUAMAP_URL = 'https://a.liveuamap.com/api';
|
||||
const WORLD_NEWS_URL = 'https://api.worldnewsapi.com/search-news';
|
||||
// 100 Liveuamap items run to about 500 KB.
|
||||
const LIVEUAMAP_COUNT = 500;
|
||||
const LIVEUAMAP_MAX_BYTES = 16 * 1024 * 1024;
|
||||
// One search per country: a point, and a hundredth of one per article, so
|
||||
// about 1.5 points a read (the free plan has 50 points a day).
|
||||
const WORLD_NEWS_ARTICLES = 50;
|
||||
const WORLD_NEWS_MAX_BYTES = 8 * 1024 * 1024;
|
||||
// World News API's categories that are not the news of the day. Its
|
||||
// `categories` filter is not asked for: it classifies few articles outside
|
||||
// English, so it would drop most of a country's own outlets; an article it
|
||||
// left unclassified is kept.
|
||||
export const WORLD_NEWS_LEFT_OUT = Object.freeze(
|
||||
new Set(['sports', 'entertainment', 'lifestyle', 'travel']),
|
||||
);
|
||||
const DAY_MS = 86_400_000;
|
||||
const TIMEOUT_MS = 20_000;
|
||||
|
||||
const ENTITIES = {
|
||||
amp: '&',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
nbsp: ' ',
|
||||
};
|
||||
|
||||
/** Text with HTML entities decoded and whitespace collapsed, or ''. */
|
||||
export function unescapeHtml(value) {
|
||||
return String(value ?? '')
|
||||
.replace(
|
||||
/&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]{2,6}));/gi,
|
||||
(all, dec, hex, name) => {
|
||||
if (name) return ENTITIES[name.toLowerCase()] ?? all;
|
||||
const code = dec ? Number(dec) : parseInt(hex, 16);
|
||||
return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : all;
|
||||
},
|
||||
)
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** An https URL without credentials, or null. */
|
||||
export function webLink(value) {
|
||||
try {
|
||||
const url = new URL(String(value ?? '').trim());
|
||||
return url.protocol === 'https:' && !url.username && !url.password
|
||||
? url.href
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** A clip a browser plays itself (an https MP4 or WebM file), or null. */
|
||||
export function playableVideo(value) {
|
||||
const url = webLink(value);
|
||||
return url && /\.(?:mp4|m4v|webm)$/i.test(new URL(url).pathname) ? url : null;
|
||||
}
|
||||
|
||||
const finiteNumber = (value) => {
|
||||
const number = typeof value === 'string' ? Number(value.trim()) : value;
|
||||
return typeof number === 'number' && Number.isFinite(number) ? number : null;
|
||||
};
|
||||
|
||||
/** Why an upstream could not be read; never carries its URL or body. */
|
||||
export class UpstreamError extends Error {
|
||||
constructor(
|
||||
message,
|
||||
{ status = null, rejected = false, quota = false, unreadable = false } = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'UpstreamError';
|
||||
this.status = status;
|
||||
// The key was refused (401 or 403): the operator's to fix.
|
||||
this.rejected = rejected;
|
||||
// The key's allowance is spent (402, or the day's reads): it answers
|
||||
// again later. A 429 is a per-second limit, not this.
|
||||
this.quota = quota;
|
||||
// The answer itself cannot be read (corrupt, or over a size cap): asking
|
||||
// again gets the same.
|
||||
this.unreadable = unreadable;
|
||||
}
|
||||
}
|
||||
|
||||
async function getText(
|
||||
fetchImpl,
|
||||
url,
|
||||
{ headers = {}, maxBytes, signal, what },
|
||||
) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
headers: { Accept: 'application/json', ...headers },
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
} catch (cause) {
|
||||
throw new UpstreamError(
|
||||
cause?.name === 'TimeoutError'
|
||||
? `${what} timed out`
|
||||
: `${what} unreachable`,
|
||||
);
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw new UpstreamError(`${what} answered ${response.status}`, {
|
||||
status: response.status,
|
||||
rejected: response.status === 401 || response.status === 403,
|
||||
quota: response.status === 402,
|
||||
});
|
||||
}
|
||||
try {
|
||||
return await readResponseTextCapped(response, maxBytes, signal);
|
||||
} catch {
|
||||
throw new UpstreamError(`${what} answer unreadable`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseJson(text, what) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
// Not the parser's message: an error answer may echo the request.
|
||||
throw new UpstreamError(`${what} answer unreadable`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Liveuamap's items with a headline, a point and a page:
|
||||
* `{id, title, lat, lon, place, publishedAt, url, image, video}`.
|
||||
*/
|
||||
export function parseLiveuamap(payload) {
|
||||
if (!Array.isArray(payload?.venues))
|
||||
throw new UpstreamError('Liveuamap answer unreadable');
|
||||
const items = [];
|
||||
for (const venue of payload.venues) {
|
||||
const id = finiteNumber(venue?.id);
|
||||
const title = unescapeHtml(venue?.name);
|
||||
const lat = finiteNumber(venue?.lat);
|
||||
const lon = finiteNumber(venue?.lng);
|
||||
const seconds = finiteNumber(venue?.timestamp);
|
||||
const url = webLink(venue?.link);
|
||||
if (
|
||||
id === null ||
|
||||
!title ||
|
||||
lat === null ||
|
||||
lon === null ||
|
||||
Math.abs(lat) > 90 ||
|
||||
Math.abs(lon) > 180 ||
|
||||
(lat === 0 && lon === 0) ||
|
||||
seconds === null ||
|
||||
!url
|
||||
)
|
||||
continue;
|
||||
items.push({
|
||||
id: Math.trunc(id),
|
||||
title,
|
||||
lat,
|
||||
lon,
|
||||
place: unescapeHtml(venue.location) || null,
|
||||
publishedAt: new Date(seconds * 1000).toISOString(),
|
||||
url,
|
||||
image: webLink(venue.img_share),
|
||||
video: playableVideo(venue.video),
|
||||
});
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
/** The worldwide feed of the last day, as of `now`. */
|
||||
export async function fetchLiveuamap({ key, fetchImpl, now, signal }) {
|
||||
const url = new URL(LIVEUAMAP_URL);
|
||||
url.search = new URLSearchParams({
|
||||
a: 'mpts',
|
||||
resid: '1',
|
||||
time: String(Math.floor((now - DAY_MS) / 1000)),
|
||||
count: String(LIVEUAMAP_COUNT),
|
||||
key,
|
||||
}).toString();
|
||||
const text = await getText(fetchImpl, url, {
|
||||
maxBytes: LIVEUAMAP_MAX_BYTES,
|
||||
signal,
|
||||
what: 'Liveuamap',
|
||||
});
|
||||
return parseLiveuamap(parseJson(text, 'Liveuamap'));
|
||||
}
|
||||
|
||||
// World News API writes times as "2026-10-30 01:31:00", in UTC.
|
||||
const worldNewsTime = (ms) =>
|
||||
new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
|
||||
function worldNewsDate(value) {
|
||||
const match = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)$/.exec(
|
||||
String(value ?? ''),
|
||||
);
|
||||
const ms = match ? Date.parse(`${match[1]}T${match[2]}Z`) : NaN;
|
||||
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A search's articles with a title, a link and a time, one per title (the
|
||||
* same story from several outlets is one), sport and entertainment left
|
||||
* out, newest first:
|
||||
* `{id, title, url, publisher, language, category, publishedAt, image, video}`.
|
||||
*/
|
||||
export function parseWorldNews(payload) {
|
||||
if (!Array.isArray(payload?.news))
|
||||
throw new UpstreamError('World News API answer unreadable');
|
||||
const seen = new Set();
|
||||
const articles = [];
|
||||
for (const article of payload.news) {
|
||||
const id = finiteNumber(article?.id);
|
||||
const title = unescapeHtml(article?.title);
|
||||
const url = webLink(article?.url);
|
||||
const publishedAt = worldNewsDate(article?.publish_date);
|
||||
const category =
|
||||
typeof article?.category === 'string'
|
||||
? article.category.toLowerCase()
|
||||
: null;
|
||||
if (id === null || !title || !url || !publishedAt) continue;
|
||||
if (WORLD_NEWS_LEFT_OUT.has(category)) continue;
|
||||
const same = title.toLowerCase();
|
||||
if (seen.has(same)) continue;
|
||||
seen.add(same);
|
||||
articles.push({
|
||||
id: Math.trunc(id),
|
||||
title,
|
||||
url,
|
||||
publisher: new URL(url).hostname.replace(/^www\./, ''),
|
||||
language:
|
||||
typeof article.language === 'string' &&
|
||||
/^[a-z]{2}$/i.test(article.language)
|
||||
? article.language.toLowerCase()
|
||||
: null,
|
||||
category,
|
||||
publishedAt,
|
||||
image: webLink(article.image),
|
||||
video: playableVideo(article.video),
|
||||
});
|
||||
}
|
||||
return articles.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt));
|
||||
}
|
||||
|
||||
/** A country's everyday news of the last day, as of `now`. */
|
||||
export async function fetchWorldNews({ key, country, fetchImpl, now, signal }) {
|
||||
const url = new URL(WORLD_NEWS_URL);
|
||||
url.search = new URLSearchParams({
|
||||
'source-country': country.toLowerCase(),
|
||||
// Both ends: sorted by time with neither, a search is cut to 31 days.
|
||||
'earliest-publish-date': worldNewsTime(now - DAY_MS),
|
||||
'latest-publish-date': worldNewsTime(now),
|
||||
sort: 'publish-time',
|
||||
'sort-direction': 'DESC',
|
||||
number: String(WORLD_NEWS_ARTICLES),
|
||||
}).toString();
|
||||
const text = await getText(fetchImpl, url, {
|
||||
headers: { 'x-api-key': key },
|
||||
maxBytes: WORLD_NEWS_MAX_BYTES,
|
||||
signal,
|
||||
what: 'World News API',
|
||||
});
|
||||
return parseWorldNews(parseJson(text, 'World News API'));
|
||||
}
|
||||
Reference in new issue
Block a user