mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
With GEV_NEWS_ORIGIN unset, Core answers the News layer's routes itself. Conflicts need no key: the last 24 h of GDELT 2.0 event exports, physical violence only (CAMEO 18-20), one report per story (copies of one agency story merged), placed where GDELT put it, a town the article's own words name winning over a bare country, and titled by the article's page title from the GKG export of the same quarter hour (the newest 3 h), else by its address words. GDELT's actor coding is never shown. Exports are read up to the newest GDELT lists; one that cannot be read is given up on without holding the rest back, and an outage of two hours answers 502 rather than an empty feed. With LIVEUAMAP_API_KEY, Liveuamap's mapped items join them. Local reads World News API by the source country at the view centre (WORLDNEWSAPI_KEY, at most WORLDNEWSAPI_DAILY_READS a day), with the Conflicts reports placed inside the area. Disputed areas (11 from Natural Earth, bundled) and territories without a country code are named as themselves and read from no country's outlets; a country's outline leaves them out. Categories, relevance and severity come from shared rules; keyed answers are cached and served while they refresh, and no key or address is logged.
275 lines
8.7 KiB
JavaScript
275 lines
8.7 KiB
JavaScript
// The keyed news APIs Core reads itself when no news server is configured:
|
|
// Liveuamap's worldwide feed (`a=mpts`) beside GDELT's events in the
|
|
// Conflicts section (gdeltEvents.js), World News API's search by source
|
|
// country for Local. Each key stays on the server: Liveuamap's rides in the query, so
|
|
// neither a URL nor an error body is ever logged or relayed.
|
|
|
|
import { readResponseTextCapped } from '../common/http.js';
|
|
|
|
const LIVEUAMAP_URL = 'https://a.liveuamap.com/api';
|
|
const WORLD_NEWS_URL = 'https://api.worldnewsapi.com/search-news';
|
|
// 100 Liveuamap items run to about 500 KB.
|
|
const LIVEUAMAP_COUNT = 500;
|
|
const LIVEUAMAP_MAX_BYTES = 16 * 1024 * 1024;
|
|
// One search per country: a point, and a hundredth of one per article, so
|
|
// about 1.5 points a read (the free plan has 50 points a day).
|
|
const WORLD_NEWS_ARTICLES = 50;
|
|
const WORLD_NEWS_MAX_BYTES = 8 * 1024 * 1024;
|
|
// World News API's categories that are not the news of the day. Its
|
|
// `categories` filter is not asked for: it classifies few articles outside
|
|
// English, so it would drop most of a country's own outlets; an article it
|
|
// left unclassified is kept.
|
|
export const WORLD_NEWS_LEFT_OUT = Object.freeze(
|
|
new Set(['sports', 'entertainment', 'lifestyle', 'travel']),
|
|
);
|
|
const DAY_MS = 86_400_000;
|
|
const TIMEOUT_MS = 20_000;
|
|
|
|
const ENTITIES = {
|
|
amp: '&',
|
|
quot: '"',
|
|
apos: "'",
|
|
lt: '<',
|
|
gt: '>',
|
|
nbsp: ' ',
|
|
};
|
|
|
|
/** Text with HTML entities decoded and whitespace collapsed, or ''. */
|
|
export function unescapeHtml(value) {
|
|
return String(value ?? '')
|
|
.replace(
|
|
/&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]{2,6}));/gi,
|
|
(all, dec, hex, name) => {
|
|
if (name) return ENTITIES[name.toLowerCase()] ?? all;
|
|
const code = dec ? Number(dec) : parseInt(hex, 16);
|
|
return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : all;
|
|
},
|
|
)
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
}
|
|
|
|
/** An https URL without credentials, or null. */
|
|
export function webLink(value) {
|
|
try {
|
|
const url = new URL(String(value ?? '').trim());
|
|
return url.protocol === 'https:' && !url.username && !url.password
|
|
? url.href
|
|
: null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** A clip a browser plays itself (an https MP4 or WebM file), or null. */
|
|
export function playableVideo(value) {
|
|
const url = webLink(value);
|
|
return url && /\.(?:mp4|m4v|webm)$/i.test(new URL(url).pathname) ? url : null;
|
|
}
|
|
|
|
const finiteNumber = (value) => {
|
|
const number = typeof value === 'string' ? Number(value.trim()) : value;
|
|
return typeof number === 'number' && Number.isFinite(number) ? number : null;
|
|
};
|
|
|
|
/** Why an upstream could not be read; never carries its URL or body. */
|
|
export class UpstreamError extends Error {
|
|
constructor(
|
|
message,
|
|
{ status = null, rejected = false, quota = false, unreadable = false } = {},
|
|
) {
|
|
super(message);
|
|
this.name = 'UpstreamError';
|
|
this.status = status;
|
|
// The key was refused (401 or 403): the operator's to fix.
|
|
this.rejected = rejected;
|
|
// The key's allowance is spent (402, or the day's reads): it answers
|
|
// again later. A 429 is a per-second limit, not this.
|
|
this.quota = quota;
|
|
// The answer itself cannot be read (corrupt, or over a size cap): asking
|
|
// again gets the same.
|
|
this.unreadable = unreadable;
|
|
}
|
|
}
|
|
|
|
async function getText(
|
|
fetchImpl,
|
|
url,
|
|
{ headers = {}, maxBytes, signal, what },
|
|
) {
|
|
let response;
|
|
try {
|
|
response = await fetchImpl(url, {
|
|
headers: { Accept: 'application/json', ...headers },
|
|
redirect: 'error',
|
|
signal: signal
|
|
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
|
: AbortSignal.timeout(TIMEOUT_MS),
|
|
});
|
|
} catch (cause) {
|
|
throw new UpstreamError(
|
|
cause?.name === 'TimeoutError'
|
|
? `${what} timed out`
|
|
: `${what} unreachable`,
|
|
);
|
|
}
|
|
if (!response.ok) {
|
|
await response.body?.cancel?.().catch?.(() => {});
|
|
throw new UpstreamError(`${what} answered ${response.status}`, {
|
|
status: response.status,
|
|
rejected: response.status === 401 || response.status === 403,
|
|
quota: response.status === 402,
|
|
});
|
|
}
|
|
try {
|
|
return await readResponseTextCapped(response, maxBytes, signal);
|
|
} catch {
|
|
throw new UpstreamError(`${what} answer unreadable`);
|
|
}
|
|
}
|
|
|
|
function parseJson(text, what) {
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch {
|
|
// Not the parser's message: an error answer may echo the request.
|
|
throw new UpstreamError(`${what} answer unreadable`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Liveuamap's items with a headline, a point and a page:
|
|
* `{id, title, lat, lon, place, publishedAt, url, image, video}`.
|
|
*/
|
|
export function parseLiveuamap(payload) {
|
|
if (!Array.isArray(payload?.venues))
|
|
throw new UpstreamError('Liveuamap answer unreadable');
|
|
const items = [];
|
|
for (const venue of payload.venues) {
|
|
const id = finiteNumber(venue?.id);
|
|
const title = unescapeHtml(venue?.name);
|
|
const lat = finiteNumber(venue?.lat);
|
|
const lon = finiteNumber(venue?.lng);
|
|
const seconds = finiteNumber(venue?.timestamp);
|
|
const url = webLink(venue?.link);
|
|
if (
|
|
id === null ||
|
|
!title ||
|
|
lat === null ||
|
|
lon === null ||
|
|
Math.abs(lat) > 90 ||
|
|
Math.abs(lon) > 180 ||
|
|
(lat === 0 && lon === 0) ||
|
|
seconds === null ||
|
|
!url
|
|
)
|
|
continue;
|
|
items.push({
|
|
id: Math.trunc(id),
|
|
title,
|
|
lat,
|
|
lon,
|
|
place: unescapeHtml(venue.location) || null,
|
|
publishedAt: new Date(seconds * 1000).toISOString(),
|
|
url,
|
|
image: webLink(venue.img_share),
|
|
video: playableVideo(venue.video),
|
|
});
|
|
}
|
|
return items;
|
|
}
|
|
|
|
/** The worldwide feed of the last day, as of `now`. */
|
|
export async function fetchLiveuamap({ key, fetchImpl, now, signal }) {
|
|
const url = new URL(LIVEUAMAP_URL);
|
|
url.search = new URLSearchParams({
|
|
a: 'mpts',
|
|
resid: '1',
|
|
time: String(Math.floor((now - DAY_MS) / 1000)),
|
|
count: String(LIVEUAMAP_COUNT),
|
|
key,
|
|
}).toString();
|
|
const text = await getText(fetchImpl, url, {
|
|
maxBytes: LIVEUAMAP_MAX_BYTES,
|
|
signal,
|
|
what: 'Liveuamap',
|
|
});
|
|
return parseLiveuamap(parseJson(text, 'Liveuamap'));
|
|
}
|
|
|
|
// World News API writes times as "2026-10-30 01:31:00", in UTC.
|
|
const worldNewsTime = (ms) =>
|
|
new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
|
|
function worldNewsDate(value) {
|
|
const match = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)$/.exec(
|
|
String(value ?? ''),
|
|
);
|
|
const ms = match ? Date.parse(`${match[1]}T${match[2]}Z`) : NaN;
|
|
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
|
}
|
|
|
|
/**
|
|
* A search's articles with a title, a link and a time, one per title (the
|
|
* same story from several outlets is one), sport and entertainment left
|
|
* out, newest first:
|
|
* `{id, title, url, publisher, language, category, publishedAt, image, video}`.
|
|
*/
|
|
export function parseWorldNews(payload) {
|
|
if (!Array.isArray(payload?.news))
|
|
throw new UpstreamError('World News API answer unreadable');
|
|
const seen = new Set();
|
|
const articles = [];
|
|
for (const article of payload.news) {
|
|
const id = finiteNumber(article?.id);
|
|
const title = unescapeHtml(article?.title);
|
|
const url = webLink(article?.url);
|
|
const publishedAt = worldNewsDate(article?.publish_date);
|
|
const category =
|
|
typeof article?.category === 'string'
|
|
? article.category.toLowerCase()
|
|
: null;
|
|
if (id === null || !title || !url || !publishedAt) continue;
|
|
if (WORLD_NEWS_LEFT_OUT.has(category)) continue;
|
|
const same = title.toLowerCase();
|
|
if (seen.has(same)) continue;
|
|
seen.add(same);
|
|
articles.push({
|
|
id: Math.trunc(id),
|
|
title,
|
|
url,
|
|
publisher: new URL(url).hostname.replace(/^www\./, ''),
|
|
language:
|
|
typeof article.language === 'string' &&
|
|
/^[a-z]{2}$/i.test(article.language)
|
|
? article.language.toLowerCase()
|
|
: null,
|
|
category,
|
|
publishedAt,
|
|
image: webLink(article.image),
|
|
video: playableVideo(article.video),
|
|
});
|
|
}
|
|
return articles.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt));
|
|
}
|
|
|
|
/** A country's everyday news of the last day, as of `now`. */
|
|
export async function fetchWorldNews({ key, country, fetchImpl, now, signal }) {
|
|
const url = new URL(WORLD_NEWS_URL);
|
|
url.search = new URLSearchParams({
|
|
'source-country': country.toLowerCase(),
|
|
// Both ends: sorted by time with neither, a search is cut to 31 days.
|
|
'earliest-publish-date': worldNewsTime(now - DAY_MS),
|
|
'latest-publish-date': worldNewsTime(now),
|
|
sort: 'publish-time',
|
|
'sort-direction': 'DESC',
|
|
number: String(WORLD_NEWS_ARTICLES),
|
|
}).toString();
|
|
const text = await getText(fetchImpl, url, {
|
|
headers: { 'x-api-key': key },
|
|
maxBytes: WORLD_NEWS_MAX_BYTES,
|
|
signal,
|
|
what: 'World News API',
|
|
});
|
|
return parseWorldNews(parseJson(text, 'World News API'));
|
|
}
|