mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
Serve the News layer without a news server: GDELT, Liveuamap and World News API
With GEV_NEWS_ORIGIN unset, Core answers the News layer's routes itself. Conflicts need no key: the last 24 h of GDELT 2.0 event exports, physical violence only (CAMEO 18-20), one report per story (copies of one agency story merged), placed where GDELT put it, a town the article's own words name winning over a bare country, and titled by the article's page title from the GKG export of the same quarter hour (the newest 3 h), else by its address words. GDELT's actor coding is never shown. Exports are read up to the newest GDELT lists; one that cannot be read is given up on without holding the rest back, and an outage of two hours answers 502 rather than an empty feed. With LIVEUAMAP_API_KEY, Liveuamap's mapped items join them. Local reads World News API by the source country at the view centre (WORLDNEWSAPI_KEY, at most WORLDNEWSAPI_DAILY_READS a day), with the Conflicts reports placed inside the area. Disputed areas (11 from Natural Earth, bundled) and territories without a country code are named as themselves and read from no country's outlets; a country's outline leaves them out. Categories, relevance and severity come from shared rules; keyed answers are cached and served while they refresh, and no key or address is logged.
This commit is contained in:
1 parent
edb5837ae5
commit
34c75ff5de
13 files changed
+4399
-32
No files matched your search
@@ -0,0 +1,543 @@
|
||||
// GDELT 2.0 events for the Conflicts section, from the 15-minute export
|
||||
// files (data.gdeltproject.org, plain downloads; no key, and not the
|
||||
// rate-limited search API). Each export is one zipped tab-separated table of
|
||||
// machine-coded events, each placed by GDELT with a point and a place type.
|
||||
//
|
||||
// A story counts when GDELT coded physical violence in it (CAMEO roots 18
|
||||
// assault, 19 fight, 20 mass violence); its place is read from all its
|
||||
// placed events (choosePlace). GDELT's actor fields are machine-coded and
|
||||
// often wrong, so no report says who did what: a report is the story (its
|
||||
// article, and the copies other outlets ran under the same address words).
|
||||
// The events carry no headline: a story's title is its page title from the
|
||||
// GKG export of the same quarter hour (read for the newest hours only, it
|
||||
// runs to 6 MB), else the words of its address. All of it is rules over
|
||||
// GDELT's own fields; nothing reads the articles.
|
||||
|
||||
import { promisify } from 'node:util';
|
||||
import { inflateRaw } from 'node:zlib';
|
||||
|
||||
import { readResponseBytesCapped } from '../common/http.js';
|
||||
import { categorizeText, relevanceOf, severityByRules } from './categorize.js';
|
||||
import { UpstreamError, unescapeHtml, webLink } from './upstream.js';
|
||||
|
||||
const EXPORTS_URL = 'https://data.gdeltproject.org/gdeltv2/';
|
||||
const STEP_MS = 15 * 60_000;
|
||||
// An events export runs to about 100 KB zipped and 500 KB unzipped; a GKG
|
||||
// export to about 6 MB and 18 MB. Unzipped off the main thread.
|
||||
const MAX_ZIP_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_TABLE_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_GKG_ZIP_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_GKG_TABLE_BYTES = 32 * 1024 * 1024;
|
||||
const inflate = promisify(inflateRaw);
|
||||
// The most words an address's words make a title of.
|
||||
const MAX_ADDRESS_WORDS = 24;
|
||||
const TIMEOUT_MS = 30_000;
|
||||
const MAX_PLACES = 3;
|
||||
|
||||
// Columns of the 2.0 events table (61).
|
||||
const COL = {
|
||||
id: 0,
|
||||
eventCode: 26,
|
||||
rootCode: 28,
|
||||
quadClass: 29,
|
||||
mentions: 31,
|
||||
geoType: 51,
|
||||
place: 52,
|
||||
lat: 56,
|
||||
lon: 57,
|
||||
added: 59,
|
||||
url: 60,
|
||||
};
|
||||
|
||||
// CAMEO event codes as the News layer's categories, and as words for an
|
||||
// article whose address carries none of its own.
|
||||
const EVENTS = [
|
||||
// Aerial weapons, artillery and tanks, bombings.
|
||||
[/^19[45]|^183/, 'strike', 'Strike reported'],
|
||||
[/^186|^185/, 'casualties', 'Killing reported'],
|
||||
[/^181/, 'humanitarian', 'Abduction reported'],
|
||||
[/^20/, 'casualties', 'Mass violence reported'],
|
||||
[/^19/, 'ground', 'Fighting reported'],
|
||||
[/^18/, 'other', 'Violence reported'],
|
||||
];
|
||||
const eventKind = (code) =>
|
||||
EVENTS.find(([pattern]) => pattern.test(code)) || EVENTS.at(-1);
|
||||
|
||||
// GDELT's place types: 1 country, 2 US state, 3 US city, 4 world city,
|
||||
// 5 world state.
|
||||
const PRECISION = {
|
||||
1: 'country',
|
||||
2: 'region',
|
||||
3: 'city',
|
||||
4: 'city',
|
||||
5: 'region',
|
||||
};
|
||||
|
||||
/** The export holding the events GDELT added in the 15 minutes before `ms`. */
|
||||
export function exportUrl(ms) {
|
||||
const at = new Date(Math.floor(ms / STEP_MS) * STEP_MS);
|
||||
const stamp = at.toISOString().replace(/[-:T]/g, '').slice(0, 14);
|
||||
return `${EXPORTS_URL}${stamp}.export.CSV.zip`;
|
||||
}
|
||||
|
||||
/** An events export's quarter hour (ms), from its name, or NaN. */
|
||||
export function exportAt(url) {
|
||||
const match = /(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)00\.export\.CSV\.zip$/.exec(
|
||||
url,
|
||||
);
|
||||
return match
|
||||
? Date.UTC(match[1], match[2] - 1, match[3], match[4], match[5])
|
||||
: NaN;
|
||||
}
|
||||
|
||||
const LASTUPDATE_URL = `${EXPORTS_URL}lastupdate.txt`;
|
||||
|
||||
/**
|
||||
* The time of the newest events export GDELT lists as published, in ms, or
|
||||
* null when its list cannot be read. Asking for one not listed yet would
|
||||
* be answered 404, and GDELT's CDN keeps that 404 for an hour.
|
||||
*/
|
||||
export async function fetchLatestExport({ fetchImpl, signal } = {}) {
|
||||
try {
|
||||
const response = await fetchImpl(LASTUPDATE_URL, {
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
return null;
|
||||
}
|
||||
const text = new TextDecoder().decode(
|
||||
await readResponseBytesCapped(response, 16 * 1024, signal),
|
||||
);
|
||||
const at = exportAt(
|
||||
/\S+\.export\.CSV\.zip/.exec(text)?.[0].replace(/^http:/, 'https:') || '',
|
||||
);
|
||||
return Number.isFinite(at) ? at : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The GKG export of the same quarter hour as an events export. */
|
||||
export const gkgUrl = (eventsUrl) =>
|
||||
eventsUrl.replace(/\.export\.CSV\.zip$/, '.gkg.csv.zip');
|
||||
|
||||
/** The day's exports up to `now`, newest first. */
|
||||
export function exportsOfDay(now, hours = 24) {
|
||||
const urls = [];
|
||||
const last = Math.floor(now / STEP_MS) * STEP_MS;
|
||||
for (let ms = last; ms > now - hours * 3_600_000; ms -= STEP_MS)
|
||||
urls.push(exportUrl(ms));
|
||||
return urls;
|
||||
}
|
||||
|
||||
/** The one table inside an export's zip, as text, up to `maxBytes`. */
|
||||
export async function unzipExport(bytes, maxBytes = MAX_TABLE_BYTES) {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const fail = () => {
|
||||
throw new UpstreamError('GDELT export unreadable', { unreadable: true });
|
||||
};
|
||||
if (bytes.length < 30 || view.getUint32(0, true) !== 0x04034b50) fail();
|
||||
// The central directory has the sizes; a local header may leave them 0.
|
||||
let eocd = -1;
|
||||
for (let i = bytes.length - 22; i >= Math.max(0, bytes.length - 0x10015); i--)
|
||||
if (view.getUint32(i, true) === 0x06054b50) {
|
||||
eocd = i;
|
||||
break;
|
||||
}
|
||||
if (eocd < 0) fail();
|
||||
const dir = view.getUint32(eocd + 16, true);
|
||||
if (dir + 46 > bytes.length || view.getUint32(dir, true) !== 0x02014b50)
|
||||
fail();
|
||||
const method = view.getUint16(dir + 10, true);
|
||||
const compressed = view.getUint32(dir + 20, true);
|
||||
const local = view.getUint32(dir + 42, true);
|
||||
if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50)
|
||||
fail();
|
||||
const start =
|
||||
local +
|
||||
30 +
|
||||
view.getUint16(local + 26, true) +
|
||||
view.getUint16(local + 28, true);
|
||||
if (start + compressed > bytes.length) fail();
|
||||
const data = bytes.subarray(start, start + compressed);
|
||||
try {
|
||||
if (method === 0 && data.length > maxBytes) fail();
|
||||
const table =
|
||||
method === 0
|
||||
? data
|
||||
: method === 8
|
||||
? await inflate(data, { maxOutputLength: maxBytes })
|
||||
: fail();
|
||||
return new TextDecoder().decode(table);
|
||||
} catch (error) {
|
||||
if (error instanceof UpstreamError) throw error;
|
||||
return fail();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An article's address as the reports carry it: https, an http one
|
||||
* upgraded (GDELT keeps the address as it found it), else null.
|
||||
*/
|
||||
export const articleLink = (value) =>
|
||||
webLink(value) || webLink(String(value ?? '').replace(/^http:/, 'https:'));
|
||||
|
||||
/**
|
||||
* The placed events of an export's table, of the stories with physical
|
||||
* violence in them: those events, and every other placed event of the same
|
||||
* stories (they help place it).
|
||||
*/
|
||||
export function parseEvents(table) {
|
||||
const all = [];
|
||||
for (const line of table.split('\n')) {
|
||||
const f = line.split('\t');
|
||||
if (f.length < 61) continue;
|
||||
const violent =
|
||||
f[COL.quadClass] === '4' && ['18', '19', '20'].includes(f[COL.rootCode]);
|
||||
const precision = PRECISION[f[COL.geoType]];
|
||||
const lat = Number(f[COL.lat]);
|
||||
const lon = Number(f[COL.lon]);
|
||||
const url = articleLink(f[COL.url]);
|
||||
const added = /^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/.exec(f[COL.added]);
|
||||
if (
|
||||
!precision ||
|
||||
!f[COL.lat] ||
|
||||
!f[COL.lon] ||
|
||||
!Number.isFinite(lat) ||
|
||||
!Number.isFinite(lon) ||
|
||||
Math.abs(lat) > 90 ||
|
||||
Math.abs(lon) > 180 ||
|
||||
(lat === 0 && lon === 0) ||
|
||||
!url ||
|
||||
!added
|
||||
)
|
||||
continue;
|
||||
all.push({
|
||||
id: f[COL.id],
|
||||
code: f[COL.eventCode],
|
||||
violent,
|
||||
lat,
|
||||
lon,
|
||||
place: f[COL.place].trim() || null,
|
||||
precision,
|
||||
mentions: Math.max(1, Number(f[COL.mentions]) || 1),
|
||||
addedAt: `${added[1]}-${added[2]}-${added[3]}T${added[4]}:${added[5]}:${added[6]}.000Z`,
|
||||
url,
|
||||
});
|
||||
}
|
||||
const stories = new Set(all.filter((e) => e.violent).map(storyKey));
|
||||
return all.filter((e) => stories.has(storyKey(e)));
|
||||
}
|
||||
|
||||
/**
|
||||
* The words of an article's address, when it is written in them
|
||||
* ("three-children-die-in-peshawar-cylinder-exp"), as a sentence; else null.
|
||||
*/
|
||||
export function addressWords(url) {
|
||||
let path;
|
||||
try {
|
||||
path = decodeURIComponent(new URL(url).pathname);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const segment = path
|
||||
.split('/')
|
||||
.map((part) => part.replace(/\.(?:s?html?|php|aspx?)$/i, ''))
|
||||
.filter((part) => /[-_]/.test(part))
|
||||
.sort((a, b) => b.length - a.length)[0];
|
||||
if (!segment) return null;
|
||||
const words = segment.split(/[-_]+/).filter(Boolean);
|
||||
const lettered = words.filter((word) => /^\p{L}+$/u.test(word));
|
||||
if (lettered.length < 4 || lettered.length < 0.7 * words.length) return null;
|
||||
// Numbers stay ("killed nearly 500"); an id does not: a word of letters
|
||||
// and digits, five digits or more, or four or more at either end.
|
||||
const kept = words.filter(
|
||||
(word, i) =>
|
||||
/^\p{L}+$/u.test(word) ||
|
||||
(/^\d{1,4}$/.test(word) &&
|
||||
!(word.length === 4 && (i === 0 || i === words.length - 1))),
|
||||
);
|
||||
const text = kept.slice(0, MAX_ADDRESS_WORDS).join(' ').toLowerCase();
|
||||
return text.charAt(0).toUpperCase() + text.slice(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Address words with the places the story is at written as names
|
||||
* ("killed in amman" → "killed in Amman"), and "u s" as "US".
|
||||
*/
|
||||
export function withNames(words, places) {
|
||||
let text = words.replace(/(?<![\p{L}])u s(?![\p{L}])/gu, 'US');
|
||||
const names = new Set(
|
||||
places.flatMap((place) =>
|
||||
String(place.place ?? '')
|
||||
.split(',')
|
||||
.map((part) => part.replace(/\(.*?\)/g, '').trim())
|
||||
.filter((part) => part.length > 2 && !/\(general\)/.test(part)),
|
||||
),
|
||||
);
|
||||
for (const name of names) {
|
||||
const pattern = new RegExp(
|
||||
`(?<![\\p{L}])${name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\p{L}])`,
|
||||
'gu',
|
||||
);
|
||||
text = text.replace(pattern, name);
|
||||
}
|
||||
return text.charAt(0).toUpperCase() + text.slice(1);
|
||||
}
|
||||
|
||||
// A page title's site part: " | The Daily Example", " - Wire Service".
|
||||
const SITE_PART = /\s+[|–—-]\s+([^|–—-]{2,60})$/u;
|
||||
|
||||
/**
|
||||
* A page title as a headline: entities decoded, the site's name after a
|
||||
* bar or dash taken off (when the headline before it is still one), or
|
||||
* null when it is too short to be one.
|
||||
*/
|
||||
export function cleanTitle(raw) {
|
||||
// An agency's tag set on with a bare hyphen ("…attack-Newswire", "…-X").
|
||||
let title = unescapeHtml(raw).replace(
|
||||
// After a word in lower case: "Jean-Pierre" keeps its name.
|
||||
/(?<=(?:^|\s)\p{Ll}\p{L}*|[\p{N}'")])-\p{Lu}\p{L}{0,15}$/u,
|
||||
'',
|
||||
);
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const match = SITE_PART.exec(title);
|
||||
if (!match) break;
|
||||
const rest = title.slice(0, match.index);
|
||||
if (rest.split(/\s+/).length < 4 || match[1].split(/\s+/).length > 6) break;
|
||||
title = rest;
|
||||
}
|
||||
title = title.trim().slice(0, 300);
|
||||
return title.length >= 12 && title.split(/\s+/).length >= 3 ? title : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* GKG page titles by article address (as articleLink writes it), for the
|
||||
* addresses in `wanted`.
|
||||
*/
|
||||
export function parseTitles(table, wanted) {
|
||||
const titles = new Map();
|
||||
for (const line of table.split('\n')) {
|
||||
const f = line.split('\t');
|
||||
const url = f[4] && articleLink(f[4]);
|
||||
if (!url || !wanted.has(url)) continue;
|
||||
const match = /<PAGE_TITLE>(.*?)<\/PAGE_TITLE>/s.exec(f.at(-1) || '');
|
||||
const title = match && cleanTitle(match[1]);
|
||||
if (title) titles.set(url, title);
|
||||
}
|
||||
return titles;
|
||||
}
|
||||
|
||||
/**
|
||||
* The story an event belongs to: its article's address words, which the
|
||||
* copies other outlets run share (an agency story), else its address.
|
||||
*/
|
||||
export function storyKey(event) {
|
||||
return addressWords(event.url)?.toLowerCase() || event.url;
|
||||
}
|
||||
|
||||
const PLACE_WORD = (name) =>
|
||||
new RegExp(
|
||||
`(?<![\\p{L}])${name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}s?(?![\\p{L}])`,
|
||||
'u',
|
||||
);
|
||||
/** Whether the words name a place's town or its country ("jordans" too). */
|
||||
function namedIn(words, place) {
|
||||
if (!words || !place) return false;
|
||||
const parts = place
|
||||
.split(',')
|
||||
.map((part) => part.replace(/\(.*?\)/g, '').trim())
|
||||
.filter((part) => part.length > 2);
|
||||
return [parts[0], parts.at(-1)].some(
|
||||
(part) => part && PLACE_WORD(part).test(words.toLowerCase()),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A story's places, best first, from all its placed events. GDELT places
|
||||
* an event at a bare country when the text only names a nationality ("a
|
||||
* Sri Lankan embassy", an Australian outlet's own country), so a town or
|
||||
* region wins over a country; of those, the ones the article's own words
|
||||
* name; then the most mentioned.
|
||||
*/
|
||||
export function choosePlaces(events, words) {
|
||||
const byPlace = new Map();
|
||||
for (const event of events) {
|
||||
const key = `${event.lat},${event.lon}`;
|
||||
const place = byPlace.get(key) || { ...event, mentions: 0 };
|
||||
place.mentions += event.mentions;
|
||||
byPlace.set(key, place);
|
||||
}
|
||||
let places = [...byPlace.values()];
|
||||
const specific = places.filter((p) => p.precision !== 'country');
|
||||
if (specific.length) places = specific;
|
||||
const named = places.filter((p) => namedIn(words, p.place));
|
||||
if (named.length) places = named;
|
||||
return places.sort((a, b) => b.mentions - a.mentions).slice(0, MAX_PLACES);
|
||||
}
|
||||
|
||||
// The earlier of two events: by the time GDELT added it, then by its id.
|
||||
const earlier = (a, b) =>
|
||||
b.addedAt < a.addedAt ||
|
||||
(b.addedAt === a.addedAt && Number(b.id) < Number(a.id))
|
||||
? b
|
||||
: a;
|
||||
|
||||
/**
|
||||
* One report per story (storyKey), its address and id the first copy's (an
|
||||
* export adding a more-mentioned event keeps both), placed by choosePlaces,
|
||||
* titled by a copy's page title (`titleOf`), else its address words, else
|
||||
* its violent event.
|
||||
*/
|
||||
export function eventReports(events, { titleOf = () => null } = {}) {
|
||||
const stories = new Map();
|
||||
for (const event of events) {
|
||||
const key = storyKey(event);
|
||||
const list = stories.get(key) || [];
|
||||
list.push(event);
|
||||
stories.set(key, list);
|
||||
}
|
||||
const reports = [];
|
||||
for (const list of stories.values()) {
|
||||
const violent = list
|
||||
.filter((e) => e.violent !== false)
|
||||
.sort(
|
||||
(a, b) => b.mentions - a.mentions || a.addedAt.localeCompare(b.addedAt),
|
||||
);
|
||||
if (!violent.length) continue;
|
||||
const [lead] = violent;
|
||||
const first = list.reduce(earlier);
|
||||
const { url } = first;
|
||||
const address = addressWords(url);
|
||||
const places = choosePlaces(list, address);
|
||||
const [, eventCategory, words] = eventKind(lead.code);
|
||||
const page = [url, ...list.map((e) => e.url)]
|
||||
.map((copy) => titleOf(copy))
|
||||
.find(Boolean);
|
||||
const own = page || (address ? withNames(address, places) : null);
|
||||
const title = own || eventTitle(words, places[0].place);
|
||||
// The article's words decide whether it is conflict, as any headline's
|
||||
// would: GDELT codes an election story as "fighting" too. GDELT's code
|
||||
// names the kind only where the words read as conflict without naming
|
||||
// one, or there are no words.
|
||||
const read = own ? categorizeText({ title: own }) : null;
|
||||
const tags = read?.tags || [];
|
||||
const relevance = read
|
||||
? relevanceOf({ category: read.category, tags, title: own })
|
||||
: 'related';
|
||||
const category =
|
||||
read && read.category !== 'other'
|
||||
? read.category
|
||||
: relevance === 'minor'
|
||||
? 'other'
|
||||
: eventCategory;
|
||||
const publisher = new URL(url).hostname.replace(/^www\./, '');
|
||||
const geo = ({ lat, lon, place, precision }) => ({
|
||||
lat,
|
||||
lon,
|
||||
place,
|
||||
precision,
|
||||
confidence: precision === 'city' ? 0.6 : 0.3,
|
||||
mappable: true,
|
||||
method: 'source',
|
||||
});
|
||||
reports.push({
|
||||
id: `gdelt-${first.id}`,
|
||||
title,
|
||||
url,
|
||||
publisher,
|
||||
source: { name: publisher, attribution: 'GDELT Project' },
|
||||
kind: 'gdelt',
|
||||
side: 'unknown',
|
||||
category,
|
||||
relevance,
|
||||
severity: Math.max(
|
||||
1,
|
||||
severityByRules({ category, tags, title: own || '' }),
|
||||
),
|
||||
severityBy: 'rules',
|
||||
publishedAt: first.addedAt,
|
||||
geo: geo(places[0]),
|
||||
locations: places.slice(1).map(geo),
|
||||
media: [],
|
||||
mentions: list.reduce((sum, e) => sum + e.mentions, 0),
|
||||
copies: new Set(list.map((e) => e.url)).size,
|
||||
// A title of the event's words and its place, when the address had
|
||||
// none: a renamed place renames it (eventTitle).
|
||||
...(own ? {} : { eventWords: words }),
|
||||
titled: page ? 'page' : own ? 'address' : 'event',
|
||||
});
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
|
||||
/** The title of a report without address words, at its (renamed) place. */
|
||||
export const eventTitle = (words, place) =>
|
||||
place ? `${words} in ${place}` : words;
|
||||
|
||||
/**
|
||||
* One export's table, or null while GDELT has not published it (GDELT
|
||||
* lists the newest in lastupdate.txt; see fetchLatestExport).
|
||||
*/
|
||||
async function fetchTable(url, { fetchImpl, signal, maxBytes, tableBytes }) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
} catch {
|
||||
throw new UpstreamError('GDELT exports unreachable');
|
||||
}
|
||||
if (response.status === 404) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw new UpstreamError(`GDELT exports answered ${response.status}`, {
|
||||
status: response.status,
|
||||
});
|
||||
}
|
||||
let bytes;
|
||||
try {
|
||||
bytes = await readResponseBytesCapped(response, maxBytes, signal);
|
||||
} catch (error) {
|
||||
// Over the cap stays over it; a read cut short may succeed again.
|
||||
throw new UpstreamError('GDELT export unreadable', {
|
||||
unreadable: error?.code === 'RESPONSE_TOO_LARGE',
|
||||
});
|
||||
}
|
||||
return unzipExport(bytes, tableBytes);
|
||||
}
|
||||
|
||||
/** One events export's events, or null while it is not published yet. */
|
||||
export async function fetchExport(url, { fetchImpl, signal } = {}) {
|
||||
const table = await fetchTable(url, {
|
||||
fetchImpl,
|
||||
signal,
|
||||
maxBytes: MAX_ZIP_BYTES,
|
||||
tableBytes: MAX_TABLE_BYTES,
|
||||
});
|
||||
return table === null ? null : parseEvents(table);
|
||||
}
|
||||
|
||||
/**
|
||||
* The page titles a GKG export holds for the addresses in `wanted`, or null
|
||||
* while it is not published yet.
|
||||
*/
|
||||
export async function fetchTitles(url, wanted, { fetchImpl, signal } = {}) {
|
||||
const table = await fetchTable(url, {
|
||||
fetchImpl,
|
||||
signal,
|
||||
maxBytes: MAX_GKG_ZIP_BYTES,
|
||||
tableBytes: MAX_GKG_TABLE_BYTES,
|
||||
});
|
||||
return table === null ? null : parseTitles(table, wanted);
|
||||
}
|
||||
Reference in new issue
Block a user