mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-06 19:34:01 +00:00
style: format extracted local services
This commit is contained in:
1 parent
233285fe12
commit
5223b6df5b
33 files changed
+1785
-830
No files matched your search
@@ -4,24 +4,39 @@ import { normalizeRegionalArticles } from '../../../src/data/regionalBrief.js';
|
||||
function decodeRssText(value) {
|
||||
return String(value || '')
|
||||
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
||||
.replace(/&/g, '&').replace(/"/g, '"').replace(/'/g, "'")
|
||||
.replace(/</g, '<').replace(/>/g, '>')
|
||||
.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function rssTag(block, tag) {
|
||||
return decodeRssText(new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`, 'i').exec(block)?.[1] || '');
|
||||
return decodeRssText(
|
||||
new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`, 'i').exec(
|
||||
block,
|
||||
)?.[1] || '',
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeRssArticles(xml, limit = 5) {
|
||||
const seen = new Set();
|
||||
const articles = [];
|
||||
for (const match of String(xml || '').matchAll(/<item>([\s\S]*?)<\/item>/gi)) {
|
||||
for (const match of String(xml || '').matchAll(
|
||||
/<item>([\s\S]*?)<\/item>/gi,
|
||||
)) {
|
||||
const item = match[1];
|
||||
const title = rssTag(item, 'title').slice(0, 180);
|
||||
const url = rssTag(item, 'link');
|
||||
let parsedUrl;
|
||||
try { parsedUrl = new URL(url); } catch { continue; }
|
||||
try {
|
||||
parsedUrl = new URL(url);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!title || !['http:', 'https:'].includes(parsedUrl.protocol)) continue;
|
||||
const source = rssTag(item, 'source');
|
||||
const signature = `${title.toLowerCase()}|${source.toLowerCase() || parsedUrl.hostname}`;
|
||||
@@ -32,7 +47,9 @@ function normalizeRssArticles(xml, limit = 5) {
|
||||
title,
|
||||
url: parsedUrl.href,
|
||||
domain: source || parsedUrl.hostname.replace(/^www\./, ''),
|
||||
publishedAt: Number.isNaN(Date.parse(rawDate)) ? null : new Date(rawDate).toISOString(),
|
||||
publishedAt: Number.isNaN(Date.parse(rawDate))
|
||||
? null
|
||||
: new Date(rawDate).toISOString(),
|
||||
sourceCountry: null,
|
||||
});
|
||||
if (articles.length >= limit) break;
|
||||
@@ -42,7 +59,8 @@ function normalizeRssArticles(xml, limit = 5) {
|
||||
|
||||
async function fetchRegionalNews(place) {
|
||||
const query = place?.locality || place?.region || place?.country;
|
||||
if (!query) return { status: 'unavailable', query: null, articles: [], source: null };
|
||||
if (!query)
|
||||
return { status: 'unavailable', query: null, articles: [], source: null };
|
||||
const rssParams = new URLSearchParams({
|
||||
q: String(query).replace(/["\\]/g, ' ').trim(),
|
||||
hl: 'en-US',
|
||||
@@ -50,13 +68,19 @@ async function fetchRegionalNews(place) {
|
||||
ceid: 'US:en',
|
||||
});
|
||||
try {
|
||||
const xml = await fetchRegionalText(`https://news.google.com/rss/search?${rssParams}`, {
|
||||
headers: { 'User-Agent': 'GodsEyeView/0.1' },
|
||||
timeoutMs: 12_000,
|
||||
});
|
||||
const xml = await fetchRegionalText(
|
||||
`https://news.google.com/rss/search?${rssParams}`,
|
||||
{
|
||||
headers: { 'User-Agent': 'GodsEyeView/0.1' },
|
||||
timeoutMs: 12_000,
|
||||
},
|
||||
);
|
||||
const articles = normalizeRssArticles(xml, 5);
|
||||
if (articles.length) return { status: 'ready', query, articles, source: 'Google News RSS' };
|
||||
} catch { /* fall through to the existing free index */ }
|
||||
if (articles.length)
|
||||
return { status: 'ready', query, articles, source: 'Google News RSS' };
|
||||
} catch {
|
||||
/* fall through to the existing free index */
|
||||
}
|
||||
const params = new URLSearchParams({
|
||||
query: `"${String(query).replace(/["\\]/g, ' ').trim()}"`,
|
||||
mode: 'artlist',
|
||||
@@ -66,12 +90,20 @@ async function fetchRegionalNews(place) {
|
||||
timespan: '48h',
|
||||
});
|
||||
try {
|
||||
const payload = await fetchRegionalJson(`https://api.gdeltproject.org/api/v2/doc/doc?${params}`, {
|
||||
headers: { 'User-Agent': 'GodsEyeView/0.1' },
|
||||
timeoutMs: 12_000,
|
||||
});
|
||||
const payload = await fetchRegionalJson(
|
||||
`https://api.gdeltproject.org/api/v2/doc/doc?${params}`,
|
||||
{
|
||||
headers: { 'User-Agent': 'GodsEyeView/0.1' },
|
||||
timeoutMs: 12_000,
|
||||
},
|
||||
);
|
||||
const articles = normalizeRegionalArticles(payload, 5);
|
||||
return { status: articles.length ? 'ready' : 'empty', query, articles, source: 'GDELT fallback' };
|
||||
return {
|
||||
status: articles.length ? 'ready' : 'empty',
|
||||
query,
|
||||
articles,
|
||||
source: 'GDELT fallback',
|
||||
};
|
||||
} catch {
|
||||
return { status: 'unavailable', query, articles: [], source: null };
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user