diff --git a/package.json b/package.json index 3850bd00f..1e4c76a35 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,9 @@ }, "./server/providers/military-installations": { "node": "./server/providers/military-installations.js" + }, + "./server/providers/regional": { + "node": "./server/providers/regional.js" } } } diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index f7d01b799..24ea28e23 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -273,5 +273,26 @@ "server/providers/overpass/transport.js" ], "external": [] + }, + "regional-provider": { + "runtime": "node", + "exports": [ + "./server/providers/regional" + ], + "modules": [ + "server/providers/common/http.js", + "server/providers/common/query.js", + "server/providers/common/rate-limit.js", + "server/providers/regional.js", + "server/providers/regional/briefing.js", + "server/providers/regional/http.js", + "server/providers/regional/news.js", + "server/providers/regional/place.js", + "server/providers/regional/query.js", + "server/providers/regional/weather-effects.js", + "server/providers/regional/weather.js", + "src/data/regionalBrief.js" + ], + "external": [] } } diff --git a/server/providers/local.js b/server/providers/local.js index 33ce8f3e0..7f9aa8d74 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -18,6 +18,10 @@ export { writeMilitaryInstallationDisk } from './military-installations/cache.js export { validMilitaryInstallationBox } from './military-installations/query.js'; export { militaryInstallationFailureReason } from './military-installations/query.js'; import { militaryInstallationsProxy } from './military-installations.js'; +export { validRegionalPoint } from './regional/query.js'; +export { regionalBriefHasAnySource } from './regional/briefing.js'; +import { regionalBriefProxy } from './regional/briefing.js'; +import { weatherEffectsProxy } from './regional/weather-effects.js'; import { cctvProxy } from './cctv.js'; import { radioBrowserProxy } from './radio.js'; export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream } from './cctv.js'; @@ -1472,372 +1476,51 @@ const GEV_REALTIME_TOOLS = [ // --------------------------------------------------------------------------- // Regional cockpit briefing proxy // --------------------------------------------------------------------------- -const REGIONAL_BRIEF_CACHE_MS = 5 * 60_000; -const REGIONAL_BRIEF_STALE_MS = 60 * 60_000; -const REGIONAL_BRIEF_MAX_CACHE = 120; -const REGIONAL_MAX_RESPONSE_BYTES = 2 * 1024 * 1024; -const _regionalBriefCache = new Map(); -const _regionalBriefInFlight = new Map(); -const _regionalBriefRateLimiter = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 90 }); -const WEATHER_EFFECTS_CACHE_MS = 5 * 60_000; -const WEATHER_EFFECTS_STALE_MS = 30 * 60_000; -const WEATHER_EFFECTS_MAX_CACHE = 180; -const WEATHER_EFFECTS_MAX_RESPONSE_BYTES = 512 * 1024; -const _weatherEffectsCache = new Map(); -const _weatherEffectsInFlight = new Map(); -const _weatherEffectsRateLimiter = makeRateLimiter({ windowMs: 60_000, max: 45, globalMax: 120 }); -let _nominatimQueue = Promise.resolve(); -let _nominatimLastRequestAt = 0; -export function validRegionalPoint(params) { - const latitude = requiredFiniteQueryNumber(params, 'latitude'); - const longitude = requiredFiniteQueryNumber(params, 'longitude'); - if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) return null; - if (latitude < -90 || latitude > 90 || longitude < -180 || longitude > 180) return null; - return { latitude, longitude }; -} -function trimRegionalBriefCache() { - while (_regionalBriefCache.size > REGIONAL_BRIEF_MAX_CACHE) { - const oldest = _regionalBriefCache.keys().next().value; - if (oldest === undefined) break; - _regionalBriefCache.delete(oldest); - } -} -function trimWeatherEffectsCache() { - while (_weatherEffectsCache.size > WEATHER_EFFECTS_MAX_CACHE) { - const oldest = _weatherEffectsCache.keys().next().value; - if (oldest === undefined) break; - _weatherEffectsCache.delete(oldest); - } -} -async function fetchRegionalJson(url, { - headers = {}, - timeoutMs = 9000, - maxBytes = REGIONAL_MAX_RESPONSE_BYTES, -} = {}) { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), timeoutMs); - try { - const response = await fetch(url, { signal: controller.signal, headers }); - if (!response.ok) throw new Error(`Upstream returned ${response.status}`); - return readResponseJsonCapped(response, maxBytes); - } finally { - clearTimeout(timeout); - } -} -async function fetchRegionalText(url, { - headers = {}, - timeoutMs = 9000, - maxBytes = REGIONAL_MAX_RESPONSE_BYTES, -} = {}) { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), timeoutMs); - try { - const response = await fetch(url, { signal: controller.signal, headers }); - if (!response.ok) throw new Error(`Upstream returned ${response.status}`); - return readResponseTextCapped(response, maxBytes); - } finally { - clearTimeout(timeout); - } -} -function decodeRssText(value) { - return String(value || '') - .replace(//g, '$1') - .replace(/&/g, '&').replace(/"/g, '"').replace(/'/g, "'") - .replace(/</g, '<').replace(/>/g, '>') - .replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); -} -function rssTag(block, tag) { - return decodeRssText(new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`, 'i').exec(block)?.[1] || ''); -} -function normalizeRssArticles(xml, limit = 5) { - const seen = new Set(); - const articles = []; - for (const match of String(xml || '').matchAll(/([\s\S]*?)<\/item>/gi)) { - const item = match[1]; - const title = rssTag(item, 'title').slice(0, 180); - const url = rssTag(item, 'link'); - let parsedUrl; - try { parsedUrl = new URL(url); } catch { continue; } - if (!title || !['http:', 'https:'].includes(parsedUrl.protocol)) continue; - const source = rssTag(item, 'source'); - const signature = `${title.toLowerCase()}|${source.toLowerCase() || parsedUrl.hostname}`; - if (seen.has(signature)) continue; - seen.add(signature); - const rawDate = rssTag(item, 'pubDate'); - articles.push({ - title, - url: parsedUrl.href, - domain: source || parsedUrl.hostname.replace(/^www\./, ''), - publishedAt: Number.isNaN(Date.parse(rawDate)) ? null : new Date(rawDate).toISOString(), - sourceCountry: null, - }); - if (articles.length >= limit) break; - } - return articles; -} -function fetchRegionalPlace(point) { - const task = _nominatimQueue.then(async () => { - const waitMs = Math.max(0, 1100 - (Date.now() - _nominatimLastRequestAt)); - if (waitMs) await new Promise((resolve) => setTimeout(resolve, waitMs)); - _nominatimLastRequestAt = Date.now(); - const params = new URLSearchParams({ - format: 'jsonv2', - lat: point.latitude.toFixed(5), - lon: point.longitude.toFixed(5), - zoom: '10', - addressdetails: '1', - 'accept-language': 'en', - }); - const payload = await fetchRegionalJson(`https://nominatim.openstreetmap.org/reverse?${params}`, { - headers: { - 'User-Agent': 'GodsEyeView/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)', - Referer: 'https://github.com/bilawalsidhu/gods-eye-view', - }, - }); - return normalizeRegionalPlace(payload); - }); - _nominatimQueue = task.catch(() => null); - return task; -} -async function fetchRegionalNews(place) { - const query = place?.locality || place?.region || place?.country; - if (!query) return { status: 'unavailable', query: null, articles: [], source: null }; - const rssParams = new URLSearchParams({ - q: String(query).replace(/["\\]/g, ' ').trim(), - hl: 'en-US', - gl: 'US', - ceid: 'US:en', - }); - try { - const xml = await fetchRegionalText(`https://news.google.com/rss/search?${rssParams}`, { - headers: { 'User-Agent': 'GodsEyeView/0.1' }, - timeoutMs: 12_000, - }); - const articles = normalizeRssArticles(xml, 5); - if (articles.length) return { status: 'ready', query, articles, source: 'Google News RSS' }; - } catch { /* fall through to the existing free index */ } - const params = new URLSearchParams({ - query: `"${String(query).replace(/["\\]/g, ' ').trim()}"`, - mode: 'artlist', - format: 'json', - maxrecords: '5', - sort: 'datedesc', - timespan: '48h', - }); - try { - const payload = await fetchRegionalJson(`https://api.gdeltproject.org/api/v2/doc/doc?${params}`, { - headers: { 'User-Agent': 'GodsEyeView/0.1' }, - timeoutMs: 12_000, - }); - const articles = normalizeRegionalArticles(payload, 5); - return { status: articles.length ? 'ready' : 'empty', query, articles, source: 'GDELT fallback' }; - } catch { - return { status: 'unavailable', query, articles: [], source: null }; - } -} -async function fetchRegionalWeather(point) { - const params = new URLSearchParams({ - latitude: point.latitude.toFixed(5), - longitude: point.longitude.toFixed(5), - current: 'temperature_2m,apparent_temperature,precipitation,weather_code,cloud_cover,wind_speed_10m,wind_direction_10m,visibility', - timezone: 'UTC', - }); - try { - const payload = await fetchRegionalJson(`https://api.open-meteo.com/v1/forecast?${params}`, { - maxBytes: WEATHER_EFFECTS_MAX_RESPONSE_BYTES, - }); - return normalizeRegionalWeather(payload); - } catch { - return null; - } -} + + + + + + + + + + + + + + + + + + + + + + + + + + + /** True when at least one regional source produced usable data. */ -export function regionalBriefHasAnySource({ place, weather, news } = {}) { - return Boolean(place || weather || (news && news.status !== 'unavailable')); -} -function regionalBriefProxy() { - async function refresh(point, key) { - const [placeResult, weatherResult] = await Promise.allSettled([ - fetchRegionalPlace(point), - fetchRegionalWeather(point), - ]); - const place = placeResult.status === 'fulfilled' ? placeResult.value : null; - const weather = weatherResult.status === 'fulfilled' ? weatherResult.value : null; - const news = await fetchRegionalNews(place); - if (!regionalBriefHasAnySource({ place, weather, news })) { - throw new Error('All regional briefing sources unavailable'); - } - const payload = { - status: place && weather && news.status !== 'unavailable' ? 'ready' : 'partial', - retrievedAt: new Date().toISOString(), - coordinates: point, - place, - placeStatus: place ? 'ready' : 'unavailable', - weather, - weatherStatus: weather ? 'ready' : 'unavailable', - newsStatus: news.status, - newsQuery: news.query, - newsSource: news.source, - articles: news.articles, - }; - _regionalBriefCache.set(key, { payload, cachedAt: Date.now() }); - trimRegionalBriefCache(); - return payload; - } - function install(middlewares) { - middlewares.use('/api/regional-brief', async (req, res) => { - if (req.method !== 'GET') { - res.writeHead(405, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'Method Not Allowed' })); - return; - } - if (!_regionalBriefRateLimiter(clientKey(req))) { - res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '10' }); - res.end(JSON.stringify({ error: 'Rate limit exceeded' })); - return; - } - const url = new URL(req.url || '', 'http://localhost'); - const point = validRegionalPoint(url.searchParams); - if (!point) { - res.writeHead(400, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'Valid latitude and longitude are required' })); - return; - } - const key = `${(Math.round(point.latitude * 10) / 10).toFixed(1)},${(Math.round(point.longitude * 10) / 10).toFixed(1)}`; - const now = Date.now(); - const cached = _regionalBriefCache.get(key); - if (cached && now - cached.cachedAt <= REGIONAL_BRIEF_CACHE_MS) { - res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=60', 'X-Regional-Brief': 'HIT' }); - res.end(JSON.stringify({ ...cached.payload, status: 'cached' })); - return; - } - const request = coalesceProxyRequest(_regionalBriefInFlight, key, () => refresh(point, key)); - try { - const payload = await request.promise; - res.writeHead(200, { - 'Content-Type': 'application/json', - 'Cache-Control': 'public, max-age=60', - 'X-Regional-Brief': request.shared ? 'INFLIGHT' : 'MISS', - }); - res.end(JSON.stringify(payload)); - } catch { - if (cached && now - cached.cachedAt <= REGIONAL_BRIEF_STALE_MS) { - res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', 'X-Regional-Brief': 'STALE' }); - res.end(JSON.stringify({ ...cached.payload, status: 'stale' })); - return; - } - res.writeHead(503, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Regional briefing is temporarily unavailable' })); - } - }); - } - return { - name: 'regional-brief-proxy', - configureServer(server) { - install(server.middlewares); - }, - configurePreviewServer(server) { - install(server.middlewares); - }, - }; -} -function weatherEffectsProxy() { - async function refresh(point, key) { - const weather = await fetchRegionalWeather(point); - if (!weather) throw new Error('Weather observation unavailable'); - const payload = { - status: 'ready', - retrievedAt: new Date().toISOString(), - coordinates: point, - weather, - }; - _weatherEffectsCache.set(key, { payload, cachedAt: Date.now() }); - trimWeatherEffectsCache(); - return payload; - } - function install(middlewares) { - middlewares.use('/api/weather-effects', async (req, res) => { - if (req.method !== 'GET') { - res.writeHead(405, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'Method Not Allowed' })); - return; - } - if (!_weatherEffectsRateLimiter(clientKey(req))) { - res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '10' }); - res.end(JSON.stringify({ error: 'Rate limit exceeded' })); - return; - } - const url = new URL(req.url || '', 'http://localhost'); - const point = validRegionalPoint(url.searchParams); - if (!point) { - res.writeHead(400, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'Valid latitude and longitude are required' })); - return; - } - const key = `${(Math.round(point.latitude * 10) / 10).toFixed(1)},${(Math.round(point.longitude * 10) / 10).toFixed(1)}`; - const now = Date.now(); - const cached = _weatherEffectsCache.get(key); - if (cached && now - cached.cachedAt <= WEATHER_EFFECTS_CACHE_MS) { - res.writeHead(200, { - 'Content-Type': 'application/json', - 'Cache-Control': 'public, max-age=60', - 'X-Weather-Effects': 'HIT', - }); - res.end(JSON.stringify({ ...cached.payload, status: 'cached' })); - return; - } - const request = coalesceProxyRequest(_weatherEffectsInFlight, key, () => refresh(point, key)); - try { - const payload = await request.promise; - res.writeHead(200, { - 'Content-Type': 'application/json', - 'Cache-Control': 'public, max-age=60', - 'X-Weather-Effects': request.shared ? 'INFLIGHT' : 'MISS', - }); - res.end(JSON.stringify(payload)); - } catch { - if (cached && now - cached.cachedAt <= WEATHER_EFFECTS_STALE_MS) { - res.writeHead(200, { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - 'X-Weather-Effects': 'STALE', - }); - res.end(JSON.stringify({ ...cached.payload, status: 'stale' })); - return; - } - res.writeHead(503, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Weather effects are temporarily unavailable' })); - } - }); - } - - return { - name: 'weather-effects-proxy', - configureServer(server) { - install(server.middlewares); - }, - configurePreviewServer(server) { - install(server.middlewares); - }, - }; -} /** * In-app key setup ("POWER UP" panel) — dev-server only. diff --git a/server/providers/regional.js b/server/providers/regional.js new file mode 100644 index 000000000..2abe49715 --- /dev/null +++ b/server/providers/regional.js @@ -0,0 +1,3 @@ +export { regionalBriefProxy, regionalBriefHasAnySource } from './regional/briefing.js'; +export { weatherEffectsProxy } from './regional/weather-effects.js'; +export { validRegionalPoint } from './regional/query.js'; diff --git a/server/providers/regional/briefing.js b/server/providers/regional/briefing.js new file mode 100644 index 000000000..217d51fd2 --- /dev/null +++ b/server/providers/regional/briefing.js @@ -0,0 +1,125 @@ +import { makeRateLimiter, clientKey } from '../common/rate-limit.js'; +import { fetchRegionalPlace } from './place.js'; +import { fetchRegionalWeather } from './weather.js'; +import { fetchRegionalNews } from './news.js'; +import { validRegionalPoint } from './query.js'; +import { coalesceProxyRequest } from '../common/http.js'; + +// --------------------------------------------------------------------------- +// Regional cockpit briefing proxy +// --------------------------------------------------------------------------- +const REGIONAL_BRIEF_CACHE_MS = 5 * 60_000; + +const REGIONAL_BRIEF_STALE_MS = 60 * 60_000; + +const REGIONAL_BRIEF_MAX_CACHE = 120; + +const _regionalBriefCache = new Map(); + +const _regionalBriefInFlight = new Map(); + +const _regionalBriefRateLimiter = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 90 }); + +function trimRegionalBriefCache() { + while (_regionalBriefCache.size > REGIONAL_BRIEF_MAX_CACHE) { + const oldest = _regionalBriefCache.keys().next().value; + if (oldest === undefined) break; + _regionalBriefCache.delete(oldest); + } +} + +/** True when at least one regional source produced usable data. */ +function regionalBriefHasAnySource({ place, weather, news } = {}) { + return Boolean(place || weather || (news && news.status !== 'unavailable')); +} + +function regionalBriefProxy() { + async function refresh(point, key) { + const [placeResult, weatherResult] = await Promise.allSettled([ + fetchRegionalPlace(point), + fetchRegionalWeather(point), + ]); + const place = placeResult.status === 'fulfilled' ? placeResult.value : null; + const weather = weatherResult.status === 'fulfilled' ? weatherResult.value : null; + const news = await fetchRegionalNews(place); + if (!regionalBriefHasAnySource({ place, weather, news })) { + throw new Error('All regional briefing sources unavailable'); + } + const payload = { + status: place && weather && news.status !== 'unavailable' ? 'ready' : 'partial', + retrievedAt: new Date().toISOString(), + coordinates: point, + place, + placeStatus: place ? 'ready' : 'unavailable', + weather, + weatherStatus: weather ? 'ready' : 'unavailable', + newsStatus: news.status, + newsQuery: news.query, + newsSource: news.source, + articles: news.articles, + }; + _regionalBriefCache.set(key, { payload, cachedAt: Date.now() }); + trimRegionalBriefCache(); + return payload; + } + + function install(middlewares) { + middlewares.use('/api/regional-brief', async (req, res) => { + if (req.method !== 'GET') { + res.writeHead(405, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Method Not Allowed' })); + return; + } + if (!_regionalBriefRateLimiter(clientKey(req))) { + res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '10' }); + res.end(JSON.stringify({ error: 'Rate limit exceeded' })); + return; + } + const url = new URL(req.url || '', 'http://localhost'); + const point = validRegionalPoint(url.searchParams); + if (!point) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Valid latitude and longitude are required' })); + return; + } + const key = `${(Math.round(point.latitude * 10) / 10).toFixed(1)},${(Math.round(point.longitude * 10) / 10).toFixed(1)}`; + const now = Date.now(); + const cached = _regionalBriefCache.get(key); + if (cached && now - cached.cachedAt <= REGIONAL_BRIEF_CACHE_MS) { + res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=60', 'X-Regional-Brief': 'HIT' }); + res.end(JSON.stringify({ ...cached.payload, status: 'cached' })); + return; + } + const request = coalesceProxyRequest(_regionalBriefInFlight, key, () => refresh(point, key)); + try { + const payload = await request.promise; + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=60', + 'X-Regional-Brief': request.shared ? 'INFLIGHT' : 'MISS', + }); + res.end(JSON.stringify(payload)); + } catch { + if (cached && now - cached.cachedAt <= REGIONAL_BRIEF_STALE_MS) { + res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', 'X-Regional-Brief': 'STALE' }); + res.end(JSON.stringify({ ...cached.payload, status: 'stale' })); + return; + } + res.writeHead(503, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Regional briefing is temporarily unavailable' })); + } + }); + } + + return { + name: 'regional-brief-proxy', + configureServer(server) { + install(server.middlewares); + }, + configurePreviewServer(server) { + install(server.middlewares); + }, + }; +} + +export { regionalBriefHasAnySource, regionalBriefProxy }; diff --git a/server/providers/regional/http.js b/server/providers/regional/http.js new file mode 100644 index 000000000..63832d493 --- /dev/null +++ b/server/providers/regional/http.js @@ -0,0 +1,37 @@ +import { readResponseJsonCapped, readResponseTextCapped } from '../common/http.js'; + +const REGIONAL_MAX_RESPONSE_BYTES = 2 * 1024 * 1024; + +async function fetchRegionalJson(url, { + headers = {}, + timeoutMs = 9000, + maxBytes = REGIONAL_MAX_RESPONSE_BYTES, +} = {}) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { signal: controller.signal, headers }); + if (!response.ok) throw new Error(`Upstream returned ${response.status}`); + return readResponseJsonCapped(response, maxBytes); + } finally { + clearTimeout(timeout); + } +} + +async function fetchRegionalText(url, { + headers = {}, + timeoutMs = 9000, + maxBytes = REGIONAL_MAX_RESPONSE_BYTES, +} = {}) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { signal: controller.signal, headers }); + if (!response.ok) throw new Error(`Upstream returned ${response.status}`); + return readResponseTextCapped(response, maxBytes); + } finally { + clearTimeout(timeout); + } +} + +export { fetchRegionalJson, fetchRegionalText }; diff --git a/server/providers/regional/news.js b/server/providers/regional/news.js new file mode 100644 index 000000000..a2cef0de3 --- /dev/null +++ b/server/providers/regional/news.js @@ -0,0 +1,80 @@ +import { fetchRegionalText, fetchRegionalJson } from './http.js'; +import { normalizeRegionalArticles } from '../../../src/data/regionalBrief.js'; + +function decodeRssText(value) { + return String(value || '') + .replace(//g, '$1') + .replace(/&/g, '&').replace(/"/g, '"').replace(/'/g, "'") + .replace(/</g, '<').replace(/>/g, '>') + .replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); +} + +function rssTag(block, tag) { + return decodeRssText(new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`, 'i').exec(block)?.[1] || ''); +} + +function normalizeRssArticles(xml, limit = 5) { + const seen = new Set(); + const articles = []; + for (const match of String(xml || '').matchAll(/([\s\S]*?)<\/item>/gi)) { + const item = match[1]; + const title = rssTag(item, 'title').slice(0, 180); + const url = rssTag(item, 'link'); + let parsedUrl; + try { parsedUrl = new URL(url); } catch { continue; } + if (!title || !['http:', 'https:'].includes(parsedUrl.protocol)) continue; + const source = rssTag(item, 'source'); + const signature = `${title.toLowerCase()}|${source.toLowerCase() || parsedUrl.hostname}`; + if (seen.has(signature)) continue; + seen.add(signature); + const rawDate = rssTag(item, 'pubDate'); + articles.push({ + title, + url: parsedUrl.href, + domain: source || parsedUrl.hostname.replace(/^www\./, ''), + publishedAt: Number.isNaN(Date.parse(rawDate)) ? null : new Date(rawDate).toISOString(), + sourceCountry: null, + }); + if (articles.length >= limit) break; + } + return articles; +} + +async function fetchRegionalNews(place) { + const query = place?.locality || place?.region || place?.country; + if (!query) return { status: 'unavailable', query: null, articles: [], source: null }; + const rssParams = new URLSearchParams({ + q: String(query).replace(/["\\]/g, ' ').trim(), + hl: 'en-US', + gl: 'US', + ceid: 'US:en', + }); + try { + const xml = await fetchRegionalText(`https://news.google.com/rss/search?${rssParams}`, { + headers: { 'User-Agent': 'GodsEyeView/0.1' }, + timeoutMs: 12_000, + }); + const articles = normalizeRssArticles(xml, 5); + if (articles.length) return { status: 'ready', query, articles, source: 'Google News RSS' }; + } catch { /* fall through to the existing free index */ } + const params = new URLSearchParams({ + query: `"${String(query).replace(/["\\]/g, ' ').trim()}"`, + mode: 'artlist', + format: 'json', + maxrecords: '5', + sort: 'datedesc', + timespan: '48h', + }); + try { + const payload = await fetchRegionalJson(`https://api.gdeltproject.org/api/v2/doc/doc?${params}`, { + headers: { 'User-Agent': 'GodsEyeView/0.1' }, + timeoutMs: 12_000, + }); + const articles = normalizeRegionalArticles(payload, 5); + return { status: articles.length ? 'ready' : 'empty', query, articles, source: 'GDELT fallback' }; + } catch { + return { status: 'unavailable', query, articles: [], source: null }; + } +} + +export { fetchRegionalNews }; diff --git a/server/providers/regional/place.js b/server/providers/regional/place.js new file mode 100644 index 000000000..4cbe8c247 --- /dev/null +++ b/server/providers/regional/place.js @@ -0,0 +1,33 @@ +import { fetchRegionalJson } from './http.js'; +import { normalizeRegionalPlace } from '../../../src/data/regionalBrief.js'; + +let _nominatimQueue = Promise.resolve(); + +let _nominatimLastRequestAt = 0; + +function fetchRegionalPlace(point) { + const task = _nominatimQueue.then(async () => { + const waitMs = Math.max(0, 1100 - (Date.now() - _nominatimLastRequestAt)); + if (waitMs) await new Promise((resolve) => setTimeout(resolve, waitMs)); + _nominatimLastRequestAt = Date.now(); + const params = new URLSearchParams({ + format: 'jsonv2', + lat: point.latitude.toFixed(5), + lon: point.longitude.toFixed(5), + zoom: '10', + addressdetails: '1', + 'accept-language': 'en', + }); + const payload = await fetchRegionalJson(`https://nominatim.openstreetmap.org/reverse?${params}`, { + headers: { + 'User-Agent': 'GodsEyeView/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)', + Referer: 'https://github.com/bilawalsidhu/gods-eye-view', + }, + }); + return normalizeRegionalPlace(payload); + }); + _nominatimQueue = task.catch(() => null); + return task; +} + +export { fetchRegionalPlace }; diff --git a/server/providers/regional/query.js b/server/providers/regional/query.js new file mode 100644 index 000000000..97db7b011 --- /dev/null +++ b/server/providers/regional/query.js @@ -0,0 +1,11 @@ +import { requiredFiniteQueryNumber } from '../common/query.js'; + +function validRegionalPoint(params) { + const latitude = requiredFiniteQueryNumber(params, 'latitude'); + const longitude = requiredFiniteQueryNumber(params, 'longitude'); + if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) return null; + if (latitude < -90 || latitude > 90 || longitude < -180 || longitude > 180) return null; + return { latitude, longitude }; +} + +export { validRegionalPoint }; diff --git a/server/providers/regional/weather-effects.js b/server/providers/regional/weather-effects.js new file mode 100644 index 000000000..9fbf7651a --- /dev/null +++ b/server/providers/regional/weather-effects.js @@ -0,0 +1,108 @@ +import { makeRateLimiter, clientKey } from '../common/rate-limit.js'; +import { fetchRegionalWeather } from './weather.js'; +import { validRegionalPoint } from './query.js'; +import { coalesceProxyRequest } from '../common/http.js'; + +const WEATHER_EFFECTS_CACHE_MS = 5 * 60_000; + +const WEATHER_EFFECTS_STALE_MS = 30 * 60_000; + +const WEATHER_EFFECTS_MAX_CACHE = 180; + +const _weatherEffectsCache = new Map(); + +const _weatherEffectsInFlight = new Map(); + +const _weatherEffectsRateLimiter = makeRateLimiter({ windowMs: 60_000, max: 45, globalMax: 120 }); + +function trimWeatherEffectsCache() { + while (_weatherEffectsCache.size > WEATHER_EFFECTS_MAX_CACHE) { + const oldest = _weatherEffectsCache.keys().next().value; + if (oldest === undefined) break; + _weatherEffectsCache.delete(oldest); + } +} + +function weatherEffectsProxy() { + async function refresh(point, key) { + const weather = await fetchRegionalWeather(point); + if (!weather) throw new Error('Weather observation unavailable'); + const payload = { + status: 'ready', + retrievedAt: new Date().toISOString(), + coordinates: point, + weather, + }; + _weatherEffectsCache.set(key, { payload, cachedAt: Date.now() }); + trimWeatherEffectsCache(); + return payload; + } + + function install(middlewares) { + middlewares.use('/api/weather-effects', async (req, res) => { + if (req.method !== 'GET') { + res.writeHead(405, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Method Not Allowed' })); + return; + } + if (!_weatherEffectsRateLimiter(clientKey(req))) { + res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '10' }); + res.end(JSON.stringify({ error: 'Rate limit exceeded' })); + return; + } + const url = new URL(req.url || '', 'http://localhost'); + const point = validRegionalPoint(url.searchParams); + if (!point) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Valid latitude and longitude are required' })); + return; + } + const key = `${(Math.round(point.latitude * 10) / 10).toFixed(1)},${(Math.round(point.longitude * 10) / 10).toFixed(1)}`; + const now = Date.now(); + const cached = _weatherEffectsCache.get(key); + if (cached && now - cached.cachedAt <= WEATHER_EFFECTS_CACHE_MS) { + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=60', + 'X-Weather-Effects': 'HIT', + }); + res.end(JSON.stringify({ ...cached.payload, status: 'cached' })); + return; + } + const request = coalesceProxyRequest(_weatherEffectsInFlight, key, () => refresh(point, key)); + try { + const payload = await request.promise; + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=60', + 'X-Weather-Effects': request.shared ? 'INFLIGHT' : 'MISS', + }); + res.end(JSON.stringify(payload)); + } catch { + if (cached && now - cached.cachedAt <= WEATHER_EFFECTS_STALE_MS) { + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'X-Weather-Effects': 'STALE', + }); + res.end(JSON.stringify({ ...cached.payload, status: 'stale' })); + return; + } + res.writeHead(503, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Weather effects are temporarily unavailable' })); + } + }); + } + + return { + name: 'weather-effects-proxy', + configureServer(server) { + install(server.middlewares); + }, + configurePreviewServer(server) { + install(server.middlewares); + }, + }; +} + +export { weatherEffectsProxy }; diff --git a/server/providers/regional/weather.js b/server/providers/regional/weather.js new file mode 100644 index 000000000..0d4fdaa0b --- /dev/null +++ b/server/providers/regional/weather.js @@ -0,0 +1,23 @@ +import { fetchRegionalJson } from './http.js'; +import { normalizeRegionalWeather } from '../../../src/data/regionalBrief.js'; + +const WEATHER_EFFECTS_MAX_RESPONSE_BYTES = 512 * 1024; + +async function fetchRegionalWeather(point) { + const params = new URLSearchParams({ + latitude: point.latitude.toFixed(5), + longitude: point.longitude.toFixed(5), + current: 'temperature_2m,apparent_temperature,precipitation,weather_code,cloud_cover,wind_speed_10m,wind_direction_10m,visibility', + timezone: 'UTC', + }); + try { + const payload = await fetchRegionalJson(`https://api.open-meteo.com/v1/forecast?${params}`, { + maxBytes: WEATHER_EFFECTS_MAX_RESPONSE_BYTES, + }); + return normalizeRegionalWeather(payload); + } catch { + return null; + } +} + +export { fetchRegionalWeather };