mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-19 17:52:21 +00:00
Merge main and resolve the CCTV batch conflicts
Main released #1228 and #1233 and deleted their changelog fragments, so they stay deleted here. Main also rewrote the tracking arrival probe in scripts/track-regression.mjs, which supersedes this branch's version; main's file is kept unchanged.
This commit is contained in:
commit
d7db1090fd
4343 files changed
+644447
-86370
No files matched your search
@@ -0,0 +1,181 @@
|
||||
import { normalizeAcartiaSightings } from '../../src/layers/wildlifeSightings/records.js';
|
||||
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { admitSameSiteKeyless } from './common/same-site.js';
|
||||
import { providerUserAgent } from './common/userAgent.js';
|
||||
|
||||
const API_URL = 'https://acartia.io/api/v1/sightings/current';
|
||||
// All views share the provider's seven-day snapshot.
|
||||
const TTL_MS = 600_000;
|
||||
const STALE_MAX_MS = 24 * 60 * 60_000;
|
||||
const MAX_RESPONSE_BYTES = 2 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 15_000;
|
||||
const COOLDOWN_MS = 60_000;
|
||||
const COOLDOWN_MIN_MS = 5000;
|
||||
const COOLDOWN_MAX_MS = 60 * 60_000;
|
||||
const FAILURE_COOLDOWN_MS = 15_000;
|
||||
|
||||
/** Bound upstream Retry-After seconds or dates to five seconds–one hour. */
|
||||
export function acartiaRetryCooldownMs(raw, nowMs) {
|
||||
const value = typeof raw === 'string' ? raw.trim() : '';
|
||||
if (value) {
|
||||
const seconds = /^\d+(?:\.\d+)?$/.test(value) ? Number(value) : NaN;
|
||||
const deadline =
|
||||
!Number.isFinite(seconds) && /[a-z]/i.test(value)
|
||||
? Date.parse(value)
|
||||
: NaN;
|
||||
const delay = Number.isFinite(seconds) ? seconds * 1000 : deadline - nowMs;
|
||||
if (Number.isFinite(delay) && delay >= 0)
|
||||
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, delay));
|
||||
}
|
||||
return COOLDOWN_MS;
|
||||
}
|
||||
|
||||
/** Salish Sea coverage. Precise coordinates and comments never enter cache. */
|
||||
export function acartiaProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
cache = new Map(),
|
||||
warn = (...args) => console.warn(...args),
|
||||
} = {}) {
|
||||
const inFlight = new Map();
|
||||
const allow = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 60,
|
||||
globalMax: 600,
|
||||
now,
|
||||
});
|
||||
let cooldownUntil = 0;
|
||||
let cooldownStatus = 429;
|
||||
let warnedClamp = false;
|
||||
|
||||
const retrySeconds = () =>
|
||||
Math.max(1, Math.ceil((cooldownUntil - now()) / 1000));
|
||||
const unavailable = (status = 502) =>
|
||||
Object.assign(new Error('acartia_unavailable'), {
|
||||
status,
|
||||
retryAfterS: retrySeconds(),
|
||||
});
|
||||
|
||||
function coolDown(status, delay) {
|
||||
const until = now() + delay;
|
||||
if (until > cooldownUntil) {
|
||||
cooldownUntil = until;
|
||||
cooldownStatus = status;
|
||||
} else if (status === 429 && now() < cooldownUntil) cooldownStatus = 429;
|
||||
}
|
||||
|
||||
async function load() {
|
||||
if (now() < cooldownUntil) throw unavailable(cooldownStatus);
|
||||
const signal = AbortSignal.timeout(TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetchImpl(API_URL, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': providerUserAgent('wildlife-sightings'),
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (response.status === 429) {
|
||||
const delay = acartiaRetryCooldownMs(
|
||||
response.headers.get('retry-after'),
|
||||
now(),
|
||||
);
|
||||
if (delay === COOLDOWN_MAX_MS && !warnedClamp) {
|
||||
warnedClamp = true;
|
||||
warn('[acartia-proxy] upstream cooldown capped at one hour');
|
||||
}
|
||||
coolDown(429, delay);
|
||||
throw unavailable(429);
|
||||
}
|
||||
throw unavailable();
|
||||
}
|
||||
// This endpoint documents a JSON array, not a results/data envelope.
|
||||
const payload = await readResponseJsonCapped(
|
||||
response,
|
||||
MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const rows = normalizeAcartiaSightings(payload, now());
|
||||
// Only a non-array is malformed. A quiet feed, or one whose rows all fall
|
||||
// outside coverage or the species checks, is a valid empty snapshot.
|
||||
if (!rows) throw new Error('invalid_snapshot');
|
||||
return { fetchedAt: now(), rows };
|
||||
} catch (error) {
|
||||
if (error.status !== 429) coolDown(502, FAILURE_COOLDOWN_MS);
|
||||
warn('[acartia-proxy] upstream refresh failed');
|
||||
throw unavailable(error.status === 429 ? 429 : 502);
|
||||
}
|
||||
}
|
||||
|
||||
function age(entry) {
|
||||
if (!entry) return Infinity;
|
||||
const elapsed = now() - entry.savedAt;
|
||||
return elapsed < 0 ? Infinity : elapsed;
|
||||
}
|
||||
|
||||
async function acquire() {
|
||||
const previous = cache.get('sightings');
|
||||
if (age(previous) < TTL_MS) return { value: previous.value, stale: false };
|
||||
if (age(previous) > STALE_MAX_MS) cache.delete('sightings');
|
||||
try {
|
||||
const { promise } = coalesceProxyRequest(
|
||||
inFlight,
|
||||
'sightings',
|
||||
async () => {
|
||||
const value = await load();
|
||||
cache.set('sightings', { value, savedAt: now() });
|
||||
return value;
|
||||
},
|
||||
);
|
||||
return { value: await promise, stale: false };
|
||||
} catch (error) {
|
||||
if (age(previous) <= STALE_MAX_MS)
|
||||
return { value: previous.value, stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (status, value, { stale = false, retryAfterS } = {}) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...(retryAfterS ? { 'Retry-After': String(retryAfterS) } : {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const pathname = (req.url || '/').split('?')[0];
|
||||
if (pathname !== '/sightings') return json(404, { error: 'unknown_route' });
|
||||
if (!allow(clientKey(req)))
|
||||
return json(429, { error: 'rate_limited' }, { retryAfterS: 60 });
|
||||
try {
|
||||
const { value, stale } = await acquire();
|
||||
return json(200, stale ? { ...value, stale: true } : value, { stale });
|
||||
} catch (error) {
|
||||
return json(
|
||||
error.status === 429 ? 429 : 502,
|
||||
{ error: 'acartia_unavailable' },
|
||||
{ retryAfterS: error.retryAfterS },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'acartia',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/acartia', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/acartia', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { acartiaProxy, acartiaRetryCooldownMs } from './acartia.js';
|
||||
|
||||
const NOW = Date.parse('2026-11-04T18:05:00Z');
|
||||
const sighting = (id = 'synthetic-1') => ({
|
||||
entry_id: id,
|
||||
latitude: 48.12345,
|
||||
longitude: -123.12345,
|
||||
created: '2026-11-04 17:00:00',
|
||||
type: 'Orca',
|
||||
no_sighted: '5',
|
||||
photo_url: 'https://provider.example/photos/synthetic.jpg',
|
||||
data_source_name: 'Synthetic Whale Network',
|
||||
data_source_entity: 'Community reports',
|
||||
trusted: true,
|
||||
data_source_comments: '[email protected] +1-555-000-0000',
|
||||
});
|
||||
|
||||
function install(options = {}, hook = 'configureServer') {
|
||||
let handler;
|
||||
const plugin = acartiaProxy({ now: () => NOW, warn() {}, ...options });
|
||||
assert.equal(plugin.name, 'acartia');
|
||||
assert.equal(typeof plugin.configureServer, 'function');
|
||||
assert.equal(typeof plugin.configurePreviewServer, 'function');
|
||||
plugin[hook]({
|
||||
middlewares: {
|
||||
use(mount, callback) {
|
||||
assert.equal(mount, '/api/acartia');
|
||||
handler = callback;
|
||||
},
|
||||
},
|
||||
});
|
||||
return async (
|
||||
url = '/sightings',
|
||||
{ method = 'GET', peer = 'local', headers = {} } = {},
|
||||
) => {
|
||||
const res = {
|
||||
headers: {},
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
this.headers = headers;
|
||||
},
|
||||
setHeader(key, value) {
|
||||
this.headers[key] = value;
|
||||
},
|
||||
end(value) {
|
||||
this.status ??= this.statusCode;
|
||||
this.body = JSON.parse(value);
|
||||
},
|
||||
};
|
||||
await handler(
|
||||
{ url, method, headers, socket: { remoteAddress: peer } },
|
||||
res,
|
||||
);
|
||||
return res;
|
||||
};
|
||||
}
|
||||
|
||||
test('cache stores only supported marine mammals inside the coverage box', async () => {
|
||||
const cache = new Map();
|
||||
const request = install({
|
||||
cache,
|
||||
fetchImpl: async () =>
|
||||
Response.json([
|
||||
sighting('orca'),
|
||||
{ ...sighting('seal'), type: 'Harbor seal' },
|
||||
{ ...sighting('unknown'), type: 'Unknown' },
|
||||
{ ...sighting('fish'), type: 'Whale shark' },
|
||||
{ ...sighting('outside'), latitude: 37.8 },
|
||||
{ ...sighting('rounded-edge'), latitude: 45.49999 },
|
||||
]),
|
||||
});
|
||||
const result = await request();
|
||||
assert.equal(result.status, 200);
|
||||
for (const rows of [result.body.rows, cache.get('sightings').value.rows]) {
|
||||
assert.deepEqual(
|
||||
rows.map((row) => [row.stableId, row.cetacean]),
|
||||
[
|
||||
['acartia:orca', true],
|
||||
['acartia:seal', false],
|
||||
],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
test(`${hook}: documented array is normalized and coarsened before caching`, async () => {
|
||||
const cache = new Map();
|
||||
let calls = 0;
|
||||
const request = install(
|
||||
{
|
||||
cache,
|
||||
fetchImpl: async (url, options) => {
|
||||
calls += 1;
|
||||
assert.equal(url, 'https://acartia.io/api/v1/sightings/current');
|
||||
assert.equal(options.redirect, 'error');
|
||||
assert.ok(options.signal instanceof AbortSignal);
|
||||
assert.equal(options.headers.Accept, 'application/json');
|
||||
assert.equal(
|
||||
options.headers['User-Agent'],
|
||||
'gods-eye-view-wildlife-sightings-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
||||
);
|
||||
return Response.json([sighting()]);
|
||||
},
|
||||
},
|
||||
hook,
|
||||
);
|
||||
assert.equal(calls, 0);
|
||||
assert.equal(cache.size, 0);
|
||||
const result = await request(
|
||||
'/sightings?url=https://attacker.example&latitude=1',
|
||||
);
|
||||
assert.equal(result.status, 200);
|
||||
assert.equal(result.headers['Cache-Control'], 'no-store');
|
||||
assert.equal(result.body.fetchedAt, NOW);
|
||||
assert.equal(result.body.rows[0].stableId, 'acartia:synthetic-1');
|
||||
assert.equal(result.body.rows[0].lat, 48.12);
|
||||
assert.equal(result.body.rows[0].lon, -123.12);
|
||||
assert.equal(result.body.rows[0].provider.name, 'Synthetic Whale Network');
|
||||
const retained = cache.get('sightings');
|
||||
assert.equal(retained.value.rows[0].lat, 48.12);
|
||||
assert.equal(retained.value.rows[0].lon, -123.12);
|
||||
for (const value of [result.body, [...cache.values()]])
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(value),
|
||||
/private-observer|555|data_source_comments|48\.12345|123\.12345/,
|
||||
);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
}
|
||||
|
||||
test('gate, method and fixed route run before cache or upstream', async () => {
|
||||
const cache = new Map();
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
cache,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json([sighting()]);
|
||||
},
|
||||
});
|
||||
for (let i = 0; i < 65; i += 1) {
|
||||
assert.equal(
|
||||
(
|
||||
await request('/sightings', {
|
||||
headers: {
|
||||
host: 'localhost:5173',
|
||||
origin: 'https://foreign.example',
|
||||
'sec-fetch-site': 'cross-site',
|
||||
},
|
||||
})
|
||||
).status,
|
||||
403,
|
||||
);
|
||||
}
|
||||
const method = await request('/sightings', { method: 'POST' });
|
||||
assert.equal(method.status, 405);
|
||||
assert.equal(method.headers.Allow, 'GET');
|
||||
for (const route of ['/', '/unknown', '/sightings/extra']) {
|
||||
const result = await request(route);
|
||||
assert.equal(result.status, 404);
|
||||
assert.equal(result.body.error, 'unknown_route');
|
||||
}
|
||||
assert.equal(calls, 0);
|
||||
assert.equal(cache.size, 0);
|
||||
const sameOrigin = await request('/sightings', {
|
||||
headers: {
|
||||
host: 'gev.example',
|
||||
origin: 'https://gev.example',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'x-forwarded-proto': 'https',
|
||||
'x-forwarded-for': '203.0.113.9',
|
||||
},
|
||||
});
|
||||
assert.equal(sameOrigin.status, 200);
|
||||
});
|
||||
|
||||
test('request 61 is locally limited even with a fresh cache', async () => {
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json([sighting()]);
|
||||
},
|
||||
});
|
||||
for (let i = 0; i < 60; i += 1) assert.equal((await request()).status, 200);
|
||||
const limited = await request();
|
||||
assert.equal(limited.status, 429);
|
||||
assert.equal(limited.body.error, 'rate_limited');
|
||||
assert.equal(limited.headers['Retry-After'], '60');
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test('one global snapshot single-flights and stays fresh for ten minutes', async () => {
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
let release;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1)
|
||||
await new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
return Response.json([sighting()]);
|
||||
},
|
||||
});
|
||||
const first = request();
|
||||
const concurrent = request();
|
||||
await new Promise(setImmediate);
|
||||
assert.equal(calls, 1);
|
||||
release();
|
||||
assert.deepEqual((await first).body, (await concurrent).body);
|
||||
at += 599_999;
|
||||
await request();
|
||||
assert.equal(calls, 1);
|
||||
at += 1;
|
||||
await request();
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
|
||||
test('a valid empty or fully filtered snapshot is a fresh success that clears old rows', async () => {
|
||||
for (const quiet of [
|
||||
[],
|
||||
[null],
|
||||
[{ ...sighting(), data_source_name: '' }],
|
||||
[{ ...sighting(), latitude: 37.8 }],
|
||||
[{ ...sighting(), type: 'Unknown' }],
|
||||
]) {
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
let empty = false;
|
||||
const cache = new Map();
|
||||
const request = install({
|
||||
cache,
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json(empty ? quiet : [sighting()]);
|
||||
},
|
||||
});
|
||||
assert.equal((await request()).body.rows.length, 1);
|
||||
empty = true;
|
||||
at += 600_001;
|
||||
const result = await request();
|
||||
assert.equal(result.status, 200);
|
||||
assert.deepEqual(result.body, { fetchedAt: at, rows: [] });
|
||||
assert.equal(result.headers['X-Data-Stale'], undefined);
|
||||
assert.equal(result.headers['Retry-After'], undefined);
|
||||
assert.deepEqual(cache.get('sightings').value.rows, []);
|
||||
// The quiet feed keeps the normal ten-minute cadence, not a failure retry.
|
||||
at += 599_999;
|
||||
assert.equal((await request()).status, 200);
|
||||
assert.equal(calls, 2);
|
||||
at += 1;
|
||||
await request();
|
||||
assert.equal(calls, 3);
|
||||
const cold = install({ fetchImpl: async () => Response.json(quiet) });
|
||||
const coldResult = await cold();
|
||||
assert.equal(coldResult.status, 200);
|
||||
assert.deepEqual(coldResult.body, { fetchedAt: NOW, rows: [] });
|
||||
}
|
||||
});
|
||||
|
||||
test('an invented wrapped snapshot never replaces the cache', async () => {
|
||||
const bad = { results: [sighting()] };
|
||||
let at = NOW;
|
||||
let fail = false;
|
||||
const cache = new Map();
|
||||
const request = install({
|
||||
cache,
|
||||
now: () => at,
|
||||
fetchImpl: async () => Response.json(fail ? bad : [sighting()]),
|
||||
});
|
||||
const fresh = await request();
|
||||
const retained = cache.get('sightings');
|
||||
fail = true;
|
||||
at += 600_001;
|
||||
const stale = await request();
|
||||
assert.equal(stale.status, 200);
|
||||
assert.equal(stale.headers['X-Data-Stale'], 'true');
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.body.fetchedAt, fresh.body.fetchedAt);
|
||||
assert.equal(cache.get('sightings'), retained);
|
||||
const coldCache = new Map();
|
||||
const cold = install({
|
||||
cache: coldCache,
|
||||
fetchImpl: async () => Response.json(bad),
|
||||
});
|
||||
const result = await cold();
|
||||
assert.equal(result.status, 502);
|
||||
assert.equal(result.body.error, 'acartia_unavailable');
|
||||
assert.equal(result.headers['Retry-After'], '15');
|
||||
assert.equal(coldCache.size, 0);
|
||||
});
|
||||
|
||||
test('failure serves stale for at most 24 hours and never reports a cold success', async () => {
|
||||
let at = NOW;
|
||||
let fail = false;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
if (fail) throw new Error('private upstream detail');
|
||||
return Response.json([sighting()]);
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
fail = true;
|
||||
at += 23 * 60 * 60_000 + 59 * 60_000;
|
||||
const stale = await request();
|
||||
assert.equal(stale.status, 200);
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.body.fetchedAt, first.body.fetchedAt);
|
||||
at += 2 * 60_000;
|
||||
const expired = await request();
|
||||
assert.equal(expired.status, 502);
|
||||
assert.deepEqual(expired.body, { error: 'acartia_unavailable' });
|
||||
});
|
||||
|
||||
test('upstream 429 honors Retry-After, caps it, and spends no requests while cooling down', async () => {
|
||||
assert.equal(acartiaRetryCooldownMs('2', NOW), 5000);
|
||||
assert.equal(acartiaRetryCooldownMs('999999', NOW), 3_600_000);
|
||||
assert.equal(
|
||||
acartiaRetryCooldownMs(new Date(NOW + 120_000).toUTCString(), NOW),
|
||||
120_000,
|
||||
);
|
||||
assert.equal(acartiaRetryCooldownMs(null, NOW), 60_000);
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
const warnings = [];
|
||||
const request = install({
|
||||
now: () => at,
|
||||
warn: (message) => warnings.push(message),
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return new Response('limit', {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '999999' },
|
||||
});
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
assert.equal(first.status, 429);
|
||||
assert.equal(first.body.error, 'acartia_unavailable');
|
||||
assert.equal(first.headers['Retry-After'], '3600');
|
||||
at += 5000;
|
||||
const cooling = await request();
|
||||
assert.equal(cooling.headers['Retry-After'], '3595');
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(
|
||||
warnings.filter((message) => message.includes('capped')).length,
|
||||
1,
|
||||
);
|
||||
at += 3_600_000;
|
||||
await request();
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(
|
||||
warnings.filter((message) => message.includes('capped')).length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test('upstream 429 also serves a prior snapshot stale', async () => {
|
||||
let at = NOW;
|
||||
let fail = false;
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return fail
|
||||
? new Response('limit', { status: 429 })
|
||||
: Response.json([sighting()]);
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
fail = true;
|
||||
at += 600_001;
|
||||
const stale = await request();
|
||||
assert.equal(stale.status, 200);
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.body.fetchedAt, first.body.fetchedAt);
|
||||
at += 1000;
|
||||
assert.equal((await request()).body.stale, true);
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
|
||||
test('bad JSON, 5xx and capped bodies start a 15 second failure cooldown', async () => {
|
||||
for (const bad of [
|
||||
() => new Response('{malformed'),
|
||||
() => new Response('private upstream message', { status: 503 }),
|
||||
() =>
|
||||
new Response('{}', {
|
||||
headers: { 'Content-Length': String(2 * 1024 * 1024 + 1) },
|
||||
}),
|
||||
]) {
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return calls === 1 ? bad() : Response.json([sighting()]);
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
assert.equal(first.status, 502);
|
||||
assert.deepEqual(first.body, { error: 'acartia_unavailable' });
|
||||
assert.equal(first.headers['Retry-After'], '15');
|
||||
at += 14_999;
|
||||
assert.equal((await request()).headers['Retry-After'], '1');
|
||||
assert.equal(calls, 1);
|
||||
at += 1;
|
||||
assert.equal((await request()).status, 200);
|
||||
assert.equal(calls, 2);
|
||||
}
|
||||
});
|
||||
@@ -1,4 +1,7 @@
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
import {
|
||||
coalesceProxyRequest,
|
||||
readResponseTextCapped,
|
||||
} from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
|
||||
@@ -19,6 +22,7 @@ import { readResponseTextCapped } from '../common/http.js';
|
||||
export function adsbLolProxy() {
|
||||
/** @type {string|null} Cached upstream JSON body. */
|
||||
let _cache = null;
|
||||
const inFlight = new Map();
|
||||
/** @type {number} Epoch-ms when the cache was populated. */
|
||||
let _cacheAt = 0;
|
||||
/** @type {number} Epoch-ms until which upstream is not contacted. */
|
||||
@@ -79,6 +83,15 @@ export function adsbLolProxy() {
|
||||
// This mount also matches its /track route, served by its own handler.
|
||||
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
|
||||
return next();
|
||||
// After the /track hand-off, so /track keeps its own method handling.
|
||||
if (req.method !== 'GET') {
|
||||
res.writeHead(405, {
|
||||
'Content-Type': 'application/json',
|
||||
Allow: 'GET',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
if (_cache && now - _cacheAt < CACHE_MS) {
|
||||
serve(res, 200, _cache, 'HIT');
|
||||
@@ -104,60 +117,73 @@ export function adsbLolProxy() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
|
||||
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
|
||||
signal,
|
||||
});
|
||||
if (upstream.ok) {
|
||||
const refresh = coalesceProxyRequest(inFlight, 'global', async () => {
|
||||
const result = (status, body, cacheStatus, extra = {}) => ({
|
||||
status,
|
||||
body,
|
||||
cacheStatus,
|
||||
extra,
|
||||
});
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
|
||||
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
|
||||
signal,
|
||||
});
|
||||
if (upstream.ok) {
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
_cache = body;
|
||||
_cacheAt = Date.now();
|
||||
_cooldownUntil = 0;
|
||||
_cooldownStatus = 0;
|
||||
return result(200, body, 'MISS');
|
||||
}
|
||||
if (upstream.status === 429 || upstream.status >= 500) {
|
||||
const failedAt = Date.now();
|
||||
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
|
||||
_cooldownStatus = upstream.status;
|
||||
console.warn(
|
||||
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
|
||||
);
|
||||
if (_cache) {
|
||||
// Do not wait for a failing response body before serving usable data.
|
||||
upstream.body?.cancel().catch(() => {});
|
||||
return result(200, _cache, 'STALE', {
|
||||
'X-ADS-B-Upstream-Status': String(upstream.status),
|
||||
'X-ADS-B-Retry-After-Seconds': String(
|
||||
Math.ceil((_cooldownUntil - Date.now()) / 1000),
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
_cache = body;
|
||||
_cacheAt = Date.now();
|
||||
_cooldownUntil = 0;
|
||||
_cooldownStatus = 0;
|
||||
serve(res, 200, body, 'MISS');
|
||||
return;
|
||||
}
|
||||
if (upstream.status === 429 || upstream.status >= 500) {
|
||||
const failedAt = Date.now();
|
||||
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
|
||||
_cooldownStatus = upstream.status;
|
||||
console.warn(
|
||||
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
|
||||
return result(
|
||||
upstream.status,
|
||||
body,
|
||||
'MISS',
|
||||
_cooldownUntil > Date.now()
|
||||
? {
|
||||
'Retry-After': String(
|
||||
Math.ceil((_cooldownUntil - Date.now()) / 1000),
|
||||
),
|
||||
}
|
||||
: {},
|
||||
);
|
||||
if (_cache) {
|
||||
// Do not wait for a failing response body before serving usable data.
|
||||
upstream.body?.cancel().catch(() => {});
|
||||
serve(res, 200, _cache, 'STALE', {
|
||||
'X-ADS-B-Upstream-Status': String(upstream.status),
|
||||
'X-ADS-B-Retry-After-Seconds': String(
|
||||
Math.ceil((_cooldownUntil - Date.now()) / 1000),
|
||||
),
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
});
|
||||
const response = await refresh.promise;
|
||||
serve(
|
||||
res,
|
||||
upstream.status,
|
||||
body,
|
||||
'MISS',
|
||||
_cooldownUntil > Date.now()
|
||||
? {
|
||||
'Retry-After': String(
|
||||
Math.ceil((_cooldownUntil - Date.now()) / 1000),
|
||||
),
|
||||
}
|
||||
: {},
|
||||
response.status,
|
||||
response.body,
|
||||
response.cacheStatus,
|
||||
response.extra,
|
||||
);
|
||||
} catch (e) {
|
||||
console.error('[adsb.lol Proxy]', e.message);
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback.js';
|
||||
import { encodedBody, gzipBody } from '../common/compress.js';
|
||||
import {
|
||||
coalesceProxyRequest,
|
||||
readResponseJsonCapped,
|
||||
@@ -16,6 +19,11 @@ let _openskyTokenExpiry = 0;
|
||||
let _openskyTokenPromise = null;
|
||||
/** @type {string|null} Cached upstream response body (JSON text). */
|
||||
let _openskyCacheBody = null;
|
||||
/**
|
||||
* @type {Buffer|null} Gzip of `_openskyCacheBody`, written together with it.
|
||||
* Null when the body is not worth compressing.
|
||||
*/
|
||||
let _openskyCacheGzip = null;
|
||||
/** @type {number} HTTP status of the cached response. */
|
||||
let _openskyCacheStatus = 0;
|
||||
/** @type {number} Epoch-ms when the response was cached. */
|
||||
@@ -67,15 +75,24 @@ async function fetchOpenSkyStates(headers) {
|
||||
try {
|
||||
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
|
||||
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
|
||||
if (upstream.status === 429) {
|
||||
if (upstream.status === 429 || upstream.status >= 500) {
|
||||
// Start the cooldown from the headers and drop the rejected body
|
||||
// unread, so a stalled or oversized 429 body cannot skip it.
|
||||
_openskyCooldownUntil =
|
||||
Date.now() + openSkyRateLimitCooldownMs(upstream);
|
||||
Date.now() +
|
||||
(upstream.status === 429
|
||||
? openSkyRateLimitCooldownMs(upstream)
|
||||
: 15000);
|
||||
_openskyCooldownStatus = upstream.status;
|
||||
void upstream.body?.cancel().catch(() => {});
|
||||
return {
|
||||
upstream,
|
||||
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
|
||||
body: JSON.stringify({
|
||||
error:
|
||||
upstream.status === 429
|
||||
? 'OpenSky rate limited.'
|
||||
: 'OpenSky upstream unavailable.',
|
||||
}),
|
||||
};
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
@@ -109,6 +126,8 @@ async function fetchOpenSkyStates(headers) {
|
||||
let _openskyTtlMs = OPENSKY_CACHE_MS;
|
||||
/** @type {number} Epoch-ms before which no upstream fetch is attempted. */
|
||||
let _openskyCooldownUntil = 0;
|
||||
let _openskyCooldownStatus = 429;
|
||||
const _openskySnapshotInFlight = new Map();
|
||||
/**
|
||||
* Picks the cache TTL from the remaining daily credit budget.
|
||||
* Client polls every 30 s, so tiers ≤30 s cost the same 480 credits/h; the
|
||||
@@ -126,6 +145,10 @@ function openskyAdaptiveTtlMs(remaining) {
|
||||
}
|
||||
/** @type {boolean} Guards duplicate auth-failure warnings in logs. */
|
||||
let _openskyAuthWarned = false;
|
||||
/** One warning per process when OPENSKY_CREDENTIALS_FILE cannot be read. */
|
||||
let _openskyCredentialsFileWarned = false;
|
||||
/** One warning per process when OPENSKY_AUTH_MODE=basic is requested. */
|
||||
let _openskyBasicRetiredWarned = false;
|
||||
/** @type {boolean} Guards duplicate invalid-auth-mode warnings. */
|
||||
let _openskyAuthModeWarned = false;
|
||||
/** Default auth mode when OPENSKY_AUTH_MODE env is unset. */
|
||||
@@ -145,6 +168,48 @@ const ADSBLOL_POINT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
// current instead of coasting a stale worldwide frame indefinitely.
|
||||
const OPENSKY_SOURCE_STALE_MS = 120_000;
|
||||
|
||||
/**
|
||||
* The OpenSky API client for OAuth: OPENSKY_CLIENT_ID and
|
||||
* OPENSKY_CLIENT_SECRET, or else the credentials.json that OpenSky's account
|
||||
* page downloads, named by OPENSKY_CREDENTIALS_FILE ("~" is the home
|
||||
* directory). The file is read on each token refresh, so a replaced file is
|
||||
* picked up without a restart.
|
||||
*
|
||||
* @param {Record<string, string|undefined>} [env]
|
||||
* @param {(path: string) => string} [readFile]
|
||||
* @returns {{ clientId: string, clientSecret: string } | null}
|
||||
*/
|
||||
export function openSkyClientCredentials(
|
||||
env = process.env,
|
||||
readFile = (path) => fs.readFileSync(path, 'utf8'),
|
||||
) {
|
||||
const clientId = String(env.OPENSKY_CLIENT_ID || '').trim();
|
||||
const clientSecret = String(env.OPENSKY_CLIENT_SECRET || '').trim();
|
||||
if (clientId && clientSecret) return { clientId, clientSecret };
|
||||
const file = String(env.OPENSKY_CREDENTIALS_FILE || '').trim();
|
||||
if (!file) return null;
|
||||
const path = file.replace(/^~(?=$|[\\/])/, os.homedir());
|
||||
let raw;
|
||||
try {
|
||||
raw = JSON.parse(readFile(path));
|
||||
} catch {
|
||||
if (!_openskyCredentialsFileWarned) {
|
||||
console.warn(
|
||||
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
|
||||
);
|
||||
_openskyCredentialsFileWarned = true;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const fileId = String(raw?.clientId ?? raw?.client_id ?? '').trim();
|
||||
const fileSecret = String(
|
||||
raw?.clientSecret ?? raw?.client_secret ?? '',
|
||||
).trim();
|
||||
return fileId && fileSecret
|
||||
? { clientId: fileId, clientSecret: fileSecret }
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Obtain a valid OpenSky OAuth2 bearer token, refreshing if needed.
|
||||
*
|
||||
@@ -163,9 +228,9 @@ export async function getOpenSkyToken() {
|
||||
// return the same promise instead of issuing a duplicate token request
|
||||
if (_openskyTokenPromise) return _openskyTokenPromise;
|
||||
|
||||
const clientId = process.env.OPENSKY_CLIENT_ID;
|
||||
const clientSecret = process.env.OPENSKY_CLIENT_SECRET;
|
||||
if (!clientId || !clientSecret) return null;
|
||||
const credentials = openSkyClientCredentials();
|
||||
if (!credentials) return null;
|
||||
const { clientId, clientSecret } = credentials;
|
||||
|
||||
// Wrap the async token fetch in a shared promise stored in _openskyTokenPromise
|
||||
_openskyTokenPromise = (async () => {
|
||||
@@ -340,8 +405,12 @@ async function fetchAdsbLolPointFallback(req) {
|
||||
ADSBLOL_POINT_MAX_RESPONSE_BYTES,
|
||||
);
|
||||
const normalized = normalizeAdsbLolPointResponse(payload);
|
||||
const body = JSON.stringify(normalized);
|
||||
const record = {
|
||||
body: JSON.stringify(normalized),
|
||||
body,
|
||||
// Compressed once per entry; ADSBLOL_POINT_CACHE_MAX bounds the
|
||||
// extra memory.
|
||||
gzip: await gzipBody(body),
|
||||
cachedAt: Date.now(),
|
||||
count: normalized.states.length,
|
||||
};
|
||||
@@ -370,6 +439,7 @@ async function fetchAdsbLolPointFallback(req) {
|
||||
async function serveAdsbLolPointFallback(req, res, requestedMode, reason) {
|
||||
const fallback = await fetchAdsbLolPointFallback(req);
|
||||
if (!fallback) return false;
|
||||
const encoded = encodedBody(req, fallback.body, fallback.gzip);
|
||||
res.writeHead(200, {
|
||||
...buildOpenSkyHeaders({
|
||||
cacheStatus: fallback.cacheStatus,
|
||||
@@ -380,8 +450,9 @@ async function serveAdsbLolPointFallback(req, res, requestedMode, reason) {
|
||||
'X-Flight-Source': 'adsb.lol',
|
||||
'X-Flight-Coverage': `${ADSBLOL_POINT_RADIUS_NM}nm regional fallback`,
|
||||
'X-Flight-Count': String(fallback.count),
|
||||
...encoded.headers,
|
||||
});
|
||||
res.end(fallback.body);
|
||||
res.end(encoded.body);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -406,8 +477,10 @@ function openSkySourceIsStale(sourceEpochMs, now = Date.now()) {
|
||||
*
|
||||
* Supports four auth modes controlled by OPENSKY_AUTH_MODE env:
|
||||
* - 'oauth' (default) — client_credentials bearer token
|
||||
* - 'basic' — HTTP Basic with OPENSKY_USERNAME / OPENSKY_PASSWORD
|
||||
* - 'auto' — try OAuth first, fall back to Basic, then anon
|
||||
* - 'basic' — retired: OpenSky stopped accepting HTTP Basic (username and
|
||||
* password) on 2026-03-18, so this requests anonymously and
|
||||
* warns once
|
||||
* - 'auto' — try OAuth first, then anon
|
||||
* - 'anon' — no credentials
|
||||
*
|
||||
* Successful responses are cached for OPENSKY_CACHE_MS (~9 s). On
|
||||
@@ -423,6 +496,15 @@ export function openSkyProxy() {
|
||||
// This mount also matches its /track route, served by its own handler.
|
||||
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
|
||||
return next();
|
||||
// After the /track hand-off, so /track keeps its own method handling.
|
||||
if (req.method !== 'GET') {
|
||||
res.writeHead(405, {
|
||||
'Content-Type': 'application/json',
|
||||
Allow: 'GET',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
res.setHeader('X-Feed-Source', 'OpenSky Network');
|
||||
const requestedMode = normalizeOpenSkyAuthMode(
|
||||
process.env.OPENSKY_AUTH_MODE,
|
||||
@@ -453,14 +535,20 @@ export function openSkyProxy() {
|
||||
reason: 'cached',
|
||||
};
|
||||
const isStale = now - _openskyCacheTime >= _openskyTtlMs;
|
||||
res.writeHead(
|
||||
_openskyCacheStatus || 200,
|
||||
buildOpenSkyHeaders({
|
||||
const encoded = encodedBody(
|
||||
req,
|
||||
_openskyCacheBody,
|
||||
_openskyCacheGzip,
|
||||
);
|
||||
res.writeHead(_openskyCacheStatus || 200, {
|
||||
...buildOpenSkyHeaders({
|
||||
cacheStatus: isStale ? 'STALE' : 'HIT',
|
||||
requestedMode: cachedMeta.requestedMode || requestedMode,
|
||||
usedMode: cachedMeta.usedMode || 'unknown',
|
||||
reason: isStale
|
||||
? 'rate_limited_serving_stale'
|
||||
? _openskyCooldownStatus === 429
|
||||
? 'rate_limited_serving_stale'
|
||||
: `upstream_http_${_openskyCooldownStatus}_serving_stale`
|
||||
: cachedMeta.reason || 'cached',
|
||||
staleSeconds: isStale
|
||||
? (now - _openskyCacheTime) / 1000
|
||||
@@ -469,8 +557,9 @@ export function openSkyProxy() {
|
||||
? (_openskyCooldownUntil - now) / 1000
|
||||
: undefined,
|
||||
}),
|
||||
);
|
||||
res.end(_openskyCacheBody);
|
||||
...encoded.headers,
|
||||
});
|
||||
res.end(encoded.body);
|
||||
return;
|
||||
}
|
||||
// Cooling down with nothing cached (cold start into a rate limit):
|
||||
@@ -487,80 +576,118 @@ export function openSkyProxy() {
|
||||
)
|
||||
return;
|
||||
res.writeHead(
|
||||
429,
|
||||
_openskyCooldownStatus,
|
||||
buildOpenSkyHeaders({
|
||||
cacheStatus: 'COOLDOWN',
|
||||
requestedMode,
|
||||
usedMode: 'none',
|
||||
reason: 'rate_limited',
|
||||
reason:
|
||||
_openskyCooldownStatus === 429
|
||||
? 'rate_limited'
|
||||
: `upstream_http_${_openskyCooldownStatus}`,
|
||||
retryAfterSeconds: (_openskyCooldownUntil - now) / 1000,
|
||||
}),
|
||||
);
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'OpenSky rate limited; proxy cooling down.',
|
||||
error: 'OpenSky upstream unavailable; proxy cooling down.',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const basicUser = process.env.OPENSKY_USERNAME || '';
|
||||
const basicPass = process.env.OPENSKY_PASSWORD || '';
|
||||
const hasBasicCreds = Boolean(basicUser && basicPass);
|
||||
const headers = { Accept: 'application/json' };
|
||||
let usedMode = 'anon';
|
||||
let reason = 'forced_anonymous';
|
||||
const refresh = coalesceProxyRequest(
|
||||
_openskySnapshotInFlight,
|
||||
'global',
|
||||
async () => {
|
||||
const headers = { Accept: 'application/json' };
|
||||
let usedMode = 'anon';
|
||||
let reason = 'forced_anonymous';
|
||||
|
||||
if (requestedMode === 'basic') {
|
||||
if (hasBasicCreds) {
|
||||
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
|
||||
usedMode = 'basic';
|
||||
reason = 'basic_credentials';
|
||||
} else {
|
||||
reason = 'missing_basic_creds';
|
||||
}
|
||||
} else if (requestedMode === 'oauth') {
|
||||
const token = await getOpenSkyToken();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
usedMode = 'oauth';
|
||||
reason = 'oauth_token';
|
||||
} else {
|
||||
reason = 'oauth_invalid_or_missing';
|
||||
}
|
||||
} else if (requestedMode === 'auto') {
|
||||
const token = await getOpenSkyToken();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
usedMode = 'oauth';
|
||||
reason = 'oauth_token';
|
||||
} else if (hasBasicCreds) {
|
||||
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
|
||||
usedMode = 'basic';
|
||||
reason = 'oauth_unavailable_fallback_basic';
|
||||
} else {
|
||||
reason = 'missing_oauth_and_basic_creds';
|
||||
}
|
||||
}
|
||||
if (requestedMode === 'basic') {
|
||||
// OpenSky no longer accepts a Basic header, so sending
|
||||
// OPENSKY_USERNAME / OPENSKY_PASSWORD would only fail; ask
|
||||
// anonymously and say once what to configure instead.
|
||||
if (!_openskyBasicRetiredWarned) {
|
||||
console.warn(
|
||||
'[OpenSky] OPENSKY_AUTH_MODE=basic: OpenSky retired username/password sign-in on 2026-03-18, so flights load anonymously (rate-limited). Use OAuth client credentials: OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
|
||||
);
|
||||
_openskyBasicRetiredWarned = true;
|
||||
}
|
||||
reason = 'basic_retired';
|
||||
} else if (requestedMode === 'oauth') {
|
||||
const token = await getOpenSkyToken();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
usedMode = 'oauth';
|
||||
reason = 'oauth_token';
|
||||
} else {
|
||||
reason = 'oauth_invalid_or_missing';
|
||||
}
|
||||
} else if (requestedMode === 'auto') {
|
||||
const token = await getOpenSkyToken();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
usedMode = 'oauth';
|
||||
reason = 'oauth_token';
|
||||
} else {
|
||||
reason = 'missing_oauth_creds';
|
||||
}
|
||||
}
|
||||
|
||||
let { upstream, body } = await fetchOpenSkyStates(headers);
|
||||
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
|
||||
if (
|
||||
(upstream.status === 401 || upstream.status === 403) &&
|
||||
requestedMode === 'auto' &&
|
||||
usedMode === 'oauth' &&
|
||||
hasBasicCreds
|
||||
) {
|
||||
const retryHeaders = {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
|
||||
};
|
||||
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
|
||||
usedMode = 'basic';
|
||||
reason = 'oauth_rejected_fallback_basic';
|
||||
}
|
||||
const { upstream, body } = await fetchOpenSkyStates(headers);
|
||||
|
||||
const sourceEpochMs = upstream.ok ? openSkySourceEpochMs(body) : null;
|
||||
const sourceEpochMs = upstream.ok
|
||||
? openSkySourceEpochMs(body)
|
||||
: null;
|
||||
// Refine the reason string to reflect the actual outcome
|
||||
if (upstream.ok && reason === 'forced_anonymous') {
|
||||
reason = 'anonymous_ok';
|
||||
} else if (
|
||||
upstream.ok &&
|
||||
usedMode === 'oauth' &&
|
||||
reason === 'oauth_token'
|
||||
) {
|
||||
reason = 'oauth_ok';
|
||||
}
|
||||
|
||||
// Only cache successful responses — error responses (401/403/429/5xx)
|
||||
// should not be served from cache on subsequent requests
|
||||
if (upstream.ok) {
|
||||
// Compress before writing either slot, so a request arriving
|
||||
// mid-refresh still reads a body with its own frame.
|
||||
const frame = await gzipBody(body);
|
||||
_openskyCacheBody = body;
|
||||
_openskyCacheGzip = frame;
|
||||
_openskyCacheStatus = upstream.status;
|
||||
_openskyCacheTime = now;
|
||||
_openskyCacheSourceEpochMs = sourceEpochMs;
|
||||
_openskyCacheMeta = {
|
||||
requestedMode,
|
||||
usedMode,
|
||||
reason,
|
||||
};
|
||||
// Credit governor: adapt the cache TTL to the remaining daily
|
||||
// budget so a continuously-open app stretches its polls instead of
|
||||
// exhausting the quota mid-day. Success also clears any cooldown.
|
||||
const remaining = Number(
|
||||
upstream.headers.get('x-rate-limit-remaining'),
|
||||
);
|
||||
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
|
||||
_openskyCooldownUntil = 0;
|
||||
}
|
||||
|
||||
return {
|
||||
upstream,
|
||||
body,
|
||||
usedMode,
|
||||
reason,
|
||||
sourceEpochMs,
|
||||
};
|
||||
},
|
||||
);
|
||||
let { upstream, body, usedMode, reason, sourceEpochMs } =
|
||||
await refresh.promise;
|
||||
if (
|
||||
upstream.ok &&
|
||||
openSkySourceIsStale(sourceEpochMs, now) &&
|
||||
@@ -571,35 +698,36 @@ export function openSkyProxy() {
|
||||
'opensky_snapshot_stale_regional_fallback',
|
||||
))
|
||||
) {
|
||||
// Keep the last global snapshot available as a fail-soft cache,
|
||||
// but do not label or render it as a fresh live result.
|
||||
_openskyCacheBody = body;
|
||||
_openskyCacheStatus = upstream.status;
|
||||
_openskyCacheTime = now;
|
||||
_openskyCacheSourceEpochMs = sourceEpochMs;
|
||||
_openskyCacheMeta = { requestedMode, usedMode, reason };
|
||||
return;
|
||||
}
|
||||
if (upstream.status === 429) {
|
||||
reason = 'rate_limited';
|
||||
if (upstream.status === 429 || upstream.status >= 500) {
|
||||
reason =
|
||||
upstream.status === 429
|
||||
? 'rate_limited'
|
||||
: `upstream_http_${upstream.status}`;
|
||||
// Credit governor: fetchOpenSkyStates already started the cooldown
|
||||
// from OpenSky's retry-after — no upstream attempts until then.
|
||||
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
|
||||
const cooldownMs = Math.max(0, _openskyCooldownUntil - Date.now());
|
||||
// Serve the last-good body instead of the 429 when we have one —
|
||||
// the layer keeps rendering (STALE-cued) instead of dying.
|
||||
if (_openskyCacheBody && _openskyCacheStatus === 200) {
|
||||
res.writeHead(
|
||||
200,
|
||||
buildOpenSkyHeaders({
|
||||
const encoded = encodedBody(
|
||||
req,
|
||||
_openskyCacheBody,
|
||||
_openskyCacheGzip,
|
||||
);
|
||||
res.writeHead(200, {
|
||||
...buildOpenSkyHeaders({
|
||||
cacheStatus: 'STALE',
|
||||
requestedMode,
|
||||
usedMode,
|
||||
reason: 'rate_limited_serving_stale',
|
||||
reason: `${reason}_serving_stale`,
|
||||
staleSeconds: (now - _openskyCacheTime) / 1000,
|
||||
retryAfterSeconds: cooldownMs / 1000,
|
||||
}),
|
||||
);
|
||||
res.end(_openskyCacheBody);
|
||||
...encoded.headers,
|
||||
});
|
||||
res.end(encoded.body);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -615,35 +743,24 @@ export function openSkyProxy() {
|
||||
}
|
||||
|
||||
if (upstream.status === 401 || upstream.status === 403) {
|
||||
if (requestedMode === 'basic' && !hasBasicCreds) {
|
||||
if (requestedMode === 'oauth' && usedMode !== 'oauth') {
|
||||
body = JSON.stringify({
|
||||
error:
|
||||
'OpenSky auth missing. Basic mode requires OPENSKY_USERNAME and OPENSKY_PASSWORD.',
|
||||
});
|
||||
reason = 'missing_basic_creds';
|
||||
} else if (requestedMode === 'oauth' && usedMode !== 'oauth') {
|
||||
body = JSON.stringify({
|
||||
error:
|
||||
'OpenSky auth invalid. OAuth mode requires valid OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET.',
|
||||
'OpenSky auth invalid. OAuth mode requires valid OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
|
||||
});
|
||||
reason = 'oauth_invalid_or_missing';
|
||||
} else if (usedMode === 'basic') {
|
||||
body = JSON.stringify({
|
||||
error: 'OpenSky auth invalid. Username/password were rejected.',
|
||||
});
|
||||
reason = 'basic_invalid_credentials';
|
||||
} else if (usedMode === 'oauth') {
|
||||
body = JSON.stringify({
|
||||
error:
|
||||
'OpenSky auth invalid. OAuth client credentials were rejected.',
|
||||
});
|
||||
reason = 'oauth_invalid_credentials';
|
||||
} else if (requestedMode === 'auto' && !hasBasicCreds) {
|
||||
} else if (requestedMode === 'auto') {
|
||||
body = JSON.stringify({
|
||||
error:
|
||||
'OpenSky auth missing. Provide basic credentials or valid OAuth client credentials.',
|
||||
'OpenSky auth missing. Provide OAuth client credentials: OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
|
||||
});
|
||||
reason = 'missing_oauth_and_basic_creds';
|
||||
reason = 'missing_oauth_creds';
|
||||
} else {
|
||||
body = JSON.stringify({
|
||||
error: 'OpenSky auth required.',
|
||||
@@ -652,55 +769,23 @@ export function openSkyProxy() {
|
||||
}
|
||||
}
|
||||
|
||||
// Refine the reason string to reflect the actual outcome
|
||||
if (upstream.ok && reason === 'forced_anonymous') {
|
||||
reason = 'anonymous_ok';
|
||||
} else if (
|
||||
upstream.ok &&
|
||||
usedMode === 'basic' &&
|
||||
reason === 'basic_credentials'
|
||||
) {
|
||||
reason = 'basic_ok';
|
||||
} else if (
|
||||
upstream.ok &&
|
||||
usedMode === 'oauth' &&
|
||||
reason === 'oauth_token'
|
||||
) {
|
||||
reason = 'oauth_ok';
|
||||
}
|
||||
|
||||
// Only cache successful responses — error responses (401/403/429/5xx)
|
||||
// should not be served from cache on subsequent requests
|
||||
if (upstream.ok) {
|
||||
_openskyCacheBody = body;
|
||||
_openskyCacheStatus = upstream.status;
|
||||
_openskyCacheTime = now;
|
||||
_openskyCacheSourceEpochMs = sourceEpochMs;
|
||||
_openskyCacheMeta = {
|
||||
requestedMode,
|
||||
usedMode,
|
||||
reason,
|
||||
};
|
||||
// Credit governor: adapt the cache TTL to the remaining daily
|
||||
// budget so a continuously-open app stretches its polls instead of
|
||||
// exhausting the quota mid-day. Success also clears any cooldown.
|
||||
const remaining = Number(
|
||||
upstream.headers.get('x-rate-limit-remaining'),
|
||||
);
|
||||
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
|
||||
_openskyCooldownUntil = 0;
|
||||
}
|
||||
|
||||
res.writeHead(
|
||||
upstream.status,
|
||||
buildOpenSkyHeaders({
|
||||
// The frame belongs to this body only if this body was cached. An
|
||||
// uncached error must not borrow the previous snapshot's frame.
|
||||
const encoded = encodedBody(
|
||||
req,
|
||||
body,
|
||||
_openskyCacheBody === body ? _openskyCacheGzip : null,
|
||||
);
|
||||
res.writeHead(upstream.status, {
|
||||
...buildOpenSkyHeaders({
|
||||
cacheStatus: 'MISS',
|
||||
requestedMode,
|
||||
usedMode,
|
||||
reason,
|
||||
}),
|
||||
);
|
||||
res.end(body);
|
||||
...encoded.headers,
|
||||
});
|
||||
res.end(encoded.body);
|
||||
} catch (e) {
|
||||
console.error('[OpenSky Proxy]', e.message);
|
||||
if (_openskyCacheBody) {
|
||||
@@ -711,17 +796,22 @@ export function openSkyProxy() {
|
||||
usedMode: 'unknown',
|
||||
reason: 'cached_stale',
|
||||
};
|
||||
res.writeHead(
|
||||
_openskyCacheStatus || 200,
|
||||
buildOpenSkyHeaders({
|
||||
const encoded = encodedBody(
|
||||
req,
|
||||
_openskyCacheBody,
|
||||
_openskyCacheGzip,
|
||||
);
|
||||
res.writeHead(_openskyCacheStatus || 200, {
|
||||
...buildOpenSkyHeaders({
|
||||
cacheStatus: 'STALE',
|
||||
requestedMode:
|
||||
cachedMeta.requestedMode || OPENSKY_AUTH_MODE_DEFAULT,
|
||||
usedMode: cachedMeta.usedMode || 'unknown',
|
||||
reason: cachedMeta.reason || 'cached_stale',
|
||||
}),
|
||||
);
|
||||
res.end(_openskyCacheBody);
|
||||
...encoded.headers,
|
||||
});
|
||||
res.end(encoded.body);
|
||||
return;
|
||||
}
|
||||
const requestedMode = normalizeOpenSkyAuthMode(
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { openSkyClientCredentials } from './opensky.js';
|
||||
|
||||
const file = (contents) => (filePath) => {
|
||||
file.read = filePath;
|
||||
if (contents instanceof Error) throw contents;
|
||||
return contents;
|
||||
};
|
||||
|
||||
test('OpenSky client env vars come first', () => {
|
||||
const read = () => assert.fail('the file is not read when env vars are set');
|
||||
assert.deepEqual(
|
||||
openSkyClientCredentials(
|
||||
{
|
||||
OPENSKY_CLIENT_ID: ' id ',
|
||||
OPENSKY_CLIENT_SECRET: 'secret',
|
||||
OPENSKY_CREDENTIALS_FILE: 'credentials.json',
|
||||
},
|
||||
read,
|
||||
),
|
||||
{ clientId: 'id', clientSecret: 'secret' },
|
||||
);
|
||||
});
|
||||
|
||||
test('OPENSKY_CREDENTIALS_FILE supplies the client when env vars are empty', () => {
|
||||
const env = {
|
||||
OPENSKY_CLIENT_ID: '',
|
||||
OPENSKY_CREDENTIALS_FILE: '~/Downloads/credentials.json',
|
||||
};
|
||||
// The file OpenSky's account page downloads uses camelCase keys.
|
||||
assert.deepEqual(
|
||||
openSkyClientCredentials(
|
||||
env,
|
||||
file('{"clientId":"cid","clientSecret":"csecret"}'),
|
||||
),
|
||||
{ clientId: 'cid', clientSecret: 'csecret' },
|
||||
);
|
||||
assert.equal(
|
||||
path.normalize(file.read),
|
||||
path.normalize(path.join(os.homedir(), 'Downloads/credentials.json')),
|
||||
);
|
||||
assert.deepEqual(
|
||||
openSkyClientCredentials(
|
||||
env,
|
||||
file('{"client_id":"cid","client_secret":"csecret"}'),
|
||||
),
|
||||
{ clientId: 'cid', clientSecret: 'csecret' },
|
||||
);
|
||||
});
|
||||
|
||||
test('a missing, unreadable or incomplete credentials file means no client', () => {
|
||||
assert.equal(openSkyClientCredentials({}, file('{}')), null);
|
||||
const env = { OPENSKY_CREDENTIALS_FILE: 'credentials.json' };
|
||||
const missing = Object.assign(new Error('nope'), { code: 'ENOENT' });
|
||||
assert.equal(openSkyClientCredentials(env, file(missing)), null);
|
||||
assert.equal(openSkyClientCredentials(env, file('not json')), null);
|
||||
assert.equal(openSkyClientCredentials(env, file('{"clientId":"cid"}')), null);
|
||||
});
|
||||
|
||||
test('credential file warnings never expose file contents or read errors', async (t) => {
|
||||
const warnings = [];
|
||||
t.mock.method(console, 'warn', (message) => warnings.push(message));
|
||||
for (const contents of [
|
||||
'{"clientId":"id","clientSecret":FAKE_SECRET_123}',
|
||||
new Error('could not read FAKE_SECRET_123'),
|
||||
]) {
|
||||
const { openSkyClientCredentials: resolve } = await import(
|
||||
`./opensky.js?warning=${warnings.length}`
|
||||
);
|
||||
const env = { OPENSKY_CREDENTIALS_FILE: '/synthetic/credentials.json' };
|
||||
assert.equal(resolve(env, file(contents)), null);
|
||||
assert.equal(resolve(env, file(contents)), null);
|
||||
}
|
||||
assert.deepEqual(warnings, [
|
||||
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
|
||||
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
|
||||
]);
|
||||
});
|
||||
@@ -7,10 +7,9 @@ import { readCappedResponseText } from '../common/http.js';
|
||||
* own credit bucket, 4 credits per call on the free tier). OAuth via the
|
||||
* shared coalesced token. 60s per-icao cache; 404/429 forwarded so the
|
||||
* client can fall back to its accumulated trail silently.
|
||||
* /api/military/track?hex=<hex> — adsb.lol tar1090 readsb trace
|
||||
* (undocumented but live; no browser CORS, hence this proxy). Up to ~24h
|
||||
* of real history per aircraft. Treat as best-effort; data is ODbL —
|
||||
* credit "adsb.lol (ODbL)" in the UI.
|
||||
* /api/military/track?hex=<hex> — compatibility refusal for the retired,
|
||||
* undocumented ADSB.lol trace route. The standalone military source uses
|
||||
* locally accumulated trails; provider-supplied history remains optional.
|
||||
*/
|
||||
export function trackBackfillProxies() {
|
||||
const TRACK_CACHE_MS = 60000;
|
||||
@@ -107,10 +106,16 @@ export function trackBackfillProxies() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
await proxyJson(
|
||||
res,
|
||||
`lol:${hex}`,
|
||||
`https://adsb.lol/data/traces/${hex.slice(-2)}/trace_full_${hex}.json`,
|
||||
res.statusCode = 503;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.setHeader('Retry-After', '1800');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error:
|
||||
'Military history unavailable; locally accumulated trails remain available',
|
||||
code: 'history_unavailable',
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
res.statusCode = 502;
|
||||
|
||||
+251
-39
@@ -1,11 +1,10 @@
|
||||
import { createCctvCatalog } from './cctv/catalog.js';
|
||||
import {
|
||||
normalizeFeedType,
|
||||
isVideoFeedType,
|
||||
toFiniteNumber,
|
||||
} from './cctv/normalize.js';
|
||||
import { quebec511VideoUrl, quebecBrowserVideoEnabled } from './cctv/quebec.js';
|
||||
import { normalizeFeedType, isVideoFeedType } from './cctv/normalize.js';
|
||||
import {
|
||||
buildSyntheticCctvSvg,
|
||||
cctvLiveMediaType,
|
||||
cctvRasterImageType,
|
||||
proxyMediaResponse,
|
||||
fetchCctvImageFromUpstream,
|
||||
fetchTxdotSnapshot,
|
||||
@@ -17,7 +16,11 @@ import {
|
||||
CCTV_MAX_SOURCES_CEILING,
|
||||
} from './cctv/constants.js';
|
||||
import { sanitizeCctvRangeHeader } from './cctv/range.js';
|
||||
import { admitSameSiteKeyless } from './common/same-site.js';
|
||||
import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs';
|
||||
import { createHlsPuller } from './cctv/stream.js';
|
||||
import { isMlitRiverAdjustmentImage } from './cctv/worldCameras.js';
|
||||
import { ottawaAuthorizedFrameUrl } from './cctv/ottawa.js';
|
||||
import { googleServerApiKey } from './places/google-key.js';
|
||||
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
|
||||
/**
|
||||
@@ -43,6 +46,120 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
const HEALTH_MAX_ENTRIES = CCTV_MAX_SOURCES_CEILING;
|
||||
/** Live HLS strategies (see ./cctv/stream.js). Shared across dev and preview. */
|
||||
const puller = createHlsPuller();
|
||||
/**
|
||||
* Bounded transient image reuse keeps cadence-limited providers (World
|
||||
* stills, Ottawa's once-a-minute API) inside their refresh floor. The byte
|
||||
* cap is the real bound; the entry cap only limits map overhead.
|
||||
*/
|
||||
const WORLD_FRAME_CACHE_MAX_ENTRIES = 1024;
|
||||
/** Retry a failed fetch sooner than a full refresh window, never under a minute. */
|
||||
const THROTTLED_FRAME_FAILURE_BACKOFF_MS = 60 * 1000;
|
||||
const WORLD_FRAME_CACHE_MAX_ENTRY_BYTES = 2 * 1024 * 1024;
|
||||
const WORLD_FRAME_CACHE_MAX_BYTES = 32 * 1024 * 1024;
|
||||
const worldFrameCache = new Map();
|
||||
const worldFrameInflight = new Map();
|
||||
const worldFrameNextAllowedAt = new Map();
|
||||
let worldFrameCacheBytes = 0;
|
||||
|
||||
function getWorldFrameCache(cameraId) {
|
||||
const entry = worldFrameCache.get(cameraId);
|
||||
if (!entry) return null;
|
||||
worldFrameCache.delete(cameraId);
|
||||
worldFrameCache.set(cameraId, entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
function storeWorldFrame(cameraId, image) {
|
||||
if (
|
||||
!Buffer.isBuffer(image?.body) ||
|
||||
image.body.length > WORLD_FRAME_CACHE_MAX_ENTRY_BYTES
|
||||
)
|
||||
return;
|
||||
const previous = worldFrameCache.get(cameraId);
|
||||
if (previous) worldFrameCacheBytes -= previous.body.length;
|
||||
const entry = {
|
||||
body: image.body,
|
||||
contentType: image.contentType,
|
||||
fetchedAt: Date.now(),
|
||||
};
|
||||
worldFrameCache.delete(cameraId);
|
||||
worldFrameCache.set(cameraId, entry);
|
||||
worldFrameCacheBytes += entry.body.length;
|
||||
while (
|
||||
worldFrameCache.size > WORLD_FRAME_CACHE_MAX_ENTRIES ||
|
||||
worldFrameCacheBytes > WORLD_FRAME_CACHE_MAX_BYTES
|
||||
) {
|
||||
const oldestId = worldFrameCache.keys().next().value;
|
||||
const oldest = worldFrameCache.get(oldestId);
|
||||
worldFrameCache.delete(oldestId);
|
||||
worldFrameCacheBytes -= oldest.body.length;
|
||||
}
|
||||
}
|
||||
|
||||
function setNextAllowedAt(cameraId, at) {
|
||||
worldFrameNextAllowedAt.delete(cameraId);
|
||||
worldFrameNextAllowedAt.set(cameraId, at);
|
||||
while (worldFrameNextAllowedAt.size > 4096) {
|
||||
worldFrameNextAllowedAt.delete(
|
||||
worldFrameNextAllowedAt.keys().next().value,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a frame from a provider with a per-camera refresh floor. Inside the
|
||||
* floor the last frame is reused; with no frame to reuse the caller gets
|
||||
* `{ throttled: true }` and serves a placeholder, so a cadence-limited camera
|
||||
* never falls through to the key-bearing Street View fallback. A failed
|
||||
* fetch backs off for a minute (capped at the floor) instead of a full
|
||||
* window.
|
||||
*/
|
||||
async function fetchThrottledFrame(source, cameraId, frameUrl) {
|
||||
const cached = getWorldFrameCache(cameraId);
|
||||
const rawRefreshMs = Number(source?.frameRefreshMs);
|
||||
const refreshMs =
|
||||
Number.isFinite(rawRefreshMs) && rawRefreshMs > 0
|
||||
? rawRefreshMs
|
||||
: 5 * 60 * 1000;
|
||||
const now = Date.now();
|
||||
if (cached && now - cached.fetchedAt < refreshMs) {
|
||||
return { ok: true, ...cached, cached: true };
|
||||
}
|
||||
|
||||
const inflight = worldFrameInflight.get(cameraId);
|
||||
if (inflight) return inflight;
|
||||
const stale = cached
|
||||
? {
|
||||
ok: true,
|
||||
body: cached.body,
|
||||
contentType: cached.contentType,
|
||||
stale: true,
|
||||
}
|
||||
: null;
|
||||
const nextAllowedAt = worldFrameNextAllowedAt.get(cameraId) || 0;
|
||||
if (now < nextAllowedAt) return stale || { ok: false, throttled: true };
|
||||
if (!frameUrl) return stale;
|
||||
setNextAllowedAt(cameraId, now + refreshMs);
|
||||
let request;
|
||||
request = fetchCctvImageFromUpstream(frameUrl)
|
||||
.then((image) => {
|
||||
if (image?.ok) {
|
||||
storeWorldFrame(cameraId, image);
|
||||
return image;
|
||||
}
|
||||
setNextAllowedAt(
|
||||
cameraId,
|
||||
Date.now() + Math.min(refreshMs, THROTTLED_FRAME_FAILURE_BACKOFF_MS),
|
||||
);
|
||||
return stale || image;
|
||||
})
|
||||
.finally(() => {
|
||||
if (worldFrameInflight.get(cameraId) === request)
|
||||
worldFrameInflight.delete(cameraId);
|
||||
});
|
||||
worldFrameInflight.set(cameraId, request);
|
||||
return request;
|
||||
}
|
||||
|
||||
/** Update the health entry for a camera, evicting the oldest entry if at capacity. */
|
||||
const setHealth = (cameraId, patch) => {
|
||||
@@ -116,8 +233,15 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' },
|
||||
signal: AbortSignal.timeout(CCTV_FRAME_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const svType = svResp.headers.get('content-type') || '';
|
||||
if (!svResp.ok || !svType.startsWith('image/')) return null;
|
||||
const svType = cctvRasterImageType(svResp.headers.get('content-type'));
|
||||
if (!svResp.ok || !svType) {
|
||||
try {
|
||||
await svResp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
@@ -135,13 +259,18 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
});
|
||||
server.middlewares.use('/api/cctv', async (req, res) => {
|
||||
try {
|
||||
const url = new URL(req.url || '/', 'http://localhost');
|
||||
// Refuse cross-site requests before any catalog or upstream work. The
|
||||
// keyless gate keeps the routes reachable behind a reverse proxy; the
|
||||
// key-bearing Street View fallback below has its own check.
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
const sources = await getCctvSources();
|
||||
const sourceById = new Map(
|
||||
sources.map((source) => [source.id, source]),
|
||||
);
|
||||
const url = new URL(req.url || '/', 'http://localhost');
|
||||
|
||||
if (url.pathname === '/sources') {
|
||||
const quebecVideo = quebecBrowserVideoEnabled();
|
||||
const body = {
|
||||
trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [],
|
||||
sources: sources.map((source) => ({
|
||||
@@ -153,6 +282,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
provider: source.provider,
|
||||
lat: source.lat,
|
||||
lon: source.lon,
|
||||
frameRefreshMs: source.frameRefreshMs,
|
||||
headingDeg: source.headingDeg,
|
||||
headingConfidence: source.headingConfidence || '',
|
||||
pitchDeg: source.pitchDeg,
|
||||
@@ -163,6 +293,13 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
feedType: normalizeFeedType(source.feedType),
|
||||
sourceKind:
|
||||
source.sourceKind || (source.url ? 'configured' : 'fallback'),
|
||||
sourceUrl: source.sourceUrl || '',
|
||||
browserMediaUrl:
|
||||
quebecVideo && source.sourceKind === 'quebec-511-browser'
|
||||
? quebec511VideoUrl(source.id)
|
||||
: '',
|
||||
accuracyM: source.accuracyM,
|
||||
positionSource: source.positionSource || '',
|
||||
poseSource: source.poseSource,
|
||||
model: source.model || '',
|
||||
manufacturer: source.manufacturer || '',
|
||||
@@ -217,8 +354,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
const mediaUrl = source?.url || '';
|
||||
const feedType = normalizeFeedType(source?.feedType || 'image');
|
||||
// Live feeds only. A still camera is served by /frame, whose fetch
|
||||
// keeps redirects on the registered origin, accepts only image/* and
|
||||
// caps the body; relaying it here would bypass all three.
|
||||
// keeps redirects on the registered origin, accepts only raster
|
||||
// images and caps the body; relaying it here would bypass all three.
|
||||
if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') {
|
||||
res.writeHead(404, {
|
||||
'Content-Type': 'application/json',
|
||||
@@ -382,7 +519,6 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
const contentType = upstream.headers.get('content-type') || '';
|
||||
if (!upstream.ok) {
|
||||
try {
|
||||
await upstream.body?.cancel();
|
||||
@@ -407,31 +543,42 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
isVideoFeedType(feedType) &&
|
||||
!(
|
||||
contentType.startsWith('video/') ||
|
||||
contentType.includes('mpegurl')
|
||||
)
|
||||
) {
|
||||
// The relayed type is the feed kind's own, checked before the
|
||||
// camera counts as healthy and before any byte is relayed.
|
||||
const contentType = cctvLiveMediaType(
|
||||
feedType,
|
||||
upstream.headers.get('content-type'),
|
||||
);
|
||||
if (!contentType) {
|
||||
try {
|
||||
await upstream.body?.cancel();
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
setHealth(cameraId, {
|
||||
status: 'degraded',
|
||||
sourceKind: 'upstream',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: `Unexpected media type ${contentType || 'unknown'}`,
|
||||
message: 'Unexpected media type',
|
||||
});
|
||||
} else {
|
||||
setHealth(cameraId, {
|
||||
status: 'ok',
|
||||
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: isVideoFeedType(feedType)
|
||||
? 'Live stream connected'
|
||||
: 'Snapshot feed connected',
|
||||
res.writeHead(502, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Media proxy failed' }));
|
||||
return;
|
||||
}
|
||||
setHealth(cameraId, {
|
||||
status: 'ok',
|
||||
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: isVideoFeedType(feedType)
|
||||
? 'Live stream connected'
|
||||
: 'Snapshot feed connected',
|
||||
});
|
||||
|
||||
await proxyMediaResponse(res, upstream, {
|
||||
contentType,
|
||||
sourceHeader: isVideoFeedType(feedType)
|
||||
? 'live-media'
|
||||
: 'upstream-image',
|
||||
@@ -505,15 +652,71 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
: '');
|
||||
|
||||
const upstreamImage =
|
||||
source?.sourceKind === 'txdot-its'
|
||||
? await fetchTxdotSnapshot(upstreamCandidate)
|
||||
: await fetchCctvImageFromUpstream(upstreamCandidate);
|
||||
source?.pack === 'world-cameras'
|
||||
? await fetchThrottledFrame(source, cameraId, upstreamCandidate)
|
||||
: source?.sourceKind === 'ottawa-traffic'
|
||||
? await fetchThrottledFrame(
|
||||
source,
|
||||
cameraId,
|
||||
ottawaAuthorizedFrameUrl(upstreamCandidate),
|
||||
)
|
||||
: source?.sourceKind === 'txdot-its'
|
||||
? await fetchTxdotSnapshot(upstreamCandidate)
|
||||
: await fetchCctvImageFromUpstream(upstreamCandidate, {
|
||||
allowMjpeg: source?.sourceKind === 'taiwan-highway-motc',
|
||||
});
|
||||
if (
|
||||
source?.sourceKind === 'mlit-river' &&
|
||||
isMlitRiverAdjustmentImage(upstreamImage)
|
||||
) {
|
||||
const svg = buildSyntheticCctvSvg({
|
||||
cameraId,
|
||||
label,
|
||||
city,
|
||||
status: 'CAMERA UNDER ADJUSTMENT',
|
||||
});
|
||||
setHealth(cameraId, {
|
||||
status: 'degraded',
|
||||
sourceKind: 'snapshot',
|
||||
label: source.provider || 'MLIT river cameras',
|
||||
message: 'Provider camera under adjustment',
|
||||
});
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'image/svg+xml',
|
||||
'Cache-Control': 'no-store',
|
||||
'X-CCTV-Source': 'provider-maintenance',
|
||||
});
|
||||
res.end(svg);
|
||||
return;
|
||||
}
|
||||
if (upstreamImage?.throttled) {
|
||||
// Inside the provider's refresh floor with no frame to reuse: a
|
||||
// placeholder, not Street View, and no health change.
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'image/svg+xml',
|
||||
'Cache-Control': 'no-store',
|
||||
'X-CCTV-Source': 'synthetic',
|
||||
});
|
||||
res.end(
|
||||
buildSyntheticCctvSvg({
|
||||
cameraId,
|
||||
label,
|
||||
city,
|
||||
status: 'WAITING FOR NEXT FRAME',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (upstreamImage?.ok) {
|
||||
setHealth(cameraId, {
|
||||
status: 'ok',
|
||||
status: upstreamImage.stale ? 'degraded' : 'ok',
|
||||
sourceKind: 'snapshot',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: 'Upstream snapshot active',
|
||||
message: upstreamImage.stale
|
||||
? 'Serving last snapshot; refresh failed'
|
||||
: upstreamImage.cached
|
||||
? 'Cached snapshot active'
|
||||
: 'Upstream snapshot active',
|
||||
});
|
||||
res.writeHead(200, {
|
||||
'Content-Type': upstreamImage.contentType,
|
||||
@@ -524,13 +727,22 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sv = await streetViewFallback({
|
||||
lat,
|
||||
lon,
|
||||
heading,
|
||||
fov,
|
||||
pitch,
|
||||
});
|
||||
// The Street View fallback spends the Google key, so a forwarded
|
||||
// request (whose client this server cannot see) gets the placeholder,
|
||||
// as if no key were configured. Any forwarding header counts, even an
|
||||
// empty one.
|
||||
const forwarded = PROXY_SIGNALS.some(
|
||||
(name) => req.headers?.[name] !== undefined,
|
||||
);
|
||||
const sv = forwarded
|
||||
? null
|
||||
: await streetViewFallback({
|
||||
lat,
|
||||
lon,
|
||||
heading,
|
||||
fov,
|
||||
pitch,
|
||||
});
|
||||
if (sv?.ok) {
|
||||
setHealth(cameraId, {
|
||||
status: 'degraded',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
CCTV_MAX_SOURCES_CEILING,
|
||||
CCTV_SHARED_SOURCE_SLOTS,
|
||||
DEFAULT_CCTV_MAX_SOURCES,
|
||||
} from './constants.js';
|
||||
|
||||
@@ -23,16 +24,27 @@ export function resolveCatalogCap(raw) {
|
||||
* exceed the catalog cap, the cap is filled ROUND-ROBIN — one camera from
|
||||
* each pack per turn, in each pack's own order — so a lowered
|
||||
* CCTV_MAX_SOURCES thins every region a little instead of deleting whichever
|
||||
* pack happened to be appended last. Duplicate ids resolve last-pack-wins so
|
||||
* pack happened to be appended last. A pack with a `reservedCount` keeps up to
|
||||
* that many rows only from the capacity ABOVE the shared floor, so a
|
||||
* reservation adds room and never shrinks the other packs' round-robin
|
||||
* share; at or below the floor reserved packs are ordinary lanes. Duplicate
|
||||
* ids resolve last-pack-wins so
|
||||
* file/env overrides keep replacing live records; the record is then counted
|
||||
* against the pack that won it.
|
||||
*
|
||||
* @param {Array<{name:string, sources:Array<object>}>} packs - Normalized
|
||||
* packs in merge order (earlier packs lose duplicate ids to later ones).
|
||||
* @param {number} maxCount - Catalog cap.
|
||||
* @param {object} [options]
|
||||
* @param {number} [options.sharedFloor=CCTV_SHARED_SOURCE_SLOTS] - Slots kept
|
||||
* for the round-robin before reservations apply.
|
||||
* @returns {{sources:Array<object>, packs:Array<{name:string, offered:number, kept:number}>}}
|
||||
*/
|
||||
export function allocateSourceCap(packs, maxCount) {
|
||||
export function allocateSourceCap(
|
||||
packs,
|
||||
maxCount,
|
||||
{ sharedFloor = CCTV_SHARED_SOURCE_SLOTS } = {},
|
||||
) {
|
||||
const owner = new Map();
|
||||
for (const pack of packs) {
|
||||
for (const source of pack.sources) {
|
||||
@@ -40,30 +52,53 @@ export function allocateSourceCap(packs, maxCount) {
|
||||
owner.set(source.id, { pack: pack.name, source });
|
||||
}
|
||||
}
|
||||
const lanes = packs.map((pack) => ({
|
||||
name: pack.name,
|
||||
const lanes = packs.map((pack) => {
|
||||
// Only the record that WON its id (the last occurrence anywhere, so also
|
||||
// the last within this pack) takes a lane slot; earlier duplicates never
|
||||
// count against the cap.
|
||||
queue: pack.sources.filter(
|
||||
const queue = pack.sources.filter(
|
||||
(source) => source?.id && owner.get(source.id)?.source === source,
|
||||
),
|
||||
next: 0,
|
||||
kept: 0,
|
||||
}));
|
||||
);
|
||||
return {
|
||||
name: pack.name,
|
||||
queue,
|
||||
reserved: Math.max(
|
||||
0,
|
||||
Math.min(queue.length, Math.floor(Number(pack.reservedCount) || 0)),
|
||||
),
|
||||
next: 0,
|
||||
kept: 0,
|
||||
};
|
||||
});
|
||||
const total = lanes.reduce((sum, lane) => sum + lane.queue.length, 0);
|
||||
const limit = Math.min(total, Math.max(0, Math.floor(maxCount)));
|
||||
|
||||
const sources = [];
|
||||
const reservedSources = [];
|
||||
if (limit >= total) {
|
||||
for (const lane of lanes) {
|
||||
sources.push(...lane.queue);
|
||||
lane.kept = lane.queue.length;
|
||||
}
|
||||
} else {
|
||||
while (sources.length < limit) {
|
||||
// Reserved packs draw only on the capacity above the shared floor, then
|
||||
// every lane (reserved ones included) shares the rest round-robin.
|
||||
const reserveBudget = Math.max(0, limit - Math.max(0, sharedFloor));
|
||||
while (reservedSources.length < reserveBudget) {
|
||||
let tookReserved = false;
|
||||
for (const lane of lanes) {
|
||||
if (sources.length >= limit) break;
|
||||
if (reservedSources.length >= reserveBudget) break;
|
||||
if (lane.kept >= lane.reserved) continue;
|
||||
reservedSources.push(lane.queue[lane.next]);
|
||||
lane.next += 1;
|
||||
lane.kept += 1;
|
||||
tookReserved = true;
|
||||
}
|
||||
if (!tookReserved) break;
|
||||
}
|
||||
while (sources.length + reservedSources.length < limit) {
|
||||
for (const lane of lanes) {
|
||||
if (sources.length + reservedSources.length >= limit) break;
|
||||
if (lane.next >= lane.queue.length) continue;
|
||||
sources.push(lane.queue[lane.next]);
|
||||
lane.next += 1;
|
||||
@@ -72,7 +107,9 @@ export function allocateSourceCap(packs, maxCount) {
|
||||
}
|
||||
}
|
||||
return {
|
||||
sources,
|
||||
// Keep the shared round-robin sequence first so the default active camera
|
||||
// remains in its established region; guaranteed regional rows follow it.
|
||||
sources: [...sources, ...reservedSources],
|
||||
packs: lanes.map((lane) => ({
|
||||
name: lane.name,
|
||||
offered: lane.queue.length,
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
|
||||
import {
|
||||
DEFAULT_CCTV_SOURCE_FILE,
|
||||
CCTV_SOURCE_CACHE_MS,
|
||||
WORLD_CCTV_SOURCE_COUNT,
|
||||
} from './constants.js';
|
||||
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
|
||||
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
|
||||
import { cameraRegion, normalizeSourceItem } from './normalize.js';
|
||||
import { loadLithuaniaSources } from './lithuania.js';
|
||||
import { loadOttawaSourcesFromOpenData } from './ottawa.js';
|
||||
import { loadQuebecSources, QUEBEC_CAMERA_SOURCE_COUNT } from './quebec.js';
|
||||
import { loadTaiwanSources } from './taiwan.js';
|
||||
import { loadWorldCameraSources } from './worldCameras.js';
|
||||
import {
|
||||
loadAustinSourcesFromOpenData,
|
||||
loadCaltransSourcesFromOpenData,
|
||||
@@ -19,6 +27,8 @@ import {
|
||||
loadNswSourcesFromOpenData,
|
||||
loadCalgarySourcesFromOpenData,
|
||||
loadCataloniaSourcesFromOpenData,
|
||||
loadIcelandSourcesFromOpenData,
|
||||
loadHkSourcesFromOpenData,
|
||||
loadDelDOTSourcesFromOpenData,
|
||||
loadKingCountySourcesFromOpenData,
|
||||
loadWsdotSourcesFromOpenData,
|
||||
@@ -26,6 +36,8 @@ import {
|
||||
loadNycSourcesFromOpenData,
|
||||
loadTorontoSourcesFromOpenData,
|
||||
loadFloridaSourcesFromOpenData,
|
||||
loadHoustonSourcesFromOpenData,
|
||||
loadA22SourcesFromOpenDataHub,
|
||||
} from './sources.js';
|
||||
|
||||
/** Env kill switch: unset or anything but "0" means enabled. */
|
||||
@@ -71,6 +83,11 @@ const LIVE_PACKS = [
|
||||
enabled: () => envEnabled('CCTV_DRIVEBC_ENABLED'),
|
||||
load: loadDriveBcSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'ottawa',
|
||||
enabled: () => envEnabled('CCTV_OTTAWA_ENABLED'),
|
||||
load: loadOttawaSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'txdot',
|
||||
enabled: () => envEnabled('CCTV_TXDOT_ENABLED'),
|
||||
@@ -106,6 +123,16 @@ const LIVE_PACKS = [
|
||||
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
|
||||
load: loadCataloniaSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'iceland',
|
||||
enabled: () => envEnabled('CCTV_ICELAND_ENABLED'),
|
||||
load: loadIcelandSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'hk',
|
||||
enabled: () => envEnabled('CCTV_HK_ENABLED'),
|
||||
load: loadHkSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'deldot',
|
||||
enabled: () => envEnabled('CCTV_DELDOT_ENABLED'),
|
||||
@@ -153,6 +180,33 @@ const LIVE_PACKS = [
|
||||
enabled: () => envOptIn('CCTV_FLORIDA_ENABLED'),
|
||||
load: loadFloridaSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'taiwan',
|
||||
enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'),
|
||||
load: loadTaiwanSources,
|
||||
},
|
||||
{
|
||||
name: 'houston',
|
||||
enabled: () => envOptIn('CCTV_HOUSTON_ENABLED'),
|
||||
load: loadHoustonSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'quebec',
|
||||
enabled: () => envEnabled('CCTV_QUEBEC_ENABLED'),
|
||||
load: loadQuebecSources,
|
||||
reservedCount: QUEBEC_CAMERA_SOURCE_COUNT,
|
||||
},
|
||||
{
|
||||
name: 'world-cameras',
|
||||
enabled: () => envEnabled('CCTV_WORLD_CAMERAS_ENABLED'),
|
||||
load: loadWorldCameraSources,
|
||||
reservedCount: WORLD_CCTV_SOURCE_COUNT,
|
||||
},
|
||||
{
|
||||
name: 'a22',
|
||||
enabled: () => envEnabled('CCTV_A22_ENABLED'),
|
||||
load: loadA22SourcesFromOpenDataHub,
|
||||
},
|
||||
];
|
||||
/**
|
||||
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
|
||||
@@ -292,7 +346,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
: [];
|
||||
// Live packs first so file/env overrides win on duplicate IDs; each pack
|
||||
// keeps its own priority order and the catalog cap is shared fairly.
|
||||
const normalizePack = (name, items) => ({
|
||||
const normalizePack = (name, items, reservedCount = 0) => ({
|
||||
name,
|
||||
sources: items
|
||||
.filter((item) => item && typeof item === 'object')
|
||||
@@ -302,6 +356,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
? items.available
|
||||
: items.length,
|
||||
region: items.region ?? null,
|
||||
reservedCount,
|
||||
});
|
||||
const packs = [
|
||||
...LIVE_PACKS.map((pack, index) =>
|
||||
@@ -310,6 +365,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
liveResults[index]?.status === 'fulfilled'
|
||||
? liveResults[index].value
|
||||
: [],
|
||||
pack.reservedCount,
|
||||
),
|
||||
),
|
||||
normalizePack('file', fromFile),
|
||||
|
||||
@@ -5,20 +5,23 @@ export const DEFAULT_AUSTIN_ROWS_URL =
|
||||
/** Default cap on Austin cameras after distance-based prioritization. */
|
||||
export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
|
||||
/**
|
||||
* Catalog-wide safety ceiling on served cameras. Each pack already caps
|
||||
* itself (nearest-to-anchor first); this bound only matters when the packs
|
||||
* together exceed it, and it is then filled round-robin across packs (see
|
||||
* cap.js) so no region is silently dropped. The default per-pack caps
|
||||
* together can exceed it, so a default install may be thinned evenly here.
|
||||
* Slots every pack shares round-robin before any reservation applies. A
|
||||
* reserved pack (Québec, World) only takes capacity above this floor, so a
|
||||
* cap at or below it thins every region evenly (see cap.js).
|
||||
*/
|
||||
export const DEFAULT_CCTV_MAX_SOURCES = 4000;
|
||||
export const CCTV_SHARED_SOURCE_SLOTS = 4000;
|
||||
/**
|
||||
* Catalog-wide default: the 4,000 shared slots plus room reserved for the
|
||||
* 1,120 Québec and 2,230 World image locations.
|
||||
*/
|
||||
export const DEFAULT_CCTV_MAX_SOURCES = 7350;
|
||||
/** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */
|
||||
export const CCTV_MAX_SOURCES_CEILING = 5000;
|
||||
export const CCTV_MAX_SOURCES_CEILING = 8000;
|
||||
/** Reference point for Austin camera prioritization (Congress & 6th). */
|
||||
export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 };
|
||||
/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */
|
||||
/** TxDOT ITS: one keyless JSON catalog per district (25 districts statewide). */
|
||||
export const TXDOT_ORIGIN = 'https://its.txdot.gov';
|
||||
const TXDOT_ORIGIN = 'https://its.txdot.gov';
|
||||
export const TXDOT_CCTV_STATUS_URL = (district) =>
|
||||
`${TXDOT_ORIGIN}/its/DistrictIts/GetCctvStatusListByDistrict?districtCode=${encodeURIComponent(district)}`;
|
||||
/** Per-camera frame. Returns JSON `{snippet:<base64 jpeg>}`, not an image body;
|
||||
@@ -197,6 +200,39 @@ export const DRIVEBC_ANCHORS = [
|
||||
{ lat: 49.2827, lon: -123.1207 }, // Vancouver
|
||||
{ lat: 48.4284, lon: -123.3656 }, // Victoria
|
||||
];
|
||||
/** City of Ottawa traffic cameras: documented JSON list (keyless). */
|
||||
export const OTTAWA_CAMERAS_URL =
|
||||
'https://traffic.ottawa.ca/map/service/camera';
|
||||
/**
|
||||
* Keyless canonical frame URL for one camera number. The city's Traffic Camera
|
||||
* Images API needs a certificate (OTTAWA_CAMERA_CERTIFICATE), which the frame
|
||||
* proxy adds at request time so it never appears in a catalog response.
|
||||
*/
|
||||
export const OTTAWA_IMAGE_URL = (cameraNumber) =>
|
||||
`https://traffic.ottawa.ca/opendata/camera?c=${encodeURIComponent(cameraNumber)}`;
|
||||
/** Pinned frame directory, for redirect containment in the media proxy. */
|
||||
export const OTTAWA_IMAGE_ORIGIN = 'https://traffic.ottawa.ca/opendata/';
|
||||
/** The API lets one certificate and user id fetch a camera once a minute. */
|
||||
export const OTTAWA_FRAME_REFRESH_MS = 60 * 1000;
|
||||
/** Alphanumeric user id this server sends with its certificate. */
|
||||
export const OTTAWA_API_USER_ID = 'gev';
|
||||
export const DEFAULT_OTTAWA_MAX_SOURCES = 250;
|
||||
/** Prioritization anchor: downtown Ottawa (Parliament Hill). */
|
||||
export const OTTAWA_ANCHORS = [{ lat: 45.4215, lon: -75.6972 }];
|
||||
/**
|
||||
* Taiwan Highway Bureau MJPEG servers. Each camera is one root path segment,
|
||||
* so the pin is the host root: a redirect must stay on the same server.
|
||||
*/
|
||||
export const TAIWAN_HIGHWAY_IMAGE_ORIGINS = Object.freeze(
|
||||
[1, 2, 3, 4, 5, 6, 7, 8].map((n) => `https://cctv-ss0${n}.thb.gov.tw/`),
|
||||
);
|
||||
/** Direct still-image directories from the World cams add-on's eligible packs. */
|
||||
export const TRIPCHECK_IMAGE_ORIGIN =
|
||||
'https://www.tripcheck.com/roadcams/cams/';
|
||||
export const MLIT_RIVER_IMAGE_ORIGIN = 'https://cam.river.go.jp/cam/now/';
|
||||
export const DEFAULT_WORLD_CCTV_SOURCE_FILE =
|
||||
'config/cctv_sources.world-image-cameras.json';
|
||||
export const WORLD_CCTV_SOURCE_COUNT = 2230;
|
||||
/** Tallinn intersection cameras: curated catalog + public stills on ristmikud.tallinn.ee. */
|
||||
export const DEFAULT_TALLINN_SOURCE_FILE = 'config/cctv_sources.tallinn.json';
|
||||
export const DEFAULT_TALLINN_MAX_SOURCES = 255;
|
||||
@@ -309,6 +345,67 @@ export const CATALONIA_ANCHORS = [
|
||||
{ lat: 41.5636, lon: 2.0111 }, // Terrassa
|
||||
];
|
||||
|
||||
/**
|
||||
* Icelandic Road and Coastal Administration (Vegagerðin, IRCA) road cameras:
|
||||
* one keyless JSON list, one row per camera view (about 500 views at roughly
|
||||
* 170 stations, verified 2026-11-06). Frames are stills on the IRCA host, each
|
||||
* rewritten by the publisher every few minutes. Camera images are published
|
||||
* under CC BY 4.0 (IRCA terms of use, "Open data" section).
|
||||
*/
|
||||
export const ICELAND_CAMERAS_URL =
|
||||
'https://gagnaveita.vegagerdin.is/api/vefmyndavelar2014_1';
|
||||
/** The only directory Iceland camera frames may come from. */
|
||||
export const ICELAND_IMAGE_ORIGIN =
|
||||
'https://www.vegagerdin.is/vgdata/vefmyndavelar/';
|
||||
/** Byte ceiling for the Iceland catalog read (~150 KB observed 2026-11-06). */
|
||||
export const ICELAND_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
|
||||
/** Ground-elevation prior in metres: the feed carries none, and the country
|
||||
* runs from sea level to highland passes. The client's ground snap corrects
|
||||
* this on 3D-tile stacks. */
|
||||
export const ICELAND_GROUND_ELEVATION_M = 60;
|
||||
/** The feed lists about 500 views; the default keeps half so the shared
|
||||
* catalog cap still leaves room for denser regions. */
|
||||
export const DEFAULT_ICELAND_MAX_SOURCES = 250;
|
||||
/** Prioritization anchors: Reykjavík and the regional centres, so a cap keeps
|
||||
* every region's nearest cameras instead of only the capital's. */
|
||||
export const ICELAND_ANCHORS = [
|
||||
{ lat: 64.1466, lon: -21.9426 }, // Reykjavík
|
||||
{ lat: 65.6885, lon: -18.1262 }, // Akureyri
|
||||
{ lat: 65.2639, lon: -14.3948 }, // Egilsstaðir
|
||||
{ lat: 66.0749, lon: -23.1351 }, // Ísafjörður
|
||||
{ lat: 63.9331, lon: -20.9971 }, // Selfoss
|
||||
];
|
||||
|
||||
/** Hong Kong Transport Department traffic snapshot cameras: keyless English
|
||||
* locations XML on DATA.GOV.HK; frames are stills on the TD CCTV host. */
|
||||
export const DEFAULT_HK_CAMERAS_URL =
|
||||
'https://static.data.gov.hk/td/traffic-snapshot-images/code/Traffic_Camera_Locations_En.xml';
|
||||
/** The only origin HK camera frames may come from. Verified 2026-10-11: still
|
||||
* URLs answer 200 on this host with no redirect hop. */
|
||||
export const HK_IMAGE_ORIGIN = 'https://tdcctv.data.one.gov.hk/';
|
||||
/** Above the full TD English catalog (~1,013 cameras) so a default install
|
||||
* keeps nearly all of them. Catalog-wide round-robin still applies when the
|
||||
* sum of packs exceeds DEFAULT_CCTV_MAX_SOURCES (Norway-style). */
|
||||
export const DEFAULT_HK_MAX_SOURCES = 1100;
|
||||
/** Prioritization anchors across Hong Kong Island, Kowloon and the New
|
||||
* Territories so a lowered cap keeps territory-wide coverage rather than
|
||||
* only the harbour corridor. */
|
||||
export const HK_ANCHORS = [
|
||||
{ lat: 22.2819, lon: 114.158 }, // Central
|
||||
{ lat: 22.2976, lon: 114.1722 }, // Tsim Sha Tsui
|
||||
{ lat: 22.3827, lon: 114.19 }, // Sha Tin
|
||||
{ lat: 22.3701, lon: 114.114 }, // Tsuen Wan
|
||||
{ lat: 22.4503, lon: 114.1688 }, // Tai Po
|
||||
{ lat: 22.493, lon: 114.139 }, // Fanling
|
||||
{ lat: 22.391, lon: 113.977 }, // Tuen Mun
|
||||
{ lat: 22.4445, lon: 114.022 }, // Yuen Long
|
||||
{ lat: 22.307, lon: 114.26 }, // Tseung Kwan O
|
||||
];
|
||||
/** Hard ceiling on the locations XML body. The live English list is ~400 KB
|
||||
* for ~1,000 cameras; this only exists so an unbounded upstream cannot be
|
||||
* buffered without limit. */
|
||||
export const HK_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
/** Statens vegvesen (Norway) road cameras: the keyless OGC API Features view
|
||||
* of the official DATEX 3.1 CCTV site table, one GeoJSON request for the whole
|
||||
* country (~900 cameras). The raw DATEX node needs an account; this view does
|
||||
@@ -413,6 +510,70 @@ export const NYC_ANCHORS = [
|
||||
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
|
||||
];
|
||||
|
||||
/** Houston TranStar cameras: one keyless JSON list for the metro (~1,000
|
||||
* cameras with a numeric id). Frames are stills synthesized from that id on
|
||||
* the city's snapshot host — never taken from the payload. The catalog is
|
||||
* served as `application/javascript`; asking for `application/json` gets HTTP
|
||||
* 406, so the loader accepts any media type. */
|
||||
export const HOUSTON_CAMERAS_URL =
|
||||
'https://traffic.houstontranstar.org/data/layers/cctvSnapshots_json.js';
|
||||
/** The only origin Houston camera frames may come from. The catalog's `url`
|
||||
* is a `showWin(...)` viewer helper and is ignored; the registered path is
|
||||
* always this origin plus `/snapshots/cctv/<id>.jpg`. */
|
||||
export const HOUSTON_IMAGE_ORIGIN = 'https://www.houstontranstar.org/';
|
||||
export const HOUSTON_IMAGE_URL = (id) =>
|
||||
`${HOUSTON_IMAGE_ORIGIN}snapshots/cctv/${id}.jpg`;
|
||||
/** Numeric TranStar snapshot ids: 3–5 digits, no leading zero. Id `0` is a
|
||||
* different (non-snapshot) camera class in the same list. */
|
||||
export const HOUSTON_CAMERA_ID_RE = /^[1-9]\d{2,4}$/;
|
||||
export const DEFAULT_HOUSTON_MAX_SOURCES = 250;
|
||||
export const HOUSTON_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
/** Ground-elevation prior in metres. The TranStar payload carries none, and
|
||||
* Houston sits a few metres above sea level. */
|
||||
export const HOUSTON_GROUND_ELEVATION_M = 15;
|
||||
/** Downtown Houston (Main @ Texas): the primary prioritization anchor. */
|
||||
export const HOUSTON_DOWNTOWN = { lat: 29.7604, lon: -95.3698 };
|
||||
export const HOUSTON_ANCHORS = [
|
||||
{ lat: 29.7604, lon: -95.3698 }, // Downtown
|
||||
{ lat: 29.739, lon: -95.461 }, // Uptown / Galleria
|
||||
{ lat: 29.71, lon: -95.398 }, // Texas Medical Center
|
||||
{ lat: 29.9902, lon: -95.3368 }, // IAH
|
||||
{ lat: 29.3013, lon: -94.7977 }, // Galveston
|
||||
];
|
||||
|
||||
/** Autostrada del Brennero (A22, Brenner–Modena) webcams: the keyless Open
|
||||
* Data Hub (NOI Techpark, South Tyrol) tourism API republishes Autobrennero's
|
||||
* own camera list with coordinates, one request for the whole motorway
|
||||
* (13 cameras). Frames are stills on the operator's host, refreshed about
|
||||
* once a minute. */
|
||||
export const DEFAULT_A22_CCTV_URL =
|
||||
'https://tourism.api.opendatahub.com/v1/WebcamInfo?source=a22&pagesize=200&removenullvalues=true';
|
||||
/** The only origin A22 camera frames may come from: `km<N>.jpg` per camera. */
|
||||
export const A22_IMAGE_ORIGIN = 'https://www.autobrennero.it/WebCamImg/';
|
||||
/** Hard ceiling on the catalog body. The whole motorway is ~45 KB. */
|
||||
export const A22_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
|
||||
/** Orthometric ground elevation (m) at each camera, keyed by its motorway
|
||||
* kilometre, from the Copernicus GLO-90 DEM via Open-Meteo's elevation API.
|
||||
* The feed's coordinates are 2D and the road climbs from the Po valley (~20 m)
|
||||
* to the Brenner Pass (~1,370 m), so one flat prior would bury the Alpine
|
||||
* cameras. A camera not listed here uses A22_DEFAULT_GROUND_ELEVATION_M. */
|
||||
export const A22_GROUND_ELEVATION_BY_KM = Object.freeze({
|
||||
1: 1365,
|
||||
68: 375,
|
||||
77: 301,
|
||||
100: 219,
|
||||
129: 206,
|
||||
138: 204,
|
||||
159: 175,
|
||||
205: 188,
|
||||
228: 72,
|
||||
240: 38,
|
||||
272: 16,
|
||||
285: 19,
|
||||
313: 40,
|
||||
});
|
||||
export const A22_DEFAULT_GROUND_ELEVATION_M = 200;
|
||||
|
||||
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
|
||||
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
|
||||
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
|
||||
|
||||
@@ -3,7 +3,7 @@ import path from 'node:path';
|
||||
import { poseHash } from '../../../src/data/cctvFootprint.js';
|
||||
|
||||
/** Sidecar written by scripts/precompute-cctv-heights.mjs. */
|
||||
export const DEFAULT_GROUND_HEIGHTS_FILE =
|
||||
const DEFAULT_GROUND_HEIGHTS_FILE =
|
||||
'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json';
|
||||
|
||||
/** @type {{path:string, mtimeMs:number, cameras:object}|null} */
|
||||
|
||||
+372
-12
@@ -1,13 +1,27 @@
|
||||
import { Readable } from 'node:stream';
|
||||
import { hashSeed, escapeXml } from './normalize.js';
|
||||
import { hashSeed, escapeXml, pinnedCameraUrl } from './normalize.js';
|
||||
import {
|
||||
A22_IMAGE_ORIGIN,
|
||||
CALGARY_IMAGE_ORIGIN,
|
||||
CCTV_FRAME_FETCH_TIMEOUT_MS,
|
||||
CCTV_FRAME_MAX_BODY_BYTES,
|
||||
CCTV_MEDIA_FETCH_TIMEOUT_MS,
|
||||
CCTV_MEDIA_IDLE_TIMEOUT_MS,
|
||||
CCTV_MEDIA_MAX_BODY_BYTES,
|
||||
HK_IMAGE_ORIGIN,
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
MLIT_RIVER_IMAGE_ORIGIN,
|
||||
TRIPCHECK_IMAGE_ORIGIN,
|
||||
NSW_IMAGE_ORIGIN,
|
||||
NSW_IMAGE_USER_AGENT,
|
||||
OTTAWA_IMAGE_ORIGIN,
|
||||
TALLINN_IMAGE_ORIGIN,
|
||||
TARKTEE_IMAGE_ORIGIN,
|
||||
ICELAND_IMAGE_ORIGIN,
|
||||
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
|
||||
TFL_IMAGE_ORIGIN,
|
||||
WARENDORF_IMAGE_ORIGINS,
|
||||
WSDOT_IMAGE_ORIGIN,
|
||||
} from './constants.js';
|
||||
/**
|
||||
* Generate a synthetic SVG billboard image for a CCTV camera placeholder.
|
||||
@@ -84,7 +98,7 @@ export function buildSyntheticCctvSvg({ cameraId, label, city, status }) {
|
||||
* @param {ReadableStream|NodeJS.ReadableStream|null} body
|
||||
* @returns {import('stream').Readable|null}
|
||||
*/
|
||||
export function toReadable(body) {
|
||||
function toReadable(body) {
|
||||
if (!body) return null;
|
||||
if (typeof body.pipe === 'function') return body;
|
||||
if (typeof body.getReader === 'function') {
|
||||
@@ -93,6 +107,73 @@ export function toReadable(body) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a Content-Type value into its lower-case essence and parameters.
|
||||
*
|
||||
* @param {string|null|undefined} value
|
||||
* @returns {{essence:string,params:Map<string,string>}|null} Null when the
|
||||
* value is missing or is not a `type/subtype` media type.
|
||||
*/
|
||||
function parseMediaType(value) {
|
||||
const [head, ...rest] = String(value ?? '').split(';');
|
||||
const essence = head.trim().toLowerCase();
|
||||
if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(essence)) return null;
|
||||
const params = new Map();
|
||||
for (const part of rest) {
|
||||
const eq = part.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
const name = part.slice(0, eq).trim().toLowerCase();
|
||||
let paramValue = part.slice(eq + 1).trim();
|
||||
if (/^".*"$/.test(paramValue)) paramValue = paramValue.slice(1, -1);
|
||||
if (name && !params.has(name)) params.set(name, paramValue);
|
||||
}
|
||||
return { essence, params };
|
||||
}
|
||||
|
||||
/** Still-image types a camera frame may be served as. */
|
||||
const CCTV_RASTER_IMAGE_TYPES = new Set([
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/gif',
|
||||
'image/webp',
|
||||
'image/avif',
|
||||
]);
|
||||
|
||||
/**
|
||||
* The canonical type of a raster camera frame, or null for any other type.
|
||||
*
|
||||
* @param {string|null|undefined} contentType
|
||||
* @returns {string|null}
|
||||
*/
|
||||
export function cctvRasterImageType(contentType) {
|
||||
const essence = parseMediaType(contentType)?.essence;
|
||||
return CCTV_RASTER_IMAGE_TYPES.has(essence) ? essence : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The type to relay a live feed as, or null when the upstream type does not
|
||||
* match the feed kind. MJPEG keeps its part boundary, without which the body
|
||||
* cannot be split into frames.
|
||||
*
|
||||
* @param {string} feedType - Normalized feed type (mp4, webm or mjpeg).
|
||||
* @param {string|null|undefined} contentType
|
||||
* @returns {string|null}
|
||||
*/
|
||||
export function cctvLiveMediaType(feedType, contentType) {
|
||||
const parsed = parseMediaType(contentType);
|
||||
if (!parsed) return null;
|
||||
if (feedType === 'mp4' || feedType === 'webm')
|
||||
return parsed.essence === `video/${feedType}` ? parsed.essence : null;
|
||||
if (feedType !== 'mjpeg' || parsed.essence !== 'multipart/x-mixed-replace')
|
||||
return null;
|
||||
const boundary = parsed.params.get('boundary') || '';
|
||||
if (!/^[0-9A-Za-z'()+_,./:=? -]{0,69}[0-9A-Za-z'()+_,./:=?-]$/.test(boundary))
|
||||
return null;
|
||||
return /^[0-9A-Za-z'+_.-]+$/.test(boundary)
|
||||
? `multipart/x-mixed-replace; boundary=${boundary}`
|
||||
: `multipart/x-mixed-replace; boundary="${boundary}"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipe an upstream fetch Response (image or video) to the client HTTP response.
|
||||
*
|
||||
@@ -104,6 +185,8 @@ export function toReadable(body) {
|
||||
* @param {Response} upstream - fetch() Response object.
|
||||
* @param {object} [opts]
|
||||
* @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header.
|
||||
* @param {string} [opts.contentType] - Validated type to send instead of the
|
||||
* upstream's own.
|
||||
* @param {number} [opts.idleTimeoutMs=CCTV_MEDIA_IDLE_TIMEOUT_MS] - Silence
|
||||
* allowed between upstream chunks before the stream is released. Injectable
|
||||
* only to keep the deadline unit-testable.
|
||||
@@ -114,10 +197,10 @@ export async function proxyMediaResponse(
|
||||
{
|
||||
sourceHeader = 'upstream',
|
||||
idleTimeoutMs = CCTV_MEDIA_IDLE_TIMEOUT_MS,
|
||||
contentType = upstream.headers.get('content-type') ||
|
||||
'application/octet-stream',
|
||||
} = {},
|
||||
) {
|
||||
const contentType =
|
||||
upstream.headers.get('content-type') || 'application/octet-stream';
|
||||
const cacheControl = upstream.headers.get('cache-control') || 'no-store';
|
||||
const contentLength = upstream.headers.get('content-length');
|
||||
const contentRange = upstream.headers.get('content-range');
|
||||
@@ -340,7 +423,14 @@ export async function fetchCctvMediaUpstream(
|
||||
if (downstream?.aborted) controller.abort();
|
||||
else downstream?.addEventListener?.('abort', onDownstreamAbort);
|
||||
try {
|
||||
return await fetchImpl(url, { headers, signal: controller.signal });
|
||||
// Redirects stay on the registered origin, as for frames.
|
||||
const upstream = await fetchWithinHost(
|
||||
url,
|
||||
{ headers, signal: controller.signal },
|
||||
fetchImpl,
|
||||
);
|
||||
if (!upstream) throw new Error('Media redirect refused');
|
||||
return upstream;
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
downstream?.removeEventListener?.('abort', onDownstreamAbort);
|
||||
@@ -428,9 +518,46 @@ export async function fetchTxdotSnapshot(
|
||||
/** Redirect hops the frame path will follow, and only within the same host. */
|
||||
const MAX_SAME_HOST_REDIRECTS = 2;
|
||||
|
||||
/**
|
||||
* Directories camera catalogs are pinned to, by host. A redirect on one of
|
||||
* these hosts must stay inside the pinned directory too, so a same-origin
|
||||
* redirect cannot leave it (another bucket on a shared storage host).
|
||||
*/
|
||||
const CCTV_PINNED_PREFIXES_BY_HOST = new Map();
|
||||
for (const prefix of [
|
||||
TFL_IMAGE_ORIGIN,
|
||||
TALLINN_IMAGE_ORIGIN,
|
||||
TARKTEE_IMAGE_ORIGIN,
|
||||
ICELAND_IMAGE_ORIGIN,
|
||||
...WARENDORF_IMAGE_ORIGINS,
|
||||
NSW_IMAGE_ORIGIN,
|
||||
OTTAWA_IMAGE_ORIGIN,
|
||||
CALGARY_IMAGE_ORIGIN,
|
||||
HK_IMAGE_ORIGIN,
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
WSDOT_IMAGE_ORIGIN,
|
||||
A22_IMAGE_ORIGIN,
|
||||
TRIPCHECK_IMAGE_ORIGIN,
|
||||
MLIT_RIVER_IMAGE_ORIGIN,
|
||||
...TAIWAN_HIGHWAY_IMAGE_ORIGINS,
|
||||
]) {
|
||||
const host = new URL(prefix).hostname;
|
||||
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
|
||||
...(CCTV_PINNED_PREFIXES_BY_HOST.get(host) || []),
|
||||
prefix,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Whether a redirect target stays inside its host's pinned directory. */
|
||||
function withinPinnedPrefix(url) {
|
||||
const prefixes = CCTV_PINNED_PREFIXES_BY_HOST.get(url.hostname);
|
||||
return !prefixes || prefixes.some((p) => pinnedCameraUrl(url.href, p));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a registered frame URL following redirects ONLY within the original
|
||||
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops).
|
||||
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops) and,
|
||||
* on a pinned camera host, within its pinned directory.
|
||||
* Default redirect-following would let an upstream steer a host-pinned
|
||||
* request, and its host-specific headers, to any origin, another port, or a
|
||||
* plaintext downgrade.
|
||||
@@ -471,7 +598,7 @@ export async function fetchWithinHost(url, init, fetchImpl = fetch) {
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (next.origin !== origin) return null;
|
||||
if (next.origin !== origin || !withinPinnedPrefix(next)) return null;
|
||||
current = next;
|
||||
}
|
||||
return null;
|
||||
@@ -504,6 +631,216 @@ export function cctvUpstreamUserAgent(url) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Largest single JPEG the MJPEG first-frame reader will buffer. */
|
||||
export const CCTV_MJPEG_FRAME_MAX_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Read one bounded JPEG part from an open MJPEG response, then stop the stream.
|
||||
*
|
||||
* The first part may follow a CRLF preamble, and its delimiter may be written
|
||||
* as `--<boundary>` or, when the declared boundary already starts with `--`,
|
||||
* as the boundary itself. A part with Content-Length is read to that length;
|
||||
* a part without one ends at the next delimiter, so a JPEG end marker inside
|
||||
* the image (an EXIF thumbnail) never truncates it. Bytes are copied once
|
||||
* into a preallocated buffer and every search resumes where it stopped, so a
|
||||
* trickling upstream costs linear time.
|
||||
*/
|
||||
async function readFirstMjpegFrame(upstream, maxBytes) {
|
||||
const mediaType = cctvLiveMediaType(
|
||||
'mjpeg',
|
||||
upstream.headers.get('content-type'),
|
||||
);
|
||||
const boundary = parseMediaType(mediaType)?.params.get('boundary') || '';
|
||||
const frameMax = Math.min(maxBytes, CCTV_MJPEG_FRAME_MAX_BYTES);
|
||||
if (
|
||||
!boundary ||
|
||||
!upstream.body ||
|
||||
!Number.isSafeInteger(frameMax) ||
|
||||
frameMax < 4
|
||||
) {
|
||||
try {
|
||||
await upstream.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const delimiters = [Buffer.from('--' + boundary, 'latin1')];
|
||||
if (boundary.startsWith('--') && boundary.length > 2)
|
||||
delimiters.push(Buffer.from(boundary, 'latin1'));
|
||||
const lineEnd = Buffer.from('\r\n', 'ascii');
|
||||
const headerTerminator = Buffer.from('\r\n\r\n', 'ascii');
|
||||
const jpegStart = Buffer.from([0xff, 0xd8]);
|
||||
const jpegEnd = Buffer.from([0xff, 0xd9]);
|
||||
const maxPreambleBytes = 64;
|
||||
const maxHeaderBytes = 8 * 1024;
|
||||
const maxBufferedBytes =
|
||||
maxPreambleBytes +
|
||||
delimiters[0].length +
|
||||
lineEnd.length +
|
||||
maxHeaderBytes +
|
||||
headerTerminator.length +
|
||||
frameMax +
|
||||
lineEnd.length +
|
||||
delimiters[0].length;
|
||||
const buffer = Buffer.allocUnsafe(maxBufferedBytes);
|
||||
let length = 0;
|
||||
let reader;
|
||||
let headerStart = -1;
|
||||
let delimiter = null;
|
||||
let headerScanFrom = 0;
|
||||
let bodyStart = -1;
|
||||
let declaredLength = null;
|
||||
let boundaryScanFrom = -1;
|
||||
|
||||
/** Match the opening delimiter after an optional CRLF preamble. */
|
||||
function findOpening() {
|
||||
let at = 0;
|
||||
while (at < length && (buffer[at] === 0x0d || buffer[at] === 0x0a)) at++;
|
||||
if (at > maxPreambleBytes) return { failed: true };
|
||||
for (const candidate of delimiters) {
|
||||
const line = Buffer.concat([candidate, lineEnd]);
|
||||
const need = at + line.length;
|
||||
if (length < need) {
|
||||
const seen = buffer.subarray(at, length);
|
||||
if (line.subarray(0, seen.length).equals(seen))
|
||||
return { pending: true };
|
||||
continue;
|
||||
}
|
||||
if (buffer.subarray(at, need).equals(line))
|
||||
return { start: need, delimiter: candidate };
|
||||
}
|
||||
return { failed: true };
|
||||
}
|
||||
|
||||
/** Parse the part headers; false rejects the stream. */
|
||||
function parseHeaders(headerEnd) {
|
||||
const headerText = buffer
|
||||
.subarray(headerStart, headerEnd)
|
||||
.toString('latin1');
|
||||
const headers = new Map();
|
||||
for (const line of headerText.split('\r\n')) {
|
||||
const colon = line.indexOf(':');
|
||||
if (colon <= 0) return false;
|
||||
const name = line.slice(0, colon).trim().toLowerCase();
|
||||
if (!name || headers.has(name)) return false;
|
||||
headers.set(name, line.slice(colon + 1).trim());
|
||||
}
|
||||
if (cctvRasterImageType(headers.get('content-type')) !== 'image/jpeg')
|
||||
return false;
|
||||
const declared = headers.get('content-length');
|
||||
if (declared !== undefined) {
|
||||
if (!/^\d+$/.test(declared)) return false;
|
||||
declaredLength = Number(declared);
|
||||
if (
|
||||
!Number.isSafeInteger(declaredLength) ||
|
||||
declaredLength < jpegStart.length + jpegEnd.length ||
|
||||
declaredLength > frameMax
|
||||
)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A complete JPEG body, or null. */
|
||||
function asJpeg(frame) {
|
||||
return frame.length >= jpegStart.length + jpegEnd.length &&
|
||||
frame.subarray(0, jpegStart.length).equals(jpegStart) &&
|
||||
frame.subarray(-jpegEnd.length).equals(jpegEnd)
|
||||
? Buffer.from(frame)
|
||||
: null;
|
||||
}
|
||||
|
||||
try {
|
||||
reader = upstream.body.getReader();
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) return null;
|
||||
|
||||
const chunk = Buffer.from(value);
|
||||
const room = maxBufferedBytes - length;
|
||||
const included = Math.min(room, chunk.length);
|
||||
chunk.copy(buffer, length, 0, included);
|
||||
length += included;
|
||||
const overflow = included < chunk.length;
|
||||
|
||||
if (headerStart < 0) {
|
||||
const opening = findOpening();
|
||||
if (opening.failed) return null;
|
||||
if (opening.pending) {
|
||||
if (overflow) return null;
|
||||
continue;
|
||||
}
|
||||
headerStart = opening.start;
|
||||
delimiter = opening.delimiter;
|
||||
headerScanFrom = headerStart;
|
||||
}
|
||||
|
||||
if (bodyStart < 0) {
|
||||
const headerEnd = buffer
|
||||
.subarray(0, length)
|
||||
.indexOf(headerTerminator, headerScanFrom);
|
||||
if (headerEnd < 0) {
|
||||
if (length - headerStart > maxHeaderBytes || overflow) return null;
|
||||
headerScanFrom = Math.max(
|
||||
headerStart,
|
||||
length - headerTerminator.length + 1,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (headerEnd - headerStart > maxHeaderBytes) return null;
|
||||
if (!parseHeaders(headerEnd)) return null;
|
||||
bodyStart = headerEnd + headerTerminator.length;
|
||||
boundaryScanFrom = bodyStart;
|
||||
}
|
||||
|
||||
const available = length - bodyStart;
|
||||
if (available >= jpegStart.length) {
|
||||
if (
|
||||
!buffer
|
||||
.subarray(bodyStart, bodyStart + jpegStart.length)
|
||||
.equals(jpegStart)
|
||||
)
|
||||
return null;
|
||||
}
|
||||
const nextDelimiter = Buffer.concat([lineEnd, delimiter]);
|
||||
const nextPart = buffer
|
||||
.subarray(0, length)
|
||||
.indexOf(nextDelimiter, boundaryScanFrom);
|
||||
if (declaredLength !== null) {
|
||||
if (nextPart >= bodyStart && nextPart - bodyStart < declaredLength)
|
||||
return null;
|
||||
if (available >= declaredLength)
|
||||
return asJpeg(buffer.subarray(bodyStart, bodyStart + declaredLength));
|
||||
} else if (nextPart >= bodyStart) {
|
||||
const frame = buffer.subarray(bodyStart, nextPart);
|
||||
return frame.length > frameMax ? null : asJpeg(frame);
|
||||
} else if (available > frameMax) {
|
||||
return null;
|
||||
}
|
||||
if (nextPart < 0)
|
||||
boundaryScanFrom = Math.max(
|
||||
bodyStart,
|
||||
length - nextDelimiter.length + 1,
|
||||
);
|
||||
if (overflow) return null;
|
||||
}
|
||||
} finally {
|
||||
if (reader) {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
/* already closed or aborted */
|
||||
}
|
||||
try {
|
||||
reader.releaseLock();
|
||||
} catch {
|
||||
/* already released */
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Fetch one upstream CCTV image within the frame-refresh budget.
|
||||
*
|
||||
@@ -516,6 +853,10 @@ export function cctvUpstreamUserAgent(url) {
|
||||
* @param {typeof fetch} [options.fetchImpl=fetch] - Fetch implementation.
|
||||
* @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout.
|
||||
* @param {number} [options.maxBytes=CCTV_FRAME_MAX_BODY_BYTES] - Snapshot byte cap.
|
||||
* @param {boolean} [options.allowMjpeg=false] - Accept a multipart MJPEG
|
||||
* answer and keep its first JPEG. Only sources registered as MJPEG stills
|
||||
* (the Taiwan Highway Bureau feeds) set it; any other host answering
|
||||
* multipart falls through to the next fallback.
|
||||
* @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>}
|
||||
*/
|
||||
export async function fetchCctvImageFromUpstream(
|
||||
@@ -524,6 +865,7 @@ export async function fetchCctvImageFromUpstream(
|
||||
fetchImpl = fetch,
|
||||
timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS,
|
||||
maxBytes = CCTV_FRAME_MAX_BODY_BYTES,
|
||||
allowMjpeg = false,
|
||||
} = {},
|
||||
) {
|
||||
if (!url || !/^https?:\/\//i.test(url)) return null;
|
||||
@@ -543,14 +885,32 @@ export async function fetchCctvImageFromUpstream(
|
||||
fetchImpl,
|
||||
);
|
||||
if (!upstream) return null;
|
||||
const contentType = upstream.headers.get('content-type') || '';
|
||||
if (!upstream.ok || !contentType.startsWith('image/')) {
|
||||
controller.abort();
|
||||
if (!upstream.ok) {
|
||||
try {
|
||||
await upstream.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const body = await readCappedResponseBytes(upstream, maxBytes);
|
||||
const upstreamType = upstream.headers.get('content-type');
|
||||
const contentType = cctvRasterImageType(upstreamType);
|
||||
if (contentType) {
|
||||
const body = await readCappedResponseBytes(upstream, maxBytes);
|
||||
if (!body) return null;
|
||||
return { ok: true, body, contentType };
|
||||
}
|
||||
if (!allowMjpeg || !cctvLiveMediaType('mjpeg', upstreamType)) {
|
||||
try {
|
||||
await upstream.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const body = await readFirstMjpegFrame(upstream, maxBytes);
|
||||
if (!body) return null;
|
||||
return { ok: true, body, contentType };
|
||||
return { ok: true, body, contentType: 'image/jpeg' };
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
|
||||
@@ -64,7 +64,7 @@ export function toFiniteNumber(value, fallback = NaN) {
|
||||
* @param {string} text
|
||||
* @returns {string}
|
||||
*/
|
||||
export function normalizeKey(text) {
|
||||
function normalizeKey(text) {
|
||||
return String(text || '')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
@@ -80,7 +80,7 @@ export function normalizeKey(text) {
|
||||
* @param {string} value
|
||||
* @returns {{lat:number, lon:number}}
|
||||
*/
|
||||
export function parsePointString(value) {
|
||||
function parsePointString(value) {
|
||||
const match = String(value || '').match(
|
||||
/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i,
|
||||
);
|
||||
@@ -100,7 +100,7 @@ export function parsePointString(value) {
|
||||
* @param {string|object|null} value
|
||||
* @returns {{lat:number, lon:number}}
|
||||
*/
|
||||
export function coerceLatLon(value) {
|
||||
function coerceLatLon(value) {
|
||||
if (!value) return { lat: NaN, lon: NaN };
|
||||
|
||||
if (typeof value === 'string') {
|
||||
@@ -399,6 +399,29 @@ export function isLikelyNycCoordinate(lat, lon) {
|
||||
);
|
||||
}
|
||||
|
||||
/** Hong Kong SAR extent, with slack for the outlying islands. */
|
||||
export function isLikelyHongKongCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 22.15 &&
|
||||
lat <= 22.6 &&
|
||||
lon >= 113.8 &&
|
||||
lon <= 114.45
|
||||
);
|
||||
}
|
||||
|
||||
/** Greater Houston (the Woodlands to Galveston, Katy to Baytown). Farther
|
||||
* TranStar partners (Dallas, Columbus) are out of this city pack. */
|
||||
export function isLikelyHoustonCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 29.2 &&
|
||||
lat <= 30.35 &&
|
||||
lon >= -96.05 &&
|
||||
lon <= -94.7
|
||||
);
|
||||
}
|
||||
|
||||
/** Mainland Norway, with slack for the coast and the Finnmark border. */
|
||||
export function isLikelyNorwayCoordinate(lat, lon) {
|
||||
return (
|
||||
@@ -410,6 +433,17 @@ export function isLikelyNorwayCoordinate(lat, lon) {
|
||||
);
|
||||
}
|
||||
|
||||
/** The A22 corridor, Brenner Pass to Modena, with slack either side. */
|
||||
export function isLikelyA22Coordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 44.5 &&
|
||||
lat <= 47.1 &&
|
||||
lon >= 10.5 &&
|
||||
lon <= 11.8
|
||||
);
|
||||
}
|
||||
|
||||
export function isLikelyFinlandCoordinate(lat, lon) {
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
|
||||
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
|
||||
@@ -426,6 +460,17 @@ export function isLikelyCataloniaCoordinate(lat, lon) {
|
||||
);
|
||||
}
|
||||
|
||||
/** Iceland bounding box (mainland and nearby islands, with a margin). */
|
||||
export function isLikelyIcelandCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 63.0 &&
|
||||
lat <= 66.7 &&
|
||||
lon >= -25.0 &&
|
||||
lon <= -13.0
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode numeric HTML character references, decimal ("í") and hex
|
||||
* ("í") alike, the entity form the Catalonia feed uses for accented
|
||||
@@ -605,6 +650,10 @@ export function normalizeSourceItem(item) {
|
||||
feedType: normalizeFeedType(item.feedType || item.type || ''),
|
||||
url: typeof item.url === 'string' ? item.url : '',
|
||||
snapshotUrl: typeof item.snapshotUrl === 'string' ? item.snapshotUrl : '',
|
||||
frameRefreshMs: toFiniteNumber(item.frameRefreshMs),
|
||||
sourceUrl: typeof item.sourceUrl === 'string' ? item.sourceUrl : '',
|
||||
accuracyM: toFiniteNumber(item.accuracyM),
|
||||
positionSource: String(item.positionSource || ''),
|
||||
license: String(item.license || item.licenseNote || ''),
|
||||
// Per-camera attribution for feeds a partner supplies inside a pack
|
||||
// (DriveBC: TransLink, city cameras). Shown beside the provider.
|
||||
@@ -648,3 +697,38 @@ export function isLikelyTorontoCoordinate(lat, lon) {
|
||||
lon <= -79.1
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A camera URL under a pinned directory, by parsed URL rather than string
|
||||
* prefix: the same scheme and exact host as `prefix`, no credentials, the
|
||||
* default port, and a normalized path inside the prefix's directory. Dot
|
||||
* segments (including encoded ones) and backslashes are resolved before the
|
||||
* path is compared, and an encoded slash or backslash is refused.
|
||||
*
|
||||
* @param {string} raw - Camera URL from a catalog or redirect.
|
||||
* @param {string} prefix - Pinned origin and directory, ending in `/`.
|
||||
* @returns {string|null} The normalized URL, or null when it is not pinned.
|
||||
*/
|
||||
export function pinnedCameraUrl(raw, prefix) {
|
||||
let url;
|
||||
let pin;
|
||||
try {
|
||||
url = new URL(String(raw ?? '').trim());
|
||||
pin = new URL(prefix);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
url.protocol !== pin.protocol ||
|
||||
url.hostname !== pin.hostname ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.port ||
|
||||
!url.pathname.startsWith(
|
||||
pin.pathname.endsWith('/') ? pin.pathname : `${pin.pathname}/`,
|
||||
) ||
|
||||
/%2f|%5c/i.test(url.pathname)
|
||||
)
|
||||
return null;
|
||||
return url.href;
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
import {
|
||||
CCTV_SOURCE_FETCH_TIMEOUT_MS,
|
||||
DEFAULT_OTTAWA_MAX_SOURCES,
|
||||
OTTAWA_ANCHORS,
|
||||
OTTAWA_API_USER_ID,
|
||||
OTTAWA_CAMERAS_URL,
|
||||
OTTAWA_FRAME_REFRESH_MS,
|
||||
OTTAWA_IMAGE_ORIGIN,
|
||||
OTTAWA_IMAGE_URL,
|
||||
} from './constants.js';
|
||||
import {
|
||||
fallbackHeadingFromId,
|
||||
pinnedCameraUrl,
|
||||
prioritizeSources,
|
||||
toFiniteNumber,
|
||||
} from './normalize.js';
|
||||
|
||||
let ottawaCertificateMissingLogged = false;
|
||||
|
||||
/** Test hook: let the next missing-certificate skip log again. */
|
||||
export function _resetOttawaCertificateWarningForTest() {
|
||||
ottawaCertificateMissingLogged = false;
|
||||
}
|
||||
|
||||
/** The operator's Traffic Camera Images API certificate, trimmed. */
|
||||
function ottawaCertificate(env = process.env) {
|
||||
return String(env.OTTAWA_CAMERA_CERTIFICATE ?? '').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the operator's certificate to a registered keyless Ottawa frame URL.
|
||||
* Only a URL inside the pinned `/opendata/` directory with a numeric camera
|
||||
* number is signed, so the certificate never goes to any other path or host.
|
||||
*
|
||||
* @param {string} url - Canonical frame URL from the catalog.
|
||||
* @param {Record<string,string|undefined>} [env]
|
||||
* @returns {string} The signed URL, or '' when there is no certificate or the
|
||||
* URL is not a registered Ottawa frame.
|
||||
*/
|
||||
export function ottawaAuthorizedFrameUrl(url, env = process.env) {
|
||||
const certificate = ottawaCertificate(env);
|
||||
const pinned = pinnedCameraUrl(url, OTTAWA_IMAGE_ORIGIN);
|
||||
if (!certificate || !pinned) return '';
|
||||
const parsed = new URL(pinned);
|
||||
const cameraNumber = parsed.searchParams.get('c') || '';
|
||||
if (parsed.pathname !== '/opendata/camera' || !/^\d+$/.test(cameraNumber))
|
||||
return '';
|
||||
const signed = new URL(OTTAWA_IMAGE_URL(cameraNumber));
|
||||
signed.searchParams.set('certificate', certificate);
|
||||
signed.searchParams.set('id', OTTAWA_API_USER_ID);
|
||||
return signed.href;
|
||||
}
|
||||
|
||||
/**
|
||||
* City of Ottawa envelope: Kanata to Orléans, Barrhaven to the river, with
|
||||
* slack for the MTO cameras on the 416/417 that the city feed also carries.
|
||||
*/
|
||||
const OTTAWA_BOUNDS = Object.freeze({
|
||||
minLat: 45.1,
|
||||
maxLat: 45.65,
|
||||
minLon: -76.2,
|
||||
maxLon: -75.1,
|
||||
});
|
||||
|
||||
/** Whether a point is inside the city feed's service area. */
|
||||
function isOttawaCoordinate(lat, lon) {
|
||||
return (
|
||||
Number.isFinite(lat) &&
|
||||
Number.isFinite(lon) &&
|
||||
lat >= OTTAWA_BOUNDS.minLat &&
|
||||
lat <= OTTAWA_BOUNDS.maxLat &&
|
||||
lon >= OTTAWA_BOUNDS.minLon &&
|
||||
lon <= OTTAWA_BOUNDS.maxLon
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the City of Ottawa's traffic cameras.
|
||||
*
|
||||
* Camera images come from the city's Traffic Camera Images API, which needs a
|
||||
* certificate from the city's registration form (OTTAWA_CAMERA_CERTIFICATE)
|
||||
* and allows one fetch per camera per minute for a certificate and user id.
|
||||
* Without a certificate the pack is skipped, with no request and one log line.
|
||||
* The camera list is the documented `map/service/camera` JSON: a camera
|
||||
* number, English and French names, coordinates and the owner (CITY or MTO).
|
||||
* The feed carries no bearing, no elevation and no status field, so headings
|
||||
* use the same deterministic low-confidence fallback as the other
|
||||
* heading-less packs and the client's ground snap refines the elevation
|
||||
* prior.
|
||||
*
|
||||
* @param {object} [_options]
|
||||
* @param {Record<string,string|undefined>} [env]
|
||||
* @returns {Promise<Array<object>>} Cameras nearest downtown first, or [] on
|
||||
* a missing certificate or any upstream or payload failure.
|
||||
*/
|
||||
export async function loadOttawaSourcesFromOpenData(
|
||||
_options = {},
|
||||
env = process.env,
|
||||
) {
|
||||
if (!ottawaCertificate(env)) {
|
||||
if (!ottawaCertificateMissingLogged) {
|
||||
ottawaCertificateMissingLogged = true;
|
||||
console.warn(
|
||||
'[CCTV] Ottawa cameras skipped: the city camera image API needs a certificate (set OTTAWA_CAMERA_CERTIFICATE)',
|
||||
);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const resp = await fetch(OTTAWA_CAMERAS_URL, {
|
||||
headers: { Accept: 'application/json' },
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] Ottawa camera download failed:', resp.status);
|
||||
return [];
|
||||
}
|
||||
const payload = await resp.json();
|
||||
const rows = Array.isArray(payload?.cameras) ? payload.cameras : null;
|
||||
if (!rows) return [];
|
||||
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const row of rows) {
|
||||
const number = toFiniteNumber(row?.camera_number);
|
||||
if (!Number.isSafeInteger(number) || number <= 0 || seen.has(number))
|
||||
continue;
|
||||
const lat = toFiniteNumber(row?.latitude);
|
||||
const lon = toFiniteNumber(row?.longitude);
|
||||
if (!isOttawaCoordinate(lat, lon)) continue;
|
||||
seen.add(number);
|
||||
|
||||
const cameraId = `ottawa-${number}`;
|
||||
const imageUrl = OTTAWA_IMAGE_URL(number);
|
||||
// The docs name the field camera_owner; the live feed sends cameraOwner.
|
||||
const owner = String(row?.camera_owner || row?.cameraOwner || '')
|
||||
.trim()
|
||||
.toUpperCase();
|
||||
cameras.push({
|
||||
id: cameraId,
|
||||
name: String(row?.name || '').trim() || `Ottawa camera ${number}`,
|
||||
city: 'Ottawa',
|
||||
cityId: 'ottawa',
|
||||
provider: 'City of Ottawa',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: 0,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
frameRefreshMs: OTTAWA_FRAME_REFRESH_MS,
|
||||
sourceKind: 'ottawa-traffic',
|
||||
license: 'City of Ottawa Traffic Camera Images API (registered access)',
|
||||
credit:
|
||||
owner === 'MTO'
|
||||
? 'Ministry of Transportation of Ontario (MTO) camera'
|
||||
: '',
|
||||
});
|
||||
}
|
||||
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_OTTAWA_MAX_SOURCES || DEFAULT_OTTAWA_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(500, Math.floor(maxRaw)))
|
||||
: DEFAULT_OTTAWA_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, OTTAWA_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded Ottawa camera sources: ${cameras.length} published (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] Ottawa camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { cameraDisplayCode, fallbackHeadingFromId } from './normalize.js';
|
||||
|
||||
const QUEBEC_CATALOG_PATH = 'public/quebec-cameras.json';
|
||||
export const QUEBEC_CAMERA_SOURCE_COUNT = 1120;
|
||||
|
||||
/**
|
||||
* Whether the operator opted in to the browser-direct Québec 511 video. Off by
|
||||
* default: the camera locations are CC BY 4.0, but whether Québec 511's
|
||||
* service terms allow another site to embed its clips is unconfirmed, so an
|
||||
* unset flag lists the cameras with a link to the 511 viewer page only.
|
||||
*/
|
||||
export function quebecBrowserVideoEnabled(env = process.env) {
|
||||
return String(env.CCTV_QUEBEC_VIDEO_ENABLED || '').trim() === '1';
|
||||
}
|
||||
|
||||
/** Build the direct browser-only MP4 URL for one namespaced MTQ camera id. */
|
||||
export function quebec511VideoUrl(cameraId) {
|
||||
const match = /^qc-511-(\d+)$/.exec(String(cameraId || ''));
|
||||
return match
|
||||
? `https://www.quebec511.info/Carte/Fenetres/camera.ashx?id=${match[1]}&format=mp4`
|
||||
: '';
|
||||
}
|
||||
|
||||
function safeViewerUrl(value) {
|
||||
try {
|
||||
const url = new URL(String(value || ''));
|
||||
return url.protocol === 'https:' && !url.username && !url.password
|
||||
? url.href
|
||||
: '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function sourceId(camera) {
|
||||
const id = String(camera?.id || '').trim();
|
||||
if (camera?.kind === 'traffic' && /^\d+$/.test(id)) return `qc-511-${id}`;
|
||||
if (camera?.kind === 'montreal' && /^mtl-\d+(?:-\d+)*$/.test(id))
|
||||
return `qc-mtl-${id.slice(4)}`;
|
||||
if (camera?.kind === 'webcam' && /^web-\d+$/.test(id))
|
||||
return `qc-web-${id.slice(4)}`;
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the checked-in, no-key Québec location catalog and adapt it to the
|
||||
* regular CCTV source schema. MTQ video stays browser-only; other rows link to
|
||||
* their operator page because this dataset has no direct image URL for them.
|
||||
*/
|
||||
export function loadQuebecSources({ sourceRoot = process.cwd() } = {}) {
|
||||
const filename = path.resolve(sourceRoot, QUEBEC_CATALOG_PATH);
|
||||
let cameras;
|
||||
try {
|
||||
const payload = JSON.parse(fs.readFileSync(filename, 'utf8'));
|
||||
cameras = Array.isArray(payload?.cameras) ? payload.cameras : [];
|
||||
} catch (error) {
|
||||
console.warn('[CCTV] Québec camera catalog unavailable:', error?.message);
|
||||
return [];
|
||||
}
|
||||
|
||||
const sources = [];
|
||||
const seen = new Set();
|
||||
for (const camera of cameras) {
|
||||
const id = sourceId(camera);
|
||||
const lat = Number(camera.lat);
|
||||
const lon = Number(camera.lon);
|
||||
if (
|
||||
!id ||
|
||||
seen.has(id) ||
|
||||
!Number.isFinite(lat) ||
|
||||
!Number.isFinite(lon) ||
|
||||
lat < 44.9 ||
|
||||
lat > 62.6 ||
|
||||
lon < -79.9 ||
|
||||
lon > -57.1
|
||||
)
|
||||
continue;
|
||||
seen.add(id);
|
||||
|
||||
const name = String(camera.nameEn || camera.name || id).trim();
|
||||
const isTraffic = camera.kind === 'traffic';
|
||||
const isMontreal = camera.kind === 'montreal';
|
||||
const sourceKind = isTraffic
|
||||
? 'quebec-511-browser'
|
||||
: isMontreal
|
||||
? 'quebec-montreal-osm'
|
||||
: 'quebec-webcam-link';
|
||||
const provider = isTraffic
|
||||
? 'Québec 511 / MTQ'
|
||||
: isMontreal
|
||||
? 'Ville de Montréal / OpenStreetMap'
|
||||
: String(camera.operator || 'Québec webcam');
|
||||
|
||||
sources.push({
|
||||
id,
|
||||
name: name.slice(0, 180),
|
||||
city: String(camera.region || 'Québec'),
|
||||
cityId: 'quebec',
|
||||
provider,
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(id),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 74,
|
||||
rangeM: 400,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: 0,
|
||||
feedType: 'image',
|
||||
// No frame URL is registered: the server never fetches 511 media. With
|
||||
// CCTV_QUEBEC_VIDEO_ENABLED=1 the browser plays the 511 MP4 in the DOM
|
||||
// overlay after a play click; otherwise the camera links to 511.
|
||||
url: '',
|
||||
snapshotUrl: '',
|
||||
sourceKind,
|
||||
sourceUrl: safeViewerUrl(camera.viewerUrl),
|
||||
accuracyM: Number.isFinite(Number(camera.accuracyM))
|
||||
? Number(camera.accuracyM)
|
||||
: null,
|
||||
positionSource: String(camera.positionSource || ''),
|
||||
license: isTraffic
|
||||
? 'Locations: Données Québec / MTQ, CC BY 4.0. Video: hosted by Québec 511 under its terms of use'
|
||||
: isMontreal
|
||||
? 'OpenStreetMap contributors, ODbL 1.0'
|
||||
: 'Operator terms apply; location provenance shown per camera',
|
||||
credit: isTraffic
|
||||
? 'Locations: MTQ · CC BY 4.0 · Video: Québec 511'
|
||||
: isMontreal
|
||||
? 'OpenStreetMap contributors · ODbL'
|
||||
: provider,
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
});
|
||||
}
|
||||
|
||||
return sources;
|
||||
}
|
||||
@@ -77,6 +77,11 @@ import {
|
||||
TORONTO_DOWNTOWN,
|
||||
TORONTO_GROUND_ELEVATION_M,
|
||||
TORONTO_MAX_CATALOG_BYTES,
|
||||
DEFAULT_HK_CAMERAS_URL,
|
||||
HK_IMAGE_ORIGIN,
|
||||
DEFAULT_HK_MAX_SOURCES,
|
||||
HK_ANCHORS,
|
||||
HK_MAX_CATALOG_BYTES,
|
||||
CCTV_SOURCE_FETCH_TIMEOUT_MS,
|
||||
DELDOT_CCTV_URL,
|
||||
DEFAULT_DELDOT_MAX_SOURCES,
|
||||
@@ -117,6 +122,24 @@ import {
|
||||
NYC_MAX_CATALOG_BYTES,
|
||||
NYC_GROUND_ELEVATION_M,
|
||||
NYC_ANCHORS,
|
||||
DEFAULT_A22_CCTV_URL,
|
||||
A22_IMAGE_ORIGIN,
|
||||
A22_MAX_CATALOG_BYTES,
|
||||
A22_GROUND_ELEVATION_BY_KM,
|
||||
A22_DEFAULT_GROUND_ELEVATION_M,
|
||||
HOUSTON_CAMERAS_URL,
|
||||
HOUSTON_IMAGE_URL,
|
||||
HOUSTON_CAMERA_ID_RE,
|
||||
DEFAULT_HOUSTON_MAX_SOURCES,
|
||||
HOUSTON_MAX_CATALOG_BYTES,
|
||||
HOUSTON_GROUND_ELEVATION_M,
|
||||
HOUSTON_ANCHORS,
|
||||
ICELAND_CAMERAS_URL,
|
||||
ICELAND_IMAGE_ORIGIN,
|
||||
ICELAND_MAX_CATALOG_BYTES,
|
||||
ICELAND_GROUND_ELEVATION_M,
|
||||
DEFAULT_ICELAND_MAX_SOURCES,
|
||||
ICELAND_ANCHORS,
|
||||
} from './constants.js';
|
||||
import {
|
||||
toFiniteNumber,
|
||||
@@ -128,7 +151,6 @@ import {
|
||||
fallbackHeadingFromId,
|
||||
isLikelyFinlandCoordinate,
|
||||
fintrafficCameraName,
|
||||
hashSeed,
|
||||
isPlausibleLatLon,
|
||||
isLikelyBcCoordinate,
|
||||
isLikelyTexasCoordinate,
|
||||
@@ -136,13 +158,18 @@ import {
|
||||
isLikelyCalgaryCoordinate,
|
||||
isLikelyFloridaCoordinate,
|
||||
isLikelyCataloniaCoordinate,
|
||||
isLikelyIcelandCoordinate,
|
||||
decodeNumericEntities,
|
||||
isLikelyNycCoordinate,
|
||||
isLikelyHoustonCoordinate,
|
||||
isLikelyHongKongCoordinate,
|
||||
isLikelyNorwayCoordinate,
|
||||
isLikelyTorontoCoordinate,
|
||||
isLikelyA22Coordinate,
|
||||
cameraDisplayCode,
|
||||
rowArrayToObject,
|
||||
prioritizeSources,
|
||||
pinnedCameraUrl,
|
||||
} from './normalize.js';
|
||||
import { directionToHeading } from '../../../src/data/directionText.js';
|
||||
import {
|
||||
@@ -304,9 +331,12 @@ export async function loadCaltransSourcesFromOpenData() {
|
||||
const lon = toFiniteNumber(loc.longitude);
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
|
||||
|
||||
const imageUrl = String(cctv.imageData?.static?.currentImageURL || '');
|
||||
// Official-host pin (see JSDoc). Also drops records with no still image.
|
||||
if (!imageUrl.startsWith('https://cwwp2.dot.ca.gov/')) continue;
|
||||
const imageUrl = pinnedCameraUrl(
|
||||
cctv.imageData?.static?.currentImageURL,
|
||||
'https://cwwp2.dot.ca.gov/',
|
||||
);
|
||||
if (!imageUrl) continue;
|
||||
|
||||
const locationName = String(loc.locationName || '').trim();
|
||||
// Leading token of locationName is the stable camera code ("TV102 -- I-580 : …").
|
||||
@@ -411,8 +441,8 @@ export async function loadTflSourcesFromOpenData() {
|
||||
const lat = toFiniteNumber(place?.lat);
|
||||
const lon = toFiniteNumber(place?.lon);
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
|
||||
const imageUrl = String(props.imageUrl || '');
|
||||
if (!imageUrl.startsWith(TFL_IMAGE_ORIGIN)) continue; // official-bucket pin
|
||||
const imageUrl = pinnedCameraUrl(props.imageUrl, TFL_IMAGE_ORIGIN);
|
||||
if (!imageUrl) continue; // official-bucket pin
|
||||
|
||||
// "JamCams_00002.00865" → "tfl-00002.00865" (provider-stable id).
|
||||
const rawId = String(place?.id || '').replace(/^JamCams_/, '');
|
||||
@@ -1123,13 +1153,15 @@ export function loadTallinnSourcesFromCatalog({
|
||||
// Rough Estonia/Tallinn metro sanity (allows nearby suburbs already in the pack).
|
||||
if (lat < 59.2 || lat > 59.7 || lon < 24.3 || lon > 25.4) continue;
|
||||
|
||||
const imageUrl =
|
||||
const imageUrl = pinnedCameraUrl(
|
||||
typeof item.url === 'string'
|
||||
? item.url.trim()
|
||||
? item.url
|
||||
: typeof item.snapshotUrl === 'string'
|
||||
? item.snapshotUrl.trim()
|
||||
: '';
|
||||
if (!imageUrl.startsWith(TALLINN_IMAGE_ORIGIN)) continue;
|
||||
? item.snapshotUrl
|
||||
: '',
|
||||
TALLINN_IMAGE_ORIGIN,
|
||||
);
|
||||
if (!imageUrl) continue;
|
||||
|
||||
const extractedHeading = toFiniteNumber(item.headingDeg, NaN);
|
||||
const hasHeading = Number.isFinite(extractedHeading);
|
||||
@@ -1231,8 +1263,8 @@ export function parseTarkteeDatexImages(xml) {
|
||||
body,
|
||||
);
|
||||
if (!refMatch || !urlMatch) continue;
|
||||
const url = urlMatch[1].trim();
|
||||
if (!url.startsWith(TARKTEE_IMAGE_ORIGIN)) continue;
|
||||
const url = pinnedCameraUrl(urlMatch[1], TARKTEE_IMAGE_ORIGIN);
|
||||
if (!url) continue;
|
||||
out.set(refMatch[1], url);
|
||||
}
|
||||
return out;
|
||||
@@ -1395,14 +1427,16 @@ export function loadWarendorfSourcesFromCatalog({
|
||||
for (const item of rows) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const id = typeof item.id === 'string' ? item.id.trim() : '';
|
||||
const url =
|
||||
const raw =
|
||||
typeof item.url === 'string'
|
||||
? item.url.trim()
|
||||
? item.url
|
||||
: typeof item.snapshotUrl === 'string'
|
||||
? item.snapshotUrl.trim()
|
||||
? item.snapshotUrl
|
||||
: '';
|
||||
if (!id || !WARENDORF_IMAGE_ORIGINS.some((o) => url.startsWith(o)))
|
||||
continue;
|
||||
const url = WARENDORF_IMAGE_ORIGINS.map((o) =>
|
||||
pinnedCameraUrl(raw, o),
|
||||
).find(Boolean);
|
||||
if (!id || !url) continue;
|
||||
const lat = typeof item.lat === 'number' ? item.lat : NaN;
|
||||
const lon = typeof item.lon === 'number' ? item.lon : NaN;
|
||||
if (!isPlausibleLatLon(lat, lon)) continue;
|
||||
@@ -1456,8 +1490,8 @@ export function nswCameraToSource(feature) {
|
||||
const lat = typeof coords?.[1] === 'number' ? coords[1] : NaN;
|
||||
if (!isLikelyNswCoordinate(lat, lon)) return null;
|
||||
const props = feature?.properties || {};
|
||||
const url = String(props.href || '').trim();
|
||||
if (!url.startsWith(NSW_IMAGE_ORIGIN)) return null;
|
||||
const url = pinnedCameraUrl(props.href, NSW_IMAGE_ORIGIN);
|
||||
if (!url) return null;
|
||||
// "N-E" -> "NE" for the compass lookup.
|
||||
const direction = String(props.direction || '')
|
||||
.trim()
|
||||
@@ -1551,8 +1585,7 @@ export function normalizeCalgaryImageUrl(raw) {
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
|
||||
parsed.protocol = 'https:';
|
||||
const upgraded = parsed.toString();
|
||||
return upgraded.startsWith(CALGARY_IMAGE_ORIGIN) ? upgraded : null;
|
||||
return pinnedCameraUrl(parsed.toString(), CALGARY_IMAGE_ORIGIN);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1835,7 +1868,7 @@ export function normalizeCataloniaImageUrl(rawUrl) {
|
||||
* @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted.
|
||||
* @returns {string}
|
||||
*/
|
||||
export function cataloniaCameraId(imageUrl) {
|
||||
function cataloniaCameraId(imageUrl) {
|
||||
const parsed = new URL(imageUrl);
|
||||
const slug = (text) =>
|
||||
text
|
||||
@@ -1982,6 +2015,393 @@ export async function loadCataloniaSourcesFromOpenData() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compass headings for Icelandic direction words, as they appear after "til"
|
||||
* (genitive: "til vesturs"), "í" (accusative: "í vestur") or "séð" ("seen",
|
||||
* as in "séð austur").
|
||||
*/
|
||||
const ICELAND_DIRECTION_HEADINGS = {
|
||||
norður: 0,
|
||||
norðaustur: 45,
|
||||
austur: 90,
|
||||
suðaustur: 135,
|
||||
suður: 180,
|
||||
suðvestur: 225,
|
||||
vestur: 270,
|
||||
norðvestur: 315,
|
||||
};
|
||||
const ICELAND_DIRECTION_WORD =
|
||||
'(norðaustur|norðvestur|suðaustur|suðvestur|norður|suður|austur|vestur)s?';
|
||||
/** A facing, plus a second one joined by "og" ("til austurs og vesturs"). */
|
||||
const ICELAND_DIRECTION_RE = new RegExp(
|
||||
`(?<![\\p{L}])(?:til|í|séð)\\s+${ICELAND_DIRECTION_WORD}` +
|
||||
`(?:\\s+og\\s+(?:til\\s+|í\\s+)?${ICELAND_DIRECTION_WORD})?(?![\\p{L}])`,
|
||||
'gu',
|
||||
);
|
||||
|
||||
/**
|
||||
* The compass heading an Icelandic view description states, or NaN. Only an
|
||||
* explicit "til <átt>", "í <átt>" or "séð <átt>" counts: a bare place name or
|
||||
* a road direction ("Miklabraut austur") is not a facing. A description naming
|
||||
* two different directions ("til norðurs og til suðurs", "til austurs og
|
||||
* vesturs") is ambiguous and yields NaN, and "niður á veg" (down onto the
|
||||
* road) states none.
|
||||
*
|
||||
* @param {string} text - The feed's `Skyring` field.
|
||||
* @returns {number} Heading in degrees [0..360), or NaN.
|
||||
*/
|
||||
export function icelandHeadingFromDescription(text) {
|
||||
const headings = new Set();
|
||||
for (const match of String(text || '')
|
||||
.toLowerCase()
|
||||
.matchAll(ICELAND_DIRECTION_RE)) {
|
||||
for (const word of [match[1], match[2]]) {
|
||||
if (word) headings.add(ICELAND_DIRECTION_HEADINGS[word]);
|
||||
}
|
||||
}
|
||||
return headings.size === 1 ? headings.values().next().value : NaN;
|
||||
}
|
||||
|
||||
/**
|
||||
* One Vegagerðin camera row -> one catalog source, or null. A row is one camera
|
||||
* view; a station (`Maelist_nr`) carries several. The id comes from the frame
|
||||
* file name, which is unique per view in the live feed.
|
||||
*
|
||||
* @param {object} row - Element of the `vefmyndavelar2014_1` array.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function icelandCameraToSource(row) {
|
||||
const url = pinnedCameraUrl(row?.Slod, ICELAND_IMAGE_ORIGIN);
|
||||
if (!url || !/\.jpe?g$/i.test(new URL(url).pathname)) return null;
|
||||
// Numbers only: Number(null) and Number('') are 0.
|
||||
const lat = typeof row.Breidd === 'number' ? row.Breidd : NaN;
|
||||
const lon = typeof row.Lengd === 'number' ? row.Lengd : NaN;
|
||||
if (!isLikelyIcelandCoordinate(lat, lon)) return null;
|
||||
const stem = new URL(url).pathname
|
||||
.slice(new URL(ICELAND_IMAGE_ORIGIN).pathname.length)
|
||||
.replace(/\.jpe?g$/i, '')
|
||||
.toLowerCase();
|
||||
if (!/^[a-z0-9_-]+$/.test(stem)) return null;
|
||||
const cameraId = `is-vegagerdin-${stem}`;
|
||||
const description = String(row.Skyring || '')
|
||||
.replace(/\s+/g, ' ')
|
||||
// Letter lookarounds, not \b: \b is ASCII-only and would join "Þverá
|
||||
// Árnessýslu" while missing "séð séð".
|
||||
.replace(/(?<!\p{L})(\p{L}+) \1(?!\p{L})/giu, '$1')
|
||||
.trim();
|
||||
const station = String(row.Myndavel || '').trim();
|
||||
const heading = icelandHeadingFromDescription(description);
|
||||
const hasHeading = Number.isFinite(heading);
|
||||
return {
|
||||
id: cameraId,
|
||||
name: description || station || `Vegagerðin ${row.Maelist_nr ?? stem}`,
|
||||
city: 'Iceland',
|
||||
cityId: 'iceland',
|
||||
provider: 'Vegagerðin (IRCA)',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: hasHeading ? 'high' : 'low',
|
||||
pitchDeg: hasHeading ? -24 : -18,
|
||||
fovDeg: hasHeading ? 56 : 44,
|
||||
rangeM: hasHeading ? 210 : 145,
|
||||
mountHeightM: hasHeading ? 10 : 8,
|
||||
groundElevationM: ICELAND_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url,
|
||||
snapshotUrl: url,
|
||||
sourceKind: 'iceland-vegagerdin',
|
||||
license:
|
||||
'Icelandic Road and Coastal Administration (IRCA) — road camera images, CC BY 4.0',
|
||||
code: cameraDisplayCode(station.toUpperCase() || description),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Icelandic road cameras (Vegagerðin / IRCA), keyless: one JSON list.
|
||||
* Frames are stills on the IRCA host and the loader registers only links
|
||||
* inside that host's camera directory (ICELAND_IMAGE_ORIGIN). The list decides
|
||||
* ids, coordinates, labels and which frame path is proxied, so it is fetched
|
||||
* over https, redirects are refused and the body read is capped at
|
||||
* ICELAND_MAX_CATALOG_BYTES.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadIcelandSourcesFromOpenData() {
|
||||
try {
|
||||
const resp = await fetch(ICELAND_CAMERAS_URL, {
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': 'gods-eye-view-cctv-proxy/1.0',
|
||||
},
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
// A response this loader will not read still owns its transport until the
|
||||
// body is released, so every rejection path cancels before returning.
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn(
|
||||
'[CCTV] Iceland catalog redirected; redirects are not followed',
|
||||
);
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] Iceland camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const body = await readResponseJsonCapped(resp, ICELAND_MAX_CATALOG_BYTES);
|
||||
const rows = Array.isArray(body) ? body : [];
|
||||
const cameras = rows.map(icelandCameraToSource).filter(Boolean);
|
||||
const unique = Array.from(
|
||||
new Map(cameras.map((camera) => [camera.id, camera])).values(),
|
||||
);
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_ICELAND_MAX_SOURCES || DEFAULT_ICELAND_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(600, Math.floor(maxRaw)))
|
||||
: DEFAULT_ICELAND_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(unique, maxCount, ICELAND_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded Iceland camera sources: ${unique.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] Iceland camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one XML child tag's text from an `<image>` block.
|
||||
*
|
||||
* @param {string} body
|
||||
* @param {string} tag
|
||||
* @returns {string}
|
||||
*/
|
||||
function hkXmlTag(body, tag) {
|
||||
const match = new RegExp(`<${tag}>\\s*([^<]*?)\\s*</${tag}>`, 'i').exec(
|
||||
String(body || ''),
|
||||
);
|
||||
return match ? match[1].trim() : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the Transport Department English camera-locations XML into rows.
|
||||
*
|
||||
* Schema-specific: each `<image>` carries key, description, lat/lon and a
|
||||
* still URL. No XML dependency — same regex style as the Tarktee DATEX parser.
|
||||
*
|
||||
* @param {string} xml
|
||||
* @returns {Array<object>}
|
||||
*/
|
||||
export function parseHkCameraLocationsXml(xml) {
|
||||
const rows = [];
|
||||
const blockRe = /<image>([\s\S]*?)<\/image>/gi;
|
||||
let match;
|
||||
while ((match = blockRe.exec(String(xml || ''))) !== null) {
|
||||
const body = match[1];
|
||||
rows.push({
|
||||
key: hkXmlTag(body, 'key'),
|
||||
region: hkXmlTag(body, 'region'),
|
||||
district: hkXmlTag(body, 'district'),
|
||||
description: hkXmlTag(body, 'description'),
|
||||
latitude: hkXmlTag(body, 'latitude'),
|
||||
longitude: hkXmlTag(body, 'longitude'),
|
||||
url: hkXmlTag(body, 'url'),
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a host-pinned still URL from a Transport Department camera key.
|
||||
*
|
||||
* The catalog also publishes a URL; the registered frame is always rebuilt
|
||||
* from the key so a catalog edit cannot steer the proxy off-origin.
|
||||
*
|
||||
* @param {string|null|undefined} key
|
||||
* @returns {?string}
|
||||
*/
|
||||
export function hkImageUrlFromKey(key) {
|
||||
const id = String(key ?? '')
|
||||
.trim()
|
||||
.toUpperCase();
|
||||
if (!/^[A-Z0-9]{1,32}$/.test(id)) return null;
|
||||
return `${HK_IMAGE_ORIGIN}${id}.JPG`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pin a catalog frame URL to the Transport Department still host.
|
||||
*
|
||||
* @param {string|null|undefined} raw
|
||||
* @returns {?string}
|
||||
*/
|
||||
export function normalizeHkImageUrl(raw) {
|
||||
const text = String(raw ?? '').trim();
|
||||
if (!text) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
|
||||
parsed.protocol = 'https:';
|
||||
return pinnedCameraUrl(parsed.toString(), HK_IMAGE_ORIGIN);
|
||||
}
|
||||
|
||||
/**
|
||||
* Label for one HK camera: the published English description, else the key.
|
||||
*
|
||||
* @param {object} record
|
||||
* @param {string} cameraId
|
||||
* @returns {string}
|
||||
*/
|
||||
export function hkCameraName(record, cameraId) {
|
||||
const description = String(record?.description ?? '').trim();
|
||||
if (description) return description;
|
||||
const key = String(record?.key ?? '').trim();
|
||||
if (key) return key;
|
||||
return `HK Camera ${String(cameraId).replace(/^hk-/, '')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* One Transport Department locations row -> one catalog source, or null.
|
||||
*
|
||||
* The dataset publishes no camera facing, so headings use the shared id-hash
|
||||
* fallback at low confidence and are corrected with the calibration gizmo.
|
||||
*
|
||||
* @param {object} record
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function hkCameraToSource(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
const key = String(record.key ?? '')
|
||||
.trim()
|
||||
.toUpperCase();
|
||||
if (!key) return null;
|
||||
const lat = toFiniteNumber(record.latitude);
|
||||
const lon = toFiniteNumber(record.longitude);
|
||||
if (!isLikelyHongKongCoordinate(lat, lon)) return null;
|
||||
|
||||
const imageUrl = hkImageUrlFromKey(key);
|
||||
if (!imageUrl) return null;
|
||||
// If the catalog carries a URL, it must already be on the pinned host.
|
||||
const catalogUrl = String(record.url ?? '').trim();
|
||||
if (catalogUrl) {
|
||||
const pinnedCatalog = normalizeHkImageUrl(catalogUrl);
|
||||
if (!pinnedCatalog) return null;
|
||||
}
|
||||
|
||||
const cameraId = `hk-${key.toLowerCase()}`;
|
||||
const name = hkCameraName(record, cameraId);
|
||||
const region = String(record.region ?? '').trim();
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name,
|
||||
city: region || 'Hong Kong',
|
||||
cityId: 'hk',
|
||||
provider: 'Transport Department',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
// Coastal / harbour prior; the client's one-shot ground snap corrects.
|
||||
groundElevationM: 15,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'hk-td-open-data',
|
||||
license:
|
||||
'Hong Kong Transport Department / DATA.GOV.HK — attribution required',
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Hong Kong Transport Department traffic snapshot cameras from the
|
||||
* keyless DATA.GOV.HK English locations XML. Frames are stills on
|
||||
* tdcctv.data.one.gov.hk.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadHkSourcesFromOpenData() {
|
||||
try {
|
||||
const endpoint = process.env.CCTV_HK_CAMERAS_URL || DEFAULT_HK_CAMERAS_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: 'application/xml,text/xml,*/*' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn('[CCTV] HK catalog redirected; redirects are not followed');
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] HK camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
let xml;
|
||||
try {
|
||||
xml = await readResponseTextCapped(resp, HK_MAX_CATALOG_BYTES);
|
||||
} catch (error) {
|
||||
if (error?.code === 'RESPONSE_TOO_LARGE') {
|
||||
console.warn('[CCTV] HK catalog body exceeded byte ceiling');
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const rows = parseHkCameraLocationsXml(xml);
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const record of rows) {
|
||||
const camera = hkCameraToSource(record);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_HK_MAX_SOURCES || DEFAULT_HK_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(1200, Math.floor(maxRaw)))
|
||||
: DEFAULT_HK_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, HK_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded HK camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn('[CCTV] HK camera download error:', error?.message || error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadDelDOTSourcesFromOpenData() {
|
||||
try {
|
||||
const resp = await fetch(DELDOT_CCTV_URL, {
|
||||
@@ -2176,10 +2596,16 @@ export async function loadKingCountySourcesFromOpenData() {
|
||||
// county image origin, so they are skipped.
|
||||
if (owner === 'WSDOT') continue;
|
||||
// The handler is served over HTTPS; the feed still writes http://.
|
||||
const imageUrl = String(attrs.ImageURL || '')
|
||||
.trim()
|
||||
.replace(/^http:\/\/info\.kingcounty\.gov\//, KING_COUNTY_IMAGE_ORIGIN);
|
||||
if (!imageUrl.startsWith(KING_COUNTY_IMAGE_ORIGIN)) continue;
|
||||
const imageUrl = pinnedCameraUrl(
|
||||
String(attrs.ImageURL || '')
|
||||
.trim()
|
||||
.replace(
|
||||
/^http:\/\/info\.kingcounty\.gov\//,
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
),
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
);
|
||||
if (!imageUrl) continue;
|
||||
const [lon, lat] = Array.isArray(row?.geometry?.coordinates)
|
||||
? row.geometry.coordinates
|
||||
: [];
|
||||
@@ -2339,8 +2765,8 @@ export async function loadWsdotSourcesFromOpenData() {
|
||||
const attrs = row?.attributes;
|
||||
const rawId = attrs?.CameraID;
|
||||
if (!Number.isSafeInteger(rawId) || rawId <= 0) continue;
|
||||
const imageUrl = String(attrs.ImageURL || '').trim();
|
||||
if (!imageUrl.startsWith(WSDOT_IMAGE_ORIGIN)) continue;
|
||||
const imageUrl = pinnedCameraUrl(attrs.ImageURL, WSDOT_IMAGE_ORIGIN);
|
||||
if (!imageUrl) continue;
|
||||
const x = row?.geometry?.x;
|
||||
const y = row?.geometry?.y;
|
||||
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
|
||||
@@ -3387,3 +3813,282 @@ export async function loadFloridaSourcesFromOpenData() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One Open Data Hub `WebcamInfo` item (source `a22`) -> one catalog source,
|
||||
* or null.
|
||||
*
|
||||
* The frame must be the operator's own `WebCamImg/km<N>.jpg` for the
|
||||
* camera's motorway kilometre. The feed names the place and the kilometre but
|
||||
* no bearing, so the heading uses the shared id-hash fallback at low
|
||||
* confidence, like Calgary and Vegvesen; the ground elevation comes from the
|
||||
* per-kilometre DEM table because the road spans ~1,350 m of altitude.
|
||||
*
|
||||
* @param {object} item - One element of the API's `Items` array.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function a22WebcamToSource(item) {
|
||||
if (!item || typeof item !== 'object') return null;
|
||||
if (item.Active === false) return null;
|
||||
if (String(item.Source ?? '').toLowerCase() !== 'a22') return null;
|
||||
|
||||
const gps = Array.isArray(item.GpsInfo) ? item.GpsInfo[0] : null;
|
||||
const lat = toFiniteNumber(gps?.Latitude);
|
||||
const lon = toFiniteNumber(gps?.Longitude);
|
||||
if (!isLikelyA22Coordinate(lat, lon)) return null;
|
||||
|
||||
const km = String(item.Mapping?.a22?.km ?? '').trim();
|
||||
if (!/^\d{1,3}$/.test(km)) return null;
|
||||
const imageUrl = `${A22_IMAGE_ORIGIN}km${Number(km)}.jpg`;
|
||||
let image;
|
||||
try {
|
||||
image = new URL(String(item.Webcamurl ?? ''));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (image.username || image.password || image.href !== imageUrl) return null;
|
||||
|
||||
const place =
|
||||
String(item.Shortname ?? item.Webcamname?.it ?? '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim() || `Km ${Number(km)}`;
|
||||
const cameraId = `it-a22-km${Number(km)}`;
|
||||
const ground = A22_GROUND_ELEVATION_BY_KM[Number(km)];
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name: `A22 ${place} (km ${Number(km)})`,
|
||||
// One motorway-wide category in the camera picker; the place is already
|
||||
// in the name.
|
||||
city: 'A22 Brennero',
|
||||
cityId: 'a22',
|
||||
provider: 'Autostrada del Brennero',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -15,
|
||||
fovDeg: 50,
|
||||
rangeM: 160,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: Number.isFinite(ground)
|
||||
? ground
|
||||
: A22_DEFAULT_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'a22-opendatahub',
|
||||
license:
|
||||
'Webcam images © Autostrada del Brennero S.p.A., via the Open Data Hub (NOI Techpark)',
|
||||
code: cameraDisplayCode(place.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the Autostrada del Brennero (A22) webcams from the keyless Open Data
|
||||
* Hub tourism API. Frames are stills on www.autobrennero.it.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadA22SourcesFromOpenDataHub() {
|
||||
try {
|
||||
const endpoint = process.env.CCTV_A22_URL || DEFAULT_A22_CCTV_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn('[CCTV] A22 catalog redirected; redirects are not followed');
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] A22 camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const payload = await readResponseJsonCapped(resp, A22_MAX_CATALOG_BYTES);
|
||||
const items = Array.isArray(payload?.Items) ? payload.Items : [];
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const item of items) {
|
||||
const camera = a22WebcamToSource(item);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
// North to south along the motorway, so the picker reads in road order.
|
||||
cameras.sort((a, b) => b.lat - a.lat);
|
||||
console.log(`[CCTV] Loaded A22 camera sources: ${cameras.length}`);
|
||||
return cameras;
|
||||
} catch (error) {
|
||||
console.warn('[CCTV] A22 camera download error:', error?.message || error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Label for one Houston TranStar camera: the interchange an operator
|
||||
* recognises ("10 EAST @ SAN JACINTO").
|
||||
*
|
||||
* @param {object} record - Raw TranStar camera row.
|
||||
* @param {string} cameraId - Derived stable id.
|
||||
* @returns {string}
|
||||
*/
|
||||
export function houstonCameraName(record, cameraId) {
|
||||
const name = String(record?.location ?? '').trim();
|
||||
if (name) return name;
|
||||
const short = String(cameraId || '').replace(/^houston-/, '');
|
||||
return short ? `Houston Camera ${short}` : 'Houston Camera';
|
||||
}
|
||||
|
||||
/**
|
||||
* TranStar `dir` (the camera's side of the freeway) -> the travel direction
|
||||
* of the nearest lanes, as TranStar's map script `translateCameraDirection`
|
||||
* reads it. Only these four values are mapped; anything else is unknown.
|
||||
*/
|
||||
const HOUSTON_TRAVEL_HEADING_BY_SIDE = Object.freeze({
|
||||
south: 90,
|
||||
north: 270,
|
||||
west: 180,
|
||||
east: 0,
|
||||
});
|
||||
|
||||
/**
|
||||
* One Houston TranStar camera row -> one catalog source, or null.
|
||||
*
|
||||
* Frame URLs are synthesized from a validated numeric id on the pinned
|
||||
* snapshot origin. The payload's `url` is a `showWin(...)` viewer helper
|
||||
* and is ignored so a catalog edit cannot steer the frame proxy off-host.
|
||||
* TranStar's `dir` is the side of the freeway the camera sits on, not the
|
||||
* way it faces: TranStar's own map labels it "Traffic closest to the camera
|
||||
* is traveling …" (South→Eastbound, North→Westbound, West→Southbound,
|
||||
* East→Northbound). That travel direction is only a low-confidence prior
|
||||
* for the view, so the heading stays estimated with the low-confidence
|
||||
* pose; `None`, blank or unknown values fall back to the shared id-hash
|
||||
* heading.
|
||||
*
|
||||
* @param {object} record - Raw TranStar camera row.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function houstonCameraToSource(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
|
||||
const lat = toFiniteNumber(record.lat);
|
||||
const lon = toFiniteNumber(record.lng);
|
||||
if (!isLikelyHoustonCoordinate(lat, lon)) return null;
|
||||
|
||||
const rawId = String(record.id ?? '').trim();
|
||||
if (!HOUSTON_CAMERA_ID_RE.test(rawId)) return null;
|
||||
const cameraId = `houston-${rawId}`;
|
||||
const imageUrl = HOUSTON_IMAGE_URL(rawId);
|
||||
const name = houstonCameraName(record, cameraId);
|
||||
const travelHeading =
|
||||
HOUSTON_TRAVEL_HEADING_BY_SIDE[
|
||||
String(record.dir ?? '')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
];
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name,
|
||||
city: 'Houston',
|
||||
cityId: 'houston',
|
||||
provider: 'Houston TranStar',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: Number.isFinite(travelHeading)
|
||||
? travelHeading
|
||||
: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: HOUSTON_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'houston-transtar',
|
||||
license:
|
||||
'TranStar disclaimer — redistribution and external framing need TxDOT written permission',
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Houston TranStar cameras from the keyless `cctvSnapshots_json.js`
|
||||
* catalog. Frames are stills on `www.houstontranstar.org`, registered from
|
||||
* a validated numeric id rather than from the payload URL. The catalog host
|
||||
* answers `Accept: application/json` with HTTP 406, so this request accepts
|
||||
* any media type.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadHoustonSourcesFromOpenData() {
|
||||
try {
|
||||
const endpoint =
|
||||
process.env.CCTV_HOUSTON_CAMERAS_URL || HOUSTON_CAMERAS_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: '*/*' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn(
|
||||
'[CCTV] Houston catalog redirected; redirects are not followed',
|
||||
);
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] Houston camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
resp,
|
||||
HOUSTON_MAX_CATALOG_BYTES,
|
||||
);
|
||||
const rows = Array.isArray(payload?.cameras) ? payload.cameras : [];
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const record of rows) {
|
||||
const camera = houstonCameraToSource(record);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_HOUSTON_MAX_SOURCES || DEFAULT_HOUSTON_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
|
||||
: DEFAULT_HOUSTON_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, HOUSTON_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded Houston camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] Houston camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,9 @@ export const HLS_LIMITS = Object.freeze({
|
||||
sessions: 2,
|
||||
leasesPerSession: 8,
|
||||
playlistBytes: 256 * 1024,
|
||||
segmentBytes: 4 * 1024 * 1024,
|
||||
// High-resolution cameras can exceed 4 MiB in a single ten-second TS segment.
|
||||
// Retained data still shares the independent 24 MiB session budget below.
|
||||
segmentBytes: 8 * 1024 * 1024,
|
||||
sessionBytes: 24 * 1024 * 1024,
|
||||
segments: 12,
|
||||
pollMs: 2000,
|
||||
@@ -70,7 +72,7 @@ export async function fetchHlsBytes(
|
||||
}
|
||||
}
|
||||
|
||||
export function sameOriginHlsUrl(value, base) {
|
||||
function sameOriginHlsUrl(value, base) {
|
||||
const url = new URL(value, base);
|
||||
if (
|
||||
!['https:', 'http:'].includes(url.protocol) ||
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
import {
|
||||
CCTV_SOURCE_FETCH_TIMEOUT_MS,
|
||||
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
|
||||
} from './constants.js';
|
||||
import {
|
||||
cameraDisplayCode,
|
||||
fallbackHeadingFromId,
|
||||
prioritizeSources,
|
||||
} from './normalize.js';
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/** Freeway Bureau (MOTC) CCTV static table in the national MOTC 2.0 XML
|
||||
* standard, published keyless and refreshed daily (~1,800 cameras). */
|
||||
export const TAIWAN_FREEWAY_CCTV_URL =
|
||||
'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml';
|
||||
export const TAIWAN_HIGHWAY_CCTV_URL =
|
||||
'https://cctv-maintain.thb.gov.tw/opendataCCTVs.xml';
|
||||
/** The whole network is ~1.2 MB. */
|
||||
export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024;
|
||||
const DEFAULT_TAIWAN_MAX_SOURCES = 600;
|
||||
/** After a failed download, wait this long before asking again. The catalog
|
||||
* host does not answer from some networks outside Taiwan, and every attempt
|
||||
* there holds the shared catalog refresh until the connect timeout. */
|
||||
export const TAIWAN_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000;
|
||||
/** The table is republished daily; a good copy stands in for a failed
|
||||
* download for up to this long. */
|
||||
const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
// The table publishes each camera's MJPEG stream. The same camera servers
|
||||
// answer a single JPEG for the same camera ID on a sibling path, which is what
|
||||
// the frame proxy can relay; only these exact pairs are ever registered.
|
||||
const STREAM_TO_FRAME = new Map([
|
||||
['cctvn.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
|
||||
['cctvn5.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
|
||||
['cctvc.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
|
||||
[
|
||||
'cctvs.freeway.gov.tw',
|
||||
['/live-view/mjpg/video.cgi', '/live-view/jpg/image.cgi'],
|
||||
],
|
||||
]);
|
||||
|
||||
const DIRECTIONS = { N: 'N', S: 'S', E: 'E', W: 'W' };
|
||||
const TAIWAN_HIGHWAY_CAMERA_HOSTS = new Set(
|
||||
TAIWAN_HIGHWAY_IMAGE_ORIGINS.map((origin) => new URL(origin).hostname),
|
||||
);
|
||||
|
||||
/** Prioritization anchors, north to south, so a cap keeps the whole island
|
||||
* rather than only greater Taipei. */
|
||||
const ANCHORS = [
|
||||
{ lat: 25.033, lon: 121.5654 }, // Taipei
|
||||
{ lat: 24.9936, lon: 121.301 }, // Taoyuan
|
||||
{ lat: 24.8138, lon: 120.9675 }, // Hsinchu
|
||||
{ lat: 24.7021, lon: 121.7378 }, // Yilan
|
||||
{ lat: 24.1477, lon: 120.6736 }, // Taichung
|
||||
{ lat: 22.9999, lon: 120.227 }, // Tainan
|
||||
{ lat: 22.6273, lon: 120.3014 }, // Kaohsiung
|
||||
];
|
||||
|
||||
/** Taiwan main island with slack for the coast; the freeway network has no
|
||||
* offshore-island cameras. */
|
||||
function isLikelyTaiwanCoordinate(lat, lon) {
|
||||
return (
|
||||
Number.isFinite(lat) &&
|
||||
Number.isFinite(lon) &&
|
||||
lat >= 21.8 &&
|
||||
lat <= 25.4 &&
|
||||
lon >= 119.9 &&
|
||||
lon <= 122.1
|
||||
);
|
||||
}
|
||||
|
||||
const XML_ENTITIES = {
|
||||
amp: '&',
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
};
|
||||
|
||||
function codePoint(value, radix) {
|
||||
const code = parseInt(value, radix);
|
||||
return Number.isInteger(code) && code > 0 && code <= 0x10ffff
|
||||
? String.fromCodePoint(code)
|
||||
: '';
|
||||
}
|
||||
|
||||
function decodeXml(text) {
|
||||
return String(text ?? '')
|
||||
.replace(
|
||||
/&(?:(amp|lt|gt|quot|apos)|#(\d{1,7})|#x([0-9a-fA-F]{1,6}));/g,
|
||||
(_, name, dec, hex) =>
|
||||
name ? XML_ENTITIES[name] : codePoint(dec ?? hex, dec ? 10 : 16),
|
||||
)
|
||||
.trim();
|
||||
}
|
||||
|
||||
function tag(block, name) {
|
||||
const match = new RegExp(`<${name}>([^<]*)</${name}>`).exec(block);
|
||||
return match ? decodeXml(match[1]) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the MOTC 2.0 `CCTVList` document into flat rows. Regex-based like the
|
||||
* Catalonia and Tarktee parsers: the schema is flat, and only the fields used
|
||||
* below are read.
|
||||
*
|
||||
* @param {string} xml
|
||||
* @returns {Array<object>}
|
||||
*/
|
||||
export function parseTaiwanCctvXml(xml) {
|
||||
const rows = [];
|
||||
const blockRe = /<CCTV>([\s\S]*?)<\/CCTV>/g;
|
||||
let match;
|
||||
while ((match = blockRe.exec(String(xml || ''))) !== null) {
|
||||
const block = match[1];
|
||||
rows.push({
|
||||
cctvId: tag(block, 'CCTVID'),
|
||||
streamUrl: tag(block, 'VideoStreamURL'),
|
||||
lat: Number(tag(block, 'PositionLat')),
|
||||
lon: Number(tag(block, 'PositionLon')),
|
||||
roadName: tag(block, 'RoadName'),
|
||||
direction: tag(block, 'RoadDirection'),
|
||||
sectionStart: tag(block, 'Start'),
|
||||
sectionEnd: tag(block, 'End'),
|
||||
mile: tag(block, 'LocationMile'),
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Published MJPEG stream URL -> the same camera's still, plus the host and
|
||||
* camera number that identify it, or null.
|
||||
*
|
||||
* @param {string} streamUrl
|
||||
* @returns {?{frameUrl: string, host: string, camera: string}}
|
||||
*/
|
||||
export function taiwanFrame(streamUrl) {
|
||||
let url;
|
||||
try {
|
||||
url = new URL(String(streamUrl ?? ''));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (url.protocol !== 'https:' || url.username || url.password || url.port)
|
||||
return null;
|
||||
const paths = STREAM_TO_FRAME.get(url.hostname);
|
||||
if (!paths || url.pathname !== paths[0]) return null;
|
||||
const camera = url.searchParams.get('camera') || '';
|
||||
if (!/^[A-Za-z0-9-]{1,40}$/.test(camera)) return null;
|
||||
return {
|
||||
// Reconstruct so extra query parameters cannot alter the request.
|
||||
frameUrl: `https://${url.hostname}${paths[1]}?camera=${camera}`,
|
||||
host: url.hostname.split('.')[0],
|
||||
camera,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* One parsed Freeway Bureau row -> one catalog source, or null.
|
||||
*
|
||||
* @param {object} row - Output of parseTaiwanCctvXml.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function taiwanCameraToSource(row) {
|
||||
if (!row || typeof row !== 'object') return null;
|
||||
const { lat, lon } = row;
|
||||
if (!isLikelyTaiwanCoordinate(lat, lon)) return null;
|
||||
const frame = taiwanFrame(row.streamUrl);
|
||||
if (!frame) return null;
|
||||
const imageUrl = frame.frameUrl;
|
||||
// CCTVID carries Chinese place names and a few IDs repeat across distinct
|
||||
// cameras, so the stable key is the camera server plus its camera number.
|
||||
const rawId = String(row.cctvId ?? '')
|
||||
.trim()
|
||||
.slice(0, 80);
|
||||
const cameraId = `tw-freeway-${frame.host}-${frame.camera.toLowerCase()}`;
|
||||
const road = String(row.roadName ?? '').trim();
|
||||
const direction = DIRECTIONS[String(row.direction ?? '').trim()] || '';
|
||||
const mile = String(row.mile ?? '').trim();
|
||||
const section = [row.sectionStart, row.sectionEnd]
|
||||
.map((part) => String(part ?? '').trim())
|
||||
.filter(Boolean)
|
||||
.join('→');
|
||||
// A bare carriageway letter is not a name; fall back to the published ID.
|
||||
const name =
|
||||
(road || mile || section
|
||||
? [road, direction, mile, section].filter(Boolean).join(' ')
|
||||
: '') ||
|
||||
rawId ||
|
||||
`Freeway camera ${frame.camera}`;
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name: name.slice(0, 160),
|
||||
// One island-wide category in the camera picker, like Finland and Norway.
|
||||
city: 'Taiwan / 臺灣',
|
||||
cityId: 'taiwan',
|
||||
provider: 'Freeway Bureau, MOTC',
|
||||
lat,
|
||||
lon,
|
||||
// The table gives the carriageway direction, not the camera's bearing,
|
||||
// so the pose uses the shared low-confidence prior.
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'tw-freeway-motc',
|
||||
license:
|
||||
'Freeway Bureau, MOTC open data, Open Government Data License, Taiwan (Version 1.0)',
|
||||
code: cameraDisplayCode(
|
||||
`${road} ${mile}`.trim().toUpperCase() || frame.camera,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* One Highway Bureau row -> a source whose first MJPEG frame is served as a
|
||||
* snapshot by the existing camera-frame proxy.
|
||||
*
|
||||
* @param {object} row - Output of parseTaiwanCctvXml.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function taiwanHighwayCameraToSource(row) {
|
||||
if (!row || typeof row !== 'object') return null;
|
||||
const { lat, lon } = row;
|
||||
if (!isLikelyTaiwanCoordinate(lat, lon)) return null;
|
||||
|
||||
let url;
|
||||
try {
|
||||
url = new URL(String(row.streamUrl ?? ''));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
url.protocol !== 'https:' ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.port ||
|
||||
url.search ||
|
||||
url.hash ||
|
||||
!TAIWAN_HIGHWAY_CAMERA_HOSTS.has(url.hostname) ||
|
||||
!/^\/[A-Za-z0-9._+~-]{1,100}$/.test(url.pathname) ||
|
||||
url.pathname.includes('..')
|
||||
)
|
||||
return null;
|
||||
|
||||
const rawId = String(row.cctvId ?? '').trim();
|
||||
if (!/^[A-Za-z0-9-]{1,80}$/.test(rawId)) return null;
|
||||
const cameraId = 'tw-thb-' + rawId.toLowerCase();
|
||||
const road = String(row.roadName ?? '').trim();
|
||||
const direction = DIRECTIONS[String(row.direction ?? '').trim()] || '';
|
||||
const mile = String(row.mile ?? '').trim();
|
||||
const section = [row.sectionStart, row.sectionEnd]
|
||||
.map((part) => String(part ?? '').trim())
|
||||
.filter(Boolean)
|
||||
.join('→');
|
||||
const name =
|
||||
(road || mile || section
|
||||
? [road, direction, mile, section].filter(Boolean).join(' ')
|
||||
: '') || rawId;
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name: name.slice(0, 160),
|
||||
city: 'Taiwan / 臺灣',
|
||||
cityId: 'taiwan',
|
||||
provider: 'Highway Bureau, MOTC',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
feedType: 'mjpeg',
|
||||
url: url.href,
|
||||
snapshotUrl: url.href,
|
||||
sourceKind: 'taiwan-highway-motc',
|
||||
license: 'Highway Bureau, MOTC open data',
|
||||
code: cameraDisplayCode((road + ' ' + mile).trim().toUpperCase() || rawId),
|
||||
};
|
||||
}
|
||||
let lastGood = null;
|
||||
let lastGoodAt = 0;
|
||||
let retryAfter = 0;
|
||||
|
||||
/** Forget the last good table and any failure backoff (tests only). */
|
||||
export function resetTaiwanSourcesCache() {
|
||||
lastGood = null;
|
||||
lastGoodAt = 0;
|
||||
retryAfter = 0;
|
||||
}
|
||||
|
||||
function fallbackSources() {
|
||||
return lastGood && Date.now() - lastGoodAt <= TAIWAN_LAST_GOOD_MAX_AGE_MS
|
||||
? lastGood
|
||||
: [];
|
||||
}
|
||||
|
||||
function taiwanSourceCap() {
|
||||
const rawCap = Number(
|
||||
process.env.CCTV_TAIWAN_MAX_SOURCES || DEFAULT_TAIWAN_MAX_SOURCES,
|
||||
);
|
||||
return Number.isFinite(rawCap)
|
||||
? Math.max(1, Math.min(2000, Math.floor(rawCap)))
|
||||
: DEFAULT_TAIWAN_MAX_SOURCES;
|
||||
}
|
||||
|
||||
/** Fetch the public provincial-road CCTV table as a no-key fallback. */
|
||||
async function loadTaiwanHighwaySources() {
|
||||
const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS);
|
||||
const response = await fetch(TAIWAN_HIGHWAY_CCTV_URL, {
|
||||
headers: { Accept: 'application/xml,text/xml' },
|
||||
redirect: 'manual',
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw new Error(
|
||||
response.status >= 300 && response.status < 400
|
||||
? 'Highway Bureau catalog redirected; redirects are not followed'
|
||||
: 'Highway Bureau catalog HTTP ' + response.status,
|
||||
);
|
||||
}
|
||||
const xml = await readResponseTextCapped(
|
||||
response,
|
||||
TAIWAN_MAX_CATALOG_BYTES,
|
||||
signal,
|
||||
);
|
||||
const cameras = new Map();
|
||||
for (const row of parseTaiwanCctvXml(xml)) {
|
||||
const camera = taiwanHighwayCameraToSource(row);
|
||||
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
|
||||
}
|
||||
if (!cameras.size)
|
||||
throw new Error('no usable Highway Bureau cameras in the table');
|
||||
const sources = prioritizeSources(
|
||||
[...cameras.values()],
|
||||
taiwanSourceCap(),
|
||||
ANCHORS,
|
||||
);
|
||||
console.log(
|
||||
'[CCTV] Loaded Taiwan Highway Bureau fallback: ' +
|
||||
cameras.size +
|
||||
' (using ' +
|
||||
sources.length +
|
||||
')',
|
||||
);
|
||||
return sources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Taiwan Freeway Bureau cameras from the keyless MOTC 2.0 XML table.
|
||||
* One bounded request per shared catalog refresh. A failed download serves the
|
||||
* last good table (up to a day old) and is not retried for an hour, so a host
|
||||
* that cannot reach the Freeway Bureau does not wait on it every refresh.
|
||||
*
|
||||
* @returns {Promise<Array<object>>}
|
||||
*/
|
||||
export async function loadTaiwanSources() {
|
||||
if (Date.now() < retryAfter) return fallbackSources();
|
||||
try {
|
||||
const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS);
|
||||
const response = await fetch(
|
||||
process.env.CCTV_TAIWAN_URL || TAIWAN_FREEWAY_CCTV_URL,
|
||||
{
|
||||
headers: { Accept: 'application/xml,text/xml' },
|
||||
redirect: 'manual',
|
||||
signal,
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw new Error(
|
||||
response.status >= 300 && response.status < 400
|
||||
? 'catalog redirected; redirects are not followed'
|
||||
: 'HTTP ' + response.status,
|
||||
);
|
||||
}
|
||||
const xml = await readResponseTextCapped(
|
||||
response,
|
||||
TAIWAN_MAX_CATALOG_BYTES,
|
||||
signal,
|
||||
);
|
||||
const cameras = new Map();
|
||||
for (const row of parseTaiwanCctvXml(xml)) {
|
||||
const camera = taiwanCameraToSource(row);
|
||||
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
|
||||
}
|
||||
if (!cameras.size) throw new Error('no usable cameras in the table');
|
||||
const sources = prioritizeSources(
|
||||
[...cameras.values()],
|
||||
taiwanSourceCap(),
|
||||
ANCHORS,
|
||||
);
|
||||
console.log(
|
||||
'[CCTV] Loaded Taiwan camera sources: ' +
|
||||
cameras.size +
|
||||
' (using ' +
|
||||
sources.length +
|
||||
')',
|
||||
);
|
||||
lastGood = sources;
|
||||
lastGoodAt = Date.now();
|
||||
retryAfter = 0;
|
||||
return sources;
|
||||
} catch (error) {
|
||||
retryAfter = Date.now() + TAIWAN_RETRY_AFTER_FAILURE_MS;
|
||||
const fallback = fallbackSources();
|
||||
if (fallback.length) {
|
||||
console.warn(
|
||||
'[CCTV] Taiwan Freeway camera download failed (' +
|
||||
(error?.message || error) +
|
||||
'); serving the last good ' +
|
||||
fallback.length +
|
||||
' cameras, next attempt in 1 h',
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
try {
|
||||
const sources = await loadTaiwanHighwaySources();
|
||||
lastGood = sources;
|
||||
lastGoodAt = Date.now();
|
||||
console.warn(
|
||||
'[CCTV] Taiwan Freeway camera download failed (' +
|
||||
(error?.message || error) +
|
||||
'); using ' +
|
||||
sources.length +
|
||||
' Highway Bureau cameras as fallback, next attempt in 1 h',
|
||||
);
|
||||
return sources;
|
||||
} catch (fallbackError) {
|
||||
console.warn(
|
||||
'[CCTV] Taiwan Freeway camera download failed (' +
|
||||
(error?.message || error) +
|
||||
'); Highway Bureau fallback failed (' +
|
||||
(fallbackError?.message || fallbackError) +
|
||||
'); no Taiwan cameras, next attempt in 1 h',
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
DEFAULT_WORLD_CCTV_SOURCE_FILE,
|
||||
MLIT_RIVER_IMAGE_ORIGIN,
|
||||
TRIPCHECK_IMAGE_ORIGIN,
|
||||
WORLD_CCTV_SOURCE_COUNT,
|
||||
} from './constants.js';
|
||||
import { pinnedCameraUrl } from './normalize.js';
|
||||
|
||||
const SOURCE_POLICIES = Object.freeze({
|
||||
'tripcheck-odot': {
|
||||
idPrefix: 'world-or-',
|
||||
origin: TRIPCHECK_IMAGE_ORIGIN,
|
||||
bounds: [41, 47, -125, -116],
|
||||
filename: /^\/roadcams\/cams\/[^/]+\.jpe?g$/i,
|
||||
frameRefreshMs: 5 * 60 * 1000,
|
||||
},
|
||||
'mlit-river': {
|
||||
idPrefix: 'world-jp-river-',
|
||||
origin: MLIT_RIVER_IMAGE_ORIGIN,
|
||||
bounds: [24, 46, 122, 154],
|
||||
filename: /^\/cam\/now\/\d+\.jpg$/i,
|
||||
// The provider asks clients to avoid frequent repeated image collection.
|
||||
frameRefreshMs: 10 * 60 * 1000,
|
||||
},
|
||||
});
|
||||
|
||||
// MLIT currently serves this fixed JPEG on cameras whose images are paused
|
||||
// for maintenance (verified against two Yamagata cameras on 2026-11-02).
|
||||
const MLIT_ADJUSTMENT_IMAGE_SHA256 =
|
||||
'dbd2c3790eb75edf9ca44162f7979f2a3799090e23f2c5142f63211a035cb191';
|
||||
|
||||
export function isMlitRiverAdjustmentImage(image) {
|
||||
return (
|
||||
image?.ok === true &&
|
||||
Buffer.isBuffer(image.body) &&
|
||||
createHash('sha256').update(image.body).digest('hex') ===
|
||||
MLIT_ADJUSTMENT_IMAGE_SHA256
|
||||
);
|
||||
}
|
||||
|
||||
function safeFrameUrl(value, policy) {
|
||||
const normalized = pinnedCameraUrl(value, policy.origin);
|
||||
if (!normalized) return '';
|
||||
const url = new URL(normalized);
|
||||
return !url.search && !url.hash && policy.filename.test(url.pathname)
|
||||
? url.href
|
||||
: '';
|
||||
}
|
||||
|
||||
function inBounds(lat, lon, [minLat, maxLat, minLon, maxLon]) {
|
||||
return (
|
||||
Number.isFinite(lat) &&
|
||||
Number.isFinite(lon) &&
|
||||
lat >= minLat &&
|
||||
lat <= maxLat &&
|
||||
lon >= minLon &&
|
||||
lon <= maxLon
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load only the new image-backed networks from the add-on export. Existing
|
||||
* Canadian networks, Québec entries, JMA placeholder images, and NZTA's
|
||||
* scraped Journey Planner rows are deliberately excluded. Vegagerðin (Iceland)
|
||||
* rows are excluded too: the Iceland pack loads the full live list of those
|
||||
* cameras, so a World copy would draw each one twice.
|
||||
*/
|
||||
export function loadWorldCameraSources({ sourceRoot = process.cwd() } = {}) {
|
||||
const filename = path.resolve(sourceRoot, DEFAULT_WORLD_CCTV_SOURCE_FILE);
|
||||
let rows;
|
||||
try {
|
||||
const payload = JSON.parse(fs.readFileSync(filename, 'utf8'));
|
||||
rows = Array.isArray(payload) ? payload : [];
|
||||
} catch (error) {
|
||||
console.warn('[CCTV] World camera catalog unavailable:', error?.message);
|
||||
return [];
|
||||
}
|
||||
|
||||
const seen = new Set();
|
||||
const sources = [];
|
||||
for (const row of rows) {
|
||||
const sourceKind = String(row?.sourceKind || '');
|
||||
const policy = SOURCE_POLICIES[sourceKind];
|
||||
if (!policy) continue;
|
||||
|
||||
const id = String(row.id || '').trim();
|
||||
const lat = Number(row.lat);
|
||||
const lon = Number(row.lon);
|
||||
const url = safeFrameUrl(row.url || row.snapshotUrl, policy);
|
||||
const snapshotUrl = safeFrameUrl(row.snapshotUrl || row.url, policy);
|
||||
if (
|
||||
!id.startsWith(policy.idPrefix) ||
|
||||
seen.has(id) ||
|
||||
!inBounds(lat, lon, policy.bounds) ||
|
||||
!url ||
|
||||
!snapshotUrl ||
|
||||
url !== snapshotUrl
|
||||
)
|
||||
continue;
|
||||
seen.add(id);
|
||||
sources.push({
|
||||
id,
|
||||
name: String(row.name || id)
|
||||
.trim()
|
||||
.slice(0, 180),
|
||||
city: String(row.city || row.country || 'World'),
|
||||
cityId: String(row.cityId || ''),
|
||||
provider: String(row.provider || 'World traffic camera'),
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: Number.isFinite(Number(row.headingDeg))
|
||||
? Number(row.headingDeg)
|
||||
: undefined,
|
||||
headingConfidence: String(row.headingConfidence || 'low'),
|
||||
feedType: 'image',
|
||||
sourceKind,
|
||||
url,
|
||||
snapshotUrl,
|
||||
frameRefreshMs: policy.frameRefreshMs,
|
||||
license: String(row.license || ''),
|
||||
credit: String(row.credit || row.provider || ''),
|
||||
});
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[CCTV] Loaded World camera image sources: ${sources.length}/${WORLD_CCTV_SOURCE_COUNT}`,
|
||||
);
|
||||
return sources;
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
/**
|
||||
* A bounded two-level cache for JSON answers: memory (entry count) and disk
|
||||
* (entry count, total bytes, age). Every memory insertion goes through one
|
||||
* function, so a disk hit is bounded like a fresh answer; disk writes are
|
||||
* followed by a prune that removes expired files first, then the oldest,
|
||||
* until both quotas hold.
|
||||
*
|
||||
* @param {object} options
|
||||
* @param {string|null} options.dir Disk directory; null keeps memory only.
|
||||
* @param {number} options.ttlMs Age past which an entry is not served.
|
||||
* @param {number} [options.maxMemoryEntries]
|
||||
* @param {number} [options.maxDiskEntries]
|
||||
* @param {number} [options.maxDiskBytes]
|
||||
* @param {() => number} [options.now]
|
||||
*/
|
||||
export function createBoundedCache({
|
||||
dir,
|
||||
ttlMs,
|
||||
maxMemoryEntries = 60,
|
||||
maxDiskEntries = 400,
|
||||
maxDiskBytes = 256 * 1024 * 1024,
|
||||
now = () => Date.now(),
|
||||
}) {
|
||||
const memory = new Map();
|
||||
let pruning = null;
|
||||
let prunePending = false;
|
||||
|
||||
const fileFor = (key) =>
|
||||
path.join(dir, `${createHash('sha1').update(key).digest('hex')}.json`);
|
||||
|
||||
function remember(key, entry) {
|
||||
memory.delete(key);
|
||||
memory.set(key, entry);
|
||||
while (memory.size > maxMemoryEntries)
|
||||
memory.delete(memory.keys().next().value);
|
||||
}
|
||||
|
||||
async function prune() {
|
||||
if (!dir) return;
|
||||
let names;
|
||||
try {
|
||||
names = (await fsp.readdir(dir)).filter((name) => name.endsWith('.json'));
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const files = [];
|
||||
for (const name of names) {
|
||||
const file = path.join(dir, name);
|
||||
try {
|
||||
const stat = await fsp.stat(file);
|
||||
files.push({ file, size: stat.size, mtime: stat.mtimeMs });
|
||||
} catch {
|
||||
/* removed meanwhile */
|
||||
}
|
||||
}
|
||||
const cutoff = now() - ttlMs;
|
||||
const keep = [];
|
||||
for (const entry of files) {
|
||||
if (entry.mtime < cutoff) await fsp.rm(entry.file, { force: true });
|
||||
else keep.push(entry);
|
||||
}
|
||||
keep.sort((a, b) => a.mtime - b.mtime);
|
||||
let bytes = keep.reduce((sum, entry) => sum + entry.size, 0);
|
||||
while (
|
||||
keep.length &&
|
||||
(keep.length > maxDiskEntries || bytes > maxDiskBytes)
|
||||
) {
|
||||
const oldest = keep.shift();
|
||||
bytes -= oldest.size;
|
||||
await fsp.rm(oldest.file, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Run one prune at a time; a request during a prune schedules one more. */
|
||||
function schedulePrune() {
|
||||
if (pruning) {
|
||||
prunePending = true;
|
||||
return pruning;
|
||||
}
|
||||
pruning = prune()
|
||||
.catch(() => {})
|
||||
.finally(() => {
|
||||
pruning = null;
|
||||
if (prunePending) {
|
||||
prunePending = false;
|
||||
schedulePrune();
|
||||
}
|
||||
});
|
||||
return pruning;
|
||||
}
|
||||
|
||||
return {
|
||||
/** The cached value, or undefined. A disk hit is re-inserted into memory. */
|
||||
async get(key) {
|
||||
const held = memory.get(key);
|
||||
if (held && now() - held.cachedAt <= ttlMs) return held.value;
|
||||
if (held) memory.delete(key);
|
||||
if (!dir) return undefined;
|
||||
try {
|
||||
const entry = JSON.parse(await fsp.readFile(fileFor(key), 'utf8'));
|
||||
if (!Number.isFinite(entry?.cachedAt) || now() - entry.cachedAt > ttlMs)
|
||||
return undefined;
|
||||
remember(key, entry);
|
||||
return entry.value;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
},
|
||||
/** Store a value in memory and, when a directory is set, on disk. */
|
||||
async set(key, value) {
|
||||
const entry = { value, cachedAt: now() };
|
||||
remember(key, entry);
|
||||
if (!dir) return;
|
||||
try {
|
||||
await fsp.mkdir(dir, { recursive: true });
|
||||
await fsp.writeFile(fileFor(key), JSON.stringify(entry));
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
// In the background: an answer never waits on a 400-file directory scan.
|
||||
// `prune()` returns the pending pass for callers that need it done.
|
||||
schedulePrune();
|
||||
},
|
||||
/** Run a prune and wait until no pass is running or queued. */
|
||||
async prune() {
|
||||
schedulePrune();
|
||||
while (pruning) await pruning;
|
||||
},
|
||||
get memorySize() {
|
||||
return memory.size;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { gzip } from 'node:zlib';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const gzipAsync = promisify(gzip);
|
||||
|
||||
/**
|
||||
* Opt-in gzip for text bodies a proxy already holds in memory.
|
||||
*
|
||||
* Not a middleware: `cctv/media.js` streams bodies and answers range requests,
|
||||
* and several providers forward binary upstream types that gzip only grows.
|
||||
* `encodedBody` returns the body with its headers so the two cannot disagree.
|
||||
*/
|
||||
|
||||
/** Skip compression for bodies smaller than 1 KiB. */
|
||||
const MIN_COMPRESSED_BYTES = 1024;
|
||||
|
||||
/**
|
||||
* RFC 9110 section 12.4.2 `qvalue`. Matching the whole token keeps `q=1oops`
|
||||
* and `q=2` unreadable instead of rounding them into acceptance.
|
||||
*/
|
||||
const QVALUE = /^(?:0(?:\.\d{0,3})?|1(?:\.0{0,3})?)$/;
|
||||
|
||||
/**
|
||||
* Whether `Accept-Encoding` offers gzip. Anything unreadable (no header, a bad
|
||||
* q value, `x-gzip`) reads as no: every client can decode identity.
|
||||
*
|
||||
* @param {import('node:http').IncomingMessage} request
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function acceptsGzip(request) {
|
||||
const header = request?.headers?.['accept-encoding'];
|
||||
if (typeof header !== 'string') return false;
|
||||
return header.split(',').some((part) => {
|
||||
const [token, ...parameters] = part.split(';');
|
||||
if (token.trim().toLowerCase() !== 'gzip') return false;
|
||||
const quality = parameters
|
||||
.map((parameter) => parameter.trim().toLowerCase())
|
||||
.find((parameter) => parameter.startsWith('q='));
|
||||
if (quality === undefined) return true;
|
||||
// `q=0` names gzip and refuses it.
|
||||
const value = quality.slice(2);
|
||||
return QVALUE.test(value) && Number(value) > 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Gzip a body once for a caller that caches it. Async, so the ~860 KB OpenSky
|
||||
* snapshot does not block other requests while it compresses.
|
||||
*
|
||||
* Returns null, meaning "serve identity", when the body is under 1 KiB or gzip
|
||||
* does not shrink it.
|
||||
*
|
||||
* @param {string|Buffer} body - the exact bytes the identity response sends.
|
||||
* @returns {Promise<Buffer|null>}
|
||||
*/
|
||||
export async function gzipBody(body) {
|
||||
const raw = Buffer.isBuffer(body) ? body : Buffer.from(body);
|
||||
if (raw.byteLength < MIN_COMPRESSED_BYTES) return null;
|
||||
const compressed = await gzipAsync(raw);
|
||||
return compressed.byteLength < raw.byteLength ? compressed : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the body and the headers that describe it. `Vary` is on both branches
|
||||
* so a shared cache never serves one encoding to a client that asked for the
|
||||
* other.
|
||||
*
|
||||
* @param {import('node:http').IncomingMessage} request
|
||||
* @param {string|Buffer} identity - the uncompressed body.
|
||||
* @param {Buffer|null} compressed - `gzipBody(identity)`, or null.
|
||||
* @returns {{body: string|Buffer, headers: Record<string,string>}}
|
||||
*/
|
||||
export function encodedBody(request, identity, compressed) {
|
||||
if (compressed && acceptsGzip(request)) {
|
||||
return {
|
||||
body: compressed,
|
||||
headers: { 'Content-Encoding': 'gzip', Vary: 'Accept-Encoding' },
|
||||
};
|
||||
}
|
||||
return { body: identity, headers: { Vary: 'Accept-Encoding' } };
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { isIPv6 } from 'node:net';
|
||||
import { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
export { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
|
||||
@@ -60,3 +61,46 @@ export function makeCostRateLimiter(envValue, defaultPerMin) {
|
||||
export function clientKey(req) {
|
||||
return String(req.socket?.remoteAddress || 'local');
|
||||
}
|
||||
|
||||
const IPV4_MAPPED_RE = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/i;
|
||||
|
||||
/**
|
||||
* Whether a socket peer address is this machine: IPv4 loopback in any of its
|
||||
* forms (`127.0.0.0/8`, including the IPv4-mapped `::ffff:127.x` a dual-stack
|
||||
* listener reports) or the IPv6 loopback. Non-address keys are not loopback.
|
||||
* @param {string} address - A `clientKey` value.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isLoopbackPeer(address) {
|
||||
const raw = String(address || '');
|
||||
const v4 = IPV4_MAPPED_RE.exec(raw)?.[1] ?? raw;
|
||||
return v4.startsWith('127.') || raw === '::1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate-limit key that survives address rotation, for limiters whose global
|
||||
* backstop everyone shares. IPv6 privacy extensions hand one host a fresh
|
||||
* address at will inside its /64, so keying by the full address lets a single
|
||||
* host mint per-peer quota until the global window closes for everyone else.
|
||||
* IPv6 peers are keyed by their /64 prefix instead, and an IPv4-mapped
|
||||
* address folds to its IPv4 form so one host cannot hold two buckets. Any
|
||||
* other value (IPv4, or the `local` fallback) passes through unchanged.
|
||||
* @param {import('node:http').IncomingMessage} req
|
||||
* @returns {string}
|
||||
*/
|
||||
export function clientNetworkKey(req) {
|
||||
const address = clientKey(req);
|
||||
const mapped = IPV4_MAPPED_RE.exec(address);
|
||||
if (mapped) return mapped[1];
|
||||
const bare = address.replace(/%.*$/, ''); // link-local zone id
|
||||
if (!isIPv6(bare)) return address;
|
||||
const [head, tail = ''] = bare.split('::');
|
||||
const headParts = head ? head.split(':') : [];
|
||||
const tailParts = tail ? tail.split(':') : [];
|
||||
const missing = Math.max(0, 8 - headParts.length - tailParts.length);
|
||||
const hextets = [...headParts, ...Array(missing).fill('0'), ...tailParts];
|
||||
return `${hextets
|
||||
.slice(0, 4)
|
||||
.map((h) => h.toLowerCase().replace(/^0+(?=.)/, ''))
|
||||
.join(':')}::/64`;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { clientNetworkKey, isLoopbackPeer } from './rate-limit.js';
|
||||
|
||||
const req = (remoteAddress) => ({ socket: { remoteAddress } });
|
||||
|
||||
test('clientNetworkKey keys IPv6 peers by /64 and folds IPv4-mapped peers', () => {
|
||||
assert.equal(clientNetworkKey(req('2001:db8:1:2::1')), '2001:db8:1:2::/64');
|
||||
assert.equal(
|
||||
clientNetworkKey(req('2001:0db8:0001:0002:abcd:ef01:2345:6789')),
|
||||
'2001:db8:1:2::/64',
|
||||
);
|
||||
assert.equal(clientNetworkKey(req('2001:DB8::1')), '2001:db8:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('fe80::1%en0')), 'fe80:0:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('::1')), '0:0:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('::ffff:10.0.0.5')), '10.0.0.5');
|
||||
assert.equal(clientNetworkKey(req('10.0.0.5')), '10.0.0.5');
|
||||
assert.equal(clientNetworkKey(req(undefined)), 'local');
|
||||
assert.equal(clientNetworkKey(req('not-an-address')), 'not-an-address');
|
||||
});
|
||||
|
||||
test('isLoopbackPeer recognizes every loopback form and nothing else', () => {
|
||||
for (const address of ['127.0.0.1', '127.9.9.9', '::1', '::ffff:127.0.0.1']) {
|
||||
assert.equal(isLoopbackPeer(address), true, address);
|
||||
}
|
||||
for (const address of [
|
||||
'10.0.0.1',
|
||||
'::ffff:10.0.0.1',
|
||||
'2001:db8::1',
|
||||
'::2',
|
||||
'local',
|
||||
'',
|
||||
undefined,
|
||||
]) {
|
||||
assert.equal(isLoopbackPeer(address), false, String(address));
|
||||
}
|
||||
});
|
||||
@@ -58,7 +58,7 @@ function forwardedProtocol(req) {
|
||||
|
||||
/**
|
||||
* Cross-site gate for keyless routes the map needs in order to render
|
||||
* (`/api/terrain/heights`). It refuses the same cross-site browser shapes as
|
||||
* (`/api/terrain/heights`, `/api/overpass`). It refuses the same cross-site browser shapes as
|
||||
* `admitSameSite` (a `Sec-Fetch-Site` other than same-origin/none, a foreign
|
||||
* or opaque `Origin`) but does not refuse reverse-proxy forwarding headers:
|
||||
* those protect credentials and spend, which a keyless route has neither of,
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
import { readRequestBody } from './request.js';
|
||||
import {
|
||||
clientKey,
|
||||
makeCostRateLimiter,
|
||||
resolvePerMinuteCap,
|
||||
} from './rate-limit.js';
|
||||
import { waitForSignal } from '../../../src/sources/requestDeadline.js';
|
||||
import { readResponseJsonCapped } from '../../../src/sources/httpBody.js';
|
||||
import { isGeminiRequestOriginAllowed } from '../gemini/request.js';
|
||||
import { parseUsage } from './textTools.js';
|
||||
|
||||
const MAX_BYTES = 512 * 1024;
|
||||
const FAILURE = 'Could not connect. Try again.';
|
||||
// A rejected key is not retried upstream for this long, unless the key changes.
|
||||
const REJECTED_KEY_COOLDOWN_MS = 60_000;
|
||||
|
||||
// What an upstream failure says about the key or the account. Gemini answers
|
||||
// an invalid key with 400 and an ErrorInfo reason, not 401; OpenAI answers an
|
||||
// account without credit with 429 insufficient_quota, which waiting won't fix.
|
||||
// A 403 is the key's project, region or permissions, so it cools down too but
|
||||
// does not blame the key. Only small error bodies are read.
|
||||
async function upstreamRefusal(response, signal) {
|
||||
const { status } = response;
|
||||
if (status === 401) return 'API_KEY_REJECTED';
|
||||
if (status === 403) return 'ACCESS_DENIED';
|
||||
if (![400, 429].includes(status)) return null;
|
||||
let error;
|
||||
try {
|
||||
error = (await readResponseJsonCapped(response, 16 * 1024, signal))?.error;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
status === 400 &&
|
||||
Array.isArray(error?.details) &&
|
||||
error.details.some((detail) => detail?.reason === 'API_KEY_INVALID')
|
||||
)
|
||||
return 'API_KEY_REJECTED';
|
||||
if (status === 429 && error?.code === 'insufficient_quota')
|
||||
return 'QUOTA_EXCEEDED';
|
||||
return null;
|
||||
}
|
||||
|
||||
// Pass through only a delay in whole seconds, capped at an hour.
|
||||
function upstreamRetryAfter(response) {
|
||||
const value = String(response.headers?.get?.('retry-after') ?? '').trim();
|
||||
if (!/^\d{1,6}$/.test(value)) return null;
|
||||
return String(Math.min(Math.max(Number(value), 1), 3600));
|
||||
}
|
||||
|
||||
export function createTextChatHandler({
|
||||
provider,
|
||||
prepare,
|
||||
parse,
|
||||
fetchImpl = (...args) => fetch(...args),
|
||||
resolveApiKey,
|
||||
resolveHost = () => process.env.HOST,
|
||||
resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS,
|
||||
resolveRateLimit = () =>
|
||||
process.env[`GEV_RATELIMIT_${provider.toUpperCase()}_CHAT_PER_MIN`],
|
||||
// An unset chat throttle follows the provider's own cost throttle.
|
||||
resolveProviderRateLimit = () => undefined,
|
||||
providerPerMin = 30,
|
||||
// Returns { error, code } when the route must refuse before any key is used.
|
||||
refuse = () => null,
|
||||
now = Date.now,
|
||||
timeoutMs = 60000,
|
||||
} = {}) {
|
||||
let limiter;
|
||||
let rejected = null;
|
||||
return async (req, res) => {
|
||||
if (req.aborted || res.destroyed || res.writableEnded) return;
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('X-Content-Type-Options', 'nosniff');
|
||||
const reply = (status, value, code) => {
|
||||
if (req.aborted || res.destroyed || res.writableEnded) return;
|
||||
res.statusCode = status;
|
||||
if (status >= 400) res.setHeader('Connection', 'close');
|
||||
res.end(
|
||||
JSON.stringify(
|
||||
typeof value === 'string'
|
||||
? { error: value, ...(code ? { code } : {}) }
|
||||
: value,
|
||||
),
|
||||
);
|
||||
};
|
||||
if (req.method !== 'POST') {
|
||||
res.setHeader('Allow', 'POST');
|
||||
return reply(405, 'Method not allowed');
|
||||
}
|
||||
const lifetime = new AbortController();
|
||||
const disconnect = () => {
|
||||
if (!res.writableEnded) lifetime.abort();
|
||||
};
|
||||
req.once('aborted', disconnect);
|
||||
res.once('close', disconnect);
|
||||
const timer = setTimeout(() => lifetime.abort(), timeoutMs);
|
||||
timer.unref?.();
|
||||
let response;
|
||||
let usage = null;
|
||||
try {
|
||||
if (
|
||||
!isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts())
|
||||
)
|
||||
return reply(403, FAILURE);
|
||||
const refusal = refuse();
|
||||
if (refusal) return reply(503, refusal.error, refusal.code);
|
||||
const apiKey = String(resolveApiKey() || '').trim();
|
||||
if (!apiKey)
|
||||
return reply(
|
||||
503,
|
||||
'Add an API key in Provider Settings',
|
||||
'API_KEY_MISSING',
|
||||
);
|
||||
if (rejected && (rejected.key !== apiKey || now() >= rejected.until))
|
||||
rejected = null;
|
||||
if (rejected) {
|
||||
res.setHeader(
|
||||
'Retry-After',
|
||||
String(Math.max(1, Math.ceil((rejected.until - now()) / 1000))),
|
||||
);
|
||||
return reply(502, FAILURE, rejected.code);
|
||||
}
|
||||
if (limiter === undefined)
|
||||
limiter = makeCostRateLimiter(
|
||||
resolveRateLimit(),
|
||||
resolvePerMinuteCap(resolveProviderRateLimit(), providerPerMin),
|
||||
);
|
||||
if (limiter && !limiter(clientKey(req))) {
|
||||
res.setHeader('Retry-After', '60');
|
||||
return reply(429, FAILURE, 'RATE_LIMITED');
|
||||
}
|
||||
if (
|
||||
!/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')
|
||||
)
|
||||
return reply(415, FAILURE);
|
||||
if (Number(req.headers['content-length']) > MAX_BYTES)
|
||||
return reply(413, FAILURE);
|
||||
const raw = await readRequestBody(req, MAX_BYTES, lifetime.signal);
|
||||
let input;
|
||||
try {
|
||||
input = JSON.parse(raw);
|
||||
} catch {
|
||||
return reply(400, FAILURE);
|
||||
}
|
||||
const prepared = prepare(input);
|
||||
if (!prepared) return reply(400, FAILURE);
|
||||
if (prepared.errorStatus) return reply(prepared.errorStatus, FAILURE);
|
||||
const { endpoint, body, headers } = prepared;
|
||||
lifetime.signal.throwIfAborted();
|
||||
const acquisition = Promise.resolve(
|
||||
fetchImpl(endpoint, {
|
||||
method: 'POST',
|
||||
redirect: 'error',
|
||||
signal: lifetime.signal,
|
||||
headers: { 'Content-Type': 'application/json', ...headers(apiKey) },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
);
|
||||
void acquisition.then(
|
||||
(late) => {
|
||||
response = late;
|
||||
if (lifetime.signal.aborted) void late.body?.cancel().catch(() => {});
|
||||
},
|
||||
() => {},
|
||||
);
|
||||
response = await waitForSignal(acquisition, lifetime.signal);
|
||||
lifetime.signal.throwIfAborted();
|
||||
if (response.redirected || (response.url && response.url !== endpoint))
|
||||
throw new Error('Unexpected destination');
|
||||
if (!response.ok) {
|
||||
const refusal = await upstreamRefusal(response, lifetime.signal);
|
||||
if (refusal) {
|
||||
void response.body?.cancel().catch(() => {});
|
||||
rejected = {
|
||||
key: apiKey,
|
||||
code: refusal,
|
||||
until: now() + REJECTED_KEY_COOLDOWN_MS,
|
||||
};
|
||||
return reply(502, FAILURE, refusal);
|
||||
}
|
||||
void response.body?.cancel().catch(() => {});
|
||||
if (![429, 503].includes(response.status)) return reply(502, FAILURE);
|
||||
const retryAfter = upstreamRetryAfter(response);
|
||||
if (retryAfter) res.setHeader('Retry-After', retryAfter);
|
||||
return reply(
|
||||
response.status,
|
||||
FAILURE,
|
||||
response.status === 429 ? 'RATE_LIMITED' : undefined,
|
||||
);
|
||||
}
|
||||
const data = await readResponseJsonCapped(
|
||||
response,
|
||||
MAX_BYTES,
|
||||
lifetime.signal,
|
||||
);
|
||||
lifetime.signal.throwIfAborted();
|
||||
usage = parseUsage(provider, data);
|
||||
const result = parse(data);
|
||||
if (!result || JSON.stringify(result).includes(apiKey))
|
||||
return reply(502, { error: FAILURE, usage });
|
||||
reply(200, { ...result, usage });
|
||||
} catch (error) {
|
||||
reply(
|
||||
error?.code === 'BODY_TOO_LARGE'
|
||||
? 413
|
||||
: lifetime.signal.aborted
|
||||
? 504
|
||||
: 502,
|
||||
{ error: FAILURE, usage },
|
||||
);
|
||||
} finally {
|
||||
if (lifetime.signal.aborted)
|
||||
void response?.body?.cancel().catch(() => {});
|
||||
clearTimeout(timer);
|
||||
req.off('aborted', disconnect);
|
||||
res.off('close', disconnect);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { realtimeDirectives } from '../openai/instructions.js';
|
||||
|
||||
// Voice wording in the shared directives and its written form. A test checks
|
||||
// each phrase still occurs in the directives, so rewording one there fails
|
||||
// rather than leaking spoken wording into chat.
|
||||
export const TEXT_WORDING = Object.freeze([
|
||||
['without speaking first', 'without writing a preamble'],
|
||||
['before speaking', 'before confirming'],
|
||||
['may first say ONE sentence', 'may first write ONE sentence'],
|
||||
["speak the result's say line", "write the result's say line"],
|
||||
['on the voice card: speak a caveat', 'in the tool result: include a caveat'],
|
||||
['as you talk', 'as you write'],
|
||||
['you can read aloud', 'you can report'],
|
||||
]);
|
||||
|
||||
// Shared-manual adaptation follows Darren Dignam's GEV COMMAND approach (#976).
|
||||
// Keep our Radio handoff and annotation outcome policies from the shared manual.
|
||||
// `options.overpass` matches the tools offered, as for voice.
|
||||
export function textInstructions(annotationGuidance, options) {
|
||||
return realtimeDirectives(annotationGuidance, options)
|
||||
.map((line, index) => {
|
||||
if (index === 0)
|
||||
return "You are GEV Chat, a concise text assistant for a Cesium geospatial app called God's Eye View.";
|
||||
if (index === 1)
|
||||
return 'Have a natural written conversation. This is text-only chat; no microphone or generated audio is used.';
|
||||
return TEXT_WORDING.reduce(
|
||||
(text, [spoken, written]) => text.replace(spoken, written),
|
||||
line,
|
||||
);
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { validateValue } from '../../../src/tools/schema.js';
|
||||
|
||||
export function invalidCalls(calls, tools) {
|
||||
const schemas = new Map(tools.map((tool) => [tool.name, tool.parameters]));
|
||||
return calls.flatMap((call, index) => {
|
||||
const schema = schemas.get(call.name);
|
||||
const errors = !schema
|
||||
? ['Unknown tool']
|
||||
: call.parseError
|
||||
? ['Arguments must be valid JSON']
|
||||
: validateValue(schema, call.args === undefined ? {} : call.args);
|
||||
return errors.length
|
||||
? [{ index, error: errors.slice(0, 4).join('; ').slice(0, 1000) }]
|
||||
: [];
|
||||
});
|
||||
}
|
||||
|
||||
// Missing usage stays unknown, never an invented zero-cost request.
|
||||
export function parseUsage(provider, data) {
|
||||
const raw = provider === 'gemini' ? data?.usageMetadata : data?.usage;
|
||||
const number = (value) =>
|
||||
Number.isSafeInteger(value) && value >= 0 ? value : null;
|
||||
const inputTokens = number(
|
||||
provider === 'gemini' ? raw?.promptTokenCount : raw?.prompt_tokens,
|
||||
);
|
||||
const candidateTokens = number(
|
||||
provider === 'gemini' ? raw?.candidatesTokenCount : raw?.completion_tokens,
|
||||
);
|
||||
const totalTokens = number(
|
||||
provider === 'gemini' ? raw?.totalTokenCount : raw?.total_tokens,
|
||||
);
|
||||
if (inputTokens === null || candidateTokens === null || totalTokens === null)
|
||||
return null;
|
||||
const reasoningTokens =
|
||||
number(
|
||||
provider === 'gemini'
|
||||
? raw?.thoughtsTokenCount
|
||||
: raw?.completion_tokens_details?.reasoning_tokens,
|
||||
) ?? 0;
|
||||
const outputTokens =
|
||||
candidateTokens + (provider === 'gemini' ? reasoningTokens : 0);
|
||||
return {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens,
|
||||
reasoningTokens,
|
||||
cachedInputTokens:
|
||||
number(
|
||||
provider === 'gemini'
|
||||
? raw?.cachedContentTokenCount
|
||||
: raw?.prompt_tokens_details?.cached_tokens,
|
||||
) ?? 0,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
/** Identify each provider while keeping the project contact URL consistent. */
|
||||
export function providerUserAgent(name) {
|
||||
return `gods-eye-view-${name}-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)`;
|
||||
}
|
||||
@@ -43,6 +43,11 @@ function number(value, min, max) {
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) && n >= min && n <= max ? n : null;
|
||||
}
|
||||
// CurrentStorms.json reports movementSpeed in miles per hour (intensity is in
|
||||
// knots); convert so `speedKt` means what it says.
|
||||
function knotsFromMph(mph) {
|
||||
return mph === null ? null : Math.round(mph * 0.868976);
|
||||
}
|
||||
// NHC stamps an advisory with its nominal time (e.g. 03:00Z) and publishes it
|
||||
// up to about half an hour before; allow that, but not a clock far ahead.
|
||||
const EARLY_ADVISORY_MS = 90 * 60_000;
|
||||
@@ -120,7 +125,7 @@ export function parseCycloneStatus(payload, now = Date.now()) {
|
||||
pressureHpa: number(raw.pressure, 800, 1100),
|
||||
movement: {
|
||||
directionDegrees: number(raw.movementDir, 0, 360),
|
||||
speedKt: number(raw.movementSpeed, 0, 200),
|
||||
speedKt: knotsFromMph(number(raw.movementSpeed, 0, 200)),
|
||||
},
|
||||
advisoryUrl: officialLink(forecast.url),
|
||||
outlookUrl: raw.id.startsWith('al')
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import { normalizeDroneAirspaceSnapshot } from '../../src/layers/droneAirspace/records.js';
|
||||
import { readResponseJsonCapped, coalesceProxyRequest } from './common/http.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { admitSameSiteKeyless } from './common/same-site.js';
|
||||
|
||||
// FAA Special Use Airspace (public, keyless). TYPE_CODE P (prohibited) and
|
||||
// R (restricted) only — the small, always-in-force slice of the national
|
||||
// SUA table. maxAllowableOffset trades ~100 m of boundary fidelity for a
|
||||
// payload small enough to refresh (~330 KB for the current P+R set).
|
||||
const API_URL =
|
||||
'https://services6.arcgis.com/ssFJjBXIUyZDrSYZ/arcgis/rest/services/' +
|
||||
'Special_Use_Airspace/FeatureServer/0/query?' +
|
||||
new URLSearchParams({
|
||||
where: "TYPE_CODE IN ('P','R')",
|
||||
outFields: [
|
||||
'NAME',
|
||||
'TYPE_CODE',
|
||||
'UPPER_VAL',
|
||||
'UPPER_UOM',
|
||||
'UPPER_CODE',
|
||||
'LOWER_VAL',
|
||||
'LOWER_UOM',
|
||||
'LOWER_CODE',
|
||||
'TIMESOFUSE',
|
||||
'CONT_AGENT',
|
||||
'CITY',
|
||||
'STATE',
|
||||
'OBJECTID',
|
||||
].join(','),
|
||||
maxAllowableOffset: '0.001',
|
||||
outSR: '4326',
|
||||
f: 'geojson',
|
||||
}).toString();
|
||||
|
||||
const MAX_PAGES = 5;
|
||||
const MIB = 1024 * 1024;
|
||||
const SNAPSHOT_TTL_MS = 3_600_000;
|
||||
/** Longest a failed refresh keeps serving the last snapshot, marked stale. */
|
||||
const STALE_MAX_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const exceededTransferLimit = (payload) =>
|
||||
payload?.exceededTransferLimit === true ||
|
||||
payload?.properties?.exceededTransferLimit === true;
|
||||
|
||||
/** Fixed-origin, bounded FAA SUA routes for dev and preview. */
|
||||
export function droneAirspaceProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
const allow = makeRateLimiter({ windowMs: 60_000, max: 60, globalMax: 1200 });
|
||||
|
||||
async function upstream(url, cap, timeout) {
|
||||
const signal = AbortSignal.timeout(timeout);
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('upstream_unavailable');
|
||||
}
|
||||
return readResponseJsonCapped(response, cap, signal);
|
||||
}
|
||||
|
||||
// A snapshot is accepted only once the service says it is complete. A page
|
||||
// limit reached while still flagged, a flagged page that adds nothing, or
|
||||
// no valid zone at all is a failed refresh, so the last good snapshot stays.
|
||||
async function fetchZones() {
|
||||
const features = [];
|
||||
let complete = false;
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const url =
|
||||
page === 0 ? API_URL : `${API_URL}&resultOffset=${features.length}`;
|
||||
const payload = await upstream(url, 8 * MIB, 30_000);
|
||||
if (!Array.isArray(payload?.features))
|
||||
throw new Error('invalid_snapshot');
|
||||
features.push(...payload.features);
|
||||
if (!exceededTransferLimit(payload)) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
if (!payload.features.length) break;
|
||||
}
|
||||
if (!complete) throw new Error('incomplete_snapshot');
|
||||
const rows = normalizeDroneAirspaceSnapshot({ features });
|
||||
if (!rows?.length) throw new Error('invalid_snapshot');
|
||||
return { fetchedAt: now(), rows };
|
||||
}
|
||||
|
||||
async function acquire(key, ttl, load) {
|
||||
const previous = cache.get(key);
|
||||
if (previous && now() - previous.savedAt < ttl)
|
||||
return { value: previous.value, stale: false };
|
||||
if (!inFlight.has(key) && inFlight.size >= 256)
|
||||
throw Object.assign(new Error('busy'), { status: 429 });
|
||||
try {
|
||||
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
|
||||
const value = await load();
|
||||
cache.set(key, { value, savedAt: now() });
|
||||
return value;
|
||||
});
|
||||
return { value: await promise, stale: false };
|
||||
} catch (error) {
|
||||
// A failed refresh serves the last snapshot, marked stale, for a day.
|
||||
if (previous && now() - previous.savedAt <= STALE_MAX_MS)
|
||||
return { value: previous.value, stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (status, value, stale = false) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...(status === 429 ? { 'Retry-After': '60' } : {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
// Keyless and needed to render the layer: refuse cross-site browsers
|
||||
// before they spend a client's quota or start a refresh, but keep the
|
||||
// documented reverse-proxy deployment working.
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
if (path !== '/' && path !== '')
|
||||
return json(404, { error: 'unknown_route' });
|
||||
if (!allow(clientKey(req))) return json(429, { error: 'rate_limited' });
|
||||
try {
|
||||
const { value, stale } = await acquire(
|
||||
'zones',
|
||||
SNAPSHOT_TTL_MS,
|
||||
fetchZones,
|
||||
);
|
||||
json(200, stale ? { ...value, stale: true } : value, stale);
|
||||
} catch (error) {
|
||||
json(error.status === 429 ? 429 : 502, {
|
||||
error: 'drone_airspace_unavailable',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'drone-airspace',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/drone-airspace', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/drone-airspace', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -39,6 +39,8 @@ const API_URL =
|
||||
// The service caps a single response at its maxRecordCount (2000); a peak
|
||||
// season can exceed that, so follow exceededTransferLimit with offset pages.
|
||||
const MAX_PAGES = 5;
|
||||
/** Longest a failed refresh keeps serving the last perimeter snapshot, marked stale. */
|
||||
const STALE_MAX_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const exceededTransferLimit = (payload) =>
|
||||
payload?.exceededTransferLimit === true ||
|
||||
@@ -165,7 +167,14 @@ export function firePerimetersProxy({
|
||||
});
|
||||
return { value: await promise, stale: false };
|
||||
} catch (error) {
|
||||
if (previous) return { value: previous.value, stale: true };
|
||||
// A failed refresh serves the last answer, marked stale. A perimeter
|
||||
// snapshot only for a day: older than that, it is no picture of
|
||||
// current fires. (InciWeb entries keep their previous behaviour.)
|
||||
if (
|
||||
previous &&
|
||||
(key !== 'perimeters' || now() - previous.savedAt <= STALE_MAX_MS)
|
||||
)
|
||||
return { value: previous.value, stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
+149
-15
@@ -1,16 +1,139 @@
|
||||
import path from 'node:path';
|
||||
import https from 'node:https';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
|
||||
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
|
||||
|
||||
/**
|
||||
* The per-address connection errors behind a fetch failure, or null. Node 24
|
||||
* puts the AggregateError directly on `err.cause`; Node 26's fetch wraps it in
|
||||
* a ConnectTimeoutError one level deeper.
|
||||
* @param {unknown} err
|
||||
* @returns {?Array<unknown>}
|
||||
*/
|
||||
function connectAttemptErrors(err) {
|
||||
let cause = err?.cause;
|
||||
for (let depth = 0; cause && depth < 3; depth++) {
|
||||
if (Array.isArray(cause.errors)) return cause.errors;
|
||||
cause = cause.cause;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Describe a fetch failure with its socket-level cause. Node reports every
|
||||
* connection failure as `fetch failed`; the cause carries the error code and
|
||||
* the address that failed (never the URL, which embeds the MAP_KEY).
|
||||
* @param {unknown} err
|
||||
* @returns {string}
|
||||
*/
|
||||
export function describeFetchError(err) {
|
||||
const message = err?.message || String(err);
|
||||
const cause = err?.cause;
|
||||
if (!cause) return message;
|
||||
const attempts = connectAttemptErrors(err);
|
||||
const parts = attempts?.length
|
||||
? attempts.map((inner) => inner?.message || inner?.code || String(inner))
|
||||
: [cause.message || cause.code || String(cause)];
|
||||
return `${message} (${parts.join('; ')})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a fetch failed because every address it tried failed to connect.
|
||||
* Node's network-family auto-selection gives each address a short attempt
|
||||
* timeout, so a slow IPv4 handshake is abandoned in favour of an unreachable
|
||||
* IPv6 address and the whole request fails (#68).
|
||||
* @param {unknown} err
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isAddressSelectionFailure(err) {
|
||||
return err instanceof TypeError && connectAttemptErrors(err)?.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET over IPv4 only, with one connection attempt bounded by the caller's
|
||||
* signal rather than by the auto-selection attempt timeout.
|
||||
* @param {string} url
|
||||
* @param {{signal?: AbortSignal}} [options]
|
||||
* @returns {Promise<Response>}
|
||||
*/
|
||||
export function fetchIpv4(url, { signal } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = https.request(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
signal,
|
||||
family: 4,
|
||||
autoSelectFamily: false,
|
||||
agent: false,
|
||||
},
|
||||
(response) => {
|
||||
// Nothing thrown here may escape: this callback runs outside the
|
||||
// promise, so an exception would be uncaught and crash the server.
|
||||
try {
|
||||
const status = response.statusCode || 500;
|
||||
// A `Response` only represents statuses up to 599; Node never hands
|
||||
// this callback a 1xx.
|
||||
if (status > 599)
|
||||
throw new Error(`FIRMS returned unsupported status ${status}`);
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(response.headers)) {
|
||||
if (Array.isArray(value))
|
||||
value.forEach((item) => headers.append(name, item));
|
||||
else if (value !== undefined) headers.set(name, String(value));
|
||||
}
|
||||
const nullBody = NULL_BODY_STATUSES.has(status);
|
||||
const upstream = new Response(
|
||||
nullBody ? null : Readable.toWeb(response),
|
||||
{ status, statusText: response.statusMessage || '', headers },
|
||||
);
|
||||
if (nullBody) response.resume();
|
||||
resolve(upstream);
|
||||
} catch (error) {
|
||||
response.destroy();
|
||||
reject(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
request.on('error', reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a FIRMS URL, retrying once over IPv4 when address selection fails.
|
||||
* Other failures (HTTP status, timeout, DNS) are returned or thrown as is.
|
||||
* @param {string} url
|
||||
* @param {{signal?: AbortSignal}} options
|
||||
* @param {{fetchImpl?: typeof fetch, fetchIpv4Impl?: typeof fetchIpv4, warn?: Function}} [deps]
|
||||
* @returns {Promise<Response>}
|
||||
*/
|
||||
export async function fetchFirms(
|
||||
url,
|
||||
options,
|
||||
{ fetchImpl = fetch, fetchIpv4Impl = fetchIpv4, warn = console.warn } = {},
|
||||
) {
|
||||
try {
|
||||
return await fetchImpl(url, options);
|
||||
} catch (err) {
|
||||
if (!isAddressSelectionFailure(err) || options?.signal?.aborted) throw err;
|
||||
warn(`[firms-proxy] ${describeFetchError(err)} — retrying over IPv4`);
|
||||
return fetchIpv4Impl(url, options);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* NASA FIRMS live active-fire proxy with a memory + disk cache.
|
||||
* Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
|
||||
*
|
||||
* Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent
|
||||
* satellites, no cross-source dedup) plus MODIS_NRT (combined Terra+Aqua,
|
||||
* ~1 km, numeric 0-100 confidence), fetched sequentially (quota courtesy)
|
||||
* Merges two VIIRS NRT sources (NOAA-20, NOAA-21 — independent satellites, no
|
||||
* cross-source dedup) plus MODIS_NRT (combined Terra+Aqua, ~1 km, numeric
|
||||
* 0-100 confidence), fetched sequentially (quota courtesy)
|
||||
* with `days=2` (`days=1` means "current UTC day", nearly empty just after
|
||||
* 00:00Z) and clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS
|
||||
* quota is 5,000 transactions / 10 min per MAP_KEY, so the cache is the point:
|
||||
@@ -18,6 +141,8 @@ import { readResponseTextCapped } from './common/http.js';
|
||||
* fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream
|
||||
* fetch across dev-server restarts. Pattern mirrors celestrakProxy.
|
||||
*
|
||||
* VIIRS_SNPP_NRT is left out: NASA stopped delivering Suomi NPP on 2026-11-02.
|
||||
*
|
||||
* Routes:
|
||||
* GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires}
|
||||
* GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions}
|
||||
@@ -25,17 +150,17 @@ import { readResponseTextCapped } from './common/http.js';
|
||||
* Keyless (no FIRMS_MAP_KEY): /api/firms → 503 {error:'no_key'}; status →
|
||||
* {hasKey:false}. Upstream is never touched without a key.
|
||||
*
|
||||
* /api/firms answers the trailing 24 hours to now. A source may instead
|
||||
* answer the 24 hours to a set time, naming its end as `at` (epoch ms); the
|
||||
* layer's row then shows that time rather than LIVE. This proxy never sets
|
||||
* `at`.
|
||||
*
|
||||
* @returns {import('vite').Plugin}
|
||||
*/
|
||||
export function firmsProxy() {
|
||||
const TTL_MS = 30 * 60_000;
|
||||
const STATUS_TTL_MS = 5 * 60_000;
|
||||
const SOURCES = [
|
||||
'VIIRS_NOAA20_NRT',
|
||||
'VIIRS_NOAA21_NRT',
|
||||
'VIIRS_SNPP_NRT',
|
||||
'MODIS_NRT',
|
||||
];
|
||||
const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'MODIS_NRT'];
|
||||
/**
|
||||
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
|
||||
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
|
||||
@@ -91,8 +216,12 @@ export function firmsProxy() {
|
||||
async function fetchSource(key, source) {
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
|
||||
const signal = AbortSignal.timeout(60_000);
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const res = await fetchFirms(url, { signal });
|
||||
if (!res.ok) {
|
||||
// Release the connection now rather than when the deadline fires.
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new Error(`HTTP ${res.status}`);
|
||||
}
|
||||
const records = parseFirmsCsv(
|
||||
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
|
||||
);
|
||||
@@ -121,7 +250,7 @@ export function firmsProxy() {
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`[firms-proxy] ${source} fetch failed:`,
|
||||
err?.message || err,
|
||||
describeFetchError(err),
|
||||
);
|
||||
sources.push({ source, count: 0, ok: false });
|
||||
}
|
||||
@@ -156,8 +285,13 @@ export function firmsProxy() {
|
||||
statusInflight = (async () => {
|
||||
try {
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`;
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const res = await fetchFirms(url, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new Error(`HTTP ${res.status}`);
|
||||
}
|
||||
const body = await res.json();
|
||||
const used = Number(body?.current_transactions);
|
||||
const limit = Number(body?.transaction_limit);
|
||||
@@ -167,7 +301,7 @@ export function firmsProxy() {
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[firms-proxy] mapkey status failed:',
|
||||
err?.message || err,
|
||||
describeFetchError(err),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -236,13 +236,26 @@ export function gbfsProxy({ ratePerMinute = GBFS_RATE_PER_MIN } = {}) {
|
||||
}
|
||||
|
||||
const upstream = await fetchGbfsUpstream(upstreamUrl.toString());
|
||||
// Only JSON is relayed, re-serialized under a fixed type: neither the
|
||||
// upstream's text nor its declared type reaches the browser.
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(upstream.body);
|
||||
} catch {
|
||||
res.writeHead(502, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'GBFS upstream returned non-JSON' }));
|
||||
return;
|
||||
}
|
||||
res.writeHead(upstream.status, {
|
||||
'Content-Type': upstream.contentType,
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Cache-Control': gbfsCacheControl(upstreamUrl.pathname),
|
||||
'X-GBFS-Upstream': upstreamUrl.hostname,
|
||||
'X-GBFS-Cache': 'MISS',
|
||||
});
|
||||
res.end(upstream.body);
|
||||
res.end(JSON.stringify(payload));
|
||||
} catch (error) {
|
||||
if (error?.name === 'AbortError') {
|
||||
res.writeHead(504, {
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import { createGeminiChatHandler } from './gemini/chat.js';
|
||||
import { createGeminiTokenHandler } from './gemini/realtime.js';
|
||||
|
||||
/** Mint constrained Live API credentials; the permanent key stays on Node. */
|
||||
function geminiLiveProxy({ annotationGuidance, realtime = {} } = {}) {
|
||||
function install(middlewares) {
|
||||
middlewares.use(
|
||||
'/api/gemini/chat',
|
||||
createGeminiChatHandler({ ...realtime, annotationGuidance }),
|
||||
);
|
||||
middlewares.use(
|
||||
'/api/gemini/token',
|
||||
createGeminiTokenHandler({ ...realtime, annotationGuidance }),
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import { createTextChatHandler } from '../common/textChat.js';
|
||||
import { invalidCalls } from '../common/textTools.js';
|
||||
import { textInstructions } from '../common/textInstructions.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from '../openai/tools.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
import { normalizeGeminiModel } from './config.js';
|
||||
|
||||
// The Gemini token route's throttle default (GEV_RATELIMIT_GEMINI_PER_MIN).
|
||||
export const GEMINI_DEFAULT_PER_MIN = 30;
|
||||
|
||||
const object = (value) =>
|
||||
value && typeof value === 'object' && !Array.isArray(value);
|
||||
const keys = (value, allowed) =>
|
||||
Object.keys(value).every((key) => allowed.includes(key));
|
||||
|
||||
// Only text and function exchanges may cross this route, never media or config.
|
||||
export function validChatContents(contents, names) {
|
||||
return (
|
||||
Array.isArray(contents) &&
|
||||
contents.length > 0 &&
|
||||
contents.length <= 96 &&
|
||||
contents.every(
|
||||
(content) =>
|
||||
object(content) &&
|
||||
keys(content, ['role', 'parts']) &&
|
||||
['user', 'model'].includes(content.role) &&
|
||||
Array.isArray(content.parts) &&
|
||||
content.parts.length > 0 &&
|
||||
content.parts.length <= 32 &&
|
||||
content.parts.every((part) => {
|
||||
if (
|
||||
!object(part) ||
|
||||
!keys(part, [
|
||||
'text',
|
||||
'functionCall',
|
||||
'functionResponse',
|
||||
'thoughtSignature',
|
||||
])
|
||||
)
|
||||
return false;
|
||||
if (
|
||||
part.thoughtSignature !== undefined &&
|
||||
(typeof part.thoughtSignature !== 'string' ||
|
||||
part.thoughtSignature.length > 32000 ||
|
||||
content.role !== 'model')
|
||||
)
|
||||
return false;
|
||||
if (Object.hasOwn(part, 'text'))
|
||||
return (
|
||||
keys(part, ['text', 'thoughtSignature']) &&
|
||||
typeof part.text === 'string'
|
||||
);
|
||||
const call = part.functionCall ?? part.functionResponse;
|
||||
if (
|
||||
!object(call) ||
|
||||
typeof call.name !== 'string' ||
|
||||
call.name.length > 128 ||
|
||||
(names && !names.has(call.name)) ||
|
||||
(call.id !== undefined &&
|
||||
(typeof call.id !== 'string' || call.id.length > 1024))
|
||||
)
|
||||
return false;
|
||||
if (part.functionCall)
|
||||
return (
|
||||
content.role === 'model' &&
|
||||
keys(part, ['functionCall', 'thoughtSignature']) &&
|
||||
keys(call, ['name', 'args', 'id']) &&
|
||||
(call.args === null || object(call.args ?? {}))
|
||||
);
|
||||
return (
|
||||
content.role === 'user' &&
|
||||
keys(part, ['functionResponse']) &&
|
||||
keys(call, ['name', 'response', 'id']) &&
|
||||
object(call.response)
|
||||
);
|
||||
}),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function createGeminiChatHandler({
|
||||
annotationGuidance,
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
// osm_query is offered, and the prompt names it, only with Overpass, as
|
||||
// for voice.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
resolveChatModel = () =>
|
||||
process.env.GEMINI_CHAT_MODEL || 'gemini-3.5-flash-lite',
|
||||
resolveApiKey = () => process.env.GEMINI_API_KEY,
|
||||
resolveProviderRateLimit = () => process.env.GEV_RATELIMIT_GEMINI_PER_MIN,
|
||||
...options
|
||||
} = {}) {
|
||||
return createTextChatHandler({
|
||||
...options,
|
||||
provider: 'gemini',
|
||||
resolveApiKey,
|
||||
resolveProviderRateLimit,
|
||||
providerPerMin: GEMINI_DEFAULT_PER_MIN,
|
||||
prepare(input) {
|
||||
const model = normalizeGeminiModel(resolveChatModel());
|
||||
if (!model || /live|audio|tts/i.test(model)) return { errorStatus: 503 };
|
||||
if (
|
||||
!object(input) ||
|
||||
!keys(input, ['contents']) ||
|
||||
!validChatContents(input.contents, null)
|
||||
)
|
||||
return null;
|
||||
if (!pairedContents(input.contents)) return null;
|
||||
const overpass = overpassConfigured();
|
||||
return {
|
||||
endpoint: `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
|
||||
headers: (key) => ({ 'x-goog-api-key': key }),
|
||||
body: {
|
||||
contents: input.contents,
|
||||
systemInstruction: {
|
||||
parts: [
|
||||
{ text: textInstructions(annotationGuidance, { overpass }) },
|
||||
],
|
||||
},
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: offeredTools(tools, { overpass }).map(
|
||||
({ name, description, parameters }) => ({
|
||||
name,
|
||||
description,
|
||||
parametersJsonSchema: parameters,
|
||||
}),
|
||||
),
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ['TEXT'],
|
||||
maxOutputTokens: 4096,
|
||||
thinkingConfig: { thinkingLevel: 'LOW' },
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
parse(data) {
|
||||
const candidate = data?.candidates?.[0];
|
||||
const content = candidate?.content;
|
||||
if (
|
||||
!content ||
|
||||
!validChatContents([content], null) ||
|
||||
content.role !== 'model'
|
||||
)
|
||||
return null;
|
||||
const calls = content.parts
|
||||
.filter((part) => part.functionCall)
|
||||
.map((part) => part.functionCall);
|
||||
return {
|
||||
content,
|
||||
finishReason: candidate.finishReason,
|
||||
invalidCalls: invalidCalls(
|
||||
calls,
|
||||
offeredTools(tools, { overpass: overpassConfigured() }),
|
||||
),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Never forward an orphan result or an unfinished batch in browser history.
|
||||
function pairedContents(contents) {
|
||||
let pending = [];
|
||||
for (const content of contents) {
|
||||
const calls = content.parts
|
||||
.filter((part) => part.functionCall)
|
||||
.map((part) => part.functionCall);
|
||||
const results = content.parts
|
||||
.filter((part) => part.functionResponse)
|
||||
.map((part) => part.functionResponse);
|
||||
if (pending.length) {
|
||||
if (
|
||||
calls.length ||
|
||||
results.length !== pending.length ||
|
||||
results.some(
|
||||
(result, i) =>
|
||||
result.name !== pending[i].name || result.id !== pending[i].id,
|
||||
)
|
||||
)
|
||||
return false;
|
||||
pending = [];
|
||||
} else if (results.length) return false;
|
||||
if (calls.length) pending = calls;
|
||||
}
|
||||
return !pending.length;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { realtimeInstructions } from '../openai/instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from '../openai/tools.js';
|
||||
|
||||
export const GEMINI_LIVE_MODEL_DEFAULT = 'gemini-3.8-live';
|
||||
export const GEMINI_TOKEN_ENDPOINT =
|
||||
@@ -14,22 +14,26 @@ export function normalizeGeminiModel(value) {
|
||||
return /^gemini-[a-z0-9][a-z0-9.-]{0,119}$/.test(model) ? model : null;
|
||||
}
|
||||
|
||||
/** REST BidiGenerateContentSetup fields, excluding its model resource name. */
|
||||
/**
|
||||
* REST BidiGenerateContentSetup fields, excluding its model resource name.
|
||||
* `overpass` says an operator configured Overpass, which `osm_query` needs.
|
||||
*/
|
||||
export function createGeminiLiveConfig(
|
||||
annotationGuidance,
|
||||
inputMode = 'open-mic',
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
{ overpass = false } = {},
|
||||
) {
|
||||
if (!['open-mic', 'push-to-talk'].includes(inputMode))
|
||||
throw new Error('Invalid Gemini input mode');
|
||||
const config = {
|
||||
generationConfig: { responseModalities: ['AUDIO'] },
|
||||
systemInstruction: {
|
||||
parts: [{ text: realtimeInstructions(annotationGuidance) }],
|
||||
parts: [{ text: realtimeInstructions(annotationGuidance, { overpass }) }],
|
||||
},
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: tools.map(
|
||||
functionDeclarations: offeredTools(tools, { overpass }).map(
|
||||
({ name, description, parameters }) => ({
|
||||
name,
|
||||
description,
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
isGeminiRequestOriginAllowed,
|
||||
readGeminiTokenRequest,
|
||||
} from './request.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
|
||||
const UPSTREAM_RESPONSE_BYTES = 32 * 1024;
|
||||
const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
|
||||
@@ -19,6 +20,8 @@ const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
|
||||
export function createGeminiTokenHandler({
|
||||
annotationGuidance,
|
||||
tools,
|
||||
// osm_query is offered only when an operator configured Overpass.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
fetchImpl = (...args) => fetch(...args),
|
||||
resolveApiKey = () => process.env.GEMINI_API_KEY,
|
||||
resolveModel = () =>
|
||||
@@ -60,7 +63,10 @@ export function createGeminiTokenHandler({
|
||||
return reply(403, 'Gemini voice requires a same-origin app request');
|
||||
apiKey = String(resolveApiKey() || '').trim();
|
||||
if (!apiKey)
|
||||
return reply(503, 'Add a Gemini API key in Provider Settings');
|
||||
return reply(503, {
|
||||
error: 'Add a Gemini API key in Provider Settings',
|
||||
code: 'API_KEY_MISSING',
|
||||
});
|
||||
model = normalizeGeminiModel(resolveModel());
|
||||
if (!model)
|
||||
return reply(503, 'The Gemini Live model configuration is invalid');
|
||||
@@ -104,6 +110,7 @@ export function createGeminiTokenHandler({
|
||||
annotationGuidance,
|
||||
inputMode,
|
||||
tools,
|
||||
{ overpass: overpassConfigured() },
|
||||
);
|
||||
const issuedAt = now();
|
||||
// REST uses bidiGenerateContentSetup. liveConnectConstraints is the SDK
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
GHOSTMAPS_MAX_FILE_BYTES,
|
||||
GHOSTMAPS_REPO,
|
||||
buildGhostMapsCatalog,
|
||||
ghostMapsRawUrl,
|
||||
parseGitRefAdvertisement,
|
||||
} from '../../src/layers/ghostMaps/catalog.js';
|
||||
import {
|
||||
readResponseBytesCapped,
|
||||
readResponseJsonCapped,
|
||||
readResponseTextCapped,
|
||||
} from './common/http.js';
|
||||
|
||||
// S2 Underground GhostMaps (public GitHub repository of KMZ intel products).
|
||||
// The repository is not ours, so there is no webhook: the watcher reads the
|
||||
// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the
|
||||
// REST API rate limit) every few seconds, and only a new commit costs REST
|
||||
// calls (tree + commit). Browsers hear about a new commit over server-sent
|
||||
// events and fetch only the products whose blob changed. Files are fetched by
|
||||
// commit-pinned raw URLs and kept on disk by blob sha.
|
||||
|
||||
const DEFAULT_POLL_MS = 15_000;
|
||||
const MIN_POLL_MS = 5_000;
|
||||
const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000];
|
||||
const TIMEOUT_MS = 15_000;
|
||||
const FILE_TIMEOUT_MS = 120_000;
|
||||
const HEARTBEAT_MS = 25_000;
|
||||
const MAX_SUBSCRIBERS = 64;
|
||||
const MAX_REF_BYTES = 64 * 1024;
|
||||
const MAX_TREE_BYTES = 16 * 1024 * 1024;
|
||||
const MAX_COMMIT_BYTES = 2 * 1024 * 1024;
|
||||
const USER_AGENT = 'gods-eye-view-ghostmaps/1.0';
|
||||
const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`;
|
||||
|
||||
/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */
|
||||
export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) {
|
||||
const seconds = Number(raw);
|
||||
if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS;
|
||||
return Math.max(MIN_POLL_MS, Math.round(seconds * 1000));
|
||||
}
|
||||
|
||||
/**
|
||||
* The repository watcher and file cache, independent of HTTP routing.
|
||||
* @returns {{start: Function, stop: Function, catalog: Function,
|
||||
* checkNow: Function, file: Function, subscribe: Function}}
|
||||
*/
|
||||
export function createGhostMapsWatcher({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
pollMs = ghostMapsPollMs(),
|
||||
cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'),
|
||||
token = () => String(process.env.GITHUB_TOKEN || '').trim(),
|
||||
log = (...args) => console.log(...args),
|
||||
warn = (...args) => console.warn(...args),
|
||||
} = {}) {
|
||||
let state = {
|
||||
commit: null,
|
||||
committedAt: null,
|
||||
message: null,
|
||||
products: [],
|
||||
checkedAt: null,
|
||||
changedAt: null,
|
||||
error: null,
|
||||
};
|
||||
let timer = null;
|
||||
let running = false;
|
||||
let checking = null;
|
||||
let failures = 0;
|
||||
let diskLoaded = false;
|
||||
const listeners = new Set();
|
||||
const downloads = new Map();
|
||||
|
||||
const apiHeaders = () => ({
|
||||
Accept: 'application/vnd.github+json',
|
||||
'User-Agent': USER_AGENT,
|
||||
...(token() ? { Authorization: `Bearer ${token()}` } : {}),
|
||||
});
|
||||
|
||||
async function get(url, headers, timeoutMs = TIMEOUT_MS) {
|
||||
const response = await fetchImpl(url, {
|
||||
headers,
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), {
|
||||
status: response.status,
|
||||
});
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async function readHead() {
|
||||
const response = await get(
|
||||
`${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`,
|
||||
{ 'User-Agent': USER_AGENT },
|
||||
);
|
||||
const sha = parseGitRefAdvertisement(
|
||||
await readResponseTextCapped(response, MAX_REF_BYTES),
|
||||
);
|
||||
if (!sha) throw new Error('GhostMaps branch head not advertised');
|
||||
return sha;
|
||||
}
|
||||
|
||||
async function readCommit(sha) {
|
||||
const api = `https://api.github.com/repos/${REPO_PATH}`;
|
||||
const tree = await readResponseJsonCapped(
|
||||
await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()),
|
||||
MAX_TREE_BYTES,
|
||||
);
|
||||
if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed');
|
||||
if (tree.truncated) warn('[ghostmaps] repository tree listing truncated');
|
||||
let committedAt = null;
|
||||
let message = null;
|
||||
try {
|
||||
const commit = await readResponseJsonCapped(
|
||||
await get(`${api}/commits/${sha}`, apiHeaders()),
|
||||
MAX_COMMIT_BYTES,
|
||||
);
|
||||
committedAt = Date.parse(commit?.commit?.committer?.date) || null;
|
||||
message = String(commit?.commit?.message || '')
|
||||
.split('\n')[0]
|
||||
.slice(0, 200);
|
||||
} catch {
|
||||
// The commit date is decoration; the tree is what matters.
|
||||
}
|
||||
return { products: buildGhostMapsCatalog(tree.tree), committedAt, message };
|
||||
}
|
||||
|
||||
async function loadDisk() {
|
||||
if (diskLoaded) return;
|
||||
diskLoaded = true;
|
||||
try {
|
||||
const saved = JSON.parse(
|
||||
await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'),
|
||||
);
|
||||
if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products))
|
||||
state = { ...state, ...saved, error: null };
|
||||
} catch {
|
||||
/* first run */
|
||||
}
|
||||
}
|
||||
|
||||
async function saveDisk() {
|
||||
try {
|
||||
await fsp.mkdir(cacheDir, { recursive: true });
|
||||
const { commit, committedAt, message, products, changedAt } = state;
|
||||
await fsp.writeFile(
|
||||
path.join(cacheDir, 'catalog.json'),
|
||||
JSON.stringify({ commit, committedAt, message, products, changedAt }),
|
||||
);
|
||||
} catch (error) {
|
||||
warn('[ghostmaps] could not save catalog:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
function emit() {
|
||||
const event = publicCatalog();
|
||||
for (const listener of listeners) {
|
||||
try {
|
||||
listener(event);
|
||||
} catch {
|
||||
/* a broken subscriber must not stop the rest */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function check() {
|
||||
await loadDisk();
|
||||
const head = await readHead();
|
||||
state.checkedAt = now();
|
||||
if (head === state.commit) {
|
||||
if (state.error !== null) {
|
||||
state.error = null;
|
||||
emit();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const next = await readCommit(head);
|
||||
const previous = new Map(state.products.map((p) => [p.id, p.version]));
|
||||
const changed = next.products
|
||||
.filter((p) => previous.get(p.id) !== p.version)
|
||||
.map((p) => p.id);
|
||||
state = {
|
||||
...state,
|
||||
...next,
|
||||
commit: head,
|
||||
changedAt: now(),
|
||||
error: null,
|
||||
};
|
||||
log(
|
||||
`[ghostmaps] commit ${head.slice(0, 8)}${
|
||||
next.message ? ` "${next.message}"` : ''
|
||||
}; updated: ${changed.join(', ') || 'none'}`,
|
||||
);
|
||||
await saveDisk();
|
||||
emit();
|
||||
// Warm the default products so the first browser load is local.
|
||||
for (const product of state.products)
|
||||
if (product.defaultOn) file(product.id).catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Run one check now, coalescing with one already in flight. */
|
||||
function checkNow() {
|
||||
if (!checking)
|
||||
checking = check()
|
||||
.then(
|
||||
(changed) => {
|
||||
failures = 0;
|
||||
return changed;
|
||||
},
|
||||
(error) => {
|
||||
failures += 1;
|
||||
state.error = error.message;
|
||||
state.checkedAt = now();
|
||||
warn('[ghostmaps] check failed:', error.message);
|
||||
emit();
|
||||
return false;
|
||||
},
|
||||
)
|
||||
.finally(() => {
|
||||
checking = null;
|
||||
});
|
||||
return checking;
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
clearTimeout(timer);
|
||||
if (!running) return;
|
||||
const delay = failures
|
||||
? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1]
|
||||
: pollMs;
|
||||
timer = setTimeout(async () => {
|
||||
await checkNow();
|
||||
schedule();
|
||||
}, delay);
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
/** Start watching (idempotent); the first check runs immediately. */
|
||||
function start() {
|
||||
if (running) return checking || Promise.resolve(false);
|
||||
running = true;
|
||||
const first = checkNow();
|
||||
first.then(schedule);
|
||||
return first;
|
||||
}
|
||||
|
||||
function stop() {
|
||||
running = false;
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
|
||||
function publicCatalog() {
|
||||
return {
|
||||
repo: GHOSTMAPS_REPO.url,
|
||||
branch: GHOSTMAPS_REPO.branch,
|
||||
commit: state.commit,
|
||||
committedAt: state.committedAt,
|
||||
message: state.message,
|
||||
checkedAt: state.checkedAt,
|
||||
changedAt: state.changedAt,
|
||||
pollMs,
|
||||
error: state.error,
|
||||
products: state.products.map(
|
||||
({
|
||||
id,
|
||||
title,
|
||||
defaultOn,
|
||||
fileName,
|
||||
fileTitle,
|
||||
version,
|
||||
size,
|
||||
dateMs,
|
||||
}) => ({
|
||||
id,
|
||||
title,
|
||||
defaultOn,
|
||||
fileName,
|
||||
fileTitle,
|
||||
version,
|
||||
size,
|
||||
dateMs,
|
||||
}),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The bytes of one product's current edition, from disk or a commit-pinned
|
||||
* download. Only catalog paths are ever fetched.
|
||||
*/
|
||||
async function file(id) {
|
||||
const product = state.products.find((p) => p.id === id);
|
||||
if (!product) return null;
|
||||
const target = path.join(cacheDir, 'files', `${product.version}.kmz`);
|
||||
try {
|
||||
return { product, bytes: await fsp.readFile(target) };
|
||||
} catch {
|
||||
/* not cached yet */
|
||||
}
|
||||
const commit = state.commit;
|
||||
let pending = downloads.get(product.version);
|
||||
if (!pending) {
|
||||
pending = (async () => {
|
||||
const response = await get(
|
||||
ghostMapsRawUrl(commit, product.path),
|
||||
{ 'User-Agent': USER_AGENT },
|
||||
FILE_TIMEOUT_MS,
|
||||
);
|
||||
const bytes = Buffer.from(
|
||||
await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES),
|
||||
);
|
||||
await fsp.mkdir(path.dirname(target), { recursive: true });
|
||||
const temp = `${target}.${process.pid}.tmp`;
|
||||
await fsp.writeFile(temp, bytes);
|
||||
await fsp.rename(temp, target);
|
||||
await pruneFiles();
|
||||
return bytes;
|
||||
})().finally(() => downloads.delete(product.version));
|
||||
downloads.set(product.version, pending);
|
||||
}
|
||||
return { product, bytes: await pending };
|
||||
}
|
||||
|
||||
/** Keep only files that a current product still points to. */
|
||||
async function pruneFiles() {
|
||||
const keep = new Set(state.products.map((p) => `${p.version}.kmz`));
|
||||
const dir = path.join(cacheDir, 'files');
|
||||
for (const name of await fsp.readdir(dir).catch(() => [])) {
|
||||
if (!keep.has(name) && !name.endsWith('.tmp'))
|
||||
await fsp.rm(path.join(dir, name), { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
function subscribe(listener) {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
checkNow,
|
||||
catalog: publicCatalog,
|
||||
file,
|
||||
subscribe,
|
||||
subscribers: () => listeners.size,
|
||||
};
|
||||
}
|
||||
|
||||
/** Same-origin GhostMaps routes for dev and preview. */
|
||||
export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) {
|
||||
const json = (res, status, value) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
|
||||
function events(req, res) {
|
||||
if (watcher.subscribers() >= MAX_SUBSCRIBERS)
|
||||
return json(res, 429, { error: 'too_many_subscribers' });
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-store',
|
||||
Connection: 'keep-alive',
|
||||
'X-Accel-Buffering': 'no',
|
||||
});
|
||||
const send = (catalog) =>
|
||||
res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`);
|
||||
res.write('retry: 5000\n\n');
|
||||
send(watcher.catalog());
|
||||
const unsubscribe = watcher.subscribe(send);
|
||||
const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
|
||||
heartbeat.unref?.();
|
||||
const close = () => {
|
||||
clearInterval(heartbeat);
|
||||
unsubscribe();
|
||||
};
|
||||
req.on('close', close);
|
||||
res.on('error', close);
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
if (req.method !== 'GET')
|
||||
return json(res, 405, { error: 'method_not_allowed' });
|
||||
const url = new URL(req.url || '/', 'http://local');
|
||||
const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname);
|
||||
if (!['/catalog', '/events'].includes(url.pathname) && !match)
|
||||
return json(res, 404, { error: 'unknown_route' });
|
||||
// Watching starts with the first request, so an install that never turns
|
||||
// the layer on never contacts GitHub.
|
||||
const first = watcher.start();
|
||||
if (url.pathname === '/catalog') {
|
||||
if (!watcher.catalog().commit) await first;
|
||||
return json(res, 200, watcher.catalog());
|
||||
}
|
||||
if (url.pathname === '/events') return events(req, res);
|
||||
try {
|
||||
if (!watcher.catalog().commit) await first;
|
||||
const found = await watcher.file(match[1]);
|
||||
if (!found) return json(res, 404, { error: 'unknown_product' });
|
||||
const pinned = url.searchParams.get('v') === found.product.version;
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/vnd.google-earth.kmz',
|
||||
'Content-Length': found.bytes.length,
|
||||
'X-GhostMaps-Version': found.product.version,
|
||||
// A versioned URL names one immutable blob.
|
||||
'Cache-Control': pinned
|
||||
? 'private, max-age=31536000, immutable'
|
||||
: 'no-store',
|
||||
});
|
||||
return res.end(found.bytes);
|
||||
} catch {
|
||||
return json(res, 502, { error: 'ghostmaps_unavailable' });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'ghostmaps',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/ghostmaps', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/ghostmaps', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, readdir, rm } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
createGhostMapsWatcher,
|
||||
ghostMapsPollMs,
|
||||
ghostMapsProxy,
|
||||
} from './ghostMaps.js';
|
||||
|
||||
const MASTER =
|
||||
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/Master Database/';
|
||||
const KINETIC =
|
||||
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz';
|
||||
const sha = (c) => c.repeat(40);
|
||||
|
||||
/** A fake GitHub: ref advertisement, tree, commit and raw files. */
|
||||
function fakeGitHub() {
|
||||
const repo = { head: sha('1'), trees: new Map(), calls: [] };
|
||||
repo.trees.set(sha('1'), [
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_15_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('a'),
|
||||
},
|
||||
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
|
||||
]);
|
||||
const body = (text, status = 200) =>
|
||||
new Response(text, {
|
||||
status,
|
||||
headers: { 'content-length': String(Buffer.byteLength(text)) },
|
||||
});
|
||||
const fetchImpl = async (url) => {
|
||||
repo.calls.push(url);
|
||||
if (url.endsWith('/info/refs?service=git-upload-pack'))
|
||||
return body(
|
||||
`001e# service=git-upload-pack\n0000\n003d${repo.head} refs/heads/main\n0000`,
|
||||
);
|
||||
const tree = /git\/trees\/([0-9a-f]{40})/.exec(url);
|
||||
if (tree) return body(JSON.stringify({ tree: repo.trees.get(tree[1]) }));
|
||||
if (/\/commits\//.test(url))
|
||||
return body(
|
||||
JSON.stringify({
|
||||
commit: {
|
||||
message: 'Oct 18 Update\nmore',
|
||||
committer: { date: '2026-10-18T00:00:00Z' },
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (url.startsWith('https://raw.githubusercontent.com/'))
|
||||
return body('KMZ!');
|
||||
return body('nope', 404);
|
||||
};
|
||||
return { repo, fetchImpl };
|
||||
}
|
||||
|
||||
async function withCache(fn) {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), 'ghostmaps-'));
|
||||
try {
|
||||
await fn(dir);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// checkNow announces the catalog before background warm-up finishes. Join the
|
||||
// same file reads before changing editions or removing the temporary cache.
|
||||
async function warmDefaults(watcher) {
|
||||
await Promise.all(
|
||||
watcher
|
||||
.catalog()
|
||||
.products.filter((product) => product.defaultOn)
|
||||
.map((product) => watcher.file(product.id)),
|
||||
);
|
||||
}
|
||||
|
||||
test('a new commit is announced once, naming only the products that changed', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { repo, fetchImpl } = fakeGitHub();
|
||||
const watcher = createGhostMapsWatcher({
|
||||
fetchImpl,
|
||||
cacheDir,
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
const events = [];
|
||||
watcher.subscribe((catalog) => events.push(catalog));
|
||||
|
||||
assert.equal(await watcher.checkNow(), true);
|
||||
await warmDefaults(watcher);
|
||||
const first = watcher.catalog();
|
||||
assert.equal(first.commit, sha('1'));
|
||||
assert.equal(first.message, 'Oct 18 Update');
|
||||
assert.deepEqual(
|
||||
first.products.map((p) => p.id),
|
||||
['cip', 'kinetic-30d'],
|
||||
);
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
// Same head: one ref read, no REST calls, no event.
|
||||
repo.calls.length = 0;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.equal(repo.calls.length, 1);
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
// A new upload: the CIP gets a newer edition, the 30-day pull is untouched.
|
||||
repo.head = sha('2');
|
||||
repo.trees.set(sha('2'), [
|
||||
...repo.trees.get(sha('1')),
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('c'),
|
||||
},
|
||||
]);
|
||||
assert.equal(await watcher.checkNow(), true);
|
||||
await warmDefaults(watcher);
|
||||
assert.equal(events.length, 2);
|
||||
const cip = events[1].products.find((p) => p.id === 'cip');
|
||||
assert.equal(cip.version, sha('c'));
|
||||
assert.match(cip.fileName, /October_18/);
|
||||
assert.equal(
|
||||
events[1].products.find((p) => p.id === 'kinetic-30d').version,
|
||||
sha('b'),
|
||||
);
|
||||
// Repository paths stay on the server.
|
||||
assert.equal('path' in cip, false);
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('files download by commit-pinned URL, cache by blob, and stale blobs are pruned', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { repo, fetchImpl } = fakeGitHub();
|
||||
const watcher = createGhostMapsWatcher({
|
||||
fetchImpl,
|
||||
cacheDir,
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
await watcher.checkNow();
|
||||
const found = await watcher.file('kinetic-30d');
|
||||
assert.equal(found.bytes.toString(), 'KMZ!');
|
||||
const raw = repo.calls.filter((url) =>
|
||||
url.includes('raw.githubusercontent'),
|
||||
);
|
||||
assert.ok(raw.every((url) => url.includes(`/GhostMaps/${sha('1')}/`)));
|
||||
assert.ok(
|
||||
raw.some((url) => url.endsWith('/Kinetic_Activities_Last_30_Days.kmz')),
|
||||
);
|
||||
const before = repo.calls.length;
|
||||
await watcher.file('kinetic-30d');
|
||||
assert.equal(repo.calls.length, before, 'second read comes from disk');
|
||||
assert.equal(await watcher.file('not-a-product'), null);
|
||||
|
||||
// When the edition changes, the old blob leaves the disk cache.
|
||||
await watcher.file('cip');
|
||||
repo.head = sha('2');
|
||||
repo.trees.set(sha('2'), [
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('c'),
|
||||
},
|
||||
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
|
||||
]);
|
||||
await watcher.checkNow();
|
||||
await watcher.file('cip');
|
||||
const files = (await readdir(path.join(cacheDir, 'files'))).sort();
|
||||
assert.deepEqual(files, [`${sha('b')}.kmz`, `${sha('c')}.kmz`]);
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('the catalog survives a restart from disk, and upstream failures are reported', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { fetchImpl } = fakeGitHub();
|
||||
const quiet = { log: () => {}, warn: () => {} };
|
||||
const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet });
|
||||
await online.checkNow();
|
||||
await warmDefaults(online);
|
||||
const offline = createGhostMapsWatcher({
|
||||
fetchImpl: async () => new Response('down', { status: 503 }),
|
||||
cacheDir,
|
||||
...quiet,
|
||||
});
|
||||
assert.equal(await offline.checkNow(), false);
|
||||
const catalog = offline.catalog();
|
||||
assert.equal(catalog.commit, sha('1'));
|
||||
assert.equal(catalog.products.length, 2);
|
||||
assert.match(catalog.error, /HTTP 503/);
|
||||
}));
|
||||
|
||||
test('an unchanged head clears an upstream outage and announces recovery once', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { fetchImpl } = fakeGitHub();
|
||||
let unavailable = false;
|
||||
let checkedAt = 1000;
|
||||
const requests = [];
|
||||
const watcher = createGhostMapsWatcher({
|
||||
cacheDir,
|
||||
now: () => checkedAt,
|
||||
fetchImpl: async (url, options) => {
|
||||
requests.push(url);
|
||||
if (unavailable) return new Response('down', { status: 503 });
|
||||
return fetchImpl(url, options);
|
||||
},
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
const events = [];
|
||||
watcher.subscribe((catalog) => events.push(catalog));
|
||||
await watcher.checkNow();
|
||||
await Promise.all([watcher.file('cip'), watcher.file('kinetic-30d')]);
|
||||
const healthy = watcher.catalog();
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
unavailable = true;
|
||||
checkedAt = 2000;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.match(watcher.catalog().error, /HTTP 503/);
|
||||
assert.equal(events.length, 2);
|
||||
|
||||
unavailable = false;
|
||||
checkedAt = 3000;
|
||||
requests.length = 0;
|
||||
assert.equal(await watcher.checkNow(), false, 'no new edition arrived');
|
||||
const recovered = watcher.catalog();
|
||||
assert.equal(recovered.error, null);
|
||||
assert.equal(recovered.checkedAt, checkedAt);
|
||||
assert.equal(recovered.commit, healthy.commit);
|
||||
assert.equal(recovered.changedAt, healthy.changedAt);
|
||||
assert.deepEqual(recovered.products, healthy.products);
|
||||
assert.equal(requests.length, 1, 'recovery needs only the ref read');
|
||||
assert.match(requests[0], /\/info\/refs\?service=git-upload-pack$/);
|
||||
assert.equal(events.length, 3, 'subscribers see the recovered catalog');
|
||||
assert.equal(events[2].error, null);
|
||||
assert.equal(events[2].checkedAt, checkedAt);
|
||||
|
||||
checkedAt = 4000;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.equal(events.length, 3, 'healthy unchanged polls stay silent');
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('poll interval is configurable with a 5 s floor', () => {
|
||||
assert.equal(ghostMapsPollMs(undefined), 15000);
|
||||
assert.equal(ghostMapsPollMs('30'), 30000);
|
||||
assert.equal(ghostMapsPollMs('1'), 5000);
|
||||
assert.equal(ghostMapsPollMs('soon'), 15000);
|
||||
});
|
||||
|
||||
test('routes: unknown paths and methods are refused without contacting GitHub', async () => {
|
||||
let started = 0;
|
||||
const watcher = {
|
||||
start: () => {
|
||||
started += 1;
|
||||
return Promise.resolve(false);
|
||||
},
|
||||
catalog: () => ({ commit: sha('1'), products: [] }),
|
||||
file: async () => null,
|
||||
subscribe: () => () => {},
|
||||
subscribers: () => 0,
|
||||
};
|
||||
let handler;
|
||||
ghostMapsProxy({ watcher }).configureServer({
|
||||
middlewares: { use: (_route, fn) => (handler = fn) },
|
||||
});
|
||||
const call = async (method, url) => {
|
||||
const res = {
|
||||
destroyed: false,
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
this.headers = headers;
|
||||
},
|
||||
end(body) {
|
||||
this.body = String(body ?? '');
|
||||
},
|
||||
};
|
||||
await handler({ method, url }, res);
|
||||
return res;
|
||||
};
|
||||
assert.equal((await call('GET', '/nope')).status, 404);
|
||||
assert.equal((await call('POST', '/catalog')).status, 405);
|
||||
assert.equal((await call('GET', '/file/../../etc')).status, 404);
|
||||
assert.equal(started, 0);
|
||||
assert.equal((await call('GET', '/file/cip')).status, 404);
|
||||
const catalog = await call('GET', '/catalog');
|
||||
assert.equal(catalog.status, 200);
|
||||
assert.equal(JSON.parse(catalog.body).commit, sha('1'));
|
||||
assert.equal(started, 2);
|
||||
});
|
||||
@@ -0,0 +1,370 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fs } from 'node:fs';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { normalizeInatObservations } from '../../../src/layers/wildlifeSightings/records.js';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
import { clampInt } from '../common/query.js';
|
||||
import { providerUserAgent } from '../common/userAgent.js';
|
||||
import { projectInatGrid, validInatGridTile } from './grid.js';
|
||||
import { createInatQueue } from './queue.js';
|
||||
|
||||
const DAILY_BUDGET = 8000;
|
||||
const DAY_MS = 24 * 60 * 60_000;
|
||||
// One page is capped independently of upstream Content-Length.
|
||||
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 20_000;
|
||||
const API_URL = 'https://api.inaturalist.org/v1';
|
||||
// Request only the fields used by the public observation projection.
|
||||
const OBSERVATIONS_URL = 'https://api.inaturalist.org/v2/observations';
|
||||
const OBSERVATION_FIELDS =
|
||||
'(id:!t,uri:!t,location:!t,geojson:all,geoprivacy:!t,taxon_geoprivacy:!t,' +
|
||||
'obscured:!t,quality_grade:!t,observed_on:!t,' +
|
||||
'time_observed_at:!t,faves_count:!t,' +
|
||||
'num_identification_agreements:!t,' +
|
||||
'photos:(url:!t,attribution:!t,license_code:!t),' +
|
||||
'taxon:(name:!t,preferred_common_name:!t,iconic_taxon_name:!t,' +
|
||||
'threatened:!t))';
|
||||
const COOLDOWN_MS = 60_000;
|
||||
const FAILURE_COOLDOWN_MS = 15_000;
|
||||
|
||||
/** Seconds and HTTP dates are accepted; a missing value earns one minute. */
|
||||
export function inatRetryCooldownMs(raw, nowMs) {
|
||||
const value = typeof raw === 'string' ? raw.trim() : '';
|
||||
if (value) {
|
||||
const seconds = /^\d+(?:\.\d+)?$/.test(value) ? Number(value) : NaN;
|
||||
const deadline =
|
||||
!Number.isFinite(seconds) && /[a-z]/i.test(value)
|
||||
? Date.parse(value)
|
||||
: NaN;
|
||||
const delay = Number.isFinite(seconds) ? seconds * 1000 : deadline - nowMs;
|
||||
if (Number.isFinite(delay) && delay >= 0) return delay;
|
||||
}
|
||||
return COOLDOWN_MS;
|
||||
}
|
||||
|
||||
function upstreamError(code, status, retryAfterS, upstreamStatus) {
|
||||
return Object.assign(new Error(code), {
|
||||
code,
|
||||
status,
|
||||
retryAfterS,
|
||||
upstreamStatus,
|
||||
});
|
||||
}
|
||||
|
||||
function observationFilters(params) {
|
||||
const query = new URLSearchParams({
|
||||
verifiable: 'true',
|
||||
captive: 'false',
|
||||
// d1 is an observation DATE, unlike creation-time created_d1.
|
||||
d1: params.observedD1,
|
||||
});
|
||||
if (params.taxa !== 'whales') query.set('photos', 'true');
|
||||
if (params.quality === 'research') query.set('quality_grade', 'research');
|
||||
if (params.taxa === 'whales') query.set('taxon_id', '152871');
|
||||
else if (['Aves', 'Mammalia', 'Insecta', 'Plantae'].includes(params.taxa))
|
||||
query.set('iconic_taxa', params.taxa);
|
||||
return query;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared iNaturalist governor. Only request counters are persisted; observation
|
||||
* payloads are projected immediately and are never written to disk.
|
||||
*/
|
||||
export function createInatClient({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
cacheDir = path.join(process.cwd(), '.gev-cache', 'inaturalist'),
|
||||
warn = (...args) => console.warn(...args),
|
||||
queueOptions,
|
||||
} = {}) {
|
||||
const enqueue = createInatQueue(queueOptions);
|
||||
let budget = null;
|
||||
let budgetLoaded = false;
|
||||
let budgetTail = Promise.resolve();
|
||||
let cooldownUntil = 0;
|
||||
// An unavailable UTFGrid tile must never place observation queries into a
|
||||
// 502 cooldown. Only an upstream 429 is a provider-wide restriction.
|
||||
const failures = new Map();
|
||||
|
||||
function dailyLimit() {
|
||||
return clampInt(
|
||||
process.env.INATURALIST_DAILY_BUDGET,
|
||||
1,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
DAILY_BUDGET,
|
||||
);
|
||||
}
|
||||
|
||||
function day() {
|
||||
return new Date(now()).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function currentBudget() {
|
||||
const date = day();
|
||||
if (budget?.date !== date) budget = { date, count: 0 };
|
||||
return budget;
|
||||
}
|
||||
|
||||
async function readBudgetOnce() {
|
||||
if (budgetLoaded) return;
|
||||
budgetLoaded = true;
|
||||
if (cacheDir === null) return;
|
||||
try {
|
||||
const value = JSON.parse(
|
||||
await fs.readFile(path.join(cacheDir, 'budget.json'), 'utf8'),
|
||||
);
|
||||
if (
|
||||
typeof value?.date === 'string' &&
|
||||
/^\d{4}-\d{2}-\d{2}$/.test(value.date) &&
|
||||
Number.isSafeInteger(value.count) &&
|
||||
value.count >= 0
|
||||
)
|
||||
budget = { date: value.date, count: value.count };
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT')
|
||||
warn('[inaturalist-proxy] budget read failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function persistBudget(value) {
|
||||
if (cacheDir === null) return;
|
||||
const destination = path.join(cacheDir, 'budget.json');
|
||||
const temporary = `${destination}.${randomUUID()}.tmp`;
|
||||
try {
|
||||
await fs.mkdir(cacheDir, { recursive: true });
|
||||
await fs.writeFile(temporary, JSON.stringify(value), 'utf8');
|
||||
await fs.rename(temporary, destination);
|
||||
} catch {
|
||||
await fs.rm(temporary, { force: true }).catch(() => {});
|
||||
warn('[inaturalist-proxy] budget write failed');
|
||||
}
|
||||
}
|
||||
|
||||
function retrySeconds(until = cooldownUntil) {
|
||||
return Math.max(0, Math.ceil((until - now()) / 1000));
|
||||
}
|
||||
|
||||
function checkCooldown(resource) {
|
||||
if (now() < cooldownUntil)
|
||||
throw upstreamError('inaturalist_rate_limited', 429, retrySeconds(), 429);
|
||||
const failure = failures.get(resource);
|
||||
if (failure && now() < failure.until)
|
||||
throw upstreamError(
|
||||
'inaturalist_unavailable',
|
||||
502,
|
||||
retrySeconds(failure.until),
|
||||
failure.status,
|
||||
);
|
||||
failures.delete(resource);
|
||||
}
|
||||
|
||||
// Reserve and atomically persist one attempt before issuing it. A recovered
|
||||
// tail prevents one disk/reservation failure from poisoning future calls.
|
||||
function reserveAttempt(signal, resource) {
|
||||
const result = budgetTail.then(async () => {
|
||||
signal?.throwIfAborted();
|
||||
checkCooldown(resource);
|
||||
await readBudgetOnce();
|
||||
signal?.throwIfAborted();
|
||||
checkCooldown(resource);
|
||||
const value = currentBudget();
|
||||
if (value.count >= dailyLimit()) {
|
||||
const retry = Math.max(
|
||||
1,
|
||||
Math.ceil((Date.parse(`${day()}T00:00:00Z`) + DAY_MS - now()) / 1000),
|
||||
);
|
||||
throw upstreamError('inaturalist_budget_exhausted', 503, retry);
|
||||
}
|
||||
value.count += 1;
|
||||
await persistBudget({ ...value });
|
||||
});
|
||||
budgetTail = result.catch(() => {});
|
||||
return result;
|
||||
}
|
||||
|
||||
async function requestProjected(url, project, resource, { signal } = {}) {
|
||||
signal?.throwIfAborted();
|
||||
checkCooldown(resource);
|
||||
return enqueue(() => dispatch(url, project, resource, signal), {
|
||||
signal,
|
||||
priority: resource === 'observations' ? 0 : 1,
|
||||
});
|
||||
}
|
||||
|
||||
async function dispatch(url, project, resource, signal) {
|
||||
signal?.throwIfAborted();
|
||||
await reserveAttempt(signal, resource);
|
||||
signal?.throwIfAborted();
|
||||
checkCooldown(resource);
|
||||
const timeout = AbortSignal.timeout(TIMEOUT_MS);
|
||||
const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
||||
let upstreamStatus;
|
||||
let retryDelay = FAILURE_COOLDOWN_MS;
|
||||
try {
|
||||
const response = await fetchImpl(url, {
|
||||
signal: requestSignal,
|
||||
redirect: 'error',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': providerUserAgent('wildlife-sightings'),
|
||||
},
|
||||
});
|
||||
upstreamStatus = response.status;
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (response.status === 429) {
|
||||
const delay = inatRetryCooldownMs(
|
||||
response.headers.get('retry-after'),
|
||||
now(),
|
||||
);
|
||||
cooldownUntil = Math.max(cooldownUntil, now() + delay);
|
||||
throw upstreamError(
|
||||
'inaturalist_rate_limited',
|
||||
429,
|
||||
retrySeconds(),
|
||||
429,
|
||||
);
|
||||
}
|
||||
if (response.headers.has('retry-after'))
|
||||
retryDelay = inatRetryCooldownMs(
|
||||
response.headers.get('retry-after'),
|
||||
now(),
|
||||
);
|
||||
throw upstreamError('inaturalist_unavailable', 502);
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
response,
|
||||
MAX_RESPONSE_BYTES,
|
||||
requestSignal,
|
||||
);
|
||||
return project(payload);
|
||||
} catch (error) {
|
||||
// Log only safe diagnostic categories, never a URL, body, status text or
|
||||
// exception message (each can contain request/private information).
|
||||
const reason = signal?.aborted
|
||||
? 'abort'
|
||||
: timeout.aborted || error.name === 'TimeoutError'
|
||||
? 'timeout'
|
||||
: upstreamStatus && upstreamStatus >= 400
|
||||
? 'http'
|
||||
: error.code === 'RESPONSE_TOO_LARGE'
|
||||
? 'response-too-large'
|
||||
: upstreamStatus
|
||||
? 'invalid-payload'
|
||||
: 'network';
|
||||
warn(
|
||||
`[inaturalist-proxy] upstream refresh failed resource=${resource} status=${upstreamStatus ?? 'none'} reason=${reason}`,
|
||||
);
|
||||
if (signal?.aborted) throw error;
|
||||
// Network, abort, schema and size failures are not upstream cooldown
|
||||
// signals. A later healthy request may recover immediately.
|
||||
if (upstreamStatus >= 500)
|
||||
failures.set(resource, {
|
||||
until: now() + retryDelay,
|
||||
status: upstreamStatus,
|
||||
});
|
||||
const failure = failures.get(resource);
|
||||
throw upstreamError(
|
||||
error.status === 429
|
||||
? 'inaturalist_rate_limited'
|
||||
: 'inaturalist_unavailable',
|
||||
error.status === 429 ? 429 : 502,
|
||||
error.status === 429
|
||||
? retrySeconds()
|
||||
: failure
|
||||
? retrySeconds(failure.until)
|
||||
: undefined,
|
||||
upstreamStatus === 429 || upstreamStatus >= 500
|
||||
? upstreamStatus
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Request one bounded observation sample and project its public fields. */
|
||||
async function getObservations(params, options = {}) {
|
||||
const query = observationFilters(params);
|
||||
// View samples stay stable in the route cache until their TTL expires.
|
||||
query.set('order_by', params.scope === 'view' ? 'random' : 'observed_on');
|
||||
if (params.scope !== 'view') query.set('order', 'desc');
|
||||
query.set('per_page', '200');
|
||||
query.set('fields', OBSERVATION_FIELDS);
|
||||
if (params.scope === 'view') {
|
||||
query.set('swlat', String(params.box.south));
|
||||
query.set('swlng', String(params.box.west));
|
||||
query.set('nelat', String(params.box.north));
|
||||
query.set('nelng', String(params.box.east));
|
||||
}
|
||||
return requestProjected(
|
||||
`${OBSERVATIONS_URL}?${query}`,
|
||||
(payload) => {
|
||||
if (
|
||||
!Array.isArray(payload?.results) ||
|
||||
!Number.isSafeInteger(payload.total_results) ||
|
||||
payload.total_results < 0
|
||||
)
|
||||
throw upstreamError('inaturalist_unavailable', 502);
|
||||
const rows = normalizeInatObservations(payload.results, {
|
||||
nowMs: now(),
|
||||
windowMs: (params.taxa === 'whales' ? 7 : 2) * DAY_MS,
|
||||
});
|
||||
// Empty views need total_results: 0. A wholly invalid page must not
|
||||
// replace good data. Valid rows excluded by the finer rolling-time
|
||||
// cutoff are a healthy empty result of the upstream DATE over-fetch.
|
||||
if (
|
||||
!rows ||
|
||||
(!rows.length &&
|
||||
(payload.results.length || payload.total_results) &&
|
||||
!normalizeInatObservations(payload.results)?.length)
|
||||
)
|
||||
throw upstreamError('inaturalist_unavailable', 502);
|
||||
return {
|
||||
rows,
|
||||
totalResults: payload.total_results,
|
||||
...(payload.total_results > payload.results.length
|
||||
? { partial: true, rawRowCount: payload.total_results }
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
'observations',
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
/** Decode one validated density tile without observation identifiers. */
|
||||
async function getGrid(params, options = {}) {
|
||||
if (!validInatGridTile(params)) throw new TypeError('Invalid grid tile');
|
||||
const query = observationFilters(params);
|
||||
return requestProjected(
|
||||
`${API_URL}/grid/${params.z}/${params.x}/${params.y}.grid.json?${query}`,
|
||||
(payload) => projectInatGrid(payload, params),
|
||||
'grid',
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
getObservations,
|
||||
getGrid,
|
||||
/** Current provider-wide or resource-specific retry information. */
|
||||
cooldownState(resource) {
|
||||
if (now() < cooldownUntil)
|
||||
return {
|
||||
upstreamStatus: 429,
|
||||
retryAfterS: retrySeconds(),
|
||||
};
|
||||
const failure = failures.get(resource);
|
||||
if (failure && now() < failure.until)
|
||||
return {
|
||||
upstreamStatus: failure.status,
|
||||
retryAfterS: retrySeconds(failure.until),
|
||||
};
|
||||
return undefined;
|
||||
},
|
||||
/** The current UTC day's reserved upstream attempts and configured limit. */
|
||||
budgetState() {
|
||||
const value = currentBudget();
|
||||
return { ...value, budget: dailyLimit() };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createInatClient } from './client.js';
|
||||
import { syntheticGrid } from './grid.fixture.mjs';
|
||||
|
||||
const NOW = Date.parse('2026-11-04T18:05:00Z');
|
||||
const observation = (id, time) => ({
|
||||
id,
|
||||
geojson: { coordinates: [-122.3, 47.6] },
|
||||
quality_grade: 'research',
|
||||
time_observed_at: time,
|
||||
observed_on: time.slice(0, 10),
|
||||
// Upload age does not determine whether an observation is recent.
|
||||
created_at: '2026-11-04T18:00:00Z',
|
||||
taxon: { name: 'Synthetic taxon', iconic_taxon_name: 'Aves' },
|
||||
});
|
||||
|
||||
function clientFor(results) {
|
||||
return createInatClient({
|
||||
now: () => NOW,
|
||||
cacheDir: null,
|
||||
queueOptions: { intervalMs: 0 },
|
||||
warn() {},
|
||||
fetchImpl: async () =>
|
||||
Response.json({ results, total_results: results.length }),
|
||||
});
|
||||
}
|
||||
|
||||
test('iNaturalist drops ancient and boundary-expired observations uploaded today', async () => {
|
||||
const client = clientFor([
|
||||
observation(1, '2010-11-04T18:00:00Z'),
|
||||
observation(2, '2026-11-02T18:04:59Z'),
|
||||
observation(3, '2026-11-02T18:05:00Z'),
|
||||
]);
|
||||
const result = await client.getObservations({
|
||||
scope: 'global',
|
||||
taxa: 'all',
|
||||
observedD1: '2026-11-02',
|
||||
});
|
||||
assert.deepEqual(
|
||||
result.rows.map((row) => row.stableId),
|
||||
['inat:3'],
|
||||
);
|
||||
});
|
||||
|
||||
test('a valid date-filtered page can be empty after the exact recency cutoff', async () => {
|
||||
const client = clientFor([observation(1, '2026-11-02T06:00:00Z')]);
|
||||
const result = await client.getObservations({
|
||||
scope: 'global',
|
||||
taxa: 'all',
|
||||
observedD1: '2026-11-02',
|
||||
});
|
||||
assert.deepEqual(result.rows, []);
|
||||
assert.equal(result.totalResults, 1);
|
||||
});
|
||||
|
||||
const params = {
|
||||
scope: 'view',
|
||||
box: { south: 47, west: -123, north: 48, east: -122 },
|
||||
taxa: 'all',
|
||||
observedD1: '2026-11-02',
|
||||
};
|
||||
|
||||
test('observations and tiles share pacing, and observation priority wins a cold burst', async () => {
|
||||
let at = 0;
|
||||
const calls = [];
|
||||
const client = createInatClient({
|
||||
cacheDir: null,
|
||||
now: () => NOW + at,
|
||||
queueOptions: {
|
||||
now: () => at,
|
||||
wait: async (ms) => {
|
||||
at += ms;
|
||||
},
|
||||
},
|
||||
fetchImpl: async (url) => {
|
||||
const isGrid = new URL(url).pathname.includes('/grid/');
|
||||
calls.push([isGrid ? 'grid' : 'observations', at]);
|
||||
return Response.json(
|
||||
isGrid ? syntheticGrid() : { results: [], total_results: 0 },
|
||||
);
|
||||
},
|
||||
});
|
||||
const tiles = Array.from({ length: 3 }, (_, x) =>
|
||||
client.getGrid({ ...params, z: 2, x, y: 1 }),
|
||||
);
|
||||
await Promise.all([...tiles, client.getObservations(params)]);
|
||||
assert.deepEqual(calls, [
|
||||
['observations', 0],
|
||||
['grid', 0],
|
||||
['grid', 0],
|
||||
['grid', 0],
|
||||
]);
|
||||
assert.equal(client.budgetState().count, 4);
|
||||
});
|
||||
|
||||
test('upstream status and safe failure reasons are logged without upstream messages or URLs', async () => {
|
||||
for (const [response, expected] of [
|
||||
[
|
||||
() =>
|
||||
new Response('request-secret', {
|
||||
status: 503,
|
||||
statusText: 'request-secret',
|
||||
}),
|
||||
/status=503 reason=http/,
|
||||
],
|
||||
[
|
||||
() => Promise.reject(new DOMException('request-secret', 'TimeoutError')),
|
||||
/status=none reason=timeout/,
|
||||
],
|
||||
[
|
||||
() => Response.json({ unexpected: 'request-secret' }),
|
||||
/status=200 reason=invalid-payload/,
|
||||
],
|
||||
]) {
|
||||
const warnings = [];
|
||||
const client = createInatClient({
|
||||
cacheDir: null,
|
||||
now: () => NOW,
|
||||
queueOptions: { intervalMs: 0 },
|
||||
warn: (message) => warnings.push(message),
|
||||
fetchImpl: response,
|
||||
});
|
||||
await assert.rejects(client.getObservations(params), { status: 502 });
|
||||
assert.equal(warnings.length, 1);
|
||||
assert.match(warnings[0], expected);
|
||||
assert.doesNotMatch(warnings[0], /request-secret|https?:|swlat|d1=/);
|
||||
}
|
||||
});
|
||||
|
||||
test('a dispatched abort is diagnosed, spends only its attempt, and leaves the next request healthy', async () => {
|
||||
const warnings = [];
|
||||
const controller = new AbortController();
|
||||
let started;
|
||||
const didStart = new Promise((resolve) => {
|
||||
started = resolve;
|
||||
});
|
||||
let calls = 0;
|
||||
const client = createInatClient({
|
||||
cacheDir: null,
|
||||
now: () => NOW,
|
||||
queueOptions: { intervalMs: 0 },
|
||||
warn: (message) => warnings.push(message),
|
||||
fetchImpl: async (url, { signal }) => {
|
||||
calls += 1;
|
||||
if (calls > 1) return Response.json({ results: [], total_results: 0 });
|
||||
started();
|
||||
return new Promise((resolve, reject) =>
|
||||
signal.addEventListener('abort', () => reject(signal.reason), {
|
||||
once: true,
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
const pending = client.getObservations(params, { signal: controller.signal });
|
||||
const failed = assert.rejects(pending, { name: 'AbortError' });
|
||||
await didStart;
|
||||
controller.abort();
|
||||
await failed;
|
||||
assert.match(warnings[0], /resource=observations status=none reason=abort/);
|
||||
await client.getObservations(params);
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
/** Build a synthetic UTFGrid with logical cell counts and repeated hit pixels. */
|
||||
export function syntheticGrid({
|
||||
z = 2,
|
||||
x = 1,
|
||||
y = 1,
|
||||
cells = [
|
||||
{ row: 1, column: 51, count: 6, iconic: 3 },
|
||||
{ row: 3, column: 49, count: 12, iconic: 47170 },
|
||||
],
|
||||
} = {}) {
|
||||
const pixels = Array.from({ length: 64 }, () => Array(64).fill(' '));
|
||||
const keys = [''];
|
||||
const data = {};
|
||||
cells.forEach((cell, index) => {
|
||||
const key = String(900_000_000 + index);
|
||||
keys.push(key);
|
||||
const value = index + 1;
|
||||
let code = value + 32;
|
||||
if (code >= 34) code += 1;
|
||||
if (code >= 92) code += 1;
|
||||
const character = String.fromCharCode(code);
|
||||
for (const [row, column] of [
|
||||
[cell.row, cell.column],
|
||||
[cell.row - 1, cell.column],
|
||||
[cell.row + 1, cell.column],
|
||||
[cell.row, cell.column - 1],
|
||||
[cell.row, cell.column + 1],
|
||||
]) {
|
||||
if (row >= 0 && row < 64 && column >= 0 && column < 64)
|
||||
pixels[row][column] = character;
|
||||
}
|
||||
data[key] = {
|
||||
captive: null,
|
||||
cellCount: cell.count,
|
||||
geoprivacy: null,
|
||||
id: Number(key),
|
||||
latitude:
|
||||
(Math.atan(
|
||||
Math.sinh(Math.PI * (1 - 2 * ((y + cell.row / 64) / 2 ** z))),
|
||||
) *
|
||||
180) /
|
||||
Math.PI,
|
||||
longitude: ((x + cell.column / 64) / 2 ** z) * 360 - 180,
|
||||
private_location: null,
|
||||
quality_grade: 'needs_id',
|
||||
'taxon.iconic_taxon_id': cell.iconic,
|
||||
uuid: `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`,
|
||||
};
|
||||
});
|
||||
return { grid: pixels.map((row) => row.join('')), keys, data };
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
// iNaturalist's operational /v1/grid/{z}/{x}/{y}.grid.json endpoint is not
|
||||
// covered by its v1 Swagger contract. Validate the entire UTFGrid shape before
|
||||
// accepting any cell; callers preserve the last good sample on schema drift.
|
||||
import {
|
||||
wildlifeGridCellId,
|
||||
WILDLIFE_GRID_MAX_ZOOM,
|
||||
} from '../../../src/layers/wildlifeSightings/gridViewport.js';
|
||||
|
||||
const INAT_GRID_SIZE = 64;
|
||||
export const INAT_GRID_DECODE_MEASURE = 'wildlife:server:tile-decode';
|
||||
const grades = new Set(['research', 'needs_id', 'casual']);
|
||||
const object = (value) =>
|
||||
!!value && typeof value === 'object' && !Array.isArray(value);
|
||||
|
||||
/** Accept only integer XYZ coordinates within the supported zoom range. */
|
||||
export function validInatGridTile({ z, x, y } = {}) {
|
||||
return (
|
||||
Number.isInteger(z) &&
|
||||
z >= 0 &&
|
||||
z <= WILDLIFE_GRID_MAX_ZOOM &&
|
||||
Number.isInteger(x) &&
|
||||
Number.isInteger(y) &&
|
||||
x >= 0 &&
|
||||
y >= 0 &&
|
||||
x < 2 ** z &&
|
||||
y < 2 ** z
|
||||
);
|
||||
}
|
||||
|
||||
/** Pure allow-list projection. A UTFGrid key is an observation identifier and
|
||||
* its repeated pixels form a dilated hit area, not additional occupied cells.
|
||||
* Neither those keys nor representative/private fields cross this boundary. */
|
||||
export function projectInatGrid(payload, tile) {
|
||||
const start = `${INAT_GRID_DECODE_MEASURE}:start`;
|
||||
const end = `${INAT_GRID_DECODE_MEASURE}:end`;
|
||||
performance.mark(start);
|
||||
try {
|
||||
return decodeInatGrid(payload, tile);
|
||||
} finally {
|
||||
performance.mark(end);
|
||||
// Keep the latest sample inspectable without retaining every tile forever.
|
||||
// PerformanceObservers still receive each individual decode measurement.
|
||||
performance.clearMeasures(INAT_GRID_DECODE_MEASURE);
|
||||
performance.measure(INAT_GRID_DECODE_MEASURE, start, end);
|
||||
performance.clearMarks(start);
|
||||
performance.clearMarks(end);
|
||||
}
|
||||
}
|
||||
|
||||
function decodeInatGrid(payload, tile) {
|
||||
const invalid = () => {
|
||||
throw new TypeError('Invalid iNaturalist UTFGrid');
|
||||
};
|
||||
if (
|
||||
!validInatGridTile(tile) ||
|
||||
!object(payload) ||
|
||||
!Array.isArray(payload.grid) ||
|
||||
payload.grid.length !== INAT_GRID_SIZE ||
|
||||
!payload.grid.every(
|
||||
(row) => typeof row === 'string' && row.length === INAT_GRID_SIZE,
|
||||
) ||
|
||||
!Array.isArray(payload.keys) ||
|
||||
!payload.keys.length ||
|
||||
payload.keys.length > INAT_GRID_SIZE ** 2 + 1 ||
|
||||
payload.keys[0] !== '' ||
|
||||
!payload.keys.every(
|
||||
(key) => typeof key === 'string' && key.length <= 128,
|
||||
) ||
|
||||
new Set(payload.keys).size !== payload.keys.length ||
|
||||
!object(payload.data)
|
||||
)
|
||||
invalid();
|
||||
const dataKeys = Object.keys(payload.data);
|
||||
if (dataKeys.length !== payload.keys.length - 1) invalid();
|
||||
for (const key of payload.keys.slice(1)) {
|
||||
if (!key || !Object.hasOwn(payload.data, key)) invalid();
|
||||
const value = payload.data[key];
|
||||
if (
|
||||
!object(value) ||
|
||||
!Number.isSafeInteger(value.cellCount) ||
|
||||
value.cellCount <= 0 ||
|
||||
!Number.isFinite(value.latitude) ||
|
||||
Math.abs(value.latitude) > 90 ||
|
||||
!Number.isFinite(value.longitude) ||
|
||||
Math.abs(value.longitude) > 180 ||
|
||||
!grades.has(value.quality_grade) ||
|
||||
!(
|
||||
value['taxon.iconic_taxon_id'] === null ||
|
||||
(Number.isSafeInteger(value['taxon.iconic_taxon_id']) &&
|
||||
value['taxon.iconic_taxon_id'] > 0)
|
||||
)
|
||||
)
|
||||
invalid();
|
||||
}
|
||||
const occupied = new Set();
|
||||
for (const line of payload.grid) {
|
||||
for (let column = 0; column < INAT_GRID_SIZE; column += 1) {
|
||||
let code = line.charCodeAt(column);
|
||||
if (code < 32 || code === 34 || code === 92) invalid();
|
||||
if (code >= 93) code -= 1;
|
||||
if (code >= 35) code -= 1;
|
||||
const index = code - 32;
|
||||
if (index < 0 || index >= payload.keys.length) invalid();
|
||||
if (index) occupied.add(payload.keys[index]);
|
||||
}
|
||||
}
|
||||
if (occupied.size !== dataKeys.length) invalid();
|
||||
const cells = new Map();
|
||||
for (const key of occupied) {
|
||||
const value = payload.data[key];
|
||||
const id = wildlifeGridCellId(value.latitude, value.longitude, tile.z);
|
||||
const cell = {
|
||||
id,
|
||||
cellCount: value.cellCount,
|
||||
lat: value.latitude,
|
||||
lon: value.longitude,
|
||||
};
|
||||
// Two different representatives claiming one logical cell are a contract
|
||||
// change; summing or choosing one silently would fabricate the count.
|
||||
if (cells.has(id)) invalid();
|
||||
cells.set(id, cell);
|
||||
}
|
||||
const rows = [...cells.values()];
|
||||
const totalResults = rows.reduce((total, cell) => total + cell.cellCount, 0);
|
||||
if (!Number.isSafeInteger(totalResults)) invalid();
|
||||
return { cells: rows, totalResults };
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { projectInatGrid } from './grid.js';
|
||||
import { syntheticGrid } from './grid.fixture.mjs';
|
||||
import { installInatProxy, NOW } from './proxy.fixture.mjs';
|
||||
import { createInatClient } from './client.js';
|
||||
|
||||
const tile = { z: 2, x: 1, y: 1 };
|
||||
const gridUrl = '/grid?z=2&x=1&y=1&taxa=Aves&quality=research';
|
||||
const install = (options, hook) => installInatProxy(gridUrl, options, hook);
|
||||
|
||||
test('UTFGrid projection validates the full shape and counts each logical occupied cell once', () => {
|
||||
const raw = syntheticGrid();
|
||||
const projected = projectInatGrid(raw, tile);
|
||||
assert.equal(raw.grid.length, 64);
|
||||
assert.ok(raw.grid.every((row) => row.length === 64));
|
||||
assert.equal(
|
||||
raw.grid.join('').split('!').length - 1,
|
||||
5,
|
||||
'one key is repeated by its hit area',
|
||||
);
|
||||
assert.equal(projected.cells.length, 2);
|
||||
assert.equal(projected.totalResults, 18);
|
||||
assert.deepEqual(projected.cells[0], {
|
||||
id: 'grid:2/1/1:1:51',
|
||||
cellCount: 6,
|
||||
lat: raw.data[raw.keys[1]].latitude,
|
||||
lon: raw.data[raw.keys[1]].longitude,
|
||||
});
|
||||
for (const cell of projected.cells)
|
||||
assert.deepEqual(Object.keys(cell).sort(), [
|
||||
'cellCount',
|
||||
'id',
|
||||
'lat',
|
||||
'lon',
|
||||
]);
|
||||
for (const privateField of [
|
||||
'900000000',
|
||||
'uuid',
|
||||
'geoprivacy',
|
||||
'private_location',
|
||||
'captive',
|
||||
'quality_grade',
|
||||
])
|
||||
assert.ok(!JSON.stringify(projected).includes(privateField));
|
||||
});
|
||||
|
||||
test('legitimate blank grid and null/unknown iconic ids remain safe', () => {
|
||||
assert.deepEqual(projectInatGrid(syntheticGrid({ cells: [] }), tile), {
|
||||
cells: [],
|
||||
totalResults: 0,
|
||||
});
|
||||
const raw = syntheticGrid({
|
||||
cells: [{ row: 1, column: 1, count: 1, iconic: null }],
|
||||
});
|
||||
const projected = projectInatGrid(raw, tile);
|
||||
assert.equal(projected.cells[0].cellCount, 1);
|
||||
raw.data[raw.keys[1]]['taxon.iconic_taxon_id'] = 999999;
|
||||
assert.deepEqual(projectInatGrid(raw, tile), projected);
|
||||
});
|
||||
|
||||
test('schema drift anywhere fails closed, including unused, malformed and orphan cells', () => {
|
||||
const changes = [
|
||||
(raw) => {
|
||||
raw.grid.pop();
|
||||
},
|
||||
(raw) => {
|
||||
raw.grid[0] = '';
|
||||
},
|
||||
(raw) => {
|
||||
raw.grid[0] = '~'.repeat(64);
|
||||
},
|
||||
(raw) => {
|
||||
raw.keys[0] = 'not-empty';
|
||||
},
|
||||
(raw) => {
|
||||
raw.keys.push(raw.keys[1]);
|
||||
},
|
||||
(raw) => {
|
||||
delete raw.data[raw.keys[1]];
|
||||
},
|
||||
(raw) => {
|
||||
raw.data.extra = {};
|
||||
},
|
||||
(raw) => {
|
||||
raw.data[raw.keys[1]].cellCount = -1;
|
||||
},
|
||||
(raw) => {
|
||||
raw.data[raw.keys[1]].latitude = '47';
|
||||
},
|
||||
(raw) => {
|
||||
raw.data[raw.keys[1]].longitude = 181;
|
||||
},
|
||||
(raw) => {
|
||||
raw.data[raw.keys[1]].quality_grade = 'new-grade';
|
||||
},
|
||||
(raw) => {
|
||||
delete raw.data[raw.keys[1]]['taxon.iconic_taxon_id'];
|
||||
},
|
||||
(raw) => {
|
||||
raw.data[raw.keys[1]]['taxon.iconic_taxon_id'] = 'Aves';
|
||||
},
|
||||
(raw) => {
|
||||
raw.grid = Array(64).fill(' '.repeat(64));
|
||||
},
|
||||
];
|
||||
for (const change of changes) {
|
||||
const raw = syntheticGrid();
|
||||
change(raw);
|
||||
assert.throws(
|
||||
() => projectInatGrid(raw, tile),
|
||||
/Invalid iNaturalist UTFGrid/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
test(`${hook}: fixed grid URL shares filters, caps and allow-list cache`, async () => {
|
||||
const calls = [];
|
||||
const cache = new Map();
|
||||
const request = install(
|
||||
{
|
||||
cache,
|
||||
fetchImpl: async (url, options) => {
|
||||
calls.push(new URL(url));
|
||||
assert.equal(options.redirect, 'error');
|
||||
assert.ok(options.signal instanceof AbortSignal);
|
||||
assert.match(
|
||||
options.headers['User-Agent'],
|
||||
/^gods-eye-view-wildlife-sightings-proxy/,
|
||||
);
|
||||
return Response.json(syntheticGrid());
|
||||
},
|
||||
},
|
||||
hook,
|
||||
);
|
||||
assert.equal(calls.length, 0);
|
||||
const response = await request(
|
||||
`${gridUrl}&url=https://attacker.test&per_page=999`,
|
||||
);
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.body.cells.length, 2);
|
||||
assert.equal(response.body.fetchedAt, NOW);
|
||||
assert.equal(response.headers['Cache-Control'], 'no-store');
|
||||
assert.equal(
|
||||
calls[0].origin + calls[0].pathname,
|
||||
'https://api.inaturalist.org/v1/grid/2/1/1.grid.json',
|
||||
);
|
||||
assert.deepEqual(Object.fromEntries(calls[0].searchParams), {
|
||||
verifiable: 'true',
|
||||
captive: 'false',
|
||||
d1: '2026-11-02',
|
||||
photos: 'true',
|
||||
quality_grade: 'research',
|
||||
iconic_taxa: 'Aves',
|
||||
});
|
||||
await request(gridUrl);
|
||||
assert.equal(calls.length, 1, 'fresh stable tile/filter key is cached');
|
||||
assert.ok(!JSON.stringify([...cache]).includes('private_location'));
|
||||
assert.ok(!JSON.stringify([...cache]).includes('900000000'));
|
||||
const whale = await request('/grid?z=2&x=1&y=1&taxa=whales&confirmed=true');
|
||||
assert.equal(whale.status, 200);
|
||||
assert.equal(calls[1].searchParams.get('taxon_id'), '152871');
|
||||
assert.equal(calls[1].searchParams.get('d1'), '2026-10-28');
|
||||
assert.equal(calls[1].searchParams.has('photos'), false);
|
||||
});
|
||||
}
|
||||
|
||||
test('grid admission rejects cross-site/method/path/tile before acquisition', async () => {
|
||||
const request = install({
|
||||
fetchImpl() {
|
||||
assert.fail('invalid grid request acquired');
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
(
|
||||
await request(gridUrl, {
|
||||
headers: {
|
||||
host: 'localhost:5173',
|
||||
origin: 'https://other.test',
|
||||
'sec-fetch-site': 'cross-site',
|
||||
},
|
||||
})
|
||||
).status,
|
||||
403,
|
||||
);
|
||||
assert.equal((await request('/grid', { method: 'POST' })).status, 405);
|
||||
assert.equal((await request('/grid/extra')).status, 404);
|
||||
for (const query of [
|
||||
'',
|
||||
'z=2&x=4&y=0',
|
||||
'z=-1&x=0&y=0',
|
||||
'z=23&x=0&y=0',
|
||||
'z=2&x=1.5&y=0',
|
||||
'z=2&x=&y=0',
|
||||
'z=2&x=0&y=4',
|
||||
]) {
|
||||
const result = await request(`/grid?${query}`);
|
||||
assert.equal(result.status, 400, query);
|
||||
assert.equal(result.body.error, 'invalid_tile');
|
||||
}
|
||||
});
|
||||
|
||||
test('grid refresh single-flight, stale fallback and hour-key expiry keep the original age', async () => {
|
||||
let time = NOW;
|
||||
let calls = 0;
|
||||
let resolve;
|
||||
let mode = 'pending';
|
||||
const request = install({
|
||||
now: () => time,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
if (mode === 'pending')
|
||||
return new Promise((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
if (mode === 'bad') return Response.json({ grid: [] });
|
||||
return Response.json(syntheticGrid());
|
||||
},
|
||||
});
|
||||
const first = request();
|
||||
const second = request();
|
||||
for (let i = 0; i < 100 && !resolve; i += 1) await new Promise(setImmediate);
|
||||
assert.equal(calls, 1);
|
||||
resolve(Response.json(syntheticGrid()));
|
||||
assert.equal((await first).status, 200);
|
||||
assert.equal((await second).status, 200);
|
||||
time += 900_001;
|
||||
mode = 'bad';
|
||||
const stale = await request();
|
||||
assert.equal(stale.status, 200);
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.body.fetchedAt, NOW);
|
||||
assert.equal(stale.headers['X-Data-Stale'], 'true');
|
||||
time = NOW + 3_600_001;
|
||||
const expired = await request();
|
||||
assert.equal(expired.status, 502);
|
||||
assert.equal(expired.body.error, 'inaturalist_unavailable');
|
||||
});
|
||||
|
||||
test('grid and observations share the upstream cooldown and daily reservations', async () => {
|
||||
let calls = 0;
|
||||
const client = createInatClient({
|
||||
now: () => NOW,
|
||||
cacheDir: null,
|
||||
queueOptions: { intervalMs: 0 },
|
||||
warn() {},
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return new Response('', {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '120' },
|
||||
});
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
client.getGrid({ ...tile, taxa: 'all', observedD1: '2026-11-02' }),
|
||||
(error) => error.status === 429 && error.retryAfterS === 120,
|
||||
);
|
||||
await assert.rejects(
|
||||
client.getObservations({
|
||||
scope: 'global',
|
||||
taxa: 'all',
|
||||
observedD1: '2026-11-02',
|
||||
}),
|
||||
(error) => error.status === 429,
|
||||
);
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(client.budgetState().count, 1);
|
||||
});
|
||||
|
||||
test('grid body cap and upstream failures return fixed errors; only real 5xx cools down', async () => {
|
||||
for (const [response, retryAfter] of [
|
||||
[() => new Response('x'.repeat(8 * 1024 * 1024 + 1)), undefined],
|
||||
[() => new Response('', { status: 503 }), '15'],
|
||||
]) {
|
||||
const request = install({ fetchImpl: async () => response() });
|
||||
const result = await request();
|
||||
assert.equal(result.status, 502);
|
||||
assert.deepEqual(result.body, { error: 'inaturalist_unavailable' });
|
||||
assert.equal(result.headers['Retry-After'], retryAfter);
|
||||
}
|
||||
});
|
||||
|
||||
test('a failed tile preserves last good cells and cannot send unrelated observations into 502', async () => {
|
||||
let at = NOW;
|
||||
let failGrid = false;
|
||||
const calls = [];
|
||||
const warnings = [];
|
||||
const request = install({
|
||||
now: () => at,
|
||||
warn: (message) => warnings.push(message),
|
||||
fetchImpl: async (url) => {
|
||||
const isGrid = new URL(url).pathname.includes('/grid/');
|
||||
calls.push(isGrid ? 'grid' : 'observations');
|
||||
if (isGrid && failGrid)
|
||||
return new Response('private upstream text', {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '45' },
|
||||
});
|
||||
return Response.json(
|
||||
isGrid ? syntheticGrid() : { results: [], total_results: 0 },
|
||||
);
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
at += 900_001;
|
||||
failGrid = true;
|
||||
const failed = await request();
|
||||
assert.equal(failed.status, 200);
|
||||
assert.equal(failed.headers['X-Data-Stale'], 'true');
|
||||
assert.equal(failed.body.stale, true);
|
||||
assert.equal(failed.body.fetchedAt, first.body.fetchedAt);
|
||||
assert.deepEqual(failed.body.cells, first.body.cells);
|
||||
assert.equal(
|
||||
(await request('/observations?scope=global&taxa=whales')).status,
|
||||
200,
|
||||
);
|
||||
assert.deepEqual(calls, ['grid', 'grid', 'observations']);
|
||||
assert.match(warnings[0], /resource=grid status=503 reason=http/);
|
||||
assert.doesNotMatch(warnings[0], /private upstream text/);
|
||||
// The tile cooldown also preserves its data without another upstream try.
|
||||
const cooling = await request();
|
||||
assert.equal(cooling.body.stale, true);
|
||||
assert.equal(cooling.headers['Retry-After'], '45');
|
||||
assert.equal(cooling.headers['X-Upstream-Status'], '503');
|
||||
assert.equal(calls.length, 3);
|
||||
at += 45_000;
|
||||
failGrid = false;
|
||||
const refreshing = await request();
|
||||
assert.equal(refreshing.body.stale, true);
|
||||
await new Promise(setImmediate);
|
||||
const recovered = await request();
|
||||
assert.equal(recovered.status, 200);
|
||||
assert.equal(recovered.body.stale, undefined);
|
||||
assert.equal(
|
||||
calls.length,
|
||||
4,
|
||||
'a failed queue task does not poison later tile work',
|
||||
);
|
||||
});
|
||||
|
||||
test('twelve queued tiles leave observation capacity and give the next dispatch to observations', async () => {
|
||||
const calls = [];
|
||||
let release;
|
||||
const request = install({
|
||||
fetchImpl: (url) => {
|
||||
const path = new URL(url).pathname;
|
||||
calls.push(path);
|
||||
if (calls.length === 1)
|
||||
return new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
return Response.json(
|
||||
path.includes('/grid/')
|
||||
? syntheticGrid()
|
||||
: { results: [], total_results: 0 },
|
||||
);
|
||||
},
|
||||
});
|
||||
const first = request('/grid?z=4&x=0&y=1');
|
||||
for (let i = 0; i < 100 && !release; i += 1) await new Promise(setImmediate);
|
||||
assert.ok(release);
|
||||
const tiles = Array.from({ length: 11 }, (_, i) =>
|
||||
request(`/grid?z=4&x=${i + 1}&y=1`),
|
||||
);
|
||||
const observation = request('/observations?scope=global');
|
||||
const queuedTile = request('/grid?z=4&x=12&y=1');
|
||||
release(Response.json(syntheticGrid()));
|
||||
const results = await Promise.all([first, observation, ...tiles, queuedTile]);
|
||||
assert.ok(results.every((result) => result.status === 200));
|
||||
assert.equal(calls.length, 14);
|
||||
assert.equal(calls[1], '/v2/observations');
|
||||
});
|
||||
@@ -0,0 +1,343 @@
|
||||
import { createInatClient } from './client.js';
|
||||
import { coalesceProxyRequest } from '../common/http.js';
|
||||
import { admitSameSiteKeyless } from '../common/same-site.js';
|
||||
import { requiredFiniteQueryNumber } from '../common/query.js';
|
||||
import { snapWildlifeBox } from '../../../src/layers/wildlifeSightings/bounds.js';
|
||||
import { validInatGridTile } from './grid.js';
|
||||
|
||||
// The upstream advertises five-minute freshness; global samples cost less often.
|
||||
const TTL_MS = 300_000;
|
||||
const GLOBAL_TTL_MS = 600_000;
|
||||
const INAT_GRID_CACHE_TTL_MS = 15 * 60_000;
|
||||
const STALE_MAX_MS = 3_600_000;
|
||||
const INAT_CACHE_MAX = 400;
|
||||
const HOUR_MS = 3_600_000;
|
||||
const TAXA = Object.freeze([
|
||||
'all',
|
||||
'Aves',
|
||||
'Mammalia',
|
||||
'Insecta',
|
||||
'Plantae',
|
||||
'whales',
|
||||
]);
|
||||
|
||||
function requestParams(pathname, query, nowMs) {
|
||||
const grid = pathname === '/grid';
|
||||
const scope = grid ? 'grid' : (query.get('scope') ?? 'view');
|
||||
if (!grid && scope !== 'view' && scope !== 'global')
|
||||
return { error: 'invalid_scope' };
|
||||
const quality =
|
||||
query.get('quality') === 'research' ||
|
||||
(grid && query.get('confirmed') === 'true')
|
||||
? 'research'
|
||||
: 'all';
|
||||
const taxa = TAXA.includes(query.get('taxa')) ? query.get('taxa') : 'all';
|
||||
let box = null;
|
||||
let tile;
|
||||
if (grid) {
|
||||
tile = Object.fromEntries(
|
||||
['z', 'x', 'y'].map((name) => [
|
||||
name,
|
||||
requiredFiniteQueryNumber(query, name),
|
||||
]),
|
||||
);
|
||||
if (!validInatGridTile(tile)) return { error: 'invalid_tile' };
|
||||
} else if (scope === 'view') {
|
||||
const south = requiredFiniteQueryNumber(query, 'south');
|
||||
const west = requiredFiniteQueryNumber(query, 'west');
|
||||
const north = requiredFiniteQueryNumber(query, 'north');
|
||||
const east = requiredFiniteQueryNumber(query, 'east');
|
||||
box = snapWildlifeBox({ south, west, north, east });
|
||||
if (!box) return { error: 'invalid_bbox' };
|
||||
}
|
||||
const windowMs = (taxa === 'whales' ? 7 * 24 : 48) * HOUR_MS;
|
||||
const windowKey = new Date(Math.floor((nowMs - windowMs) / HOUR_MS) * HOUR_MS)
|
||||
.toISOString()
|
||||
.replace('.000Z', 'Z');
|
||||
const bounds = box ? `${box.south},${box.west},${box.north},${box.east}` : '';
|
||||
const baseKey = grid
|
||||
? `grid:v1:${tile.z}/${tile.x}/${tile.y}:${quality}:${taxa}`
|
||||
: `v1:${scope}:${bounds}:${quality}:${taxa}`;
|
||||
return {
|
||||
scope,
|
||||
...(grid
|
||||
? { ...tile, waitForFresh: query.get('refresh') === 'wait' }
|
||||
: { box }),
|
||||
quality,
|
||||
taxa,
|
||||
observedD1: windowKey.slice(0, 10),
|
||||
baseKey,
|
||||
key: `${baseKey}:${windowKey}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** A shared-provider route for dev and preview; acquisition starts on demand. */
|
||||
export function inaturalistProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
cacheDir,
|
||||
cache = new Map(),
|
||||
warn = (...args) => console.warn(...args),
|
||||
queueOptions,
|
||||
} = {}) {
|
||||
const inFlight = new Map();
|
||||
const consumers = new Map();
|
||||
let client = null;
|
||||
|
||||
function age(entry) {
|
||||
if (!entry) return Infinity;
|
||||
const elapsed = now() - entry.savedAt;
|
||||
return elapsed < 0 ? Infinity : elapsed;
|
||||
}
|
||||
|
||||
function joinFlight(promise, group, signal) {
|
||||
group.count += 1;
|
||||
return new Promise((resolve, reject) => {
|
||||
let released = false;
|
||||
function release() {
|
||||
if (released) return;
|
||||
released = true;
|
||||
signal?.removeEventListener('abort', abort);
|
||||
group.count -= 1;
|
||||
if (!group.count && !group.background && !group.settled) {
|
||||
group.controller.abort();
|
||||
// A fresh consumer must not join an abandoned queued generation.
|
||||
if (inFlight.get(group.key) === promise) inFlight.delete(group.key);
|
||||
if (consumers.get(group.key) === group) consumers.delete(group.key);
|
||||
}
|
||||
}
|
||||
function abort() {
|
||||
release();
|
||||
reject(signal.reason);
|
||||
}
|
||||
signal?.addEventListener('abort', abort, { once: true });
|
||||
if (signal?.aborted) abort();
|
||||
promise.then(
|
||||
(value) => {
|
||||
release();
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
release();
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function refresh(params, background = false) {
|
||||
// Global samples share pending refreshes across hourly window changes.
|
||||
const flightKey = params.scope === 'global' ? params.baseKey : params.key;
|
||||
let group = consumers.get(flightKey);
|
||||
if (!group) {
|
||||
group = {
|
||||
key: flightKey,
|
||||
controller: new AbortController(),
|
||||
count: 0,
|
||||
settled: false,
|
||||
background,
|
||||
};
|
||||
consumers.set(flightKey, group);
|
||||
}
|
||||
group.background ||= background;
|
||||
const { promise, shared } = coalesceProxyRequest(
|
||||
inFlight,
|
||||
flightKey,
|
||||
async () => {
|
||||
client ??= createInatClient({
|
||||
fetchImpl,
|
||||
now,
|
||||
cacheDir,
|
||||
warn,
|
||||
queueOptions,
|
||||
});
|
||||
const options = { signal: group.controller.signal };
|
||||
const result = await (params.scope === 'grid'
|
||||
? client.getGrid(params, options)
|
||||
: client.getObservations(params, options));
|
||||
const value = {
|
||||
fetchedAt: now(),
|
||||
...(params.scope === 'grid'
|
||||
? {}
|
||||
: { scope: params.scope, box: params.box }),
|
||||
...result,
|
||||
};
|
||||
cache.delete(params.key);
|
||||
cache.set(params.key, {
|
||||
value,
|
||||
savedAt: now(),
|
||||
baseKey: params.baseKey,
|
||||
});
|
||||
while (cache.size > INAT_CACHE_MAX)
|
||||
cache.delete(cache.keys().next().value);
|
||||
return value;
|
||||
},
|
||||
);
|
||||
if (!shared) {
|
||||
const settle = () => {
|
||||
group.settled = true;
|
||||
if (consumers.get(flightKey) === group) consumers.delete(flightKey);
|
||||
};
|
||||
void promise.then(settle, settle);
|
||||
}
|
||||
return { promise, group };
|
||||
}
|
||||
|
||||
async function acquire(params, signal) {
|
||||
signal?.throwIfAborted();
|
||||
for (const [key, entry] of cache) {
|
||||
if (age(entry) > STALE_MAX_MS) cache.delete(key);
|
||||
}
|
||||
const current = cache.get(params.key);
|
||||
const ttl =
|
||||
params.scope === 'grid'
|
||||
? INAT_GRID_CACHE_TTL_MS
|
||||
: params.scope === 'global'
|
||||
? GLOBAL_TTL_MS
|
||||
: TTL_MS;
|
||||
if (current && age(current) < ttl) {
|
||||
cache.delete(params.key);
|
||||
cache.set(params.key, current);
|
||||
return { value: current.value, stale: false };
|
||||
}
|
||||
// Window-hour rollover can reuse prior cells without claiming a new age.
|
||||
let previous = current;
|
||||
let previousKey = params.key;
|
||||
if (!previous) {
|
||||
for (const [key, entry] of cache) {
|
||||
if (
|
||||
entry.baseKey === params.baseKey &&
|
||||
(!previous || entry.savedAt > previous.savedAt)
|
||||
) {
|
||||
previous = entry;
|
||||
previousKey = key;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (params.scope === 'global' && age(previous) < GLOBAL_TTL_MS)
|
||||
return { value: previous.value, stale: false };
|
||||
if (
|
||||
params.scope === 'grid' &&
|
||||
!params.waitForFresh &&
|
||||
age(previous) <= STALE_MAX_MS
|
||||
) {
|
||||
cache.delete(previousKey);
|
||||
cache.set(previousKey, previous);
|
||||
// Retain one background consumer after this HTTP response closes. A stale
|
||||
// tile is useful immediately; waiting for the token queue defeats SWR.
|
||||
const cooling = client?.cooldownState('grid');
|
||||
if (!cooling) {
|
||||
const { promise } = refresh(params, true);
|
||||
void promise.catch(() => {});
|
||||
}
|
||||
return {
|
||||
value: previous.value,
|
||||
stale: true,
|
||||
...cooling,
|
||||
};
|
||||
}
|
||||
try {
|
||||
const { promise, group } = refresh(params);
|
||||
return { value: await joinFlight(promise, group, signal), stale: false };
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
if (age(previous) <= STALE_MAX_MS)
|
||||
return {
|
||||
value: previous.value,
|
||||
stale: true,
|
||||
retryAfterS: error.retryAfterS,
|
||||
upstreamStatus: error.upstreamStatus,
|
||||
};
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (
|
||||
status,
|
||||
value,
|
||||
{ stale = false, retryAfterS, upstreamStatus } = {},
|
||||
) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...(Number.isFinite(retryAfterS)
|
||||
? { 'Retry-After': String(retryAfterS) }
|
||||
: {}),
|
||||
...(upstreamStatus
|
||||
? { 'X-Upstream-Status': String(upstreamStatus) }
|
||||
: {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const url = String(req.url || '/');
|
||||
const queryAt = url.indexOf('?');
|
||||
const pathname = queryAt < 0 ? url : url.slice(0, queryAt);
|
||||
if (pathname !== '/observations' && pathname !== '/grid')
|
||||
return json(404, { error: 'unknown_route' });
|
||||
const params = requestParams(
|
||||
pathname,
|
||||
new URLSearchParams(queryAt < 0 ? '' : url.slice(queryAt + 1)),
|
||||
now(),
|
||||
);
|
||||
if (params.error) return json(400, { error: params.error });
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort();
|
||||
// IncomingMessage.close also fires after a complete request body. The
|
||||
// response close identifies the abandoned browser fetch, while aborted
|
||||
// covers a request cut short before it reached the response stage.
|
||||
req.once?.('aborted', abort);
|
||||
res.once?.('close', abort);
|
||||
try {
|
||||
const { value, stale, retryAfterS, upstreamStatus } = await acquire(
|
||||
params,
|
||||
controller.signal,
|
||||
);
|
||||
return json(200, stale ? { ...value, stale: true } : value, {
|
||||
stale,
|
||||
retryAfterS,
|
||||
upstreamStatus,
|
||||
});
|
||||
} catch (error) {
|
||||
if (controller.signal.aborted) return;
|
||||
const status =
|
||||
error.code === 'inaturalist_budget_exhausted'
|
||||
? 503
|
||||
: error.status === 429
|
||||
? 429
|
||||
: 502;
|
||||
const code =
|
||||
error.code === 'inaturalist_budget_exhausted'
|
||||
? error.code
|
||||
: status === 429
|
||||
? 'inaturalist_rate_limited'
|
||||
: 'inaturalist_unavailable';
|
||||
return json(
|
||||
status,
|
||||
{ error: code },
|
||||
{
|
||||
retryAfterS: error.retryAfterS,
|
||||
upstreamStatus: error.upstreamStatus,
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
req.removeListener?.('aborted', abort);
|
||||
res.removeListener?.('close', abort);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'inaturalist',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/inaturalist', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/inaturalist', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,893 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { installInatProxy, NOW } from './proxy.fixture.mjs';
|
||||
import { createInatClient, inatRetryCooldownMs } from './client.js';
|
||||
|
||||
const observation = (id = 1) => ({
|
||||
id,
|
||||
uri: `https://www.inaturalist.org/observations/${id}`,
|
||||
geojson: { coordinates: [-122.351, 47.613] },
|
||||
time_observed_at: '2026-11-04T17:00:00Z',
|
||||
observed_on: '2026-11-04',
|
||||
created_at: '2026-11-04T17:01:00Z',
|
||||
quality_grade: 'research',
|
||||
license_code: 'cc-by',
|
||||
positional_accuracy: 12,
|
||||
faves_count: 2,
|
||||
num_identification_agreements: 3,
|
||||
taxon: {
|
||||
name: 'Calypte anna',
|
||||
preferred_common_name: "Anna's hummingbird",
|
||||
iconic_taxon_name: 'Aves',
|
||||
},
|
||||
photos: [
|
||||
{
|
||||
url: 'https://inaturalist-open-data.s3.amazonaws.com/photos/123/square.jpeg',
|
||||
license_code: 'cc-by',
|
||||
attribution: 'Synthetic photographer, CC BY',
|
||||
},
|
||||
],
|
||||
user: { login: 'must-never-leave-server' },
|
||||
});
|
||||
const envelope = (rows = [observation()], total = rows.length) => ({
|
||||
total_results: total,
|
||||
page: 1,
|
||||
per_page: 200,
|
||||
results: rows,
|
||||
});
|
||||
const boxUrl = (index = 0, extra = '') =>
|
||||
`/observations?south=47.51&west=${-122.49 + index * 0.2}&north=47.69&east=${-122.31 + index * 0.2}${extra}`;
|
||||
|
||||
function install(options = {}, hook = 'configureServer') {
|
||||
return installInatProxy(
|
||||
boxUrl(),
|
||||
{
|
||||
// Serial dispatch makes queued cancellation and coalescing observable.
|
||||
queueOptions: { intervalMs: 0, windowMs: 0, maxInFlight: 1 },
|
||||
...options,
|
||||
},
|
||||
hook,
|
||||
);
|
||||
}
|
||||
|
||||
async function until(check) {
|
||||
for (let i = 0; i < 100; i += 1) {
|
||||
if (check()) return;
|
||||
await new Promise(setImmediate);
|
||||
}
|
||||
assert.fail('expected fake upstream calls were not dispatched');
|
||||
}
|
||||
|
||||
for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
test(`${hook}: actual v2 envelope, fixed query, projection and privacy`, async () => {
|
||||
const calls = [];
|
||||
const cache = new Map();
|
||||
const request = install(
|
||||
{
|
||||
cache,
|
||||
fetchImpl: async (url, options) => {
|
||||
calls.push(new URL(url));
|
||||
assert.equal(options.redirect, 'error');
|
||||
assert.ok(options.signal instanceof AbortSignal);
|
||||
assert.equal(options.headers.Accept, 'application/json');
|
||||
assert.equal(
|
||||
options.headers['User-Agent'],
|
||||
'gods-eye-view-wildlife-sightings-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
||||
);
|
||||
return Response.json(envelope());
|
||||
},
|
||||
},
|
||||
hook,
|
||||
);
|
||||
assert.equal(calls.length, 0, 'construction and mounting do not acquire');
|
||||
const result = await request(
|
||||
boxUrl(0, '&url=https://attacker.example&per_page=999'),
|
||||
);
|
||||
assert.equal(result.status, 200);
|
||||
assert.equal(result.headers['Cache-Control'], 'no-store');
|
||||
assert.equal(result.body.rows[0].stableId, 'inat:1');
|
||||
assert.equal(result.body.fetchedAt, NOW);
|
||||
assert.equal(result.body.scope, 'view');
|
||||
assert.deepEqual(result.body.box, {
|
||||
south: 47.5,
|
||||
west: -122.5,
|
||||
north: 47.7,
|
||||
east: -122.3,
|
||||
});
|
||||
const query = calls[0].searchParams;
|
||||
assert.equal(
|
||||
calls[0].origin + calls[0].pathname,
|
||||
'https://api.inaturalist.org/v2/observations',
|
||||
);
|
||||
// Only the projected fields are requested, never user or identification data.
|
||||
assert.match(query.get('fields'), /^\(id:!t,uri:!t,.*photos:\(url:!t/);
|
||||
assert.doesNotMatch(query.get('fields'), /user|identifications/);
|
||||
assert.equal(query.get('swlat'), '47.5');
|
||||
assert.equal(query.get('swlng'), '-122.5');
|
||||
assert.equal(query.get('nelat'), '47.7');
|
||||
assert.equal(query.get('nelng'), '-122.3');
|
||||
assert.equal(query.get('d1'), '2026-11-02');
|
||||
assert.equal(query.has('created_d1'), false);
|
||||
assert.equal(query.get('per_page'), '200');
|
||||
assert.equal(query.get('verifiable'), 'true');
|
||||
assert.equal(query.get('photos'), 'true');
|
||||
assert.equal(query.get('captive'), 'false');
|
||||
assert.equal(query.get('order_by'), 'random');
|
||||
assert.equal(query.has('order'), false);
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(result.body),
|
||||
/must-never-leave-server|"user"/,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify([...cache.values()]),
|
||||
/must-never-leave-server|"user"/,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test('gate, method, route and query validation run before quota, cache or upstream', async () => {
|
||||
const cache = new Map();
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
cache,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json(envelope());
|
||||
},
|
||||
});
|
||||
for (let i = 0; i < 32; i += 1) {
|
||||
const response = await request('/observations?scope=global', {
|
||||
headers: {
|
||||
host: 'localhost:5173',
|
||||
origin: 'https://foreign.example',
|
||||
'sec-fetch-site': 'cross-site',
|
||||
},
|
||||
});
|
||||
assert.equal(response.status, 403);
|
||||
}
|
||||
const method = await request('/observations', { method: 'POST' });
|
||||
assert.equal(method.status, 405);
|
||||
assert.equal(method.headers.Allow, 'GET');
|
||||
for (const route of ['/', '/unknown', '/observations/extra', '//[invalid'])
|
||||
assert.equal((await request(route)).body.error, 'unknown_route');
|
||||
assert.equal(
|
||||
(await request('/observations?scope=planet')).body.error,
|
||||
'invalid_scope',
|
||||
);
|
||||
for (const query of [
|
||||
'',
|
||||
'south=NaN&west=0&north=1&east=1',
|
||||
'south=&west=0&north=1&east=1',
|
||||
'south=0&west=0&north=0&east=1',
|
||||
'south=0&west=1&north=1&east=0',
|
||||
'south=-91&west=0&north=1&east=1',
|
||||
'south=0&west=-181&north=1&east=1',
|
||||
'south=0&west=0&north=41&east=1',
|
||||
'south=0&west=0&north=1&east=41',
|
||||
]) {
|
||||
const result = await request(`/observations?${query}`);
|
||||
assert.equal(result.status, 400, query);
|
||||
assert.equal(result.body.error, 'invalid_bbox', query);
|
||||
}
|
||||
assert.equal(cache.size, 0);
|
||||
assert.equal(calls, 0);
|
||||
const allowed = await request('/observations?scope=global', {
|
||||
headers: {
|
||||
host: 'gev.example',
|
||||
origin: 'https://gev.example',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'x-forwarded-proto': 'https',
|
||||
'x-forwarded-for': '203.0.113.9',
|
||||
},
|
||||
});
|
||||
assert.equal(allowed.status, 200);
|
||||
});
|
||||
|
||||
test('cached requests never exhaust a local request admission limiter', async () => {
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json(envelope());
|
||||
},
|
||||
});
|
||||
for (let i = 0; i < 30; i += 1) assert.equal((await request()).status, 200);
|
||||
const cached = await request();
|
||||
assert.equal(cached.status, 200);
|
||||
assert.equal(cached.headers['Retry-After'], undefined);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test('proxy rejects oversized snapped spans before acquisition and preserves canonical client boxes', async () => {
|
||||
const urls = [];
|
||||
const request = install({
|
||||
fetchImpl: async (url) => {
|
||||
urls.push(new URL(url));
|
||||
return Response.json(envelope());
|
||||
},
|
||||
});
|
||||
for (const query of [
|
||||
'south=10.2&west=0&north=50.1&east=1',
|
||||
'south=0&west=10.2&north=1&east=50.1',
|
||||
]) {
|
||||
const result = await request(`/observations?${query}`);
|
||||
assert.equal(result.status, 400);
|
||||
assert.equal(result.body.error, 'invalid_bbox');
|
||||
}
|
||||
assert.equal(urls.length, 0);
|
||||
const canonical =
|
||||
'/observations?south=47.4&west=-122.3&north=47.6&east=-122.1';
|
||||
const first = await request(canonical);
|
||||
assert.equal(first.status, 200);
|
||||
assert.deepEqual(first.body.box, {
|
||||
south: 47.4,
|
||||
west: -122.3,
|
||||
north: 47.6,
|
||||
east: -122.1,
|
||||
});
|
||||
assert.equal(urls[0].searchParams.get('swlat'), '47.4');
|
||||
assert.equal(urls[0].searchParams.get('swlng'), '-122.3');
|
||||
const second = await request(canonical);
|
||||
assert.deepEqual(second.body, first.body);
|
||||
assert.equal(urls.length, 1, 'canonical boxes share their cache key');
|
||||
});
|
||||
|
||||
test('view snapping, whales, research and global queries obey the requested policy', async () => {
|
||||
const urls = [];
|
||||
const request = install({
|
||||
fetchImpl: async (url) => {
|
||||
urls.push(new URL(url));
|
||||
return Response.json(envelope());
|
||||
},
|
||||
});
|
||||
await request(
|
||||
'/observations?south=40.2&west=-123.7&north=46.1&east=-121.4&quality=research&taxa=Aves',
|
||||
);
|
||||
assert.equal(urls[0].searchParams.get('swlat'), '40');
|
||||
assert.equal(urls[0].searchParams.get('nelat'), '47');
|
||||
assert.equal(urls[0].searchParams.get('per_page'), '200');
|
||||
assert.equal(urls[0].searchParams.get('quality_grade'), 'research');
|
||||
assert.equal(urls[0].searchParams.get('iconic_taxa'), 'Aves');
|
||||
await request(boxUrl(0, '&taxa=whales'));
|
||||
const whales = urls[1].searchParams;
|
||||
assert.equal(whales.get('taxon_id'), '152871');
|
||||
assert.equal(whales.get('d1'), '2026-10-28');
|
||||
assert.equal(whales.has('created_d1'), false);
|
||||
assert.equal(whales.has('photos'), false);
|
||||
assert.equal(whales.has('iconic_taxa'), false);
|
||||
await request('/observations?scope=global&quality=typo&taxa=typo&south=bad');
|
||||
const global = urls[2].searchParams;
|
||||
for (const field of [
|
||||
'swlat',
|
||||
'swlng',
|
||||
'nelat',
|
||||
'nelng',
|
||||
'quality_grade',
|
||||
'iconic_taxa',
|
||||
])
|
||||
assert.equal(global.has(field), false, field);
|
||||
assert.equal(global.get('per_page'), '200');
|
||||
assert.equal(global.get('d1'), '2026-11-02');
|
||||
assert.equal(global.has('created_d1'), false);
|
||||
assert.equal(global.get('order_by'), 'observed_on');
|
||||
assert.equal(global.get('order'), 'desc');
|
||||
await request('/observations?scope=global&taxa=whales');
|
||||
const globalWhales = urls[3].searchParams;
|
||||
assert.equal(globalWhales.get('d1'), '2026-10-28');
|
||||
assert.equal(globalWhales.has('created_d1'), false);
|
||||
assert.equal(globalWhales.get('order_by'), 'observed_on');
|
||||
assert.equal(globalWhales.get('order'), 'desc');
|
||||
});
|
||||
|
||||
test('fresh cache hits and global ten-minute TTL spend no upstream quota', async () => {
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json(envelope());
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
at += 299_999;
|
||||
assert.deepEqual((await request()).body, first.body);
|
||||
at += 1;
|
||||
await request();
|
||||
assert.equal(calls, 2);
|
||||
await request('/observations?scope=global');
|
||||
at += 599_999;
|
||||
await request('/observations?scope=global');
|
||||
assert.equal(calls, 3);
|
||||
at += 1;
|
||||
await request('/observations?scope=global');
|
||||
assert.equal(calls, 4);
|
||||
});
|
||||
|
||||
test('the same snapped view and window hour returns one cached random sample until expiry', async () => {
|
||||
let at = NOW;
|
||||
let draws = 0;
|
||||
const cache = new Map();
|
||||
const request = install({
|
||||
cache,
|
||||
now: () => at,
|
||||
fetchImpl: async (url) => {
|
||||
const query = new URL(url).searchParams;
|
||||
assert.equal(query.get('order_by'), 'random');
|
||||
assert.equal(query.get('per_page'), '200');
|
||||
return Response.json(envelope([observation(++draws)]));
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
for (const elapsed of [1, 1000, 60_000, 299_999]) {
|
||||
at = NOW + elapsed;
|
||||
const next = await request();
|
||||
assert.deepEqual(next.body, first.body);
|
||||
}
|
||||
assert.equal(draws, 1);
|
||||
assert.match(cache.keys().next().value, /2026-11-02T18:00:00Z$/);
|
||||
at = NOW + 300_000;
|
||||
const refresh = await request();
|
||||
assert.equal(refresh.body.rows[0].stableId, 'inat:2');
|
||||
assert.equal(draws, 2);
|
||||
});
|
||||
|
||||
test('disconnected queued views are dropped while a remaining coalesced consumer keeps its request', async () => {
|
||||
const calls = [];
|
||||
const releases = [];
|
||||
const cache = new Map();
|
||||
const request = install({
|
||||
cache,
|
||||
fetchImpl: (url, { signal }) => {
|
||||
calls.push({ url: new URL(url), signal });
|
||||
return new Promise((resolve) => releases.push(resolve));
|
||||
},
|
||||
});
|
||||
const first = request(boxUrl(0));
|
||||
await until(() => releases.length === 1);
|
||||
const superseded = new AbortController();
|
||||
const oldView = request(boxUrl(1), { signal: superseded.signal });
|
||||
const oneConsumer = new AbortController();
|
||||
const one = request(boxUrl(2), { signal: oneConsumer.signal });
|
||||
const other = request(boxUrl(2));
|
||||
superseded.abort();
|
||||
oneConsumer.abort();
|
||||
await Promise.all([oldView, one]);
|
||||
// Rejoining the abandoned key creates a healthy generation; the old
|
||||
// coalescing promise must not later delete or cancel this replacement.
|
||||
const replacement = request(boxUrl(1));
|
||||
releases[0](Response.json(envelope()));
|
||||
await until(() => releases.length === 2);
|
||||
assert.equal(calls[1].url.searchParams.get('swlng'), '-122.1');
|
||||
assert.equal(calls[1].signal.aborted, false);
|
||||
releases[1](Response.json(envelope([observation(2)])));
|
||||
await until(() => releases.length === 3);
|
||||
assert.equal(calls[2].url.searchParams.get('swlng'), '-122.3');
|
||||
releases[2](Response.json(envelope([observation(3)])));
|
||||
const results = await Promise.all([first, other, replacement]);
|
||||
assert.ok(results.every((result) => result.status === 200));
|
||||
assert.equal(
|
||||
calls.length,
|
||||
3,
|
||||
'the abandoned queued generation never dispatched',
|
||||
);
|
||||
assert.equal(cache.size, 3);
|
||||
assert.equal(results[1].body.rows[0].stableId, 'inat:2');
|
||||
assert.equal(results[2].body.rows[0].stableId, 'inat:3');
|
||||
});
|
||||
|
||||
test('observation windows reject old sightings uploaded today and keep recent observations', async () => {
|
||||
for (const [taxa, recentTime, expectedD1] of [
|
||||
['all', '2026-11-04T17:00:00Z', '2026-11-02'],
|
||||
['whales', '2026-10-30T17:00:00Z', '2026-10-28'],
|
||||
]) {
|
||||
let upstreamQuery;
|
||||
const request = install({
|
||||
fetchImpl: async (url) => {
|
||||
upstreamQuery = new URL(url).searchParams;
|
||||
return Response.json(
|
||||
envelope([
|
||||
{
|
||||
...observation(80),
|
||||
time_observed_at: '2010-06-01T12:00:00Z',
|
||||
observed_on: '2010-06-01',
|
||||
},
|
||||
{
|
||||
...observation(81),
|
||||
time_observed_at: recentTime,
|
||||
observed_on: recentTime.slice(0, 10),
|
||||
},
|
||||
]),
|
||||
);
|
||||
},
|
||||
});
|
||||
const result = await request(boxUrl(0, `&taxa=${taxa}`));
|
||||
assert.equal(result.status, 200, taxa);
|
||||
assert.equal(upstreamQuery.get('d1'), expectedD1);
|
||||
assert.equal(upstreamQuery.has('created_d1'), false);
|
||||
assert.equal(upstreamQuery.get('order_by'), 'random');
|
||||
assert.deepEqual(
|
||||
result.body.rows.map((row) => row.stableId),
|
||||
['inat:81'],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('same-key requests single-flight and a seventeenth distinct in-flight key queues', async () => {
|
||||
const releases = [];
|
||||
const request = install({
|
||||
fetchImpl: () => new Promise((resolve) => releases.push(resolve)),
|
||||
});
|
||||
const a = request();
|
||||
const b = request();
|
||||
await until(() => releases.length === 1);
|
||||
releases.shift()(Response.json(envelope()));
|
||||
assert.deepEqual((await a).body, (await b).body);
|
||||
const pending = Array.from({ length: 16 }, (_, i) => request(boxUrl(i + 1)));
|
||||
await until(() => releases.length === 1);
|
||||
pending.push(request(boxUrl(18)));
|
||||
for (let i = 0; i < 17; i += 1) {
|
||||
await until(() => releases.length > i);
|
||||
releases[i](Response.json(envelope()));
|
||||
}
|
||||
assert.ok((await Promise.all(pending)).every(({ status }) => status === 200));
|
||||
});
|
||||
|
||||
test('a fresh global sample survives the hourly observation-window rollover without another fetch', async () => {
|
||||
let at = Date.parse('2026-11-04T18:59:00Z');
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json(envelope());
|
||||
},
|
||||
});
|
||||
const first = await request('/observations?scope=global');
|
||||
at += 60_000;
|
||||
const nextHour = await request('/observations?scope=global');
|
||||
assert.deepEqual(nextHour.body, first.body);
|
||||
assert.equal(nextHour.body.stale, undefined);
|
||||
assert.equal(calls, 1);
|
||||
at += 9 * 60_000;
|
||||
await request('/observations?scope=global');
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
|
||||
test('pending global refresh coalesces across the hourly observation-window rollover with a busy queue', async () => {
|
||||
let at = Date.parse('2026-11-04T18:59:59Z');
|
||||
const releases = [];
|
||||
const cache = new Map();
|
||||
const request = install({
|
||||
cache,
|
||||
now: () => at,
|
||||
fetchImpl: (url) =>
|
||||
new Promise((resolve) =>
|
||||
releases.push({
|
||||
resolve,
|
||||
url: new URL(url),
|
||||
}),
|
||||
),
|
||||
});
|
||||
const first = request(
|
||||
'/observations?scope=global&quality=research&taxa=Aves',
|
||||
);
|
||||
const views = Array.from({ length: 15 }, (_, i) => request(boxUrl(i)));
|
||||
await until(() => releases.length === 1);
|
||||
assert.equal(releases[0].url.searchParams.get('d1'), '2026-11-02');
|
||||
at = Date.parse('2026-11-04T19:00:01Z');
|
||||
const nextHour = request(
|
||||
'/observations?scope=global&quality=research&taxa=Aves',
|
||||
);
|
||||
await new Promise(setImmediate);
|
||||
assert.equal(
|
||||
releases.length,
|
||||
1,
|
||||
'the same global query joins the pending refresh',
|
||||
);
|
||||
const distinct = request(
|
||||
'/observations?scope=global&quality=research&taxa=Mammalia',
|
||||
);
|
||||
for (let i = 0; i < 17; i += 1) {
|
||||
await until(() => releases.length > i);
|
||||
releases[i].resolve(Response.json(envelope()));
|
||||
}
|
||||
const [original, joined] = await Promise.all([first, nextHour]);
|
||||
assert.equal(original.status, 200);
|
||||
assert.equal((await distinct).status, 200);
|
||||
assert.deepEqual(joined.body, original.body);
|
||||
assert.ok((await Promise.all(views)).every(({ status }) => status === 200));
|
||||
const globalEntries = [...cache.entries()].filter(
|
||||
([, entry]) => entry.value.scope === 'global',
|
||||
);
|
||||
assert.equal(globalEntries.length, 2);
|
||||
assert.match(globalEntries[0][0], /2026-11-02T18:00:00Z$/);
|
||||
assert.equal(
|
||||
(await request('/observations?scope=global&quality=research&taxa=Aves'))
|
||||
.status,
|
||||
200,
|
||||
);
|
||||
assert.equal(
|
||||
releases.length,
|
||||
17,
|
||||
'the accepted global answer keeps its ten-minute freshness',
|
||||
);
|
||||
});
|
||||
|
||||
test('view refreshes retain distinct hourly keys while a previous-hour request is pending', async () => {
|
||||
let at = Date.parse('2026-11-04T18:59:59Z');
|
||||
const releases = [];
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: () => new Promise((resolve) => releases.push(resolve)),
|
||||
});
|
||||
const first = request();
|
||||
await until(() => releases.length === 1);
|
||||
at = Date.parse('2026-11-04T19:00:01Z');
|
||||
const nextHour = request();
|
||||
await new Promise(setImmediate);
|
||||
assert.equal(
|
||||
releases.length,
|
||||
1,
|
||||
'the next view queues behind the current call',
|
||||
);
|
||||
releases[0](Response.json(envelope()));
|
||||
await until(() => releases.length === 2);
|
||||
releases[1](Response.json(envelope()));
|
||||
assert.ok(
|
||||
(await Promise.all([first, nextHour])).every(
|
||||
({ status }) => status === 200,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('upstream minute ceiling waits for request 61 across client addresses', async () => {
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
queueOptions: {
|
||||
now: () => at,
|
||||
intervalMs: 0,
|
||||
wait: async (ms) => {
|
||||
at += ms;
|
||||
},
|
||||
},
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json(envelope());
|
||||
},
|
||||
});
|
||||
for (let i = 0; i < 60; i += 1)
|
||||
assert.equal(
|
||||
(await request(boxUrl(i), { peer: `10.0.0.${i}` })).status,
|
||||
200,
|
||||
);
|
||||
const queued = await request(boxUrl(60), { peer: '10.0.1.1' });
|
||||
assert.equal(queued.status, 200);
|
||||
assert.equal(queued.headers['Retry-After'], undefined);
|
||||
assert.equal(at, NOW + 60_000);
|
||||
assert.equal(calls, 61);
|
||||
});
|
||||
|
||||
test('views refresh after many prior requests without fabricating an upstream limit', async () => {
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json(envelope());
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
at += 300_001;
|
||||
for (let i = 1; i <= 60; i += 1)
|
||||
await request(boxUrl(i), { peer: `10.0.0.${i}` });
|
||||
const stale = await request();
|
||||
assert.equal(stale.status, 200);
|
||||
assert.equal(stale.body.stale, undefined);
|
||||
assert.notEqual(stale.body.fetchedAt, first.body.fetchedAt);
|
||||
assert.equal(calls, 62);
|
||||
});
|
||||
|
||||
test('bounded samples preserve partial metadata, and well-formed empty views are valid', async () => {
|
||||
let at = NOW;
|
||||
let fail = false;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () =>
|
||||
fail
|
||||
? new Response('unavailable', { status: 503 })
|
||||
: Response.json(envelope([observation()], 500)),
|
||||
});
|
||||
const first = await request();
|
||||
assert.equal(first.body.partial, true);
|
||||
assert.equal(first.body.rawRowCount, 500);
|
||||
fail = true;
|
||||
at += 59 * 60_000;
|
||||
const stale = await request();
|
||||
assert.equal(
|
||||
stale.status,
|
||||
200,
|
||||
'hourly window changes still permit stale fallback',
|
||||
);
|
||||
assert.equal(stale.headers['X-Data-Stale'], 'true');
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.body.fetchedAt, first.body.fetchedAt);
|
||||
assert.equal(stale.body.partial, true);
|
||||
at += 2 * 60_000;
|
||||
const expired = await request();
|
||||
assert.equal(expired.status, 502);
|
||||
assert.equal(expired.body.error, 'inaturalist_unavailable');
|
||||
const empty = install({
|
||||
fetchImpl: async () => Response.json(envelope([], 0)),
|
||||
});
|
||||
const result = await empty();
|
||||
assert.equal(result.status, 200);
|
||||
assert.deepEqual(result.body.rows, []);
|
||||
});
|
||||
|
||||
test('bad envelopes, JSON and capped bodies fail closed without inventing a cooldown', async () => {
|
||||
for (const bad of [
|
||||
() => Response.json({ total_results: 1, results: {} }),
|
||||
() => Response.json({ total_results: 1, observations: [observation()] }),
|
||||
() => Response.json({ total_results: '1', results: [observation()] }),
|
||||
() => Response.json(envelope([], 1)),
|
||||
() => Response.json(envelope([{ ...observation(), id: null }], 1)),
|
||||
() => new Response('{malformed'),
|
||||
() =>
|
||||
new Response('{}', {
|
||||
headers: { 'Content-Length': String(8 * 1024 * 1024 + 1) },
|
||||
}),
|
||||
]) {
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return calls === 1 ? bad() : Response.json(envelope());
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
assert.equal(first.status, 502);
|
||||
assert.equal(first.body.error, 'inaturalist_unavailable');
|
||||
assert.equal(first.headers['Retry-After'], undefined);
|
||||
assert.equal((await request(boxUrl(1))).status, 200);
|
||||
assert.equal(calls, 2);
|
||||
}
|
||||
});
|
||||
|
||||
test('a wholly invalid iNat page cannot replace a previously valid snapshot', async () => {
|
||||
let at = NOW;
|
||||
let fail = false;
|
||||
const cache = new Map();
|
||||
const request = install({
|
||||
cache,
|
||||
now: () => at,
|
||||
fetchImpl: async () =>
|
||||
Response.json(
|
||||
fail
|
||||
? envelope([{ ...observation(), quality_grade: 'casual' }])
|
||||
: envelope(),
|
||||
),
|
||||
});
|
||||
const first = await request();
|
||||
const original = cache.values().next().value;
|
||||
fail = true;
|
||||
at += 300_001;
|
||||
const stale = await request();
|
||||
assert.equal(stale.status, 200);
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.body.fetchedAt, first.body.fetchedAt);
|
||||
assert.equal(cache.values().next().value, original);
|
||||
});
|
||||
|
||||
test('429 cooldown honors seconds, HTTP dates, and long upstream deadlines', async () => {
|
||||
assert.equal(inatRetryCooldownMs(null, NOW), 60_000);
|
||||
assert.equal(inatRetryCooldownMs('0', NOW), 0);
|
||||
assert.equal(inatRetryCooldownMs('2', NOW), 2000);
|
||||
assert.equal(inatRetryCooldownMs('999999', NOW), 999_999_000);
|
||||
assert.equal(inatRetryCooldownMs('7200', NOW), 7_200_000);
|
||||
assert.equal(
|
||||
inatRetryCooldownMs(new Date(NOW + 7_200_000).toUTCString(), NOW),
|
||||
7_200_000,
|
||||
);
|
||||
let at = NOW;
|
||||
const releases = [];
|
||||
const warnings = [];
|
||||
const request = install({
|
||||
now: () => at,
|
||||
warn: (message) => warnings.push(message),
|
||||
fetchImpl: () => new Promise((resolve) => releases.push(resolve)),
|
||||
});
|
||||
const first = request();
|
||||
const later = request(boxUrl(1));
|
||||
await until(() => releases.length === 1);
|
||||
releases[0](
|
||||
new Response('limit', {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '999999' },
|
||||
}),
|
||||
);
|
||||
assert.equal((await first).headers['Retry-After'], '999999');
|
||||
assert.equal((await later).headers['Retry-After'], '999999');
|
||||
at += 1000;
|
||||
assert.equal((await request(boxUrl(1))).headers['Retry-After'], '999998');
|
||||
at += 5000;
|
||||
const cooling = await request(boxUrl(2));
|
||||
assert.equal(cooling.status, 429);
|
||||
assert.equal(cooling.headers['Retry-After'], '999993');
|
||||
assert.equal(
|
||||
releases.length,
|
||||
1,
|
||||
'queued calls do not dispatch during cooldown',
|
||||
);
|
||||
assert.equal(
|
||||
warnings.filter((message) => message.includes('capped')).length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
for (const [upstreamStatus, expectedStatus, code, retryAfterS] of [
|
||||
[429, 429, 'inaturalist_rate_limited', 120],
|
||||
[503, 502, 'inaturalist_unavailable', 120],
|
||||
]) {
|
||||
test(`concurrent reservations honor an immediate upstream ${upstreamStatus} before dispatch`, async () => {
|
||||
let calls = 0;
|
||||
const warnings = [];
|
||||
const client = createInatClient({
|
||||
now: () => 0,
|
||||
cacheDir: null,
|
||||
queueOptions: { intervalMs: 0 },
|
||||
warn: (message) => warnings.push(message),
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return new Response('synthetic failure', {
|
||||
status: upstreamStatus,
|
||||
headers: { 'Retry-After': '120' },
|
||||
});
|
||||
},
|
||||
});
|
||||
const params = {
|
||||
scope: 'global',
|
||||
quality: 'all',
|
||||
taxa: 'all',
|
||||
observedD1: '1969-12-30',
|
||||
};
|
||||
const results = await Promise.allSettled(
|
||||
Array.from({ length: 3 }, () => client.getObservations(params)),
|
||||
);
|
||||
assert.equal(
|
||||
calls,
|
||||
1,
|
||||
'pending reservations must not dispatch during cooldown',
|
||||
);
|
||||
for (const result of results) {
|
||||
assert.equal(result.status, 'rejected');
|
||||
assert.equal(result.reason.status, expectedStatus);
|
||||
assert.equal(result.reason.code, code);
|
||||
assert.equal(result.reason.retryAfterS, retryAfterS);
|
||||
}
|
||||
assert.equal(
|
||||
warnings.length,
|
||||
1,
|
||||
'only the dispatched refresh logs a failure',
|
||||
);
|
||||
await assert.rejects(client.getObservations(params), {
|
||||
status: expectedStatus,
|
||||
code,
|
||||
retryAfterS,
|
||||
});
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
}
|
||||
|
||||
test('budget is read lazily, persisted as counters only, enforced across restart and UTC rollover', async (t) => {
|
||||
const parent = await mkdtemp(path.join(os.tmpdir(), 'gev-inat-budget-'));
|
||||
t.after(() => rm(parent, { recursive: true, force: true }));
|
||||
const cacheDir = path.join(parent, 'not-created');
|
||||
const before = process.env.INATURALIST_DAILY_BUDGET;
|
||||
process.env.INATURALIST_DAILY_BUDGET = '2';
|
||||
t.after(() =>
|
||||
before === undefined
|
||||
? delete process.env.INATURALIST_DAILY_BUDGET
|
||||
: (process.env.INATURALIST_DAILY_BUDGET = before),
|
||||
);
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
const options = {
|
||||
now: () => at,
|
||||
cacheDir,
|
||||
warn() {},
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return Response.json(envelope());
|
||||
},
|
||||
};
|
||||
const client = createInatClient(options);
|
||||
assert.deepEqual(
|
||||
await readdir(parent),
|
||||
[],
|
||||
'construction performs no directory write',
|
||||
);
|
||||
const params = {
|
||||
scope: 'global',
|
||||
quality: 'all',
|
||||
taxa: 'all',
|
||||
observedD1: '2026-11-02',
|
||||
};
|
||||
await Promise.all([
|
||||
client.getObservations(params),
|
||||
client.getObservations(params),
|
||||
]);
|
||||
assert.deepEqual(
|
||||
JSON.parse(await readFile(path.join(cacheDir, 'budget.json'), 'utf8')),
|
||||
{ date: '2026-11-04', count: 2 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
await readdir(cacheDir),
|
||||
['budget.json'],
|
||||
'atomic temporary files are retired',
|
||||
);
|
||||
const restarted = createInatClient(options);
|
||||
await assert.rejects(restarted.getObservations(params), {
|
||||
status: 503,
|
||||
code: 'inaturalist_budget_exhausted',
|
||||
});
|
||||
assert.equal(calls, 2);
|
||||
at = Date.parse('2026-11-05T00:00:00Z');
|
||||
await restarted.getObservations(params);
|
||||
assert.deepEqual(restarted.budgetState(), {
|
||||
date: '2026-11-05',
|
||||
count: 1,
|
||||
budget: 2,
|
||||
});
|
||||
assert.deepEqual(
|
||||
JSON.parse(await readFile(path.join(cacheDir, 'budget.json'), 'utf8')),
|
||||
{ date: '2026-11-05', count: 1 },
|
||||
);
|
||||
assert.equal(calls, 3);
|
||||
});
|
||||
|
||||
test('memory cache is an LRU bounded to 400 projected entries', async () => {
|
||||
let at = NOW;
|
||||
const cache = new Map();
|
||||
const request = install({
|
||||
cache,
|
||||
now: () => at,
|
||||
fetchImpl: async () => Response.json(envelope()),
|
||||
});
|
||||
for (let i = 0; i < 400; i += 1) {
|
||||
if (i && i % 20 === 0) at += 61_000;
|
||||
await request(boxUrl(i), { peer: String(i) });
|
||||
}
|
||||
assert.equal(cache.size, 400);
|
||||
const firstKey = cache.keys().next().value;
|
||||
const secondKey = [...cache.keys()][1];
|
||||
await request(boxUrl(0));
|
||||
await request(boxUrl(400));
|
||||
assert.equal(cache.size, 400);
|
||||
assert.equal(cache.has(firstKey), true);
|
||||
assert.equal(cache.has(secondKey), false);
|
||||
});
|
||||
|
||||
test('aborted client calls spend no budget and never fetch', async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const client = createInatClient({
|
||||
now: () => NOW,
|
||||
cacheDir: null,
|
||||
queueOptions: { intervalMs: 0 },
|
||||
fetchImpl: () => assert.fail('aborted call fetched'),
|
||||
});
|
||||
await assert.rejects(
|
||||
client.getObservations({}, { signal: controller.signal }),
|
||||
{ name: 'AbortError' },
|
||||
);
|
||||
assert.equal(client.budgetState().count, 0);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { inaturalistProxy } from './observations.js';
|
||||
|
||||
export const NOW = Date.parse('2026-11-04T18:05:00Z');
|
||||
|
||||
/** Mount the proxy over synthetic HTTP objects, without opening a socket. */
|
||||
export function installInatProxy(
|
||||
defaultUrl,
|
||||
options = {},
|
||||
hook = 'configureServer',
|
||||
) {
|
||||
let handler;
|
||||
const plugin = inaturalistProxy({
|
||||
now: () => NOW,
|
||||
cacheDir: null,
|
||||
queueOptions: { intervalMs: 0, windowMs: 0 },
|
||||
warn() {},
|
||||
...options,
|
||||
});
|
||||
assert.equal(plugin.name, 'inaturalist');
|
||||
assert.equal(typeof plugin.configureServer, 'function');
|
||||
assert.equal(typeof plugin.configurePreviewServer, 'function');
|
||||
plugin[hook]({
|
||||
middlewares: {
|
||||
use(mount, callback) {
|
||||
assert.equal(mount, '/api/inaturalist');
|
||||
handler = callback;
|
||||
},
|
||||
},
|
||||
});
|
||||
return async (
|
||||
url = defaultUrl,
|
||||
{ method = 'GET', peer = 'local', headers = {}, signal } = {},
|
||||
) => {
|
||||
const res = Object.assign(new EventEmitter(), {
|
||||
headers: {},
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
Object.assign(this.headers, headers);
|
||||
},
|
||||
setHeader(key, value) {
|
||||
this.headers[key] = value;
|
||||
},
|
||||
end(value) {
|
||||
this.status ??= this.statusCode;
|
||||
this.body = JSON.parse(value);
|
||||
},
|
||||
});
|
||||
const abort = () => {
|
||||
res.destroyed = true;
|
||||
res.emit('close');
|
||||
};
|
||||
signal?.addEventListener('abort', abort, { once: true });
|
||||
try {
|
||||
await handler(
|
||||
Object.assign(new EventEmitter(), {
|
||||
url,
|
||||
method,
|
||||
headers,
|
||||
socket: { remoteAddress: peer },
|
||||
}),
|
||||
res,
|
||||
);
|
||||
} finally {
|
||||
signal?.removeEventListener('abort', abort);
|
||||
}
|
||||
return res;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
|
||||
// A four-request burst keeps a cold view responsive; thereafter every provider
|
||||
// call shares the same one-per-second token supply. The rolling minute ceiling
|
||||
// queues the few requests a token bucket alone could allow above 60/min.
|
||||
const INAT_INTERVAL_MS = 1000;
|
||||
const INAT_BURST = 4;
|
||||
const INAT_WINDOW_MS = 60_000;
|
||||
const INAT_MAX_PER_WINDOW = 60;
|
||||
const INAT_MAX_IN_FLIGHT = 4;
|
||||
|
||||
/** Observation queries outrank tiles waiting for a token. Aborted consumers
|
||||
* leave the queue before an attempt or daily-budget reservation is spent. */
|
||||
export function createInatQueue({
|
||||
now = () => performance.now(),
|
||||
wait = (ms) => delay(ms),
|
||||
intervalMs = INAT_INTERVAL_MS,
|
||||
burst = INAT_BURST,
|
||||
windowMs = INAT_WINDOW_MS,
|
||||
maxInFlight = INAT_MAX_IN_FLIGHT,
|
||||
} = {}) {
|
||||
const pending = [];
|
||||
const dispatchedAt = [];
|
||||
let tokens = burst;
|
||||
let refilledAt = null;
|
||||
let running = false;
|
||||
let inFlight = 0;
|
||||
let slotFreed = null;
|
||||
|
||||
function refill() {
|
||||
const at = now();
|
||||
if (refilledAt !== null && intervalMs > 0)
|
||||
tokens = Math.min(
|
||||
burst,
|
||||
tokens + Math.max(0, at - refilledAt) / intervalMs,
|
||||
);
|
||||
if (intervalMs <= 0) tokens = burst;
|
||||
refilledAt = at;
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
if (running) return;
|
||||
running = true;
|
||||
// A microtask also lets same-turn observation queries take priority.
|
||||
void Promise.resolve().then(drain);
|
||||
}
|
||||
|
||||
async function drain() {
|
||||
try {
|
||||
while (pending.length) {
|
||||
if (inFlight >= maxInFlight) {
|
||||
await new Promise((resolve) => (slotFreed = resolve));
|
||||
continue;
|
||||
}
|
||||
refill();
|
||||
const at = now();
|
||||
while (dispatchedAt.length && dispatchedAt[0] <= at - windowMs)
|
||||
dispatchedAt.shift();
|
||||
const tokenWait = tokens < 1 ? (1 - tokens) * intervalMs : 0;
|
||||
const windowWait =
|
||||
windowMs > 0 && dispatchedAt.length >= INAT_MAX_PER_WINDOW
|
||||
? dispatchedAt[0] + windowMs - at
|
||||
: 0;
|
||||
if (tokenWait > 0 || windowWait > 0) {
|
||||
await wait(Math.ceil(Math.max(tokenWait, windowWait)));
|
||||
continue;
|
||||
}
|
||||
// Stable sorting preserves FIFO order within each priority.
|
||||
pending.sort((a, b) => a.priority - b.priority);
|
||||
const job = pending.shift();
|
||||
job.signal?.removeEventListener('abort', job.abort);
|
||||
if (job.signal?.aborted) {
|
||||
job.reject(job.signal.reason);
|
||||
continue;
|
||||
}
|
||||
tokens -= 1;
|
||||
if (windowMs > 0) dispatchedAt.push(at);
|
||||
inFlight += 1;
|
||||
// A rejected task releases its slot without stopping the queue.
|
||||
void new Promise((resolve) => resolve(job.task()))
|
||||
.then(job.resolve, job.reject)
|
||||
.finally(() => {
|
||||
inFlight -= 1;
|
||||
const wake = slotFreed;
|
||||
slotFreed = null;
|
||||
wake?.();
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
running = false;
|
||||
}
|
||||
}
|
||||
|
||||
return function enqueue(task, { signal, priority = 1 } = {}) {
|
||||
if (signal?.aborted) return Promise.reject(signal.reason);
|
||||
return new Promise((resolve, reject) => {
|
||||
const job = { task, signal, priority, resolve, reject };
|
||||
job.abort = () => {
|
||||
const index = pending.indexOf(job);
|
||||
if (index >= 0) pending.splice(index, 1);
|
||||
signal.removeEventListener('abort', job.abort);
|
||||
reject(signal.reason);
|
||||
};
|
||||
signal?.addEventListener('abort', job.abort, { once: true });
|
||||
pending.push(job);
|
||||
schedule();
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createInatQueue } from './queue.js';
|
||||
|
||||
async function until(check) {
|
||||
for (let i = 0; i < 100; i += 1) {
|
||||
if (check()) return;
|
||||
await new Promise(setImmediate);
|
||||
}
|
||||
assert.fail('queue did not reach its expected state');
|
||||
}
|
||||
|
||||
test('shared governor paces a four-call burst within the rolling minute ceiling', async () => {
|
||||
let at = 0;
|
||||
const waits = [];
|
||||
const queue = createInatQueue({
|
||||
now: () => at,
|
||||
wait: async (ms) => {
|
||||
waits.push(ms);
|
||||
at += ms;
|
||||
},
|
||||
});
|
||||
const dispatched = [];
|
||||
await Promise.all(
|
||||
Array.from({ length: 65 }, () => queue(() => dispatched.push(at))),
|
||||
);
|
||||
assert.equal(dispatched.length, 65);
|
||||
assert.deepEqual(dispatched.slice(0, 7), [0, 0, 0, 0, 1000, 2000, 3000]);
|
||||
assert.deepEqual(waits.slice(0, 3), [1000, 1000, 1000]);
|
||||
for (const time of dispatched)
|
||||
assert.ok(
|
||||
dispatched.filter((other) => other > time - 60_000 && other <= time)
|
||||
.length <= 60,
|
||||
);
|
||||
assert.equal(dispatched[60], 60_000);
|
||||
});
|
||||
|
||||
test('an observation arriving during the pacing wait overtakes queued tiles', async () => {
|
||||
let at = 0;
|
||||
const waits = [];
|
||||
const queue = createInatQueue({
|
||||
burst: 1,
|
||||
now: () => at,
|
||||
wait: (ms) => new Promise((resolve) => waits.push({ ms, resolve })),
|
||||
});
|
||||
const dispatched = [];
|
||||
await queue(() => dispatched.push('first tile'));
|
||||
const tile = queue(() => dispatched.push('waiting tile'));
|
||||
await until(() => waits.length === 1);
|
||||
const observation = queue(() => dispatched.push('observation'), {
|
||||
priority: 0,
|
||||
});
|
||||
at += waits[0].ms;
|
||||
waits[0].resolve();
|
||||
await observation;
|
||||
assert.deepEqual(dispatched, ['first tile', 'observation']);
|
||||
await until(() => waits.length === 2);
|
||||
at += waits[1].ms;
|
||||
waits[1].resolve();
|
||||
await tile;
|
||||
assert.deepEqual(dispatched, ['first tile', 'observation', 'waiting tile']);
|
||||
});
|
||||
|
||||
test('superseded tasks leave before dispatch, and a rejection cannot poison the tail', async () => {
|
||||
const queue = createInatQueue({ intervalMs: 0 });
|
||||
const dispatched = [];
|
||||
let release;
|
||||
const first = queue(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
release = resolve;
|
||||
}),
|
||||
);
|
||||
await until(() => release);
|
||||
const controller = new AbortController();
|
||||
const superseded = queue(() => assert.fail('superseded tile dispatched'), {
|
||||
signal: controller.signal,
|
||||
});
|
||||
const dropped = assert.rejects(superseded, { name: 'AbortError' });
|
||||
controller.abort();
|
||||
await dropped;
|
||||
const failed = queue(() => {
|
||||
dispatched.push('failed');
|
||||
throw new Error('synthetic');
|
||||
});
|
||||
const failure = assert.rejects(failed, /synthetic/);
|
||||
const succeeding = queue(() => dispatched.push('recovered'));
|
||||
release();
|
||||
await Promise.all([first, failure, succeeding]);
|
||||
assert.deepEqual(dispatched, ['failed', 'recovered']);
|
||||
});
|
||||
|
||||
test('a slow call does not hold later calls; at most four run at once', async () => {
|
||||
const queue = createInatQueue({ intervalMs: 0, windowMs: 0 });
|
||||
const releases = [];
|
||||
const started = [];
|
||||
const slow = (id) => () =>
|
||||
new Promise((resolve) => {
|
||||
started.push(id);
|
||||
releases.push(() => resolve(id));
|
||||
});
|
||||
const jobs = [1, 2, 3, 4, 5].map((id) => queue(slow(id)));
|
||||
await until(() => started.length === 4);
|
||||
assert.deepEqual(started, [1, 2, 3, 4], 'four start without waiting');
|
||||
releases[1]();
|
||||
assert.equal(await jobs[1], 2, 'a later call finishes before the first');
|
||||
await until(() => started.length === 5);
|
||||
assert.deepEqual(started, [1, 2, 3, 4, 5], 'a freed slot starts the next');
|
||||
for (const release of releases) release();
|
||||
assert.deepEqual(await Promise.all(jobs), [1, 2, 3, 4, 5]);
|
||||
});
|
||||
@@ -0,0 +1,218 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createInatClient } from './client.js';
|
||||
import { installInatProxy, NOW } from './proxy.fixture.mjs';
|
||||
import { syntheticGrid } from './grid.fixture.mjs';
|
||||
import { INAT_GRID_DECODE_MEASURE } from './grid.js';
|
||||
|
||||
const gridUrl = '/grid?z=2&x=1&y=1&taxa=Aves';
|
||||
const params = {
|
||||
scope: 'global',
|
||||
taxa: 'all',
|
||||
observedD1: '2026-11-02',
|
||||
};
|
||||
const tick = () => new Promise(setImmediate);
|
||||
|
||||
const install = (options, hook) => installInatProxy(gridUrl, options, hook);
|
||||
|
||||
test('tile cache lasts fifteen minutes, then returns stale before revalidation finishes', async () => {
|
||||
let at = NOW;
|
||||
let calls = 0;
|
||||
let release;
|
||||
const request = install({
|
||||
now: () => at,
|
||||
fetchImpl: () => {
|
||||
calls += 1;
|
||||
return calls === 1
|
||||
? Response.json(syntheticGrid())
|
||||
: new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
at += 899_999;
|
||||
const freshPromise = request();
|
||||
await tick();
|
||||
assert.equal(calls, 1, 'a fresh tile never spends another upstream attempt');
|
||||
assert.deepEqual((await freshPromise).body, first.body);
|
||||
at += 1;
|
||||
const stalePromise = request();
|
||||
let resolved = false;
|
||||
void stalePromise.then(() => {
|
||||
resolved = true;
|
||||
});
|
||||
await tick();
|
||||
try {
|
||||
assert.equal(
|
||||
resolved,
|
||||
true,
|
||||
'stale cells publish while refresh is pending',
|
||||
);
|
||||
const stale = await stalePromise;
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.body.fetchedAt, NOW);
|
||||
assert.deepEqual(stale.body.cells, first.body.cells);
|
||||
await request();
|
||||
assert.equal(calls, 2, 'all stale consumers share one background refresh');
|
||||
const refreshPromise = request(`${gridUrl}&refresh=wait`);
|
||||
let refreshed = false;
|
||||
void refreshPromise.then(() => {
|
||||
refreshed = true;
|
||||
});
|
||||
await tick();
|
||||
assert.equal(
|
||||
refreshed,
|
||||
false,
|
||||
'a background consumer can await the same refresh',
|
||||
);
|
||||
release(Response.json(syntheticGrid()));
|
||||
const fresh = await refreshPromise;
|
||||
assert.equal(fresh.body.stale, undefined);
|
||||
assert.equal(fresh.body.fetchedAt, at);
|
||||
assert.equal(
|
||||
calls,
|
||||
2,
|
||||
'waiting for fresh cells never duplicates upstream work',
|
||||
);
|
||||
release = null;
|
||||
} finally {
|
||||
release?.(Response.json(syntheticGrid()));
|
||||
}
|
||||
});
|
||||
|
||||
test('more than sixteen queued views and thirty local requests never report local rate limiting', async () => {
|
||||
let release;
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1)
|
||||
await new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
return Response.json(syntheticGrid());
|
||||
},
|
||||
});
|
||||
const pending = Array.from({ length: 40 }, (_, x) =>
|
||||
request(`/grid?z=6&x=${x}&y=1`),
|
||||
);
|
||||
await tick();
|
||||
release();
|
||||
const results = await Promise.all(pending);
|
||||
assert.ok(results.every((result) => result.status === 200));
|
||||
assert.equal(calls, 40);
|
||||
});
|
||||
|
||||
test('only real upstream 429 or 5xx failures start cooldown', async () => {
|
||||
for (const bad of [
|
||||
() => Promise.reject(new Error('network')),
|
||||
() => Response.json({ malformed: true }),
|
||||
() => new Response('', { status: 404 }),
|
||||
]) {
|
||||
let calls = 0;
|
||||
const client = createInatClient({
|
||||
now: () => NOW,
|
||||
cacheDir: null,
|
||||
queueOptions: { intervalMs: 0, windowMs: 0 },
|
||||
warn() {},
|
||||
fetchImpl: () => {
|
||||
calls += 1;
|
||||
return calls === 1
|
||||
? bad()
|
||||
: Response.json({ results: [], total_results: 0 });
|
||||
},
|
||||
});
|
||||
await assert.rejects(client.getObservations(params), { status: 502 });
|
||||
await client.getObservations(params);
|
||||
assert.equal(
|
||||
calls,
|
||||
2,
|
||||
'a non-cooldown failure must not hold the next attempt',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('actual upstream cooldown is marked separately from a local budget stop', async () => {
|
||||
for (const upstream of [429, 503]) {
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: () => {
|
||||
calls += 1;
|
||||
return new Response('', {
|
||||
status: upstream,
|
||||
headers: { 'Retry-After': '120' },
|
||||
});
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
assert.equal(first.headers['X-Upstream-Status'], String(upstream));
|
||||
assert.equal(first.headers['Retry-After'], '120');
|
||||
await request();
|
||||
assert.equal(calls, 1);
|
||||
}
|
||||
});
|
||||
|
||||
test('upstream Retry-After zero survives the HTTP response and permits an immediate next request', async () => {
|
||||
for (const upstream of [429, 503]) {
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: () => {
|
||||
calls += 1;
|
||||
return calls === 1
|
||||
? new Response('', {
|
||||
status: upstream,
|
||||
headers: { 'Retry-After': '0' },
|
||||
})
|
||||
: Response.json(syntheticGrid());
|
||||
},
|
||||
});
|
||||
const first = await request();
|
||||
assert.equal(first.headers['X-Upstream-Status'], String(upstream));
|
||||
assert.equal(first.headers['Retry-After'], '0');
|
||||
assert.equal((await request()).status, 200);
|
||||
assert.equal(calls, 2);
|
||||
}
|
||||
});
|
||||
|
||||
test('a local daily budget stop has a distinct code and no upstream cooldown marker', async (t) => {
|
||||
const before = process.env.INATURALIST_DAILY_BUDGET;
|
||||
process.env.INATURALIST_DAILY_BUDGET = '1';
|
||||
t.after(() => {
|
||||
if (before === undefined) delete process.env.INATURALIST_DAILY_BUDGET;
|
||||
else process.env.INATURALIST_DAILY_BUDGET = before;
|
||||
});
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: () => {
|
||||
calls += 1;
|
||||
return Response.json(syntheticGrid());
|
||||
},
|
||||
});
|
||||
assert.equal((await request()).status, 200);
|
||||
const stopped = await request('/grid?z=2&x=0&y=1');
|
||||
assert.equal(stopped.status, 503);
|
||||
assert.equal(stopped.body.error, 'inaturalist_budget_exhausted');
|
||||
assert.equal(stopped.headers['X-Upstream-Status'], undefined);
|
||||
assert.equal((await request()).status, 200, 'cached data remains available');
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test('cached tiles avoid UTFGrid decode and decode timing entries stay bounded', async () => {
|
||||
const request = install({
|
||||
fetchImpl: () => Response.json(syntheticGrid()),
|
||||
});
|
||||
await request();
|
||||
const sample = performance.getEntriesByName(INAT_GRID_DECODE_MEASURE)[0];
|
||||
assert.ok(sample.duration >= 0);
|
||||
for (let i = 0; i < 10; i += 1) await request();
|
||||
assert.equal(
|
||||
performance.getEntriesByName(INAT_GRID_DECODE_MEASURE)[0],
|
||||
sample,
|
||||
);
|
||||
for (let x = 0; x < 4; x += 1) await request(`/grid?z=3&x=${x}&y=1`);
|
||||
assert.equal(
|
||||
performance.getEntriesByName(INAT_GRID_DECODE_MEASURE).length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
import { LIBRARY_ENTRIES } from '../../src/library/catalog.js';
|
||||
import { createLibraryHandler } from '../../src/library/proxy.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { admitSameSite, admitSameSiteKeyless } from './common/same-site.js';
|
||||
|
||||
export {
|
||||
libraryAvailability,
|
||||
libraryKeyValue,
|
||||
parseLibraryTilePath,
|
||||
} from '../../src/library/proxy.js';
|
||||
|
||||
/** `content-type` as `Content-Type`: a fetch Response's names, as Node writes them. */
|
||||
const headerName = (name) =>
|
||||
name.replace(
|
||||
/(^|-)([a-z])/g,
|
||||
(_, dash, letter) => dash + letter.toUpperCase(),
|
||||
);
|
||||
|
||||
/**
|
||||
* The Data Library's one server route on the dev and preview servers: the
|
||||
* portable handler (`src/library/proxy.js`) behind this server's same-site
|
||||
* gate and rate limits.
|
||||
*/
|
||||
export function libraryProxy({
|
||||
entries = LIBRARY_ENTRIES,
|
||||
env = process.env,
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
} = {}) {
|
||||
const byId = new Map(entries.map((entry) => [entry.id, entry]));
|
||||
const handle = createLibraryHandler({ entries, env, fetchImpl, now });
|
||||
const allowData = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 30,
|
||||
globalMax: 600,
|
||||
});
|
||||
const allowTiles = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 600,
|
||||
globalMax: 6000,
|
||||
});
|
||||
|
||||
function refuse(res, status, error) {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...(status === 429 ? { 'Retry-After': '60' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify({ error }));
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
if (req.method !== 'GET') return refuse(res, 405, 'method_not_allowed');
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
const [route, id] = path.split('/').filter(Boolean);
|
||||
const entry = byId.get(id);
|
||||
if (route === 'catalog' || !entry) {
|
||||
if (admitSameSiteKeyless(req, res)) return undefined;
|
||||
} else {
|
||||
// A keyed entry spends the operator's credential: refuse forwarded
|
||||
// requests like the other cost-bearing routes. Keyless entries keep
|
||||
// working behind a reverse proxy, like terrain heights.
|
||||
if (
|
||||
entry.source?.key
|
||||
? admitSameSite(req, res)
|
||||
: admitSameSiteKeyless(req, res)
|
||||
)
|
||||
return undefined;
|
||||
const allow = route === 'tile' ? allowTiles : allowData;
|
||||
if (!allow(clientKey(req))) return refuse(res, 429, 'rate_limited');
|
||||
}
|
||||
const response = await handle(
|
||||
new Request(`http://library.invalid/api/library${req.url || '/'}`, {
|
||||
headers: { Accept: String(req.headers?.accept || '*/*') },
|
||||
}),
|
||||
);
|
||||
if (res.destroyed) return undefined;
|
||||
const headers = {};
|
||||
for (const [name, value] of response.headers)
|
||||
headers[headerName(name)] = value;
|
||||
const body = Buffer.from(await response.arrayBuffer());
|
||||
res.writeHead(response.status, headers);
|
||||
res.end(
|
||||
String(headers['Content-Type']).startsWith('application/json')
|
||||
? body.toString('utf8')
|
||||
: body,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'data-library',
|
||||
handler,
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/library', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/library', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
libraryAvailability,
|
||||
libraryProxy,
|
||||
parseLibraryTilePath,
|
||||
} from './library.js';
|
||||
import { validateLibraryCatalog } from '../../src/library/manifest.js';
|
||||
import { libraryEntry } from '../../src/library/catalog.js';
|
||||
|
||||
const base = {
|
||||
family: 'events',
|
||||
provider: 'Fixture',
|
||||
description: 'Fixture entry.',
|
||||
maintainer: 'test',
|
||||
coverage: { global: true },
|
||||
licence: { name: 'Public domain', url: 'https://example.org/licence' },
|
||||
attribution: { text: 'Fixture' },
|
||||
status: 'community',
|
||||
};
|
||||
const { entries, errors } = validateLibraryCatalog([
|
||||
{
|
||||
...base,
|
||||
id: 'proxied-points',
|
||||
kind: 'dataset',
|
||||
title: 'Proxied points',
|
||||
source: {
|
||||
format: 'geojson',
|
||||
url: 'https://data.example.org/points.json',
|
||||
fetch: 'proxied',
|
||||
maxBytes: 4096,
|
||||
},
|
||||
display: { label: 'title' },
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'direct-points',
|
||||
kind: 'dataset',
|
||||
title: 'Direct points',
|
||||
source: {
|
||||
format: 'geojson',
|
||||
url: 'https://data.example.org/direct.json',
|
||||
fetch: 'browser-direct',
|
||||
},
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'keyed-points',
|
||||
kind: 'dataset',
|
||||
title: 'Keyed points',
|
||||
source: {
|
||||
format: 'geojson',
|
||||
url: 'https://keyed.example.org/points.json',
|
||||
fetch: 'proxied',
|
||||
key: { id: 'firms', in: 'query', name: 'MAP_KEY' },
|
||||
},
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'opt-in-points',
|
||||
kind: 'dataset',
|
||||
title: 'Opt-in points',
|
||||
source: {
|
||||
format: 'geojson',
|
||||
url: 'https://data.example.org/opt.json',
|
||||
fetch: 'proxied',
|
||||
},
|
||||
licence: { ...base.licence, operatorOptIn: true },
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'proxied-tiles',
|
||||
kind: 'tiles',
|
||||
title: 'Proxied tiles',
|
||||
source: {
|
||||
format: 'xyz',
|
||||
url: 'https://tiles.example.org/{z}/{x}/{y}.png',
|
||||
fetch: 'proxied',
|
||||
minZoom: 2,
|
||||
maxZoom: 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'fixture-pack',
|
||||
kind: 'pack',
|
||||
family: 'cameras',
|
||||
title: 'Fixture cameras',
|
||||
source: { format: 'host', url: 'https://cams.example.org/' },
|
||||
pack: { host: 'cctv', env: 'CCTV_FIXTURE_ENABLED' },
|
||||
licence: { ...base.licence, operatorOptIn: true },
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(errors, []);
|
||||
|
||||
const collection = (features) =>
|
||||
JSON.stringify({ type: 'FeatureCollection', features });
|
||||
const point = (title) => ({
|
||||
type: 'Feature',
|
||||
geometry: { type: 'Point', coordinates: [1, 2] },
|
||||
properties: { title },
|
||||
});
|
||||
|
||||
function install({ env = {}, fetchImpl, now, catalog = entries } = {}) {
|
||||
const plugin = libraryProxy({ entries: catalog, env, fetchImpl, now });
|
||||
let handler;
|
||||
plugin.configurePreviewServer({
|
||||
middlewares: {
|
||||
use(path, callback) {
|
||||
assert.equal(path, '/api/library');
|
||||
handler = callback;
|
||||
},
|
||||
},
|
||||
});
|
||||
return async (url, headers = {}) => {
|
||||
const res = {
|
||||
headers: {},
|
||||
writeHead(status, values) {
|
||||
this.status = status;
|
||||
this.headers = values;
|
||||
},
|
||||
setHeader(name, value) {
|
||||
this.headers[name] = value;
|
||||
},
|
||||
end(body) {
|
||||
this.raw = body;
|
||||
try {
|
||||
this.body = JSON.parse(body);
|
||||
} catch {
|
||||
this.body = null;
|
||||
}
|
||||
},
|
||||
set statusCode(value) {
|
||||
this.status = value;
|
||||
},
|
||||
};
|
||||
await handler(
|
||||
{ url, method: 'GET', headers, socket: { remoteAddress: '127.0.0.1' } },
|
||||
res,
|
||||
);
|
||||
return res;
|
||||
};
|
||||
}
|
||||
|
||||
test('availability is decided by the operator environment and names no secret', () => {
|
||||
const none = libraryAvailability(entries, {});
|
||||
assert.deepEqual(none.get('proxied-points'), {
|
||||
available: true,
|
||||
reason: null,
|
||||
});
|
||||
assert.equal(none.get('keyed-points').reason, 'key-missing');
|
||||
assert.equal(none.get('opt-in-points').reason, 'operator-opt-in');
|
||||
assert.equal(none.get('fixture-pack').reason, 'operator-opt-in');
|
||||
const configured = libraryAvailability(entries, {
|
||||
FIRMS_MAP_KEY: 'secret',
|
||||
GEV_LIBRARY_ENABLE: 'opt-in-points',
|
||||
GEV_LIBRARY_DISABLE: 'proxied-points',
|
||||
CCTV_FIXTURE_ENABLED: '1',
|
||||
});
|
||||
assert.equal(configured.get('keyed-points').available, true);
|
||||
assert.equal(configured.get('opt-in-points').available, true);
|
||||
assert.equal(configured.get('proxied-points').reason, 'operator-disabled');
|
||||
assert.equal(configured.get('fixture-pack').available, true);
|
||||
assert.doesNotMatch(JSON.stringify([...configured]), /secret/);
|
||||
});
|
||||
|
||||
test('the catalog route lists every entry with its availability', async () => {
|
||||
const request = install();
|
||||
const res = await request('/catalog');
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.body.entries.length, entries.length);
|
||||
assert.deepEqual(
|
||||
res.body.entries.find(({ id }) => id === 'opt-in-points'),
|
||||
{ id: 'opt-in-points', available: false, reason: 'operator-opt-in' },
|
||||
);
|
||||
});
|
||||
|
||||
test('a proxied dataset reaches only its manifest URL, refuses redirects and is normalized', async () => {
|
||||
const calls = [];
|
||||
const request = install({
|
||||
now: () => 42,
|
||||
fetchImpl: async (url, options) => {
|
||||
calls.push({ url, options });
|
||||
return new Response(collection([point('Storm'), { geometry: null }]));
|
||||
},
|
||||
});
|
||||
const res = await request('/data/proxied-points?url=https://evil.example');
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].url, 'https://data.example.org/points.json');
|
||||
assert.equal(calls[0].options.redirect, 'error');
|
||||
assert.ok(calls[0].options.signal instanceof AbortSignal);
|
||||
assert.equal(res.body.features.length, 1);
|
||||
assert.equal(res.body.features[0].properties.name, 'Storm');
|
||||
assert.equal(res.body.dropped, 1);
|
||||
assert.equal(res.body.fetchedAt, 42);
|
||||
// A fresh cache answers without another upstream request.
|
||||
await request('/data/proxied-points');
|
||||
assert.equal(calls.length, 1);
|
||||
});
|
||||
|
||||
test('browser-direct, unknown, opt-in and wrong-kind entries are not proxied', async () => {
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: async () => {
|
||||
calls++;
|
||||
return new Response(collection([]));
|
||||
},
|
||||
});
|
||||
assert.equal((await request('/data/direct-points')).status, 404);
|
||||
assert.equal((await request('/data/nope-nope')).status, 404);
|
||||
assert.equal((await request('/data/proxied-tiles')).status, 404);
|
||||
assert.equal((await request('/tile/proxied-points/2/0/0')).status, 404);
|
||||
const optIn = await request('/data/opt-in-points');
|
||||
assert.equal(optIn.status, 403);
|
||||
assert.equal(optIn.body.error, 'operator-opt-in');
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
test('a keyed entry gets the operator key server-side and refuses forwarded requests', async () => {
|
||||
const calls = [];
|
||||
const request = install({
|
||||
env: { FIRMS_MAP_KEY: 'k3y' },
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(new URL(url));
|
||||
return new Response(collection([point('Fire')]));
|
||||
},
|
||||
});
|
||||
const forwarded = await request('/data/keyed-points', {
|
||||
'x-forwarded-for': '203.0.113.9',
|
||||
});
|
||||
assert.equal(forwarded.status, 403);
|
||||
assert.equal(calls.length, 0);
|
||||
const res = await request('/data/keyed-points');
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(calls[0].searchParams.get('MAP_KEY'), 'k3y');
|
||||
assert.doesNotMatch(res.raw, /k3y/);
|
||||
});
|
||||
|
||||
test('an oversized or failing upstream answers 502 without its text, then serves stale', async () => {
|
||||
let clock = 0;
|
||||
let mode = 'ok';
|
||||
const request = install({
|
||||
now: () => clock,
|
||||
fetchImpl: async () => {
|
||||
if (mode === 'big') return new Response('x'.repeat(8192));
|
||||
if (mode === 'fail')
|
||||
return new Response('secret upstream text', { status: 500 });
|
||||
return new Response(collection([point('A')]));
|
||||
},
|
||||
});
|
||||
mode = 'big';
|
||||
const big = await request('/data/proxied-points');
|
||||
assert.equal(big.status, 502);
|
||||
assert.doesNotMatch(big.raw, /x{20}/);
|
||||
mode = 'ok';
|
||||
assert.equal((await request('/data/proxied-points')).status, 200);
|
||||
clock = 10 * 3600 * 1000;
|
||||
mode = 'fail';
|
||||
const stale = await request('/data/proxied-points');
|
||||
assert.equal(stale.status, 200);
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.headers['X-Data-Stale'], 'true');
|
||||
assert.doesNotMatch(stale.raw, /secret upstream text/);
|
||||
});
|
||||
|
||||
test('cross-site browser requests are refused before any upstream request', async () => {
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: async () => {
|
||||
calls++;
|
||||
return new Response(collection([]));
|
||||
},
|
||||
});
|
||||
const res = await request('/data/proxied-points', {
|
||||
host: 'localhost:5173',
|
||||
origin: 'https://evil.example',
|
||||
'sec-fetch-site': 'cross-site',
|
||||
});
|
||||
assert.equal(res.status, 403);
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
test('tiles: bounded z/x/y, image types only, cached', async () => {
|
||||
const calls = [];
|
||||
const request = install({
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(url);
|
||||
if (url.includes('/3/'))
|
||||
return new Response('<html>', {
|
||||
headers: { 'content-type': 'text/html' },
|
||||
});
|
||||
return new Response(new Uint8Array([137, 80, 78, 71]), {
|
||||
headers: { 'content-type': 'image/png' },
|
||||
});
|
||||
},
|
||||
});
|
||||
assert.equal((await request('/tile/proxied-tiles/1/0/0')).status, 400);
|
||||
assert.equal((await request('/tile/proxied-tiles/2/4/0')).status, 400);
|
||||
assert.equal((await request('/tile/proxied-tiles/2/a/0')).status, 400);
|
||||
const ok = await request('/tile/proxied-tiles/2/1/3');
|
||||
assert.equal(ok.status, 200);
|
||||
assert.equal(ok.headers['Content-Type'], 'image/png');
|
||||
assert.equal(calls[0], 'https://tiles.example.org/2/1/3.png');
|
||||
await request('/tile/proxied-tiles/2/1/3');
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal((await request('/tile/proxied-tiles/3/1/1')).status, 502);
|
||||
});
|
||||
|
||||
test('tile paths are parsed against the zoom range', () => {
|
||||
const source = { minZoom: 0, maxZoom: 3 };
|
||||
assert.deepEqual(parseLibraryTilePath(['3', '7', '7'], source), {
|
||||
z: 3,
|
||||
x: 7,
|
||||
y: 7,
|
||||
});
|
||||
assert.equal(parseLibraryTilePath(['3', '8', '0'], source), null);
|
||||
assert.equal(parseLibraryTilePath(['4', '0', '0'], source), null);
|
||||
assert.equal(parseLibraryTilePath(['1', '0'], source), null);
|
||||
});
|
||||
|
||||
test('only GET is served', async () => {
|
||||
const plugin = libraryProxy({ entries, env: {} });
|
||||
const res = {
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
this.headers = headers;
|
||||
},
|
||||
end() {},
|
||||
};
|
||||
await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res);
|
||||
assert.equal(res.status, 405);
|
||||
});
|
||||
|
||||
test('a partition index is cached only when complete; a complete one outlives failed rebuilds', async () => {
|
||||
const epc = libraryEntry('epc-england-wales');
|
||||
let clock = 0;
|
||||
let footers = 'ok';
|
||||
let listings = 0;
|
||||
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
|
||||
const fetchImpl = async (url) => {
|
||||
if (url.includes('list-type')) {
|
||||
listings += 1;
|
||||
return new Response(listing);
|
||||
}
|
||||
return footers === 'ok'
|
||||
? new Response(new Uint8Array(100))
|
||||
: new Response('', { status: 503 });
|
||||
};
|
||||
const request = install({ catalog: [epc], now: () => clock, fetchImpl });
|
||||
const first = await request('/index/epc-england-wales');
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(first.body.complete, true);
|
||||
const builtAt = first.body.builtAt;
|
||||
|
||||
// Past cacheSeconds the rebuild fails: the complete index is still served,
|
||||
// and no rebuild runs again until the partial pause has passed.
|
||||
clock += (epc.source.cacheSeconds + 1) * 1000;
|
||||
footers = 'fail';
|
||||
const kept = await request('/index/epc-england-wales');
|
||||
assert.equal(kept.body.complete, true);
|
||||
assert.equal(kept.body.builtAt, builtAt);
|
||||
const listed = listings;
|
||||
await request('/index/epc-england-wales');
|
||||
assert.equal(listings, listed);
|
||||
|
||||
// A server that never had a complete index serves the partial one as such,
|
||||
// and rebuilds after the pause.
|
||||
const fresh = install({ catalog: [epc], now: () => clock, fetchImpl });
|
||||
const partial = await fresh('/index/epc-england-wales');
|
||||
assert.equal(partial.status, 200);
|
||||
assert.equal(partial.body.complete, false);
|
||||
const before = listings;
|
||||
await fresh('/index/epc-england-wales');
|
||||
assert.equal(listings, before);
|
||||
clock += 6 * 60_000;
|
||||
footers = 'ok';
|
||||
const rebuilt = await fresh('/index/epc-england-wales');
|
||||
assert.equal(rebuilt.body.complete, true);
|
||||
assert.equal(listings, before + 1);
|
||||
});
|
||||
@@ -27,13 +27,13 @@ import { normalizeDump1090Aircraft } from '../../src/sources/adsbRecords.js';
|
||||
*/
|
||||
|
||||
export const LOCAL_RECEIVERS_ROUTE = '/api/local-receivers/aircraft';
|
||||
export const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
|
||||
export const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
|
||||
export const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
|
||||
export const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
|
||||
export const LOCAL_RECEIVER_CACHE_MS = 1_000;
|
||||
const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
|
||||
const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
|
||||
const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
|
||||
const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
|
||||
const LOCAL_RECEIVER_CACHE_MS = 1_000;
|
||||
/** A feed whose own `now` is older than this is reported stale. */
|
||||
export const LOCAL_RECEIVER_STALE_MS = 10_000;
|
||||
const LOCAL_RECEIVER_STALE_MS = 10_000;
|
||||
const MAX_FEEDS = 8;
|
||||
const LOG_PREFIX = '[local-receivers]';
|
||||
|
||||
@@ -139,7 +139,7 @@ class FeedError extends Error {
|
||||
* @param {string} address Resolved IP address.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isLocalReceiverAddress(address) {
|
||||
function isLocalReceiverAddress(address) {
|
||||
const value = String(address || '').toLowerCase();
|
||||
if (value === '::1') return true;
|
||||
const mapped = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(value);
|
||||
@@ -152,7 +152,7 @@ export function isLocalReceiverAddress(address) {
|
||||
* @param {typeof dnsLookup} lookupImpl `dns.promises.lookup`-compatible.
|
||||
* @returns {Promise<Array<{address:string, family:number}>>}
|
||||
*/
|
||||
export async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
|
||||
async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
|
||||
let resolved;
|
||||
try {
|
||||
resolved = await lookupImpl(hostname, { all: true, verbatim: true });
|
||||
@@ -274,7 +274,7 @@ function fetchLocalReceiverFeed(url, options) {
|
||||
* @param {object} options
|
||||
* @returns {Promise<object>} Parsed JSON document.
|
||||
*/
|
||||
export async function fetchLocalReceiverDocument(
|
||||
async function fetchLocalReceiverDocument(
|
||||
url,
|
||||
{
|
||||
fetchImpl = fetchLocalReceiverFeed,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import path from 'node:path';
|
||||
import { openSkyProxy } from './aircraft/opensky.js';
|
||||
import { celestrakProxy, rocketLaunchesProxy } from './space.js';
|
||||
import { tomtomProxy } from './traffic.js';
|
||||
@@ -7,7 +8,7 @@ import { adsbdbProxy } from './aircraft/enrichment.js';
|
||||
import { overpassProxy } from './overpass.js';
|
||||
import { militaryInstallationsProxy } from './military-installations.js';
|
||||
import { regionalBriefProxy } from './regional/briefing.js';
|
||||
import { geocodeProxy } from './regional/place.js';
|
||||
import { geocodeProxy, NOMINATIM_CACHE_DIR } from './regional/place.js';
|
||||
import { weatherEffectsProxy } from './regional/weather-effects.js';
|
||||
import { cctvProxy } from './cctv.js';
|
||||
import { defaultSourceRoot } from './common/source-root.js';
|
||||
@@ -15,6 +16,7 @@ import { radioBrowserProxy } from './radio.js';
|
||||
import { gbfsProxy } from './gbfs.js';
|
||||
import { localReceiversProxy } from './local-receivers.js';
|
||||
import { transitProxy } from './transit.js';
|
||||
import { railProxy } from './rail.js';
|
||||
import { adsbLolProxy } from './aircraft/adsb-lol.js';
|
||||
import { aisLiveProxy } from './vessels/ais-live.js';
|
||||
import { trackBackfillProxies } from './aircraft/tracks.js';
|
||||
@@ -26,14 +28,33 @@ import { keySetupEndpoint } from '../standalone/key-setup.js';
|
||||
import { weatherProxy } from './weather.js';
|
||||
import { firePerimetersProxy } from './firePerimeters.js';
|
||||
import { gnssIntegrityProxy } from './gnssIntegrity.js';
|
||||
import { ghostMapsProxy } from './ghostMaps.js';
|
||||
import { shipwrecksProxy } from './shipwrecks.js';
|
||||
import { droneAirspaceProxy } from './droneAirspace.js';
|
||||
import { newsProxy } from './news.js';
|
||||
import { inaturalistProxy } from './inaturalist/observations.js';
|
||||
import { acartiaProxy } from './acartia.js';
|
||||
import { cycloneProxy } from './cyclones.js';
|
||||
import { windProxy } from './wind.js';
|
||||
import { libraryProxy } from './library.js';
|
||||
|
||||
/**
|
||||
* Construct the local provider plugins in their established order.
|
||||
* `realtime` configures OpenAI; `gemini` explicitly opts into Gemini Live.
|
||||
* `nominatimStorageDir` binds an owned absolute directory before construction;
|
||||
* omission retains the durable per-source cache and policy state.
|
||||
*/
|
||||
function localProviderPlugins({ realtime, gemini } = {}) {
|
||||
function localProviderPlugins({
|
||||
realtime,
|
||||
gemini,
|
||||
nominatimStorageDir = NOMINATIM_CACHE_DIR,
|
||||
} = {}) {
|
||||
if (
|
||||
typeof nominatimStorageDir !== 'string' ||
|
||||
!nominatimStorageDir.trim() ||
|
||||
!path.isAbsolute(nominatimStorageDir)
|
||||
)
|
||||
throw new TypeError('nominatimStorageDir must be an absolute directory');
|
||||
return [
|
||||
openSkyProxy(),
|
||||
celestrakProxy(),
|
||||
@@ -45,13 +66,14 @@ function localProviderPlugins({ realtime, gemini } = {}) {
|
||||
overpassProxy(),
|
||||
militaryInstallationsProxy(),
|
||||
regionalBriefProxy(),
|
||||
geocodeProxy(),
|
||||
geocodeProxy({ storageDir: nominatimStorageDir }),
|
||||
weatherEffectsProxy(),
|
||||
cctvProxy({ sourceRoot: defaultSourceRoot }),
|
||||
radioBrowserProxy(),
|
||||
gbfsProxy(),
|
||||
localReceiversProxy(),
|
||||
transitProxy(),
|
||||
railProxy(),
|
||||
adsbLolProxy(),
|
||||
aisLiveProxy(),
|
||||
trackBackfillProxies(),
|
||||
@@ -64,6 +86,13 @@ function localProviderPlugins({ realtime, gemini } = {}) {
|
||||
mapillaryProxy(),
|
||||
firePerimetersProxy(),
|
||||
gnssIntegrityProxy(),
|
||||
libraryProxy(),
|
||||
ghostMapsProxy(),
|
||||
shipwrecksProxy(),
|
||||
droneAirspaceProxy(),
|
||||
newsProxy(),
|
||||
inaturalistProxy(),
|
||||
acartiaProxy(),
|
||||
keySetupEndpoint(),
|
||||
];
|
||||
}
|
||||
|
||||
+101
-14
@@ -1,19 +1,106 @@
|
||||
import { installMapillaryRoutes } from './mapillary/routes.js';
|
||||
import { sameSiteGated } from './common/same-site.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import {
|
||||
fetchTile,
|
||||
mapillaryToken,
|
||||
parseTilePath,
|
||||
retryAfterWait,
|
||||
TileUpstreamError,
|
||||
} from './mapillary/tiles.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: cached Mapillary coverage tiles (token added server-side) and a
|
||||
* status route that says whether a token is configured.
|
||||
* Tile requests per client IP per minute, cache hits included: over 60 new
|
||||
* 9-tile views a minute, more than a person flying the camera reaches.
|
||||
*/
|
||||
function mapillaryProxy() {
|
||||
return {
|
||||
name: 'mapillary-proxy',
|
||||
configureServer(server) {
|
||||
installMapillaryRoutes(server.middlewares);
|
||||
},
|
||||
configurePreviewServer(server) {
|
||||
installMapillaryRoutes(server.middlewares);
|
||||
},
|
||||
};
|
||||
export const TILE_ROUTE_MAX_PER_MIN = 600;
|
||||
|
||||
function sendJson(res, status, payload) {
|
||||
res.statusCode = status;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
export { mapillaryProxy };
|
||||
/** GET /api/mapillary/status — whether a token is configured, never its value. */
|
||||
function handleStatus(req, res) {
|
||||
if (req.method !== 'GET')
|
||||
return sendJson(res, 405, { error: 'Method not allowed' });
|
||||
sendJson(res, 200, { configured: Boolean(mapillaryToken()) });
|
||||
}
|
||||
|
||||
/** GET /api/mapillary/tiles/coverage/{z}/{x}/{y} — trimmed protobuf tile. */
|
||||
async function handleTile(req, res, allow) {
|
||||
if (req.method !== 'GET')
|
||||
return sendJson(res, 405, { error: 'Method not allowed' });
|
||||
if (!allow(clientKey(req))) {
|
||||
res.setHeader('Retry-After', '5');
|
||||
return sendJson(res, 429, {
|
||||
error: 'Too many tile requests',
|
||||
retryAfter: 5,
|
||||
});
|
||||
}
|
||||
const address = parseTilePath((req.url || '').split('?')[0]);
|
||||
if (!address)
|
||||
return sendJson(res, 400, {
|
||||
error: 'Tile path must be /coverage/{z}/{x}/{y} with z 11–14',
|
||||
});
|
||||
if (!mapillaryToken())
|
||||
return sendJson(res, 503, { error: 'no_key', keyRequired: true });
|
||||
// A client that leaves (a tile the camera moved past) cancels its fetch.
|
||||
const left = new AbortController();
|
||||
res.on?.('close', () => {
|
||||
if (!res.writableEnded) left.abort();
|
||||
});
|
||||
try {
|
||||
const { bytes, source } = await fetchTile(address, { signal: left.signal });
|
||||
res.statusCode = bytes.length ? 200 : 204;
|
||||
res.setHeader('Content-Type', 'application/x-protobuf');
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
res.setHeader('X-Gev-Cache', source);
|
||||
res.end(bytes.length ? bytes : undefined);
|
||||
} catch (error) {
|
||||
if (left.signal.aborted) return;
|
||||
const status = error instanceof TileUpstreamError ? error.status : 0;
|
||||
// A rejected token is a key problem the panel can name, not a fault.
|
||||
if (status === 401 || status === 403)
|
||||
return sendJson(res, 403, {
|
||||
error: 'Mapillary rejected the access token',
|
||||
keyRejected: true,
|
||||
});
|
||||
if (status === 429) {
|
||||
const retryAfter = retryAfterWait(error.retryAfter);
|
||||
res.setHeader('Retry-After', String(retryAfter));
|
||||
return sendJson(res, 429, {
|
||||
error: 'Mapillary is rate-limiting tile requests',
|
||||
retryAfter,
|
||||
});
|
||||
}
|
||||
// An upstream 400 must not read as a malformed request to this proxy.
|
||||
sendJson(res, 502, { error: error?.message || 'Tile fetch failed' });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Vite plugin: Mapillary coverage tiles with the token added server-side, and
|
||||
* a status route. Both refuse cross-site requests so another page cannot
|
||||
* spend the token.
|
||||
*/
|
||||
export function mapillaryProxy() {
|
||||
const install = (middlewares) => {
|
||||
const allow = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: TILE_ROUTE_MAX_PER_MIN,
|
||||
globalMax: TILE_ROUTE_MAX_PER_MIN * 4,
|
||||
});
|
||||
middlewares.use('/api/mapillary/status', sameSiteGated(handleStatus));
|
||||
middlewares.use(
|
||||
'/api/mapillary/tiles',
|
||||
sameSiteGated((req, res) => handleTile(req, res, allow)),
|
||||
);
|
||||
};
|
||||
return {
|
||||
name: 'mapillary-proxy',
|
||||
configureServer: (server) => install(server.middlewares),
|
||||
configurePreviewServer: (server) => install(server.middlewares),
|
||||
};
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
import path from 'node:path';
|
||||
|
||||
/** Mapillary vector tile root; the path template is /{layer}/2/{z}/{x}/{y}. */
|
||||
export const MAPILLARY_TILE_HOST = 'https://tiles.mapillary.com/maps/vtp';
|
||||
|
||||
/** Public tile layer names this proxy exposes, mapped to Mapillary's ids. */
|
||||
export const TILE_LAYERS = Object.freeze({
|
||||
// Region-sized z6–10 tiles are refused; the app never asks for them. The
|
||||
// z14 `image` layer is ~98% of a tile and unused (positions come from the
|
||||
// graph API), so it is dropped in transit.
|
||||
coverage: Object.freeze({
|
||||
upstream: 'mly1_public',
|
||||
zoomRanges: Object.freeze([Object.freeze([0, 5]), Object.freeze([11, 14])]),
|
||||
dropLayers: Object.freeze(['image']),
|
||||
}),
|
||||
});
|
||||
|
||||
/** Disk cache root, alongside the other providers' caches. */
|
||||
export const MAPILLARY_CACHE_DIR = path.join(
|
||||
process.cwd(),
|
||||
'.gev-cache',
|
||||
'mapillary',
|
||||
);
|
||||
export const TILE_DISK_DIR = path.join(MAPILLARY_CACHE_DIR, 'tiles');
|
||||
|
||||
/**
|
||||
* Longer than the 15-minute upstream Cache-Control: tiles change only when new
|
||||
* imagery is processed, and a day avoids re-downloading a city per camera move.
|
||||
*/
|
||||
export const TILE_DISK_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* A tile write starts a background sweep at most this often: expired files go
|
||||
* first, then the oldest until the cache fits in TILE_DISK_MAX_BYTES.
|
||||
*/
|
||||
export const TILE_DISK_SWEEP_INTERVAL_MS = 15 * 60 * 1000;
|
||||
export const TILE_DISK_MAX_BYTES = 1024 * 1024 * 1024;
|
||||
/** Per-file cost on top of its size, so empty tiles still count. */
|
||||
export const TILE_DISK_FILE_OVERHEAD_BYTES = 4096;
|
||||
|
||||
/** A z14 image tile over a dense city is ~11 MB; anything past this is wrong. */
|
||||
export const TILE_MAX_BYTES = 48 * 1024 * 1024;
|
||||
|
||||
export const TILE_MEMORY_BUDGET_BYTES = 96 * 1024 * 1024;
|
||||
/** Per-entry cost on top of its bytes, so empty tiles still count. */
|
||||
export const TILE_MEMORY_ENTRY_OVERHEAD_BYTES = 1024;
|
||||
export const TILE_FETCH_TIMEOUT_MS = 60_000;
|
||||
/**
|
||||
* Upstream fetches at once; further misses queue. A view needs at most 25
|
||||
* tiles, and the cap stops a burst of misses opening thousands of requests.
|
||||
*/
|
||||
export const TILE_UPSTREAM_CONCURRENCY = 6;
|
||||
/**
|
||||
* Tile requests per client IP per minute, cache hits included. 600 is 24 new
|
||||
* 25-tile views a minute, more than a person flying the camera reaches, but it
|
||||
* stops one page draining the shared token into a Mapillary rate limit.
|
||||
*/
|
||||
export const TILE_ROUTE_MAX_PER_MIN = 600;
|
||||
/**
|
||||
* How long a 401/403 answers tile misses before asking Mapillary again. A
|
||||
* changed token is tried at once.
|
||||
*/
|
||||
export const TILE_KEY_REJECTED_HOLD_MS = 5 * 60 * 1000;
|
||||
/** After a 429 without a usable Retry-After, hold tile misses this long. */
|
||||
export const TILE_RATE_LIMIT_HOLD_MS = 60_000;
|
||||
/** The longest Retry-After honoured, so a bad header cannot stall coverage. */
|
||||
export const TILE_RATE_LIMIT_MAX_HOLD_MS = 10 * 60 * 1000;
|
||||
|
||||
/** The client token lives in the browser by design; the server adds it to tile URLs too. */
|
||||
export function mapillaryToken() {
|
||||
return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim();
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
import { sameSiteGated } from '../common/same-site.js';
|
||||
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { fetchTile, TileRequestError, TileUpstreamError } from './tiles.js';
|
||||
import { mapillaryToken, TILE_ROUTE_MAX_PER_MIN } from './constants.js';
|
||||
|
||||
function sendJson(res, status, payload) {
|
||||
res.statusCode = status;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
/** GET /api/mapillary/status — whether a token is configured, never its value. */
|
||||
function handleStatus(req, res) {
|
||||
if (req.method !== 'GET')
|
||||
return sendJson(res, 405, { error: 'Method not allowed' });
|
||||
sendJson(res, 200, { configured: Boolean(mapillaryToken()) });
|
||||
}
|
||||
|
||||
/** GET /api/mapillary/tiles/coverage/{z}/{x}/{y} — cached protobuf tile. */
|
||||
async function handleTile(req, res, allow) {
|
||||
if (req.method !== 'GET')
|
||||
return sendJson(res, 405, { error: 'Method not allowed' });
|
||||
if (!allow(clientKey(req))) {
|
||||
res.setHeader('Retry-After', '5');
|
||||
return sendJson(res, 429, {
|
||||
error: 'Too many tile requests',
|
||||
retryAfter: 5,
|
||||
});
|
||||
}
|
||||
const match = /^\/(coverage)\/(\d{1,2})\/(\d{1,6})\/(\d{1,6})$/.exec(
|
||||
(req.url || '').split('?')[0],
|
||||
);
|
||||
if (!match)
|
||||
return sendJson(res, 400, {
|
||||
error: 'Tile path must be /coverage/{z}/{x}/{y}',
|
||||
});
|
||||
if (!mapillaryToken())
|
||||
return sendJson(res, 503, { error: 'no_key', keyRequired: true });
|
||||
const abandoned = new AbortController();
|
||||
req.on?.('aborted', () => abandoned.abort());
|
||||
res.on?.('close', () => abandoned.abort());
|
||||
try {
|
||||
const { bytes, source } = await fetchTile(
|
||||
{ layer: match[1], z: match[2], x: match[3], y: match[4] },
|
||||
{ signal: abandoned.signal },
|
||||
);
|
||||
if (res.writableEnded) return;
|
||||
res.statusCode = bytes.length ? 200 : 204;
|
||||
res.setHeader('Content-Type', 'application/x-protobuf');
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
res.setHeader('X-Gev-Cache', source);
|
||||
res.end(bytes.length ? bytes : undefined);
|
||||
} catch (error) {
|
||||
if (res.writableEnded || abandoned.signal.aborted) return;
|
||||
if (error instanceof TileRequestError)
|
||||
return sendJson(res, error.status, { error: error.message });
|
||||
if (error instanceof TileUpstreamError) {
|
||||
// A rejected token is a key problem the panel can name, not a fault.
|
||||
if (error.keyRejected)
|
||||
return sendJson(res, 403, {
|
||||
error: 'Mapillary rejected the access token',
|
||||
keyRejected: true,
|
||||
});
|
||||
if (error.status === 429) {
|
||||
const retryAfter = error.retryAfterSec || 60;
|
||||
res.setHeader('Retry-After', String(retryAfter));
|
||||
return sendJson(res, 429, {
|
||||
error: 'Mapillary is rate-limiting tile requests',
|
||||
retryAfter,
|
||||
});
|
||||
}
|
||||
// An upstream 400 must not read as a malformed request to this proxy.
|
||||
return sendJson(res, 502, { error: error.message });
|
||||
}
|
||||
sendJson(res, 502, { error: error?.message || 'Tile fetch failed' });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach the Mapillary routes. Both refuse cross-site requests so another page
|
||||
* cannot spend the token or provoke a 429 that holds every tile miss.
|
||||
*/
|
||||
export function installMapillaryRoutes(middlewares) {
|
||||
const allow = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: TILE_ROUTE_MAX_PER_MIN,
|
||||
globalMax: TILE_ROUTE_MAX_PER_MIN * 4,
|
||||
});
|
||||
middlewares.use('/api/mapillary/status', sameSiteGated(handleStatus));
|
||||
middlewares.use(
|
||||
'/api/mapillary/tiles',
|
||||
sameSiteGated((req, res) => handleTile(req, res, allow)),
|
||||
);
|
||||
}
|
||||
+216
-469
@@ -1,518 +1,265 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { readResponseBytesCapped } from '../common/http.js';
|
||||
import { stripTileLayers } from './trim.js';
|
||||
import {
|
||||
MAPILLARY_TILE_HOST,
|
||||
TILE_LAYERS,
|
||||
TILE_DISK_DIR,
|
||||
TILE_DISK_TTL_MS,
|
||||
TILE_DISK_SWEEP_INTERVAL_MS,
|
||||
TILE_DISK_MAX_BYTES,
|
||||
TILE_DISK_FILE_OVERHEAD_BYTES,
|
||||
TILE_MAX_BYTES,
|
||||
TILE_MEMORY_BUDGET_BYTES,
|
||||
TILE_MEMORY_ENTRY_OVERHEAD_BYTES,
|
||||
TILE_FETCH_TIMEOUT_MS,
|
||||
TILE_UPSTREAM_CONCURRENCY,
|
||||
TILE_KEY_REJECTED_HOLD_MS,
|
||||
TILE_RATE_LIMIT_HOLD_MS,
|
||||
TILE_RATE_LIMIT_MAX_HOLD_MS,
|
||||
mapillaryToken,
|
||||
} from './constants.js';
|
||||
|
||||
/** Log without the stack; the token is never in scope here. */
|
||||
function warn(what, error) {
|
||||
console.warn(`[Mapillary Proxy] ${what}:`, error?.message || error);
|
||||
/** Mapillary vector tiles, /{layer}/2/{z}/{x}/{y}: the only host the token goes to. */
|
||||
const TILE_HOST = 'https://tiles.mapillary.com/maps/vtp';
|
||||
/** Only the street zooms the app draws are proxied. */
|
||||
const MIN_ZOOM = 11;
|
||||
const MAX_ZOOM = 14;
|
||||
/**
|
||||
* The z14 `image` layer is ~98% of a tile (12 MB over a dense city) and unused
|
||||
* (positions come from the Graph API), so it is dropped in transit.
|
||||
*/
|
||||
const DROP_LAYERS = ['image'];
|
||||
/** A z14 tile with its image layer is ~12 MB; anything past this is wrong. */
|
||||
export const TILE_MAX_BYTES = 48 * 1024 * 1024;
|
||||
const FETCH_TIMEOUT_MS = 60_000;
|
||||
/** Upstream fetches at once; more wait their turn. */
|
||||
export const TILE_UPSTREAM_CONCURRENCY = 6;
|
||||
/** Tiles change only when new imagery is processed. */
|
||||
export const TILE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
/** Trimmed tiles are ~80 KB: a dozen 9-tile views, well under 100 MB. */
|
||||
export const TILE_MEMORY_MAX_TILES = 120;
|
||||
/** A trimmed tile is ~80 KB; a body this big failed to trim and is not kept. */
|
||||
export const TILE_MEMORY_MAX_TILE_BYTES = 4 * 1024 * 1024;
|
||||
/** The longest upstream Retry-After honoured, in seconds. */
|
||||
export const MAX_RETRY_AFTER_SEC = 600;
|
||||
|
||||
/** The client token lives in the browser by design; the server adds it to tile URLs too. */
|
||||
export function mapillaryToken() {
|
||||
return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim();
|
||||
}
|
||||
|
||||
/** Thrown for a request this proxy refuses before contacting Mapillary. */
|
||||
export class TileRequestError extends Error {
|
||||
constructor(message, status = 400) {
|
||||
super(message);
|
||||
this.name = 'TileRequestError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
/** Mapillary answered with an error status; 401/403 set `keyRejected`. */
|
||||
/** Mapillary answered with an error status (or a size this proxy refuses). */
|
||||
export class TileUpstreamError extends Error {
|
||||
constructor(status, message, { retryAfterSec = null } = {}) {
|
||||
constructor(status, message, retryAfter = null) {
|
||||
super(message || `Mapillary tiles HTTP ${status}`);
|
||||
this.name = 'TileUpstreamError';
|
||||
this.status = status;
|
||||
this.keyRejected = status === 401 || status === 403;
|
||||
this.retryAfterSec = retryAfterSec;
|
||||
/** The upstream Retry-After header, as sent. */
|
||||
this.retryAfter = retryAfter;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A rejected token or rate limit that answers every tile miss until it expires
|
||||
* (or the token changes). Cached tiles are still served meanwhile.
|
||||
* @type {{status: number, until: number, token: string}|null}
|
||||
*/
|
||||
let _upstreamHold = null;
|
||||
|
||||
/** Milliseconds to wait from a Retry-After header (seconds or an HTTP date). */
|
||||
function retryAfterMs(header, now = Date.now()) {
|
||||
/** Seconds a Retry-After header asks for (an integer or an HTTP date), else null. */
|
||||
function retryAfterSeconds(header, now = Date.now()) {
|
||||
const text = String(header ?? '').trim();
|
||||
let ms = NaN;
|
||||
if (/^\d+$/.test(text)) ms = Number(text) * 1000;
|
||||
else if (text) ms = Date.parse(text) - now;
|
||||
if (!Number.isFinite(ms) || ms <= 0) return TILE_RATE_LIMIT_HOLD_MS;
|
||||
return Math.min(ms, TILE_RATE_LIMIT_MAX_HOLD_MS);
|
||||
if (!text) return null;
|
||||
const seconds = /^\d+$/.test(text)
|
||||
? Number(text)
|
||||
: (Date.parse(text) - now) / 1000;
|
||||
return Number.isFinite(seconds) && seconds > 0 ? Math.ceil(seconds) : null;
|
||||
}
|
||||
|
||||
function heldRefusal(token) {
|
||||
const hold = _upstreamHold;
|
||||
if (!hold) return null;
|
||||
const left = hold.until - Date.now();
|
||||
if (left <= 0 || hold.token !== token) {
|
||||
_upstreamHold = null;
|
||||
/** The wait a 429 asks for, bounded; a missing or malformed header means a minute. */
|
||||
export function retryAfterWait(header, now = Date.now()) {
|
||||
return Math.min(retryAfterSeconds(header, now) ?? 60, MAX_RETRY_AFTER_SEC);
|
||||
}
|
||||
|
||||
/**
|
||||
* While Mapillary rate-limits, tile misses are refused here until the wait is
|
||||
* over, so the rest of a view (and the fetches queued behind the first 429)
|
||||
* do not go on asking. Scoped to the token that was refused.
|
||||
* @type {{until: number, token: string}|null}
|
||||
*/
|
||||
let refusal = null;
|
||||
|
||||
/** The refusal still in force for `token`, as the error to answer with. */
|
||||
function refusedNow(token) {
|
||||
if (!refusal) return null;
|
||||
const left = refusal.until - Date.now();
|
||||
if (left <= 0 || refusal.token !== token) {
|
||||
refusal = null;
|
||||
return null;
|
||||
}
|
||||
return new TileUpstreamError(hold.status, undefined, {
|
||||
retryAfterSec: Math.ceil(left / 1000),
|
||||
});
|
||||
return new TileUpstreamError(429, undefined, String(Math.ceil(left / 1000)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a tile address against the layer's zoom ranges.
|
||||
* @returns {{layer:string, upstream:string, z:number, x:number, y:number, key:string}}
|
||||
* Parse `/coverage/{z}/{x}/{y}`; null unless z is 11–14 and x/y are on the grid.
|
||||
* @returns {{z: number, x: number, y: number, key: string}|null}
|
||||
*/
|
||||
export function normalizeTileAddress({ layer, z, x, y }) {
|
||||
const spec = Object.hasOwn(TILE_LAYERS, layer) ? TILE_LAYERS[layer] : null;
|
||||
if (!spec) throw new TileRequestError(`Unknown tile layer: ${layer}`);
|
||||
const zi = Number(z);
|
||||
const xi = Number(x);
|
||||
const yi = Number(y);
|
||||
if (
|
||||
![zi, xi, yi].every((v) => Number.isInteger(v) && v >= 0) ||
|
||||
!spec.zoomRanges.some(([min, max]) => zi >= min && zi <= max)
|
||||
)
|
||||
throw new TileRequestError(
|
||||
`Tile zoom for ${layer} must be ${spec.zoomRanges
|
||||
.map(([min, max]) => `${min}–${max}`)
|
||||
.join(' or ')}`,
|
||||
);
|
||||
const n = 2 ** zi;
|
||||
if (xi >= n || yi >= n)
|
||||
throw new TileRequestError('Tile address out of range');
|
||||
return {
|
||||
layer,
|
||||
upstream: spec.upstream,
|
||||
dropLayers: spec.dropLayers || [],
|
||||
z: zi,
|
||||
x: xi,
|
||||
y: yi,
|
||||
key: `${layer}/${zi}/${xi}/${yi}`,
|
||||
};
|
||||
export function parseTilePath(pathname) {
|
||||
const match = /^\/coverage\/(\d{1,2})\/(\d{1,6})\/(\d{1,6})$/.exec(pathname);
|
||||
if (!match) return null;
|
||||
const [z, x, y] = match.slice(1).map(Number);
|
||||
if (z < MIN_ZOOM || z > MAX_ZOOM || x >= 2 ** z || y >= 2 ** z) return null;
|
||||
return { z, x, y, key: `${z}/${x}/${y}` };
|
||||
}
|
||||
|
||||
/** @type {Map<string, {bytes: Buffer, at: number}>} insertion-ordered LRU */
|
||||
const _memory = new Map();
|
||||
let _memoryBytes = 0;
|
||||
const memoryCost = (bytes) => bytes.length + TILE_MEMORY_ENTRY_OVERHEAD_BYTES;
|
||||
/**
|
||||
* Upstream fetches shared by every request for the same tile; the last waiter
|
||||
* to leave cancels it (see `joinFlight`).
|
||||
* @type {Map<string, {controller: AbortController, promise: Promise<Buffer>, waiters: number}>}
|
||||
*/
|
||||
const _inFlight = new Map();
|
||||
|
||||
function memoryGet(key) {
|
||||
const hit = _memory.get(key);
|
||||
if (!hit) return null;
|
||||
// Same life as the disk copy, so a long-running server never serves stale
|
||||
// coverage.
|
||||
if (Date.now() - hit.at > TILE_DISK_TTL_MS) {
|
||||
_memory.delete(key);
|
||||
_memoryBytes -= memoryCost(hit.bytes);
|
||||
return null;
|
||||
}
|
||||
_memory.delete(key);
|
||||
_memory.set(key, hit);
|
||||
return hit.bytes;
|
||||
}
|
||||
|
||||
/** `at` is when Mapillary served the bytes (a disk tile's mtime). */
|
||||
function memoryPut(key, bytes, at = Date.now()) {
|
||||
if (memoryCost(bytes) > TILE_MEMORY_BUDGET_BYTES / 2) return;
|
||||
const existing = _memory.get(key);
|
||||
if (existing) _memoryBytes -= memoryCost(existing.bytes);
|
||||
_memory.set(key, { bytes, at });
|
||||
_memoryBytes += memoryCost(bytes);
|
||||
while (_memoryBytes > TILE_MEMORY_BUDGET_BYTES && _memory.size) {
|
||||
const [oldest, entry] = _memory.entries().next().value;
|
||||
_memory.delete(oldest);
|
||||
_memoryBytes -= memoryCost(entry.bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/** Disk cache root; tests point it at a temporary directory. */
|
||||
let _diskDir = TILE_DISK_DIR;
|
||||
|
||||
function diskPath({ layer, z, x, y }, root = _diskDir) {
|
||||
return path.join(root, layer, String(z), `${x}-${y}.pbf`);
|
||||
}
|
||||
|
||||
async function readDisk(address) {
|
||||
const file = diskPath(address);
|
||||
function trim(bytes) {
|
||||
try {
|
||||
const stat = await fsp.stat(file);
|
||||
if (Date.now() - stat.mtimeMs > TILE_DISK_TTL_MS) return null;
|
||||
return { bytes: await fsp.readFile(file), at: stat.mtimeMs };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Unsettled background writes and sweeps; requests never wait on them. */
|
||||
const _background = new Set();
|
||||
|
||||
function track(task) {
|
||||
const settled = task
|
||||
.catch(() => {})
|
||||
.finally(() => _background.delete(settled));
|
||||
_background.add(settled);
|
||||
return settled;
|
||||
}
|
||||
|
||||
/** @type {Map<string, number>} cache file to its newest write's sequence */
|
||||
const _diskWrites = new Map();
|
||||
let _diskWriteSeq = 0;
|
||||
|
||||
function diskWritePending(address) {
|
||||
return _diskWrites.has(diskPath(address));
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a tile in the background. `at` keeps a rewrite's original fetch time
|
||||
* so it does not extend the tile's life. An overtaken write steps aside.
|
||||
*/
|
||||
function writeDisk(address, bytes, at = null) {
|
||||
// Sweep this write's root even if the cache dir has moved since.
|
||||
const root = _diskDir;
|
||||
const file = diskPath(address, root);
|
||||
const seq = ++_diskWriteSeq;
|
||||
const tmp = `${file}.${process.pid}-${seq}.tmp`;
|
||||
const newest = () => _diskWrites.get(file) === seq;
|
||||
_diskWrites.set(file, seq);
|
||||
return track(
|
||||
fsp
|
||||
.mkdir(path.dirname(file), { recursive: true })
|
||||
.then(() => fsp.writeFile(tmp, bytes))
|
||||
.then(() => at != null && fsp.utimes(tmp, new Date(), new Date(at)))
|
||||
.then(() => {
|
||||
if (!newest()) return fsp.rm(tmp, { force: true });
|
||||
return fsp.rename(tmp, file).then(() => scheduleSweep(root));
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await fsp.rm(tmp, { force: true }).catch(() => {});
|
||||
warn('tile cache write failed', error);
|
||||
})
|
||||
.finally(() => {
|
||||
if (newest()) _diskWrites.delete(file);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
let _lastSweepAt = 0;
|
||||
let _sweeping = null;
|
||||
let _sweepWarned = false;
|
||||
|
||||
function scheduleSweep(root) {
|
||||
if (_sweeping || Date.now() - _lastSweepAt < TILE_DISK_SWEEP_INTERVAL_MS)
|
||||
return;
|
||||
_lastSweepAt = Date.now();
|
||||
const sweep = sweepTileDisk({ root })
|
||||
.catch((error) => {
|
||||
if (_sweepWarned) return;
|
||||
_sweepWarned = true;
|
||||
warn('tile cache sweep failed', error);
|
||||
})
|
||||
.finally(() => {
|
||||
if (_sweeping === tracked) _sweeping = null;
|
||||
});
|
||||
const tracked = track(sweep);
|
||||
_sweeping = tracked;
|
||||
}
|
||||
|
||||
/** Every file under `dir` with size and mtime; a missing dir is empty. */
|
||||
async function listCacheFiles(dir) {
|
||||
let entries;
|
||||
try {
|
||||
entries = await fsp.readdir(dir, { withFileTypes: true });
|
||||
return stripTileLayers(bytes, DROP_LAYERS);
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return [];
|
||||
throw error;
|
||||
}
|
||||
const files = [];
|
||||
for (const entry of entries) {
|
||||
const file = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) files.push(...(await listCacheFiles(file)));
|
||||
else if (entry.isFile()) {
|
||||
const stat = await fsp.stat(file).catch(() => null);
|
||||
if (stat) files.push({ file, size: stat.size, at: stat.mtimeMs });
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete expired tiles, then the oldest until the rest fit in `maxBytes`.
|
||||
* @returns {Promise<{removed: number, bytes: number}>} files deleted, charged bytes kept
|
||||
*/
|
||||
export async function sweepTileDisk({
|
||||
root = _diskDir,
|
||||
maxBytes = TILE_DISK_MAX_BYTES,
|
||||
} = {}) {
|
||||
const now = Date.now();
|
||||
const files = (await listCacheFiles(root)).sort((a, b) => a.at - b.at);
|
||||
const cost = (size) => size + TILE_DISK_FILE_OVERHEAD_BYTES;
|
||||
let bytes = files.reduce((sum, { size }) => sum + cost(size), 0);
|
||||
let removed = 0;
|
||||
for (const { file, size, at } of files) {
|
||||
if (now - at <= TILE_DISK_TTL_MS && bytes <= maxBytes) break;
|
||||
await fsp.rm(file, { force: true });
|
||||
bytes -= cost(size);
|
||||
removed++;
|
||||
}
|
||||
return { removed, bytes };
|
||||
}
|
||||
|
||||
/** The one origin the tile token may be sent to, at every redirect hop. */
|
||||
const TILE_ORIGIN = new URL(MAPILLARY_TILE_HOST).origin;
|
||||
export const TILE_MAX_REDIRECTS = 3;
|
||||
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
||||
|
||||
/**
|
||||
* Follow redirects by hand so every hop stays HTTPS on the tile origin and the
|
||||
* token in the query never reaches another host. A bad redirect is a 502.
|
||||
*/
|
||||
async function fetchPinned(url, init) {
|
||||
let next = url;
|
||||
for (let hop = 0; ; hop++) {
|
||||
const response = await fetch(next, { ...init, redirect: 'manual' });
|
||||
if (!REDIRECT_STATUSES.has(response.status)) return response;
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (hop >= TILE_MAX_REDIRECTS)
|
||||
throw new TileUpstreamError(502, 'Mapillary tile redirected too often');
|
||||
let target = null;
|
||||
try {
|
||||
target = new URL(response.headers.get('location') ?? '', next);
|
||||
} catch {
|
||||
/* unparseable Location */
|
||||
}
|
||||
if (
|
||||
!response.headers.get('location') ||
|
||||
target?.protocol !== 'https:' ||
|
||||
target.origin !== TILE_ORIGIN ||
|
||||
target.username ||
|
||||
target.password
|
||||
)
|
||||
throw new TileUpstreamError(
|
||||
502,
|
||||
'Mapillary tile redirect left the tile origin',
|
||||
);
|
||||
next = target.href;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchUpstream(address, signal) {
|
||||
const token = mapillaryToken();
|
||||
if (!token) throw new TileRequestError('Mapillary token not configured', 503);
|
||||
const held = heldRefusal(token);
|
||||
if (held) throw held;
|
||||
const url = `${MAPILLARY_TILE_HOST}/${address.upstream}/2/${address.z}/${address.x}/${address.y}?access_token=${encodeURIComponent(token)}`;
|
||||
const timeout = AbortSignal.timeout(TILE_FETCH_TIMEOUT_MS);
|
||||
const oversize = new AbortController();
|
||||
const response = await fetchPinned(url, {
|
||||
signal: AbortSignal.any([signal, timeout, oversize.signal].filter(Boolean)),
|
||||
headers: { Accept: 'application/x-protobuf' },
|
||||
});
|
||||
if (response.status === 404 || response.status === 204) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
const error = new TileUpstreamError(response.status);
|
||||
if (error.keyRejected)
|
||||
_upstreamHold = {
|
||||
status: response.status,
|
||||
until: Date.now() + TILE_KEY_REJECTED_HOLD_MS,
|
||||
token,
|
||||
};
|
||||
else if (response.status === 429) {
|
||||
const wait = retryAfterMs(response.headers.get('retry-after'));
|
||||
_upstreamHold = { status: 429, until: Date.now() + wait, token };
|
||||
error.retryAfterSec = Math.ceil(wait / 1000);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
// Chunked or compressed bodies have no Content-Length: the cap applies as
|
||||
// the body streams, and an oversized one also aborts the request.
|
||||
try {
|
||||
const bytes = await readResponseBytesCapped(
|
||||
response,
|
||||
TILE_MAX_BYTES,
|
||||
signal,
|
||||
);
|
||||
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
} catch (error) {
|
||||
if (error?.code !== 'RESPONSE_TOO_LARGE') throw error;
|
||||
oversize.abort();
|
||||
throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap');
|
||||
}
|
||||
}
|
||||
|
||||
/** @type {{active: number, queue: Array<() => void>}} upstream slots */
|
||||
let _slots = { active: 0, queue: [] };
|
||||
|
||||
/**
|
||||
* Resolve with a release function once a slot is free. An abort while
|
||||
* queued leaves the queue.
|
||||
* @returns {Promise<() => void>}
|
||||
*/
|
||||
function acquireUpstreamSlot(signal) {
|
||||
const slots = _slots;
|
||||
return new Promise((resolve, reject) => {
|
||||
if (signal.aborted) return reject(signal.reason);
|
||||
const onAbort = () => {
|
||||
const index = slots.queue.indexOf(grant);
|
||||
if (index !== -1) slots.queue.splice(index, 1);
|
||||
reject(signal.reason);
|
||||
};
|
||||
function grant() {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
slots.active++;
|
||||
let released = false;
|
||||
resolve(() => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
slots.active--;
|
||||
slots.queue.shift()?.();
|
||||
});
|
||||
}
|
||||
if (slots.active < TILE_UPSTREAM_CONCURRENCY) return grant();
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
slots.queue.push(grant);
|
||||
});
|
||||
}
|
||||
|
||||
function trim(address, bytes) {
|
||||
if (!address.dropLayers.length || !bytes.length) return bytes;
|
||||
try {
|
||||
return stripTileLayers(bytes, address.dropLayers);
|
||||
} catch (error) {
|
||||
warn('tile trim failed, serving raw', error);
|
||||
console.warn('[Mapillary Proxy] tile trim failed:', error?.message);
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch one tile from memory, disk, a shared flight, then Mapillary. Empty
|
||||
* bytes mean no coverage.
|
||||
* @param {{layer:string,z:number|string,x:number|string,y:number|string}} request
|
||||
* @param {{signal?: AbortSignal}} [options]
|
||||
* @returns {Promise<{bytes: Buffer, source: 'memory'|'disk'|'inflight'|'upstream', address: object}>}
|
||||
*/
|
||||
export async function fetchTile(request, { signal } = {}) {
|
||||
const address = normalizeTileAddress(request);
|
||||
const memory = memoryGet(address.key);
|
||||
if (memory) return { bytes: memory, source: 'memory', address };
|
||||
const disk = await readDisk(address);
|
||||
if (disk) {
|
||||
// Older cache files may be untrimmed: rewrite once, keeping the fetch time.
|
||||
const bytes = trim(address, disk.bytes);
|
||||
if (bytes !== disk.bytes && !diskWritePending(address))
|
||||
writeDisk(address, bytes, disk.at);
|
||||
memoryPut(address.key, bytes, disk.at);
|
||||
return { bytes, source: 'disk', address };
|
||||
/** One tile from Mapillary, trimmed; empty bytes mean no coverage. */
|
||||
async function fetchUpstream({ z, x, y }, signal) {
|
||||
const token = mapillaryToken();
|
||||
const refused = refusedNow(token);
|
||||
if (refused) throw refused;
|
||||
const response = await fetch(
|
||||
`${TILE_HOST}/mly1_public/2/${z}/${x}/${y}?access_token=${encodeURIComponent(token)}`,
|
||||
{
|
||||
signal: AbortSignal.any([signal, AbortSignal.timeout(FETCH_TIMEOUT_MS)]),
|
||||
// The token is in the query: a redirect is never followed (a 3xx is a 502).
|
||||
redirect: 'manual',
|
||||
headers: { Accept: 'application/x-protobuf' },
|
||||
},
|
||||
);
|
||||
if (!response.ok || response.status === 204) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (response.status === 204 || response.status === 404)
|
||||
return Buffer.alloc(0);
|
||||
const retryAfter = response.headers.get('retry-after');
|
||||
if (response.status === 429)
|
||||
refusal = {
|
||||
token,
|
||||
until: Date.now() + retryAfterWait(retryAfter) * 1000,
|
||||
};
|
||||
throw new TileUpstreamError(response.status, undefined, retryAfter);
|
||||
}
|
||||
try {
|
||||
const bytes = await readResponseBytesCapped(response, TILE_MAX_BYTES);
|
||||
return trim(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length));
|
||||
} catch (error) {
|
||||
if (error?.code !== 'RESPONSE_TOO_LARGE') throw error;
|
||||
throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap');
|
||||
}
|
||||
// A caller gone while the disk was read must not start a fetch nobody joins.
|
||||
signal?.throwIfAborted();
|
||||
let flight = _inFlight.get(address.key);
|
||||
const joined = Boolean(flight) && !flight.controller.signal.aborted;
|
||||
if (!joined) flight = startFlight(address);
|
||||
const bytes = await joinFlight(flight, signal);
|
||||
return { bytes, source: joined ? 'inflight' : 'upstream', address };
|
||||
}
|
||||
|
||||
/** Start the shared upstream fetch; it holds a slot until the body is read. */
|
||||
let upstreamActive = 0;
|
||||
/** @type {Array<() => void>} fetches waiting for a slot, oldest first */
|
||||
const upstreamWaiting = [];
|
||||
|
||||
/**
|
||||
* Run `task` in an upstream slot, waiting for one unless `signal` aborts
|
||||
* first. A freed slot passes straight to the oldest waiter, still counted, so
|
||||
* a request arriving meanwhile cannot take it first.
|
||||
*/
|
||||
async function withUpstreamSlot(signal, task) {
|
||||
if (upstreamActive < TILE_UPSTREAM_CONCURRENCY) upstreamActive++;
|
||||
else
|
||||
await new Promise((resolve, reject) => {
|
||||
const wake = () => {
|
||||
signal.removeEventListener('abort', leave);
|
||||
resolve();
|
||||
};
|
||||
const leave = () => {
|
||||
const at = upstreamWaiting.indexOf(wake);
|
||||
if (at !== -1) upstreamWaiting.splice(at, 1);
|
||||
reject(signal.reason);
|
||||
};
|
||||
upstreamWaiting.push(wake);
|
||||
signal.addEventListener('abort', leave, { once: true });
|
||||
});
|
||||
try {
|
||||
// Handed a slot after leaving: it goes on to the next waiter.
|
||||
signal.throwIfAborted();
|
||||
return await task();
|
||||
} finally {
|
||||
const next = upstreamWaiting.shift();
|
||||
if (next) next();
|
||||
else upstreamActive--;
|
||||
}
|
||||
}
|
||||
|
||||
/** `promise`, or a rejection as soon as `signal` aborts. */
|
||||
function unlessAborted(promise, signal) {
|
||||
if (!signal) return promise;
|
||||
signal.throwIfAborted();
|
||||
return new Promise((resolve, reject) => {
|
||||
const leave = () => reject(signal.reason);
|
||||
signal.addEventListener('abort', leave, { once: true });
|
||||
promise
|
||||
.then(resolve, reject)
|
||||
.finally(() => signal.removeEventListener('abort', leave));
|
||||
});
|
||||
}
|
||||
|
||||
/** @type {Map<string, {bytes: Buffer, at: number}>} least recently used first */
|
||||
const memory = new Map();
|
||||
/**
|
||||
* Upstream fetches shared per tile, with the requests still waiting on each.
|
||||
* @type {Map<string, {promise: Promise<Buffer>, controller: AbortController, waiters: number}>}
|
||||
*/
|
||||
const inFlight = new Map();
|
||||
|
||||
function remember(key, bytes, at = Date.now()) {
|
||||
if (bytes.length > TILE_MEMORY_MAX_TILE_BYTES) return;
|
||||
memory.delete(key);
|
||||
memory.set(key, { bytes, at });
|
||||
for (const oldest of memory.keys()) {
|
||||
if (memory.size <= TILE_MEMORY_MAX_TILES) break;
|
||||
memory.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
function recall(key) {
|
||||
const hit = memory.get(key);
|
||||
if (!hit) return null;
|
||||
if (Date.now() - hit.at > TILE_TTL_MS) {
|
||||
memory.delete(key);
|
||||
return null;
|
||||
}
|
||||
remember(key, hit.bytes, hit.at); // most recently used again
|
||||
return hit.bytes;
|
||||
}
|
||||
|
||||
/** Start the upstream fetch for a tile, shared until its last request leaves. */
|
||||
function startFlight(address) {
|
||||
const controller = new AbortController();
|
||||
const flight = { controller, waiters: 0, promise: null };
|
||||
flight.promise = acquireUpstreamSlot(controller.signal)
|
||||
.then((release) =>
|
||||
fetchUpstream(address, controller.signal).finally(release),
|
||||
)
|
||||
.then((raw) => trim(address, raw))
|
||||
const flight = { controller, waiters: 0 };
|
||||
flight.promise = withUpstreamSlot(controller.signal, () =>
|
||||
fetchUpstream(address, controller.signal),
|
||||
)
|
||||
.then((bytes) => {
|
||||
memoryPut(address.key, bytes);
|
||||
writeDisk(address, bytes);
|
||||
remember(address.key, bytes);
|
||||
return bytes;
|
||||
})
|
||||
.finally(() => {
|
||||
if (_inFlight.get(address.key) === flight) _inFlight.delete(address.key);
|
||||
if (inFlight.get(address.key) === flight) inFlight.delete(address.key);
|
||||
});
|
||||
// Every waiter may have left before it settles; nobody awaits it then.
|
||||
// Every request may have left; the rejection is then nobody's.
|
||||
flight.promise.catch(() => {});
|
||||
_inFlight.set(address.key, flight);
|
||||
inFlight.set(address.key, flight);
|
||||
return flight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait on a shared flight. A caller's abort rejects only that caller; the
|
||||
* fetch is cancelled when the last waiter leaves.
|
||||
* One tile from memory, a shared in-flight fetch, or Mapillary. `signal`
|
||||
* aborts when the request's client leaves; the fetch stops once all have.
|
||||
* @param {{z: number, x: number, y: number, key: string}} address from parseTilePath
|
||||
* @param {{signal?: AbortSignal}} [options]
|
||||
* @returns {Promise<{bytes: Buffer, source: 'memory'|'inflight'|'upstream'}>}
|
||||
*/
|
||||
function joinFlight(flight, signal) {
|
||||
signal?.throwIfAborted();
|
||||
export async function fetchTile(address, { signal } = {}) {
|
||||
const cached = recall(address.key);
|
||||
if (cached) return { bytes: cached, source: 'memory' };
|
||||
let flight = inFlight.get(address.key);
|
||||
const shared = Boolean(flight);
|
||||
flight ||= startFlight(address);
|
||||
flight.waiters++;
|
||||
return new Promise((resolve, reject) => {
|
||||
let done = false;
|
||||
const leave = () => {
|
||||
if (done) return false;
|
||||
done = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
flight.waiters--;
|
||||
return true;
|
||||
};
|
||||
const onAbort = () => {
|
||||
if (!leave()) return;
|
||||
if (flight.waiters === 0) flight.controller.abort();
|
||||
reject(signal.reason);
|
||||
};
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
flight.promise.then(
|
||||
(bytes) => leave() && resolve(bytes),
|
||||
(error) => leave() && reject(error),
|
||||
);
|
||||
});
|
||||
try {
|
||||
const bytes = await unlessAborted(flight.promise, signal);
|
||||
return { bytes, source: shared ? 'inflight' : 'upstream' };
|
||||
} finally {
|
||||
if (--flight.waiters === 0) {
|
||||
// A later request for the tile must not join a cancelled fetch.
|
||||
if (inFlight.get(address.key) === flight) inFlight.delete(address.key);
|
||||
flight.controller.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Test seam: forget every cached tile held in memory. */
|
||||
export function _resetTileMemoryForTest() {
|
||||
_memory.clear();
|
||||
_memoryBytes = 0;
|
||||
_inFlight.clear();
|
||||
_upstreamHold = null;
|
||||
_slots = { active: 0, queue: [] };
|
||||
}
|
||||
|
||||
/** Test seam: how many tiles memory holds and the bytes charged for them. */
|
||||
export function _tileMemoryForTest() {
|
||||
return { entries: _memory.size, bytes: _memoryBytes };
|
||||
}
|
||||
|
||||
/** Test seam: wait for every background write and the sweeps they start. */
|
||||
export async function _settleTileWritesForTest() {
|
||||
while (_background.size) await Promise.allSettled([..._background]);
|
||||
}
|
||||
|
||||
/** Test seam: keep the disk cache in `dir` (null restores the default). */
|
||||
export function _setTileCacheDirForTest(dir) {
|
||||
_diskDir = dir || TILE_DISK_DIR;
|
||||
_lastSweepAt = 0;
|
||||
/** Test seam: forget every cached and in-flight tile. */
|
||||
export function _resetTileCacheForTest() {
|
||||
memory.clear();
|
||||
inFlight.clear();
|
||||
refusal = null;
|
||||
upstreamActive = 0;
|
||||
upstreamWaiting.length = 0;
|
||||
}
|
||||
@@ -41,16 +41,3 @@ export function stripTileLayers(bytes, dropNames) {
|
||||
}, null);
|
||||
return dropped ? Buffer.from(writer.finish()) : bytes;
|
||||
}
|
||||
|
||||
/** Names of the layers inside a vector tile, in order. */
|
||||
export function listTileLayers(bytes) {
|
||||
const names = [];
|
||||
if (!bytes?.length) return names;
|
||||
new PbfReader(bytes).readFields((tag, _result, pbf) => {
|
||||
if (tag !== TILE_LAYER_FIELD) return;
|
||||
const end = pbf.readVarint() + pbf.pos;
|
||||
names.push(layerName(bytes.subarray(pbf.pos, end)));
|
||||
pbf.pos = end;
|
||||
}, null);
|
||||
return names;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { admitSameSite } from './common/same-site.js';
|
||||
import {
|
||||
MILITARY_INSTALLATION_ELEMENT_CAP,
|
||||
MILITARY_INSTALLATION_MAX_RESPONSE_BYTES,
|
||||
@@ -97,6 +98,7 @@ function militaryInstallationsProxy() {
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
if (admitSameSite(req, res)) return;
|
||||
const url = new URL(req.url, 'http://localhost');
|
||||
const requested = validMilitaryInstallationBox(url.searchParams);
|
||||
if (!requested) {
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import { newsQueryAllowed } from '../../src/layers/news/source.js';
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
import { sameSiteGated } from './common/same-site.js';
|
||||
import { createKeyedNews } from './news/keyed.js';
|
||||
|
||||
// News (/api/news/*). With GEV_NEWS_ORIGIN set (a news server such as
|
||||
// https://news.example), this forwards exactly the GET requests the news
|
||||
// layer sends there and relays the JSON. Without it (or with a value that
|
||||
// is no plain origin, warned of once), Core answers the same routes itself
|
||||
// (news/keyed.js): Conflicts from GDELT's event exports with no key, and
|
||||
// Liveuamap's items beside them with LIVEUAMAP_API_KEY; Local from World
|
||||
// News API, "key required" while WORLDNEWSAPI_KEY is unset. A host that
|
||||
// serves news only from its own server mounts it with `builtIn: false`.
|
||||
|
||||
const TIMEOUT_MS = 15_000;
|
||||
const MAX_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* The configured news server's origin, or null: http(s) only, no
|
||||
* credentials, query or fragment; a path is ignored.
|
||||
* @param {string|undefined} raw
|
||||
*/
|
||||
export function newsOrigin(raw = process.env.GEV_NEWS_ORIGIN) {
|
||||
const text = String(raw ?? '').trim();
|
||||
if (!text) return null;
|
||||
try {
|
||||
const url = new URL(text);
|
||||
if (
|
||||
!['http:', 'https:'].includes(url.protocol) ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.search ||
|
||||
url.hash
|
||||
)
|
||||
return null;
|
||||
return url.origin;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Text for the log: no address (a key may ride in one), one short line. */
|
||||
const logText = (error) =>
|
||||
`${error?.name || 'Error'}: ${String(error?.message ?? '')}`
|
||||
.replace(/[a-z][a-z0-9+.-]*:\/\/\S*/gi, '<url>')
|
||||
.replace(/\s+/g, ' ')
|
||||
.slice(0, 200);
|
||||
|
||||
/** GEV_NEWS_ORIGIN as newsOrigin reads it; a value it refuses is warned of once. */
|
||||
function configuredOrigin(env, log) {
|
||||
let warned = null;
|
||||
return () => {
|
||||
const raw = String(env().GEV_NEWS_ORIGIN ?? '').trim();
|
||||
const base = newsOrigin(raw);
|
||||
if (!base && raw && warned !== raw) {
|
||||
warned = raw;
|
||||
// Not the value: it may carry credentials.
|
||||
log(
|
||||
'[news] GEV_NEWS_ORIGIN is not a plain http(s) origin; answering with the built-in news backend',
|
||||
);
|
||||
}
|
||||
return base;
|
||||
};
|
||||
}
|
||||
|
||||
/** A 503's known key or configuration state, rebuilt from its fields; else null. */
|
||||
async function upstreamState(upstream) {
|
||||
let body = null;
|
||||
try {
|
||||
body = JSON.parse(await readResponseTextCapped(upstream, 16 * 1024));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (body?.error === 'news_not_configured')
|
||||
return { error: body.error, unavailable: true, retryable: false };
|
||||
if (
|
||||
['no_key', 'key_rejected', 'key_quota'].includes(body?.error) &&
|
||||
typeof body.key === 'string' &&
|
||||
/^[a-z][a-z0-9-]{0,31}$/.test(body.key)
|
||||
)
|
||||
return { error: body.error, key: body.key };
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A thin, validated GET proxy for the news layer's three read routes, or
|
||||
* their keyed answers without a news server. A host that serves news only
|
||||
* from its own server passes `builtIn: false`: without GEV_NEWS_ORIGIN the
|
||||
* routes then answer "not configured" and Core reads nothing itself.
|
||||
* @param {{env?: () => object, log?: (message: string) => void, origin?: () => string|null, fetchImpl?: typeof fetch, timeoutMs?: number, builtIn?: boolean, keyed?: {answer: (path: string, params: URLSearchParams) => Promise<{status: number, body: object}>}|null}} [options]
|
||||
*/
|
||||
export function newsProxy({
|
||||
env = () => process.env,
|
||||
log = (message) => console.warn(message),
|
||||
origin = configuredOrigin(env, log),
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
timeoutMs = TIMEOUT_MS,
|
||||
builtIn = true,
|
||||
keyed = builtIn ? createKeyedNews({ env, fetchImpl, log }) : null,
|
||||
} = {}) {
|
||||
const json = (res, status, value) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
});
|
||||
res.end(typeof value === 'string' ? value : JSON.stringify(value));
|
||||
};
|
||||
|
||||
async function handler(req, res) {
|
||||
if (req.method !== 'GET')
|
||||
return json(res, 405, { error: 'method_not_allowed' });
|
||||
let local, target;
|
||||
try {
|
||||
// `//[` and the like are no URL: a 400, not an unhandled rejection.
|
||||
local = new URL(req.url || '/', 'http://local');
|
||||
target = new URL(
|
||||
`/api/news${local.pathname}${local.search}`,
|
||||
'http://local',
|
||||
);
|
||||
} catch {
|
||||
return json(res, 400, { error: 'invalid_news_request' });
|
||||
}
|
||||
if (!newsQueryAllowed(target))
|
||||
return json(res, 400, { error: 'invalid_news_request' });
|
||||
const base = origin();
|
||||
if (!base) {
|
||||
if (!keyed)
|
||||
return json(res, 503, {
|
||||
error: 'news_not_configured',
|
||||
unavailable: true,
|
||||
retryable: false,
|
||||
});
|
||||
try {
|
||||
const { status, body } = await keyed.answer(
|
||||
local.pathname,
|
||||
target.searchParams,
|
||||
);
|
||||
return json(res, status, body);
|
||||
} catch (error) {
|
||||
log(`[news] built-in news answer failed: ${logText(error)}`);
|
||||
return json(res, 502, { error: 'news_unavailable' });
|
||||
}
|
||||
}
|
||||
try {
|
||||
const upstream = await fetchImpl(
|
||||
new URL(`${target.pathname}${target.search}`, base),
|
||||
{
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
},
|
||||
);
|
||||
if (upstream.status === 503) {
|
||||
// The news server's own key and configuration states reach the
|
||||
// panel as they are, so it can say which key is missing.
|
||||
const state = await upstreamState(upstream);
|
||||
if (state) return json(res, 503, state);
|
||||
}
|
||||
if (!upstream.ok) {
|
||||
await upstream.body?.cancel().catch(() => {});
|
||||
return json(res, 502, {
|
||||
error: 'news_upstream_error',
|
||||
status: upstream.status,
|
||||
});
|
||||
}
|
||||
const text = await readResponseTextCapped(upstream, MAX_BYTES);
|
||||
// Relayed only as JSON; the browser validates every record.
|
||||
JSON.parse(text);
|
||||
return json(res, 200, text);
|
||||
} catch (error) {
|
||||
return json(res, error?.name === 'TimeoutError' ? 504 : 502, {
|
||||
error: 'news_unavailable',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// The routes spend keys and upstream reads: no other site's page, and no
|
||||
// proxied request, may drive them (SECURITY.md, cross-site gate).
|
||||
const gated = sameSiteGated(handler);
|
||||
return {
|
||||
name: 'news',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/news', gated);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/news', gated);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { newsOrigin, newsProxy } from './news.js';
|
||||
import { createKeyedNews } from './news/keyed.js';
|
||||
|
||||
/** Mount the proxy on a fake middleware stack and call it like Connect does. */
|
||||
function mount(options) {
|
||||
let handler = null;
|
||||
newsProxy(options).configureServer({
|
||||
middlewares: {
|
||||
use(prefix, fn) {
|
||||
assert.equal(prefix, '/api/news');
|
||||
handler = fn;
|
||||
},
|
||||
},
|
||||
});
|
||||
return async (url, method = 'GET', headers = {}) => {
|
||||
const res = {
|
||||
destroyed: false,
|
||||
status: 0,
|
||||
headers: {},
|
||||
body: '',
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
this.headers = headers;
|
||||
},
|
||||
setHeader(name, value) {
|
||||
this.headers[name] = value;
|
||||
},
|
||||
end(body) {
|
||||
if (this.statusCode) this.status = this.statusCode;
|
||||
this.body = body;
|
||||
},
|
||||
};
|
||||
await handler({ url, method, headers }, res);
|
||||
return { ...res, json: res.body ? JSON.parse(res.body) : null };
|
||||
};
|
||||
}
|
||||
|
||||
test('the origin must be a plain http(s) origin', () => {
|
||||
assert.equal(
|
||||
newsOrigin('http://news.example:8080'),
|
||||
'http://news.example:8080',
|
||||
);
|
||||
assert.equal(
|
||||
newsOrigin('https://news.example/base/'),
|
||||
'https://news.example',
|
||||
);
|
||||
for (const bad of [
|
||||
undefined,
|
||||
'',
|
||||
' ',
|
||||
'ftp://x',
|
||||
'http://u:p@x',
|
||||
'http://x/?a=1',
|
||||
'http://x/#f',
|
||||
'not a url',
|
||||
])
|
||||
assert.equal(newsOrigin(bad), null, String(bad));
|
||||
});
|
||||
|
||||
test('without GEV_NEWS_ORIGIN Core answers itself: Conflicts from GDELT, Local saying it needs its key', async () => {
|
||||
const hosts = [];
|
||||
const fetchImpl = async (url) => {
|
||||
hosts.push(new URL(String(url)).host);
|
||||
throw new Error('offline');
|
||||
};
|
||||
const call = mount({
|
||||
origin: () => null,
|
||||
fetchImpl,
|
||||
keyed: createKeyedNews({ env: () => ({}), fetchImpl, log: () => {} }),
|
||||
});
|
||||
// GDELT's exports need no key; offline, the read fails like any upstream.
|
||||
const hotspots = await call('/hotspots');
|
||||
assert.equal(hotspots.status, 502);
|
||||
assert.deepEqual(hotspots.json, { error: 'news_upstream_error' });
|
||||
const local = await call('/feed?hotspot=all&at=31.95,35.93');
|
||||
assert.equal(local.status, 503);
|
||||
assert.deepEqual(local.json, { error: 'no_key', key: 'worldnewsapi' });
|
||||
// The area is read from the bundled countries, with no key.
|
||||
const area = await call('/outline?at=31.95,35.93&v=4');
|
||||
assert.equal(area.status, 200);
|
||||
assert.equal(area.json.area.name, 'Jordan');
|
||||
// The client's allowlist still holds.
|
||||
assert.equal((await call('/feed?side=UA')).status, 400);
|
||||
assert.ok(hosts.length > 0);
|
||||
assert.ok(hosts.every((host) => host === 'data.gdeltproject.org'));
|
||||
});
|
||||
|
||||
test('forwards only the client routes, by GET, and relays the JSON', async () => {
|
||||
const calls = [];
|
||||
const call = mount({
|
||||
origin: () => 'https://news.example',
|
||||
fetchImpl: async (url, init) => {
|
||||
calls.push({ url: String(url), init });
|
||||
return new Response('{"reports":[],"nextBefore":null}', { status: 200 });
|
||||
},
|
||||
});
|
||||
const ok = await call('/feed?hotspot=all&limit=60&today=1&events=1');
|
||||
assert.equal(ok.status, 200);
|
||||
assert.deepEqual(ok.json, { reports: [], nextBefore: null });
|
||||
assert.equal(ok.headers['Cache-Control'], 'no-store');
|
||||
assert.equal(
|
||||
calls[0].url,
|
||||
'https://news.example/api/news/feed?hotspot=all&limit=60&today=1&events=1',
|
||||
);
|
||||
assert.equal(calls[0].init.redirect, 'error');
|
||||
assert.ok(calls[0].init.signal instanceof AbortSignal);
|
||||
|
||||
for (const url of [
|
||||
'/air-alerts',
|
||||
'/feed?side=UA',
|
||||
'/feed?limit=9999',
|
||||
'/../admin',
|
||||
'/hotspots?x=1',
|
||||
]) {
|
||||
const refused = await call(url);
|
||||
assert.equal(refused.status, 400, url);
|
||||
}
|
||||
const post = await call('/hotspots', 'POST');
|
||||
assert.equal(post.status, 405);
|
||||
assert.equal(post.headers.Allow, 'GET');
|
||||
assert.equal(calls.length, 1);
|
||||
});
|
||||
|
||||
test('upstream failures, bad JSON and timeouts become bounded errors', async () => {
|
||||
const failing = mount({
|
||||
origin: () => 'https://news.example',
|
||||
fetchImpl: async () => new Response('down', { status: 500 }),
|
||||
});
|
||||
const down = await failing('/hotspots');
|
||||
assert.equal(down.status, 502);
|
||||
assert.deepEqual(down.json, { error: 'news_upstream_error', status: 500 });
|
||||
|
||||
const garbled = mount({
|
||||
origin: () => 'https://news.example',
|
||||
fetchImpl: async () => new Response('<html>', { status: 200 }),
|
||||
});
|
||||
assert.equal((await garbled('/hotspots')).status, 502);
|
||||
|
||||
const slow = mount({
|
||||
origin: () => 'https://news.example',
|
||||
timeoutMs: 10,
|
||||
// AbortSignal.timeout does not hold the event loop open: the hanging
|
||||
// fetch does, until its signal aborts, so the run cannot end before it.
|
||||
fetchImpl: (_url, { signal }) =>
|
||||
new Promise((_, reject) => {
|
||||
const alive = setTimeout(() => {}, 60_000);
|
||||
signal.addEventListener('abort', () => {
|
||||
clearTimeout(alive);
|
||||
reject(signal.reason);
|
||||
});
|
||||
}),
|
||||
});
|
||||
const timedOut = await slow('/hotspots');
|
||||
assert.equal(timedOut.status, 504);
|
||||
assert.deepEqual(timedOut.json, { error: 'news_unavailable' });
|
||||
});
|
||||
|
||||
test('a GEV_NEWS_ORIGIN that is no plain origin is warned of once, and Core answers itself', async () => {
|
||||
const logs = [];
|
||||
const asked = [];
|
||||
const call = mount({
|
||||
env: () => ({ GEV_NEWS_ORIGIN: 'https://user:[email protected]' }),
|
||||
log: (line) => logs.push(line),
|
||||
fetchImpl: async () => assert.fail('nothing is forwarded'),
|
||||
keyed: {
|
||||
answer: async (path) => {
|
||||
asked.push(path);
|
||||
return { status: 200, body: { reports: [], nextBefore: null } };
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal((await call('/feed?hotspot=all')).status, 200);
|
||||
assert.equal((await call('/feed?hotspot=all')).status, 200);
|
||||
assert.deepEqual(asked, ['/feed', '/feed']);
|
||||
assert.equal(logs.length, 1);
|
||||
assert.doesNotMatch(logs[0], /secret|news\.example/);
|
||||
});
|
||||
|
||||
test('a built-in answer that throws is a 502, logged in one line without addresses', async () => {
|
||||
const logs = [];
|
||||
const call = mount({
|
||||
env: () => ({}),
|
||||
log: (line) => logs.push(line),
|
||||
keyed: {
|
||||
answer: async () => {
|
||||
throw new TypeError('bad https://api.example/?key=abc\nmore');
|
||||
},
|
||||
},
|
||||
});
|
||||
const failed = await call('/hotspots');
|
||||
assert.equal(failed.status, 502);
|
||||
assert.deepEqual(failed.json, { error: 'news_unavailable' });
|
||||
assert.equal(logs.length, 1);
|
||||
assert.match(logs[0], /TypeError/);
|
||||
assert.doesNotMatch(logs[0], /key=abc|\n/);
|
||||
});
|
||||
|
||||
test('a host that serves news only from its own server turns the built-in backend off', async () => {
|
||||
let fetches = 0;
|
||||
const call = mount({
|
||||
origin: () => null,
|
||||
builtIn: false,
|
||||
fetchImpl: async () => {
|
||||
fetches++;
|
||||
throw new Error('unexpected');
|
||||
},
|
||||
});
|
||||
for (const url of [
|
||||
'/hotspots',
|
||||
'/feed?hotspot=all',
|
||||
'/outline?at=31.95,35.93&v=4',
|
||||
]) {
|
||||
const answer = await call(url);
|
||||
assert.equal(answer.status, 503, url);
|
||||
assert.deepEqual(answer.json, {
|
||||
error: 'news_not_configured',
|
||||
unavailable: true,
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
assert.equal(fetches, 0);
|
||||
});
|
||||
|
||||
test('a request line that is no URL is a 400, not a crash', async () => {
|
||||
const call = mount({
|
||||
origin: () => null,
|
||||
keyed: { answer: async () => assert.fail('nothing is answered') },
|
||||
});
|
||||
// `GET /api/news//[`: Connect strips the mount prefix and leaves `//[`.
|
||||
const bad = await call('//[');
|
||||
assert.equal(bad.status, 400);
|
||||
assert.deepEqual(bad.json, { error: 'invalid_news_request' });
|
||||
});
|
||||
|
||||
test('cross-site and proxied requests are refused before anything is read', async () => {
|
||||
let answered = 0;
|
||||
const call = mount({
|
||||
origin: () => null,
|
||||
keyed: {
|
||||
answer: async () => {
|
||||
answered++;
|
||||
return { status: 200, body: { reports: [], nextBefore: null } };
|
||||
},
|
||||
},
|
||||
});
|
||||
const host = 'localhost:5173';
|
||||
for (const headers of [
|
||||
{ host, origin: 'https://evil.example' },
|
||||
{ host, origin: 'null' },
|
||||
{ host, 'sec-fetch-site': 'cross-site' },
|
||||
{ host, 'x-forwarded-for': '203.0.113.9' },
|
||||
]) {
|
||||
const refused = await call('/feed?hotspot=all', 'GET', headers);
|
||||
assert.equal(refused.status, 403, JSON.stringify(headers));
|
||||
}
|
||||
assert.equal(answered, 0);
|
||||
for (const headers of [
|
||||
{ host },
|
||||
{ host, origin: `http://${host}`, 'sec-fetch-site': 'same-origin' },
|
||||
]) {
|
||||
const passed = await call('/feed?hotspot=all', 'GET', headers);
|
||||
assert.equal(passed.status, 200, JSON.stringify(headers));
|
||||
}
|
||||
assert.equal(answered, 2);
|
||||
});
|
||||
|
||||
test("a news server's key and configuration states are relayed as they are", async () => {
|
||||
for (const body of [
|
||||
{ error: 'no_key', key: 'worldnewsapi' },
|
||||
{ error: 'key_rejected', key: 'worldnewsapi' },
|
||||
{ error: 'key_quota', key: 'worldnewsapi' },
|
||||
{ error: 'news_not_configured', unavailable: true, retryable: false },
|
||||
]) {
|
||||
const call = mount({
|
||||
origin: () => 'https://news.example',
|
||||
fetchImpl: async () =>
|
||||
new Response(JSON.stringify(body), { status: 503 }),
|
||||
});
|
||||
const relayed = await call('/feed?hotspot=all&at=31.95,35.93');
|
||||
assert.equal(relayed.status, 503, body.error);
|
||||
assert.deepEqual(relayed.json, body);
|
||||
}
|
||||
// Any other 503 stays a bounded upstream error.
|
||||
const other = mount({
|
||||
origin: () => 'https://news.example',
|
||||
fetchImpl: async () =>
|
||||
new Response('{"error":"maintenance","detail":"x"}', { status: 503 }),
|
||||
});
|
||||
const down = await other('/hotspots');
|
||||
assert.equal(down.status, 502);
|
||||
assert.deepEqual(down.json, { error: 'news_upstream_error', status: 503 });
|
||||
});
|
||||
@@ -0,0 +1,232 @@
|
||||
// Areas for the News layer, from the bundled Natural Earth packs (public
|
||||
// domain): the disputed area or else the country at a point, its outline in
|
||||
// the /api/news/outline form, and a point inside a country to pin its
|
||||
// national news at. A disputed area is checked first and named as itself,
|
||||
// so no one country's name stands for it; so is a territory the countries
|
||||
// pack gives no code (Somaliland), rather than read as sea. Read once, on
|
||||
// first use.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
import {
|
||||
containsAnchor,
|
||||
geometryOf,
|
||||
pointInRing,
|
||||
} from '../../../src/data/adminBoundaries.js';
|
||||
|
||||
const PACKS = '../../../src/data/local_data/natural_earth/';
|
||||
const PACK = new URL(`${PACKS}countries.json`, import.meta.url);
|
||||
const DISPUTED = new URL(`${PACKS}disputed_areas.json`, import.meta.url);
|
||||
const ISO2 = /^[A-Z]{2}$/;
|
||||
|
||||
let countries = null;
|
||||
let territories = [];
|
||||
|
||||
/**
|
||||
* One entry per ISO code: its name and every Country part's geometry. The
|
||||
* Country features without a code ("-99": Somaliland, Northern Cyprus,
|
||||
* sovereign base areas, no man's lands) are kept apart as territories.
|
||||
*/
|
||||
function loadCountries(read = () => JSON.parse(readFileSync(PACK, 'utf8'))) {
|
||||
if (countries) return countries;
|
||||
const pack = read();
|
||||
const decimals = pack.meta?.decimals ?? 3;
|
||||
const byCode = new Map();
|
||||
const codeless = [];
|
||||
for (const feature of pack.features || []) {
|
||||
// The constituent countries of the United Kingdom repeat its part.
|
||||
if (feature.type !== 'Country') continue;
|
||||
if (!ISO2.test(feature.iso2 || '')) {
|
||||
if (feature.name)
|
||||
codeless.push({ name: feature.name, feature, decimals });
|
||||
continue;
|
||||
}
|
||||
const entry = byCode.get(feature.iso2) || {
|
||||
code: feature.iso2,
|
||||
name: null,
|
||||
size: 0,
|
||||
parts: [],
|
||||
};
|
||||
entry.parts.push({ feature, decimals });
|
||||
// Named after its largest feature: Kazakhstan, not the Baikonur lease.
|
||||
const size = JSON.stringify(feature.polygons).length;
|
||||
if (size > entry.size) {
|
||||
entry.name = feature.name;
|
||||
entry.size = size;
|
||||
}
|
||||
byCode.set(feature.iso2, entry);
|
||||
}
|
||||
countries = byCode;
|
||||
territories = codeless;
|
||||
return countries;
|
||||
}
|
||||
|
||||
/** Test seam: read another pack (or the bundled one again). */
|
||||
export function resetCountries(read) {
|
||||
countries = null;
|
||||
if (read) loadCountries(read);
|
||||
}
|
||||
|
||||
const geometries = (entry) => entry.parts.map((part) => geometryOf(part));
|
||||
|
||||
let disputed = null;
|
||||
const slug = (name) =>
|
||||
name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
|
||||
/** The disputed areas: `{key, name, kind, heldBy, claimedBy, rings}`. */
|
||||
function loadDisputed() {
|
||||
if (disputed) return disputed;
|
||||
const pack = JSON.parse(readFileSync(DISPUTED, 'utf8'));
|
||||
disputed = (pack.areas || []).map((area) => ({
|
||||
key: `disputed-${slug(area.name)}`,
|
||||
name: area.name,
|
||||
kind: area.kind,
|
||||
heldBy: area.heldBy ?? null,
|
||||
claimedBy: area.claimedBy ?? null,
|
||||
rings: area.rings,
|
||||
}));
|
||||
return disputed;
|
||||
}
|
||||
|
||||
/** The disputed area at a point, or null. */
|
||||
export function disputedAt(lat, lon) {
|
||||
return (
|
||||
loadDisputed().find((area) =>
|
||||
area.rings.some((ring) => pointInRing(ring, lat, lon)),
|
||||
) || null
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The area a point is in, as the News layer names it: a disputed area, or
|
||||
* a territory without a code, as itself (`kind: 'disputed'`, `code` null),
|
||||
* else its country (`kind: 'country'`, with `country` the countryAt
|
||||
* entry); null at sea.
|
||||
*/
|
||||
export function areaAt(lat, lon) {
|
||||
const area = disputedAt(lat, lon);
|
||||
if (area)
|
||||
return {
|
||||
...area,
|
||||
kind: 'disputed',
|
||||
held: area.kind,
|
||||
code: null,
|
||||
polygons: area.rings.map((outer) => ({ outer, holes: [] })),
|
||||
};
|
||||
const country = countryAt(lat, lon);
|
||||
if (country)
|
||||
return {
|
||||
key: country.code,
|
||||
name: country.name,
|
||||
kind: 'country',
|
||||
code: country.code,
|
||||
country,
|
||||
};
|
||||
const territory = territoryAt(lat, lon);
|
||||
return territory
|
||||
? {
|
||||
key: `disputed-${slug(territory.name)}`,
|
||||
name: territory.name,
|
||||
kind: 'disputed',
|
||||
held: null,
|
||||
code: null,
|
||||
polygons: geometryOf(territory).polygons.map(([outer, ...holes]) => ({
|
||||
outer,
|
||||
holes,
|
||||
})),
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Whether a point lies in a geometry (geometryOf), across the antimeridian. */
|
||||
function inGeometry(geometry, lat, lon) {
|
||||
const [west, south, east, north] = geometry.bbox;
|
||||
const x = lon - 360 * Math.round((lon - (west + east) / 2) / 360);
|
||||
if (x < west || x > east || lat < south || lat > north) return false;
|
||||
return containsAnchor(geometry.polygons, lat, lon);
|
||||
}
|
||||
|
||||
/** The country at a point, or null at sea or in a territory without a code. */
|
||||
export function countryAt(lat, lon) {
|
||||
for (const entry of loadCountries().values())
|
||||
for (const geometry of geometries(entry))
|
||||
if (inGeometry(geometry, lat, lon)) return entry;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The territory without a code at a point (Somaliland), or null. */
|
||||
function territoryAt(lat, lon) {
|
||||
loadCountries();
|
||||
return (
|
||||
territories.find((part) => inGeometry(geometryOf(part), lat, lon)) || null
|
||||
);
|
||||
}
|
||||
|
||||
/** A country by its ISO code, or null. */
|
||||
export const countryByCode = (code) => loadCountries().get(code) || null;
|
||||
|
||||
/**
|
||||
* A country's outline as /api/news/outline answers it (`{outer, holes}`),
|
||||
* the disputed areas inside it left out as holes (Russia's without Crimea).
|
||||
*/
|
||||
export function countryPolygons(entry) {
|
||||
const areas = loadDisputed();
|
||||
return geometries(entry).flatMap((geometry) =>
|
||||
geometry.polygons.map(([outer, ...holes]) => ({
|
||||
outer,
|
||||
holes: [
|
||||
...holes,
|
||||
...areas
|
||||
.flatMap((area) => area.rings)
|
||||
// By a point inside it: its corners lie on the neighbours' borders.
|
||||
.filter((ring) => {
|
||||
const inside = ringInterior(ring);
|
||||
return inside && pointInRing(outer, inside.lat, inside.lon);
|
||||
}),
|
||||
],
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A point inside the country's largest part, for news placed in the country
|
||||
* alone: its label point when that lies inside, else the middle of the
|
||||
* widest span across the part at its middle latitude.
|
||||
*/
|
||||
export function countryAnchor(entry) {
|
||||
const largest = geometries(entry).sort((a, b) => b.areaKm2 - a.areaKm2)[0];
|
||||
const { label, polygons, bbox } = largest;
|
||||
if (containsAnchor(polygons, label.lat, label.lon)) return label;
|
||||
const [outer] = polygons.reduce((a, b) =>
|
||||
b[0].length > a[0].length ? b : a,
|
||||
);
|
||||
return ringInterior(outer, (bbox[1] + bbox[3]) / 2) || label;
|
||||
}
|
||||
|
||||
/**
|
||||
* A point inside a ring: the middle of its widest span at `lat` (its middle
|
||||
* latitude by default); null when the ring is not crossed there.
|
||||
*/
|
||||
function ringInterior(outer, lat = middleLat(outer)) {
|
||||
const crossings = [];
|
||||
for (let i = 0, j = outer.length - 1; i < outer.length; j = i++) {
|
||||
const [x1, y1] = outer[j];
|
||||
const [x2, y2] = outer[i];
|
||||
if (y1 > lat !== y2 > lat)
|
||||
crossings.push(x1 + ((lat - y1) * (x2 - x1)) / (y2 - y1));
|
||||
}
|
||||
crossings.sort((a, b) => a - b);
|
||||
let best = null;
|
||||
for (let i = 0; i + 1 < crossings.length; i += 2)
|
||||
if (!best || crossings[i + 1] - crossings[i] > best[1] - best[0])
|
||||
best = [crossings[i], crossings[i + 1]];
|
||||
return best ? { lat, lon: (best[0] + best[1]) / 2 } : null;
|
||||
}
|
||||
|
||||
const middleLat = (ring) => {
|
||||
const lats = ring.map(([, y]) => y);
|
||||
return (Math.min(...lats) + Math.max(...lats)) / 2;
|
||||
};
|
||||
@@ -0,0 +1,872 @@
|
||||
// Rule-based report categories, relevance and severity: the News layer's
|
||||
// base reading, shared by every news server. Multilingual stems (en, uk, ru,
|
||||
// ar, he and more) matched at a word start on normalised text; a trailing `$`
|
||||
// makes a stem whole-word. A title hit counts double. Ties go to the earlier
|
||||
// category in CATEGORY_ORDER, which runs from most to least specific.
|
||||
//
|
||||
// A news server may replace these readings with its own; the rules stay
|
||||
// as the fallback.
|
||||
|
||||
/** Text as the rules match it: folded case, one apostrophe, no vowel marks. */
|
||||
function normaliseForMatch(text) {
|
||||
return String(text || '')
|
||||
.normalize('NFKC')
|
||||
.toLowerCase()
|
||||
.replace(/ё/g, 'е')
|
||||
.replace(/[’ʼ`´]/g, "'")
|
||||
.replace(/[ً-ٰٟ֑-ׇ]/g, '')
|
||||
.replace(/[أإآ]/g, 'ا');
|
||||
}
|
||||
|
||||
export const CATEGORY_ORDER = Object.freeze([
|
||||
'strike',
|
||||
'alert',
|
||||
'air-defence',
|
||||
'ground',
|
||||
'casualties',
|
||||
'humanitarian',
|
||||
'infrastructure',
|
||||
'disaster',
|
||||
'protest',
|
||||
'political',
|
||||
// Business and trade: the rules' reading, or World News API's business.
|
||||
'economy',
|
||||
]);
|
||||
|
||||
// [stem, weight]; weight defaults to 1.
|
||||
const RULES = {
|
||||
strike: [
|
||||
'strike',
|
||||
'airstrike',
|
||||
'air strike',
|
||||
'missile',
|
||||
'rocket',
|
||||
'drone',
|
||||
'shelling',
|
||||
'shelled',
|
||||
'bomb',
|
||||
'explosion',
|
||||
'blast',
|
||||
'artillery',
|
||||
'shahed',
|
||||
['attack', 0.5],
|
||||
'удар',
|
||||
'ракет',
|
||||
'дрон',
|
||||
'бпла',
|
||||
'обстріл',
|
||||
'обстрел',
|
||||
'вибух',
|
||||
'взрыв',
|
||||
'шахед',
|
||||
'бомб',
|
||||
['атак', 0.5],
|
||||
'غارة',
|
||||
'غارات',
|
||||
'قصف',
|
||||
'صاروخ',
|
||||
'صواريخ',
|
||||
'مسيرة',
|
||||
'انفجار',
|
||||
['استهداف', 0.7],
|
||||
['هجوم', 0.5],
|
||||
'תקיפה',
|
||||
'טיל',
|
||||
'רקט',
|
||||
'כטב"מ',
|
||||
'פיצוץ',
|
||||
'הפצצ',
|
||||
],
|
||||
'air-defence': [
|
||||
'air defen',
|
||||
'intercept',
|
||||
'shot down',
|
||||
'downed',
|
||||
'air raid',
|
||||
'ппо',
|
||||
'збит',
|
||||
'сбит',
|
||||
'перехоп',
|
||||
'перехват',
|
||||
'повітряна тривога',
|
||||
'воздушная тревога',
|
||||
'الدفاع الجوي',
|
||||
'اعتراض',
|
||||
'יירוט',
|
||||
'יורט',
|
||||
'כיפת ברזל',
|
||||
'אזעק',
|
||||
],
|
||||
ground: [
|
||||
'advance$',
|
||||
'advances$',
|
||||
'advancing',
|
||||
'offensive',
|
||||
'captur',
|
||||
'liberat',
|
||||
'assault',
|
||||
'clash',
|
||||
'fighting',
|
||||
'counteroffensive',
|
||||
'frontline',
|
||||
'front line',
|
||||
'battle',
|
||||
'encircl',
|
||||
['troops', 0.5],
|
||||
'наступ',
|
||||
'штурм',
|
||||
'звільн',
|
||||
'освобо',
|
||||
'бої$',
|
||||
'бои$',
|
||||
'контрнаступ',
|
||||
'захоп',
|
||||
'захват',
|
||||
'оточ',
|
||||
'окруж',
|
||||
'продвиж',
|
||||
'просуван',
|
||||
'اشتباك',
|
||||
'معارك',
|
||||
'تقدم',
|
||||
'سيطر',
|
||||
'اقتحام',
|
||||
'توغل',
|
||||
'לחימה',
|
||||
'קרב',
|
||||
'פשיט',
|
||||
'תמרון',
|
||||
],
|
||||
casualties: [
|
||||
'killed',
|
||||
'dead$',
|
||||
'deaths',
|
||||
'wounded',
|
||||
'injured',
|
||||
'casualt',
|
||||
'death toll',
|
||||
'загин',
|
||||
'поранен',
|
||||
'жертв',
|
||||
'вбит',
|
||||
'погиб',
|
||||
'ранен',
|
||||
'убит',
|
||||
'قتلى',
|
||||
'شهداء',
|
||||
'شهيد',
|
||||
'جرحى',
|
||||
'مقتل',
|
||||
'اصاب',
|
||||
'הרוג',
|
||||
'נהרג',
|
||||
'פצוע',
|
||||
'נפצע',
|
||||
],
|
||||
humanitarian: [
|
||||
'evacuat',
|
||||
'aid$',
|
||||
'refugee',
|
||||
'displac',
|
||||
'humanitarian',
|
||||
'famine',
|
||||
'hunger',
|
||||
'convoy',
|
||||
'shelter',
|
||||
'евакуац',
|
||||
'эвакуац',
|
||||
'гуманітар',
|
||||
'гуманитар',
|
||||
'біженц',
|
||||
'беженц',
|
||||
'انسانية',
|
||||
'نازح',
|
||||
'نزوح',
|
||||
'مساعدات',
|
||||
'اجلاء',
|
||||
'مجاعة',
|
||||
'הומניטרי',
|
||||
'פינוי',
|
||||
'סיוע',
|
||||
],
|
||||
infrastructure: [
|
||||
'power outage',
|
||||
'blackout',
|
||||
'electricity',
|
||||
'energy',
|
||||
'grid$',
|
||||
'substation',
|
||||
'water supply',
|
||||
'bridge',
|
||||
'railway',
|
||||
'refinery',
|
||||
'port$',
|
||||
'dam$',
|
||||
'power plant',
|
||||
'енерг',
|
||||
'энерг',
|
||||
'знеструм',
|
||||
'обесточ',
|
||||
'підстанц',
|
||||
'подстанц',
|
||||
'залізниц',
|
||||
'железн',
|
||||
'нпз',
|
||||
'الكهرباء',
|
||||
'محطة',
|
||||
'مياه',
|
||||
'جسر',
|
||||
'חשמל',
|
||||
'תשתית',
|
||||
],
|
||||
disaster: [
|
||||
'earthquake',
|
||||
'flood',
|
||||
'wildfire',
|
||||
'landslide',
|
||||
'cyclone',
|
||||
'hurricane',
|
||||
'typhoon',
|
||||
'tsunami',
|
||||
'eruption',
|
||||
'glof$',
|
||||
'glacial lake',
|
||||
'storm$',
|
||||
'lightning',
|
||||
'землетрус',
|
||||
'землетрясен',
|
||||
'повін',
|
||||
'паводк',
|
||||
'наводнен',
|
||||
'زلزال',
|
||||
'فيضان',
|
||||
'سيول',
|
||||
'רעידת אדמה',
|
||||
'שיטפון',
|
||||
],
|
||||
protest: [
|
||||
'protest',
|
||||
'demonstrat',
|
||||
'rally$',
|
||||
'rallies',
|
||||
'riot',
|
||||
// Not a bare "протест": "протестировал" is "tested".
|
||||
'протест$',
|
||||
'протеста',
|
||||
'протесту',
|
||||
'протестом',
|
||||
'протесте$',
|
||||
'протесты',
|
||||
'протестов',
|
||||
'протестн',
|
||||
'протестую',
|
||||
'протестова',
|
||||
'мітинг',
|
||||
'митинг',
|
||||
'احتجاج',
|
||||
'مظاهر',
|
||||
'تظاهر',
|
||||
'הפגנ',
|
||||
'מחאה',
|
||||
],
|
||||
political: [
|
||||
'talks$',
|
||||
'negotiat',
|
||||
'sanction',
|
||||
'summit',
|
||||
'ceasefire',
|
||||
'truce',
|
||||
'peace',
|
||||
'election',
|
||||
'parliament',
|
||||
['president', 0.5],
|
||||
['minister', 0.5],
|
||||
['statement', 0.5],
|
||||
// A state visit ("Emir of Qatar arrives in Amman").
|
||||
['visit', 0.5],
|
||||
['delegation', 0.5],
|
||||
['agreement', 0.5],
|
||||
'memorandum',
|
||||
'diplomat',
|
||||
'bilateral',
|
||||
'ambassador',
|
||||
['визит', 0.5],
|
||||
['делегац', 0.5],
|
||||
'дипломат',
|
||||
'переговор',
|
||||
'санкц',
|
||||
'перемир',
|
||||
'вибори',
|
||||
'выборы',
|
||||
['президент', 0.5],
|
||||
['міністр', 0.5],
|
||||
['министр', 0.5],
|
||||
'مفاوضات',
|
||||
'عقوبات',
|
||||
'هدنة',
|
||||
'وقف اطلاق النار',
|
||||
['وزير', 0.5],
|
||||
['رئيس', 0.5],
|
||||
'משא ומתן',
|
||||
'הפסקת אש',
|
||||
'סנקצי',
|
||||
['ראש הממשלה', 0.5],
|
||||
],
|
||||
// Elections and party politics ("How Gina Hinojosa Captured Texas" is a
|
||||
// governor's race, not a battle). They count toward political only when
|
||||
// nothing physical happened (categorizeText): a strike in an election week
|
||||
// is still a strike.
|
||||
election: [
|
||||
'midterm',
|
||||
'electoral',
|
||||
'elect$',
|
||||
'ballot',
|
||||
'voter',
|
||||
'vote$',
|
||||
'votes$',
|
||||
'voting',
|
||||
'polls$',
|
||||
'candidate',
|
||||
'senate race',
|
||||
'referendum',
|
||||
'inaugurat',
|
||||
// Plurals only: "Democratic Republic of Congo", "Republican Guard".
|
||||
'democrats$',
|
||||
'republicans$',
|
||||
'gop$',
|
||||
'избира',
|
||||
'голосован',
|
||||
'виборц',
|
||||
'голосуван',
|
||||
'انتخاب',
|
||||
'בחירות',
|
||||
],
|
||||
};
|
||||
|
||||
// Business, trade, industry and energy, counted only when nothing physical
|
||||
// happened (categorizeText): an oil depot struck is a strike. English,
|
||||
// Russian, Ukrainian, Turkish and the European stems.
|
||||
RULES.economy = [
|
||||
'econom',
|
||||
'gdp$',
|
||||
'inflation',
|
||||
'investment',
|
||||
'investor',
|
||||
'export',
|
||||
'imports$',
|
||||
'import$',
|
||||
'tariff',
|
||||
'trade deal',
|
||||
'trade turnover',
|
||||
'central bank',
|
||||
'stock market',
|
||||
'budget',
|
||||
'oil and gas',
|
||||
'oilfield',
|
||||
'oil field',
|
||||
'gas field',
|
||||
'exploration',
|
||||
'drilling',
|
||||
'mining',
|
||||
'industr',
|
||||
'manufactur',
|
||||
'business',
|
||||
'company',
|
||||
'companies',
|
||||
'revenue',
|
||||
'profit',
|
||||
'merger',
|
||||
'tourism',
|
||||
'agricultur',
|
||||
'harvest',
|
||||
'экономи',
|
||||
'инвест',
|
||||
'экспорт',
|
||||
'импорт',
|
||||
'инфляц',
|
||||
'ввп$',
|
||||
'бюджет',
|
||||
'промышлен',
|
||||
'бизнес',
|
||||
'компани',
|
||||
'товарооборот',
|
||||
'економі',
|
||||
'інвест',
|
||||
'експорт',
|
||||
'промисло',
|
||||
'ekonomi',
|
||||
'yatırım',
|
||||
'ihracat',
|
||||
'ithalat',
|
||||
'enflasyon',
|
||||
'sanayi',
|
||||
'şirket',
|
||||
'wirtschaft',
|
||||
'konjunktur',
|
||||
'unternehm',
|
||||
'inversi',
|
||||
'investiss',
|
||||
'investiment',
|
||||
'empresa',
|
||||
'entreprise',
|
||||
'impresa',
|
||||
];
|
||||
|
||||
// Attack words beyond PHYSICAL's harm, which keep a report off economy.
|
||||
const ATTACK = new RegExp(
|
||||
String.raw`(?<![\p{L}])(?:attack|assault|target|hit(?![\p{L}])|fire|plot|war(?![\p{L}])|wartime|атак|удар|пораж|пожар|обстр|війн|войн|هجوم|استهداف|حرب|מתקפ|ataque|attaque|attacco|angriff|saldır)`,
|
||||
'u',
|
||||
);
|
||||
|
||||
function escapeRegex(text) {
|
||||
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
// Spanish, French, Portuguese, Italian and German stems, for outlets that
|
||||
// report in their own language (a train crash in Spain, in Spanish).
|
||||
const EUROPEAN = {
|
||||
strike: [
|
||||
'bombarde',
|
||||
'misil',
|
||||
'míssil',
|
||||
'dron$',
|
||||
'explosión',
|
||||
'explosão',
|
||||
'esplosi',
|
||||
'ataque',
|
||||
'attaque',
|
||||
'attacco',
|
||||
'angriff',
|
||||
'rakete',
|
||||
'luftangriff',
|
||||
],
|
||||
'air-defence': ['antiaére', 'défense aérienne', 'flugabwehr', 'abatid'],
|
||||
ground: [
|
||||
'combate',
|
||||
'combats',
|
||||
'enfrentamiento',
|
||||
'affrontement',
|
||||
'scontri',
|
||||
'gefecht',
|
||||
],
|
||||
casualties: [
|
||||
'muert',
|
||||
'fallecid',
|
||||
'herid',
|
||||
'morts$',
|
||||
'tué',
|
||||
'blessé',
|
||||
'morti',
|
||||
'ferit',
|
||||
'mortos',
|
||||
'ferido',
|
||||
'muere',
|
||||
'mueren',
|
||||
'fallece',
|
||||
'meurt',
|
||||
'morre',
|
||||
'tote$',
|
||||
'verletzt',
|
||||
'víctimas',
|
||||
'victimes',
|
||||
'vittime',
|
||||
],
|
||||
humanitarian: [
|
||||
'evacua',
|
||||
'évacu',
|
||||
'refugiad',
|
||||
'réfugi',
|
||||
'flüchtling',
|
||||
'desplazad',
|
||||
],
|
||||
infrastructure: [
|
||||
'apagón',
|
||||
'panne',
|
||||
'blackout',
|
||||
'descarril',
|
||||
'déraill',
|
||||
'deragliament',
|
||||
'entgleis',
|
||||
'accidente',
|
||||
'accident',
|
||||
'incidente',
|
||||
'choque',
|
||||
'collision',
|
||||
'collisione',
|
||||
'zusammenstoß',
|
||||
'incendio',
|
||||
'incendie',
|
||||
],
|
||||
disaster: [
|
||||
'terremoto',
|
||||
'séisme',
|
||||
'tremblement',
|
||||
'erdbeben',
|
||||
'inundaci',
|
||||
'inondation',
|
||||
'alluvion',
|
||||
'überschwemm',
|
||||
'huracán',
|
||||
'ouragan',
|
||||
'uragano',
|
||||
'tormenta',
|
||||
'tempête',
|
||||
'tempesta',
|
||||
'sturm',
|
||||
'cicl',
|
||||
'erupci',
|
||||
'éruption',
|
||||
'eruzione',
|
||||
],
|
||||
protest: [
|
||||
'protesta',
|
||||
'manifesta',
|
||||
'disturbio',
|
||||
'émeute',
|
||||
'huelga',
|
||||
'grève',
|
||||
'sciopero',
|
||||
'streik',
|
||||
'demonstration',
|
||||
],
|
||||
};
|
||||
for (const [category, stems] of Object.entries(EUROPEAN))
|
||||
RULES[category] = [...(RULES[category] || []), ...stems];
|
||||
|
||||
const COMPILED = Object.entries(RULES).map(([category, stems]) => ({
|
||||
category,
|
||||
rules: stems.map((entry) => {
|
||||
const [stem, weight = 1] = Array.isArray(entry) ? entry : [entry];
|
||||
const whole = stem.endsWith('$');
|
||||
const body = escapeRegex(
|
||||
normaliseForMatch(whole ? stem.slice(0, -1) : stem),
|
||||
).replace(/\s+/g, '\\s+');
|
||||
const arabic = /\p{Script=Arabic}/u.test(stem) ? '(?:[وفبلك]?(?:ال)?)' : '';
|
||||
const hebrew = /\p{Script=Hebrew}/u.test(stem) ? '(?:[ובלמהשכ]{0,2})' : '';
|
||||
return {
|
||||
weight,
|
||||
regex: new RegExp(
|
||||
`(?<![\\p{L}\\p{N}])${arabic}${hebrew}${body}${whole ? '(?![\\p{L}\\p{N}])' : ''}`,
|
||||
'u',
|
||||
),
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
const EVENT_CATEGORIES = new Set([
|
||||
'strike',
|
||||
'air-defence',
|
||||
'ground',
|
||||
'casualties',
|
||||
'disaster',
|
||||
]);
|
||||
// Words that tie a report to armed conflict or crisis, whatever its category
|
||||
// (en, fr, ru, uk, ar, he, es).
|
||||
const CONFLICT_TERMS = new RegExp(
|
||||
[
|
||||
'war',
|
||||
'army',
|
||||
'military',
|
||||
'militar',
|
||||
'troops',
|
||||
'soldier',
|
||||
'militant',
|
||||
'jihad',
|
||||
'terror',
|
||||
'insurg',
|
||||
'rebel',
|
||||
'ceasefire',
|
||||
'truce',
|
||||
'hostage',
|
||||
'sanction',
|
||||
'shell',
|
||||
'drone',
|
||||
'missile',
|
||||
'airstrike',
|
||||
'frappe',
|
||||
'armée',
|
||||
'soldat',
|
||||
'djihad',
|
||||
'fama',
|
||||
'jnim',
|
||||
'houthi',
|
||||
'hezbollah',
|
||||
'hamas',
|
||||
'irgc',
|
||||
'rsf',
|
||||
'wagner',
|
||||
'africa corps',
|
||||
'войн',
|
||||
'війн',
|
||||
'армі',
|
||||
'арми',
|
||||
'всу',
|
||||
'зсу',
|
||||
'бпла',
|
||||
'обстр',
|
||||
'фронт',
|
||||
'حرب',
|
||||
'الجيش',
|
||||
'هجوم',
|
||||
'قصف',
|
||||
'الحوثي',
|
||||
'مسلح',
|
||||
'מלחמה',
|
||||
'צה"ל',
|
||||
'guerra',
|
||||
'ejército',
|
||||
// Violence, coercion and their aftermath.
|
||||
'massacre',
|
||||
'atrocit',
|
||||
'genocide',
|
||||
'arms',
|
||||
'weapon',
|
||||
'detain',
|
||||
'abduct',
|
||||
'kidnap',
|
||||
'air raid',
|
||||
'alert',
|
||||
'jamming',
|
||||
'тривог',
|
||||
'тревог',
|
||||
'відбій',
|
||||
'отбой',
|
||||
'occupied',
|
||||
'occupation',
|
||||
'territor',
|
||||
'convoy',
|
||||
'demining',
|
||||
'forces',
|
||||
'raid',
|
||||
'conflict',
|
||||
'putin',
|
||||
'zelensk',
|
||||
'окуп',
|
||||
'оккуп',
|
||||
// Diplomacy and strategic stakes of the conflict.
|
||||
'ambassador',
|
||||
'mediat',
|
||||
'election',
|
||||
'talks',
|
||||
'negotiat',
|
||||
'envoy',
|
||||
'diplomat',
|
||||
'foreign minister',
|
||||
'condemn',
|
||||
'nuclear',
|
||||
'hormuz',
|
||||
'blockade',
|
||||
'négociation',
|
||||
'переговор',
|
||||
'перемир',
|
||||
'مفاوضات',
|
||||
'هدنة',
|
||||
]
|
||||
// Short words are whole words ("war" is not "warning" or "Warsaw").
|
||||
.map((w) => `(?<![\\p{L}])${w}${w.length < 5 ? '(?![\\p{L}])' : ''}`)
|
||||
.join('|'),
|
||||
'iu',
|
||||
);
|
||||
|
||||
/**
|
||||
* How much a report matters to its hotspot, by rule: `core` for a physical conflict or crisis event, `related` for
|
||||
* news tied to the conflict (by wording, or because the source covers
|
||||
* nothing else), `minor` for the rest of what a broad outlet publishes
|
||||
* (prices, sport, health, culture).
|
||||
*/
|
||||
const termPatterns = new Map();
|
||||
/** A word-start pattern for a hotspot term ("pow" must not match "power"). */
|
||||
function termPattern(term) {
|
||||
if (!termPatterns.has(term))
|
||||
termPatterns.set(
|
||||
term,
|
||||
new RegExp(
|
||||
// Short terms are whole words ("pow" is not "power"); longer ones are
|
||||
// stems ("mobiliz" → mobilized, mobilization).
|
||||
`(?<![\\p{L}])${normaliseForMatch(term).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}${term.length < 5 ? '(?![\\p{L}])' : ''}`,
|
||||
'u',
|
||||
),
|
||||
);
|
||||
return termPatterns.get(term);
|
||||
}
|
||||
|
||||
export function relevanceOf({
|
||||
category,
|
||||
tags = [],
|
||||
title = '',
|
||||
summary = '',
|
||||
dedicated = false,
|
||||
relatedTerms = [],
|
||||
floor = 'minor',
|
||||
}) {
|
||||
const text = normaliseForMatch(`${title} ${summary}`);
|
||||
// A hotspot's own vocabulary counts as conflict wording there (Iran:
|
||||
// "Qalibaf", "IAEA"; Israel: "settler"; Ukraine: "prisoner exchange").
|
||||
const conflict =
|
||||
CONFLICT_TERMS.test(text) ||
|
||||
relatedTerms.some((term) => termPattern(term).test(text));
|
||||
// Any event reading counts, even when another category wins the title
|
||||
// ("the accident toll rises to 74": infrastructure first, casualties too).
|
||||
if (
|
||||
EVENT_CATEGORIES.has(category) ||
|
||||
tags.some((tag) => EVENT_CATEGORIES.has(tag))
|
||||
)
|
||||
return 'core';
|
||||
if (conflict && ['humanitarian', 'infrastructure'].includes(category))
|
||||
return 'core';
|
||||
if (conflict || dedicated || category === 'humanitarian') return 'related';
|
||||
return floor;
|
||||
}
|
||||
|
||||
// Alerts: air-raid alerts and reports of drones or missiles on their way
|
||||
// ("Реактивний БпЛА курсом на Полтаву", "КАБи на Донеччину", "drone heading
|
||||
// for Kyiv", "➡️"). Something is in the air; nothing has hit yet. Most of an
|
||||
// air force's channel is these, and as "strikes" they would merge into real
|
||||
// strike events and inflate them.
|
||||
// A direction counts only after a weapon ("БпЛА … з півдня"), so a convoy
|
||||
// leaving "in the direction of Oleshky" is not an alert.
|
||||
const WEAPON =
|
||||
'(?:бпла|дрон|шахед|герань|ракет|каб|балістик|крилат|боєприпас|боеприпас|uav|drone|missile|shahed|ballistic|cruise|munition)';
|
||||
const DIRECTION =
|
||||
'(?:курсом на|у напрямку|в напрямку|в направлении|у бік|в сторону|повз |з (?:північ|півден|схід|сход|захід|заход)|с (?:север|юг|восток|запад)|на |над |heading |on course|toward)';
|
||||
const ALERT = new RegExp(
|
||||
`${WEAPON}[^.!?]{0,40}${DIRECTION}|^[^\\p{L}]*курсом на|тривог|тревог|загроза застосування|угроза применения|air[- ]raid|threat of (?:missile|drone|ballistic|uav)|[➡⬅⬆⬇↗↘↙↖]`,
|
||||
'u',
|
||||
);
|
||||
// Wording that something did hit: then it is a strike (or air defence), not an alert.
|
||||
const IMPACT =
|
||||
/влуч|вибух|уражен|пошкодж|загин|поранен|зруйн|пожеж|атак|удар по|збит|знищ|попадан|взрыв|поражен|поврежд|погиб|ранен|разруш|сбит|уничтож|\bhit\b|\bhits\b|struck|explosion|blast|killed|injur|wounded|damage|destroy|fire\b|shot down|intercept|attack/u;
|
||||
|
||||
/** Whether a report's title is an alert: a threat on its way, nothing hit yet. */
|
||||
function isAlertText(title = '', summary = '') {
|
||||
const t = normaliseForMatch(`${title} ${summary}`.slice(0, 400));
|
||||
return ALERT.test(t) && !IMPACT.test(t);
|
||||
}
|
||||
|
||||
// "Hit" or "struck" says nothing on its own: what hit decides (a missile is a
|
||||
// strike, a flood or lightning a disaster), and those words score by
|
||||
// themselves. A few phrases use a category's word for another thing.
|
||||
const PHRASES = [
|
||||
[/storm shadow/gu, 'missile'],
|
||||
[/lightning strikes?/gu, 'lightning'],
|
||||
[
|
||||
/(?:general|hunger|labou?r|workers?|nationwide|national|transport|rail|teachers?|doctors?|nurses?) strikes?|on strike|strike action/gu,
|
||||
'protest',
|
||||
],
|
||||
];
|
||||
const rephrase = (text) =>
|
||||
PHRASES.reduce((out, [re, word]) => out.replace(re, word), text);
|
||||
|
||||
// An election story borrows the words of war and disaster ("‘An Earthquake’:
|
||||
// A Democratic Rout", "captured Texas", "a bloodbath for Republicans"). With
|
||||
// election wording and nothing physical (no weapon, no one hurt, no quake's
|
||||
// magnitude), the event categories give way to political.
|
||||
const ELECTION =
|
||||
/(?<![\p{L}])(?:election|midterm|electoral|ballot|voters?(?![\p{L}])|votes?(?![\p{L}])|voting|polls(?![\p{L}])|senate race|referendum|democrats|republicans|gop(?![\p{L}])|[\p{L}]+-elect(?![\p{L}])|выбор|избира|вибор|голосув|голосов|انتخاب|בחירות)/u;
|
||||
const PHYSICAL =
|
||||
/(?<![\p{L}])(?:kill|dead(?![\p{L}])|fighting|clash|air ?strike|strikes?(?![\p{L}])|struck|destroy|downed|ablaze|on fire|catch(?:es)? fire|caught fire|naval mine|landmine|shot down|sunk|sank|deaths?|injur|wounded|casualt|troops|soldier|missile|drone|rocket|shell(?:ing|ed)|artiller|explosi|blast|bomb|gunfire|shot(?![\p{L}])|shooting|magnitude|richter|tremor|evacuat|flood(?:ed|ing|waters)|убит|погиб|ранен|ракет|дрон|обстрел|загин|поранен|обстріл|قتل|جرح|صاروخ|הרג|פצוע|טיל)/u;
|
||||
/** The categories of a physical event (an alert is a threat, not yet one). */
|
||||
const EVENT_KINDS = new Set([
|
||||
'strike',
|
||||
'air-defence',
|
||||
'ground',
|
||||
'casualties',
|
||||
'disaster',
|
||||
'infrastructure',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Categorise a report from its title and summary.
|
||||
* @returns {{category: string, tags: string[]}}
|
||||
*/
|
||||
export function categorizeText({ title = '', summary = '' } = {}) {
|
||||
if (isAlertText(title)) return { category: 'alert', tags: ['alert'] };
|
||||
const t = rephrase(normaliseForMatch(title));
|
||||
const s = rephrase(normaliseForMatch(summary));
|
||||
const scores = new Map();
|
||||
for (const { category, rules } of COMPILED) {
|
||||
let score = 0;
|
||||
for (const { regex, weight } of rules) {
|
||||
if (regex.test(t)) score += 2 * weight;
|
||||
else if (regex.test(s)) score += weight;
|
||||
}
|
||||
if (score > 0) scores.set(category, score);
|
||||
}
|
||||
const all = `${t} ${s}`;
|
||||
const election = scores.get('election') || 0;
|
||||
scores.delete('election');
|
||||
const physical = PHYSICAL.test(all);
|
||||
// Business words and no harm or attack: "advance exploration" is no
|
||||
// front-line advance; "attack an industrial facility" stays a strike.
|
||||
if (physical || ATTACK.test(all)) scores.delete('economy');
|
||||
else if (scores.has('economy'))
|
||||
for (const kind of EVENT_KINDS) scores.delete(kind);
|
||||
if (!physical) {
|
||||
if (election)
|
||||
scores.set('political', (scores.get('political') || 0) + election);
|
||||
if (ELECTION.test(all)) {
|
||||
for (const kind of EVENT_KINDS) scores.delete(kind);
|
||||
scores.set('political', Math.max(scores.get('political') || 0, 1));
|
||||
}
|
||||
}
|
||||
if (!scores.size) return { category: 'other', tags: [] };
|
||||
const ranked = CATEGORY_ORDER.filter((c) => scores.has(c)).sort(
|
||||
(a, b) => scores.get(b) - scores.get(a),
|
||||
);
|
||||
return { category: ranked[0], tags: ranked };
|
||||
}
|
||||
|
||||
// Severity by rule, for reports without a severity of their own (0 none,
|
||||
// 1 minor incident, 2 significant: deaths, injuries or real damage, 3 major:
|
||||
// mass casualties). Wording in en, uk, ru, ar, he, es, fr, de.
|
||||
const SEVERITY_EVENTS = new Set([...EVENT_CATEGORIES, 'infrastructure']);
|
||||
const MAJOR_HARM =
|
||||
/mass casualt|massacre|dozens (?:of \p{L}+ )?(?:killed|dead)|hundreds (?:of \p{L}+ )?(?:killed|dead)|масов\p{L}* (?:жертв|загибел|гибел)|десятки (?:загибл|погибш|убит|вбит|людей загин|человек погиб)|مجزرة|عشرات (?:القتلى|الشهداء)|טבח/u;
|
||||
const HARM_WORDS =
|
||||
'kill|dead|died|death|injur|wound|casualt|загин|загибл|погиб|убит|вбит|поранен|ранен|постражд|пострадал|травм|قتل|قتيل|شهيد|شهداء|جرح|جريح|إصاب|اصاب|נהרג|הרוג|פצוע|muert|herid|tué|mort|bless|getötet|tote|verletz';
|
||||
const HARM = new RegExp(String.raw`(?<![\p{L}])(?:${HARM_WORDS})`, 'u');
|
||||
// "No casualties", "без жертв": the harm word is there, the harm is not.
|
||||
const NO_HARM =
|
||||
/(?:no|without|zero) (?:casualt|injur|deaths|victims|one (?:was )?(?:hurt|injured))|без (?:жертв|постраждал|пострадавш|загибл|погибш)|не постраждал|не пострадал|لا (?:إصابات|اصابات)|אין נפגעים|sin (?:víctimas|heridos)|sans (?:victime|blessé)/u;
|
||||
const DAMAGE =
|
||||
/destroy|damag|ablaze|on fire|fire broke|explosion|collapse|зруйн|разруш|пошкодж|поврежд|пожеж|пожар|вибух|взрыв|دمار|تدمير|انفجار|حريق|הרס|פיצוץ|שריפה|destru|incendi|explos|zerstör/u;
|
||||
// An age ("a 71-year-old woman", "71-річна", "71-летний") is not a count.
|
||||
const NUMBER = String.raw`(\d{1,3}(?:[,.\s]\d{3})+|\d{1,5})(?![\d,.]*[-\s]*(?:year|yr|річн|рок|летн|год))`;
|
||||
const COUNTS = [
|
||||
// "14 killed", "three people dead" is left to the words; "14 civilians dead".
|
||||
new RegExp(String.raw`${NUMBER}\s+(?:\p{L}+\s+){0,2}(?:${HARM_WORDS})`, 'gu'),
|
||||
// "killing 14", "death toll rises to 74", "kills at least 20".
|
||||
new RegExp(
|
||||
String.raw`(?:kill\p{L}*|toll\p{L}*(?:\s+\p{L}+){0,3}|загинул\p{L}*|погибл\p{L}*|жертв\p{L}*)\s+(?:at least\s+|не менее\s+|щонайменше\s+|до\s+|to\s+)?${NUMBER}`,
|
||||
'gu',
|
||||
),
|
||||
];
|
||||
|
||||
function largestCount(text) {
|
||||
let most = 0;
|
||||
for (const pattern of COUNTS)
|
||||
for (const match of text.matchAll(pattern)) {
|
||||
const n = Number(match[1].replace(/[,.\s]/g, ''));
|
||||
// A year ("since 2022 killed…") is not a toll.
|
||||
if (n < 1900 || n > 2100) most = Math.max(most, n);
|
||||
}
|
||||
return most;
|
||||
}
|
||||
|
||||
/** A report's severity by rule, 0–3. */
|
||||
export function severityByRules({
|
||||
category,
|
||||
tags = [],
|
||||
title = '',
|
||||
summary = '',
|
||||
}) {
|
||||
const text = normaliseForMatch(`${title} ${summary}`);
|
||||
const harmed = HARM.test(text) && !NO_HARM.test(text);
|
||||
if (MAJOR_HARM.test(text) || (harmed && largestCount(text) >= 10)) return 3;
|
||||
const event =
|
||||
SEVERITY_EVENTS.has(category) ||
|
||||
tags.some((tag) => SEVERITY_EVENTS.has(tag));
|
||||
if (harmed && (event || category === 'humanitarian')) return 2;
|
||||
if (event && DAMAGE.test(text)) return 2;
|
||||
return event ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,546 @@
|
||||
// GDELT 2.0 events for the Conflicts section, from the 15-minute export
|
||||
// files (data.gdeltproject.org, plain downloads; no key, and not the
|
||||
// rate-limited search API). Each export is one zipped tab-separated table of
|
||||
// machine-coded events, each placed by GDELT with a point and a place type.
|
||||
//
|
||||
// A story counts when GDELT coded physical violence in it (CAMEO roots 18
|
||||
// assault, 19 fight, 20 mass violence); its place is read from all its
|
||||
// placed events (choosePlace). GDELT's actor fields are machine-coded and
|
||||
// often wrong, so no report says who did what: a report is the story (its
|
||||
// article, and the copies other outlets ran under the same address words).
|
||||
// The events carry no headline: a story's title is its page title from the
|
||||
// GKG export of the same quarter hour (read for the newest hours only, it
|
||||
// runs to 6 MB), else the words of its address. All of it is rules over
|
||||
// GDELT's own fields; nothing reads the articles.
|
||||
|
||||
import { promisify } from 'node:util';
|
||||
import { inflateRaw } from 'node:zlib';
|
||||
|
||||
import { readResponseBytesCapped } from '../common/http.js';
|
||||
import { categorizeText, relevanceOf, severityByRules } from './categorize.js';
|
||||
import { UpstreamError, unescapeHtml, webLink } from './upstream.js';
|
||||
|
||||
const EXPORTS_URL = 'https://data.gdeltproject.org/gdeltv2/';
|
||||
const STEP_MS = 15 * 60_000;
|
||||
// An events export runs to about 100 KB zipped and 500 KB unzipped; a GKG
|
||||
// export to about 6 MB and 18 MB. Unzipped off the main thread.
|
||||
const MAX_ZIP_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_TABLE_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_GKG_ZIP_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_GKG_TABLE_BYTES = 32 * 1024 * 1024;
|
||||
const inflate = promisify(inflateRaw);
|
||||
// The most words an address's words make a title of.
|
||||
const MAX_ADDRESS_WORDS = 24;
|
||||
const TIMEOUT_MS = 30_000;
|
||||
const MAX_PLACES = 3;
|
||||
|
||||
// Columns of the 2.0 events table (61).
|
||||
const COL = {
|
||||
id: 0,
|
||||
eventCode: 26,
|
||||
rootCode: 28,
|
||||
quadClass: 29,
|
||||
mentions: 31,
|
||||
geoType: 51,
|
||||
place: 52,
|
||||
lat: 56,
|
||||
lon: 57,
|
||||
added: 59,
|
||||
url: 60,
|
||||
};
|
||||
|
||||
// CAMEO event codes as the News layer's categories, and as words for an
|
||||
// article whose address carries none of its own.
|
||||
const EVENTS = [
|
||||
// Aerial weapons, artillery and tanks, bombings.
|
||||
[/^19[45]|^183/, 'strike', 'Strike reported'],
|
||||
[/^186|^185/, 'casualties', 'Killing reported'],
|
||||
[/^181/, 'humanitarian', 'Abduction reported'],
|
||||
[/^20/, 'casualties', 'Mass violence reported'],
|
||||
[/^19/, 'ground', 'Fighting reported'],
|
||||
[/^18/, 'other', 'Violence reported'],
|
||||
];
|
||||
const eventKind = (code) =>
|
||||
EVENTS.find(([pattern]) => pattern.test(code)) || EVENTS.at(-1);
|
||||
|
||||
// GDELT's place types: 1 country, 2 US state, 3 US city, 4 world city,
|
||||
// 5 world state.
|
||||
const PRECISION = {
|
||||
1: 'country',
|
||||
2: 'region',
|
||||
3: 'city',
|
||||
4: 'city',
|
||||
5: 'region',
|
||||
};
|
||||
|
||||
/** The export holding the events GDELT added in the 15 minutes before `ms`. */
|
||||
export function exportUrl(ms) {
|
||||
const at = new Date(Math.floor(ms / STEP_MS) * STEP_MS);
|
||||
const stamp = at.toISOString().replace(/[-:T]/g, '').slice(0, 14);
|
||||
return `${EXPORTS_URL}${stamp}.export.CSV.zip`;
|
||||
}
|
||||
|
||||
/** An events export's quarter hour (ms), from its name, or NaN. */
|
||||
export function exportAt(url) {
|
||||
const match = /(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)00\.export\.CSV\.zip$/.exec(
|
||||
url,
|
||||
);
|
||||
return match
|
||||
? Date.UTC(match[1], match[2] - 1, match[3], match[4], match[5])
|
||||
: NaN;
|
||||
}
|
||||
|
||||
const LASTUPDATE_URL = `${EXPORTS_URL}lastupdate.txt`;
|
||||
|
||||
/**
|
||||
* The time of the newest events export GDELT lists as published, in ms, or
|
||||
* null when its list cannot be read. Asking for one not listed yet would
|
||||
* be answered 404, and GDELT's CDN keeps that 404 for an hour.
|
||||
*/
|
||||
export async function fetchLatestExport({ fetchImpl, signal } = {}) {
|
||||
try {
|
||||
const response = await fetchImpl(LASTUPDATE_URL, {
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
return null;
|
||||
}
|
||||
const text = new TextDecoder().decode(
|
||||
await readResponseBytesCapped(response, 16 * 1024, signal),
|
||||
);
|
||||
const at = exportAt(
|
||||
/\S+\.export\.CSV\.zip/.exec(text)?.[0].replace(/^http:/, 'https:') || '',
|
||||
);
|
||||
return Number.isFinite(at) ? at : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The GKG export of the same quarter hour as an events export. */
|
||||
export const gkgUrl = (eventsUrl) =>
|
||||
eventsUrl.replace(/\.export\.CSV\.zip$/, '.gkg.csv.zip');
|
||||
|
||||
/** The day's exports up to `now`, newest first. */
|
||||
export function exportsOfDay(now, hours = 24) {
|
||||
const urls = [];
|
||||
const last = Math.floor(now / STEP_MS) * STEP_MS;
|
||||
for (let ms = last; ms > now - hours * 3_600_000; ms -= STEP_MS)
|
||||
urls.push(exportUrl(ms));
|
||||
return urls;
|
||||
}
|
||||
|
||||
/** The one table inside an export's zip, as text, up to `maxBytes`. */
|
||||
export async function unzipExport(bytes, maxBytes = MAX_TABLE_BYTES) {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const fail = () => {
|
||||
throw new UpstreamError('GDELT export unreadable', { unreadable: true });
|
||||
};
|
||||
if (bytes.length < 30 || view.getUint32(0, true) !== 0x04034b50) fail();
|
||||
// The central directory has the sizes; a local header may leave them 0.
|
||||
let eocd = -1;
|
||||
for (let i = bytes.length - 22; i >= Math.max(0, bytes.length - 0x10015); i--)
|
||||
if (view.getUint32(i, true) === 0x06054b50) {
|
||||
eocd = i;
|
||||
break;
|
||||
}
|
||||
if (eocd < 0) fail();
|
||||
const dir = view.getUint32(eocd + 16, true);
|
||||
if (dir + 46 > bytes.length || view.getUint32(dir, true) !== 0x02014b50)
|
||||
fail();
|
||||
const method = view.getUint16(dir + 10, true);
|
||||
const compressed = view.getUint32(dir + 20, true);
|
||||
const local = view.getUint32(dir + 42, true);
|
||||
if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50)
|
||||
fail();
|
||||
const start =
|
||||
local +
|
||||
30 +
|
||||
view.getUint16(local + 26, true) +
|
||||
view.getUint16(local + 28, true);
|
||||
if (start + compressed > bytes.length) fail();
|
||||
const data = bytes.subarray(start, start + compressed);
|
||||
try {
|
||||
if (method === 0 && data.length > maxBytes) fail();
|
||||
const table =
|
||||
method === 0
|
||||
? data
|
||||
: method === 8
|
||||
? await inflate(data, { maxOutputLength: maxBytes })
|
||||
: fail();
|
||||
return new TextDecoder().decode(table);
|
||||
} catch (error) {
|
||||
if (error instanceof UpstreamError) throw error;
|
||||
return fail();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An article's address as the reports carry it: https, an http one
|
||||
* upgraded (GDELT keeps the address as it found it), else null.
|
||||
*/
|
||||
export const articleLink = (value) =>
|
||||
webLink(value) || webLink(String(value ?? '').replace(/^http:/, 'https:'));
|
||||
|
||||
/**
|
||||
* The placed events of an export's table, of the stories with physical
|
||||
* violence in them: those events, and every other placed event of the same
|
||||
* stories (they help place it).
|
||||
*/
|
||||
export function parseEvents(table) {
|
||||
const all = [];
|
||||
for (const line of table.split('\n')) {
|
||||
const f = line.split('\t');
|
||||
if (f.length < 61) continue;
|
||||
const violent =
|
||||
f[COL.quadClass] === '4' && ['18', '19', '20'].includes(f[COL.rootCode]);
|
||||
const precision = PRECISION[f[COL.geoType]];
|
||||
const lat = Number(f[COL.lat]);
|
||||
const lon = Number(f[COL.lon]);
|
||||
const url = articleLink(f[COL.url]);
|
||||
const added = /^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/.exec(f[COL.added]);
|
||||
if (
|
||||
!precision ||
|
||||
!f[COL.lat] ||
|
||||
!f[COL.lon] ||
|
||||
!Number.isFinite(lat) ||
|
||||
!Number.isFinite(lon) ||
|
||||
Math.abs(lat) > 90 ||
|
||||
Math.abs(lon) > 180 ||
|
||||
(lat === 0 && lon === 0) ||
|
||||
!url ||
|
||||
!added
|
||||
)
|
||||
continue;
|
||||
all.push({
|
||||
id: f[COL.id],
|
||||
code: f[COL.eventCode],
|
||||
violent,
|
||||
lat,
|
||||
lon,
|
||||
place: f[COL.place].trim() || null,
|
||||
precision,
|
||||
mentions: Math.max(1, Number(f[COL.mentions]) || 1),
|
||||
addedAt: `${added[1]}-${added[2]}-${added[3]}T${added[4]}:${added[5]}:${added[6]}.000Z`,
|
||||
url,
|
||||
});
|
||||
}
|
||||
const stories = new Set(all.filter((e) => e.violent).map(storyKey));
|
||||
return all.filter((e) => stories.has(storyKey(e)));
|
||||
}
|
||||
|
||||
/**
|
||||
* The words of an article's address, when it is written in them
|
||||
* ("three-children-die-in-peshawar-cylinder-exp"), as a sentence; else null.
|
||||
*/
|
||||
export function addressWords(url) {
|
||||
let path;
|
||||
try {
|
||||
path = decodeURIComponent(new URL(url).pathname);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const segment = path
|
||||
.split('/')
|
||||
.map((part) => part.replace(/\.(?:s?html?|php|aspx?)$/i, ''))
|
||||
.filter((part) => /[-_]/.test(part))
|
||||
.sort((a, b) => b.length - a.length)[0];
|
||||
if (!segment) return null;
|
||||
const words = segment.split(/[-_]+/).filter(Boolean);
|
||||
const lettered = words.filter((word) => /^\p{L}+$/u.test(word));
|
||||
if (lettered.length < 4 || lettered.length < 0.7 * words.length) return null;
|
||||
// Numbers stay ("killed nearly 500"); an id does not: a word of letters
|
||||
// and digits, five digits or more, or four or more at either end.
|
||||
const kept = words.filter(
|
||||
(word, i) =>
|
||||
/^\p{L}+$/u.test(word) ||
|
||||
(/^\d{1,4}$/.test(word) &&
|
||||
!(word.length === 4 && (i === 0 || i === words.length - 1))),
|
||||
);
|
||||
const text = kept.slice(0, MAX_ADDRESS_WORDS).join(' ').toLowerCase();
|
||||
return text.charAt(0).toUpperCase() + text.slice(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Address words with the places the story is at written as names
|
||||
* ("killed in amman" → "killed in Amman"), and "u s" as "US".
|
||||
*/
|
||||
export function withNames(words, places) {
|
||||
let text = words.replace(/(?<![\p{L}])u s(?![\p{L}])/gu, 'US');
|
||||
const names = new Set(
|
||||
places.flatMap((place) =>
|
||||
String(place.place ?? '')
|
||||
.split(',')
|
||||
.map((part) => part.replace(/\(.*?\)/g, '').trim())
|
||||
.filter((part) => part.length > 2 && !/\(general\)/.test(part)),
|
||||
),
|
||||
);
|
||||
for (const name of names) {
|
||||
const pattern = new RegExp(
|
||||
`(?<![\\p{L}])${name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\p{L}])`,
|
||||
'gu',
|
||||
);
|
||||
text = text.replace(pattern, name);
|
||||
}
|
||||
return text.charAt(0).toUpperCase() + text.slice(1);
|
||||
}
|
||||
|
||||
// A page title's site part: " | The Daily Example", " - Wire Service".
|
||||
const SITE_PART = /\s+[|–—-]\s+([^|–—-]{2,60})$/u;
|
||||
|
||||
/**
|
||||
* A page title as a headline: entities decoded, the site's name after a
|
||||
* bar or dash taken off (when the headline before it is still one), or
|
||||
* null when it is too short to be one.
|
||||
*/
|
||||
export function cleanTitle(raw) {
|
||||
// An agency's tag set on with a bare hyphen ("…attack-Newswire", "…-X").
|
||||
let title = unescapeHtml(raw).replace(
|
||||
// After a word in lower case: "Jean-Pierre" keeps its name.
|
||||
/(?<=(?:^|\s)\p{Ll}\p{L}*|[\p{N}'")])-\p{Lu}\p{L}{0,15}$/u,
|
||||
'',
|
||||
);
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const match = SITE_PART.exec(title);
|
||||
if (!match) break;
|
||||
const rest = title.slice(0, match.index);
|
||||
if (rest.split(/\s+/).length < 4 || match[1].split(/\s+/).length > 6) break;
|
||||
title = rest;
|
||||
}
|
||||
title = title.trim().slice(0, 300);
|
||||
return title.length >= 12 && title.split(/\s+/).length >= 3 ? title : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* GKG page titles by article address (as articleLink writes it), for the
|
||||
* addresses in `wanted`.
|
||||
*/
|
||||
export function parseTitles(table, wanted) {
|
||||
const titles = new Map();
|
||||
for (const line of table.split('\n')) {
|
||||
const f = line.split('\t');
|
||||
const url = f[4] && articleLink(f[4]);
|
||||
if (!url || !wanted.has(url)) continue;
|
||||
const match = /<PAGE_TITLE>(.*?)<\/PAGE_TITLE>/s.exec(f.at(-1) || '');
|
||||
const title = match && cleanTitle(match[1]);
|
||||
if (title) titles.set(url, title);
|
||||
}
|
||||
return titles;
|
||||
}
|
||||
|
||||
/**
|
||||
* The story an event belongs to: its article's address words, which the
|
||||
* copies other outlets run share (an agency story), else its address.
|
||||
*/
|
||||
export function storyKey(event) {
|
||||
return addressWords(event.url)?.toLowerCase() || event.url;
|
||||
}
|
||||
|
||||
const PLACE_WORD = (name) =>
|
||||
new RegExp(
|
||||
`(?<![\\p{L}])${name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}s?(?![\\p{L}])`,
|
||||
'u',
|
||||
);
|
||||
/** Whether the words name a place's town or its country ("jordans" too). */
|
||||
function namedIn(words, place) {
|
||||
if (!words || !place) return false;
|
||||
const parts = place
|
||||
.split(',')
|
||||
.map((part) => part.replace(/\(.*?\)/g, '').trim())
|
||||
.filter((part) => part.length > 2);
|
||||
return [parts[0], parts.at(-1)].some(
|
||||
(part) => part && PLACE_WORD(part).test(words.toLowerCase()),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A story's places, best first, from all its placed events. GDELT places
|
||||
* an event at a bare country when the text only names a nationality ("a
|
||||
* Sri Lankan embassy", an Australian outlet's own country), so a town or
|
||||
* region wins over a country; of those, the ones the article's own words
|
||||
* name; then the most mentioned.
|
||||
*/
|
||||
export function choosePlaces(events, words) {
|
||||
const byPlace = new Map();
|
||||
for (const event of events) {
|
||||
const key = `${event.lat},${event.lon}`;
|
||||
const place = byPlace.get(key) || { ...event, mentions: 0 };
|
||||
place.mentions += event.mentions;
|
||||
byPlace.set(key, place);
|
||||
}
|
||||
let places = [...byPlace.values()];
|
||||
const specific = places.filter((p) => p.precision !== 'country');
|
||||
if (specific.length) places = specific;
|
||||
const named = places.filter((p) => namedIn(words, p.place));
|
||||
if (named.length) places = named;
|
||||
return places.sort((a, b) => b.mentions - a.mentions).slice(0, MAX_PLACES);
|
||||
}
|
||||
|
||||
// The earlier of two events: by the time GDELT added it, then by its id.
|
||||
const earlier = (a, b) =>
|
||||
b.addedAt < a.addedAt ||
|
||||
(b.addedAt === a.addedAt && Number(b.id) < Number(a.id))
|
||||
? b
|
||||
: a;
|
||||
|
||||
/**
|
||||
* One report per story (storyKey), its address and id the first copy's (an
|
||||
* export adding a more-mentioned event keeps both), placed by choosePlaces,
|
||||
* titled by a copy's page title (`titleOf`), else its address words, else
|
||||
* its violent event.
|
||||
*/
|
||||
export function eventReports(events, { titleOf = () => null } = {}) {
|
||||
const stories = new Map();
|
||||
for (const event of events) {
|
||||
const key = storyKey(event);
|
||||
const list = stories.get(key) || [];
|
||||
list.push(event);
|
||||
stories.set(key, list);
|
||||
}
|
||||
const reports = [];
|
||||
for (const list of stories.values()) {
|
||||
const violent = list
|
||||
.filter((e) => e.violent !== false)
|
||||
.sort(
|
||||
(a, b) => b.mentions - a.mentions || a.addedAt.localeCompare(b.addedAt),
|
||||
);
|
||||
if (!violent.length) continue;
|
||||
const [lead] = violent;
|
||||
const first = list.reduce(earlier);
|
||||
const { url } = first;
|
||||
const address = addressWords(url);
|
||||
const places = choosePlaces(list, address);
|
||||
const [, eventCategory, words] = eventKind(lead.code);
|
||||
const page = [url, ...list.map((e) => e.url)]
|
||||
.map((copy) => titleOf(copy))
|
||||
.find(Boolean);
|
||||
const own = page || (address ? withNames(address, places) : null);
|
||||
const title = own || eventTitle(words, places[0].place);
|
||||
// The article's words decide whether it is conflict, as any headline's
|
||||
// would: GDELT codes an election story as "fighting" too. GDELT's code
|
||||
// names the kind only where the words read as conflict without naming
|
||||
// one. A story with no words at all has GDELT's code alone to go by,
|
||||
// which is not enough to list it under Conflicts: it is minor (kept
|
||||
// for Local) until a page title comes in.
|
||||
const read = own ? categorizeText({ title: own }) : null;
|
||||
const tags = read?.tags || [];
|
||||
const relevance = read
|
||||
? relevanceOf({ category: read.category, tags, title: own })
|
||||
: 'minor';
|
||||
const category = !read
|
||||
? eventCategory
|
||||
: read.category !== 'other'
|
||||
? read.category
|
||||
: relevance === 'minor'
|
||||
? 'other'
|
||||
: eventCategory;
|
||||
const publisher = new URL(url).hostname.replace(/^www\./, '');
|
||||
const geo = ({ lat, lon, place, precision }) => ({
|
||||
lat,
|
||||
lon,
|
||||
place,
|
||||
precision,
|
||||
confidence: precision === 'city' ? 0.6 : 0.3,
|
||||
mappable: true,
|
||||
method: 'source',
|
||||
});
|
||||
reports.push({
|
||||
id: `gdelt-${first.id}`,
|
||||
title,
|
||||
url,
|
||||
publisher,
|
||||
source: { name: publisher, attribution: 'GDELT Project' },
|
||||
kind: 'gdelt',
|
||||
side: 'unknown',
|
||||
category,
|
||||
relevance,
|
||||
severity: Math.max(
|
||||
1,
|
||||
severityByRules({ category, tags, title: own || '' }),
|
||||
),
|
||||
severityBy: 'rules',
|
||||
publishedAt: first.addedAt,
|
||||
geo: geo(places[0]),
|
||||
locations: places.slice(1).map(geo),
|
||||
media: [],
|
||||
mentions: list.reduce((sum, e) => sum + e.mentions, 0),
|
||||
copies: new Set(list.map((e) => e.url)).size,
|
||||
// A title of the event's words and its place, when the address had
|
||||
// none: a renamed place renames it (eventTitle).
|
||||
...(own ? {} : { eventWords: words }),
|
||||
titled: page ? 'page' : own ? 'address' : 'event',
|
||||
});
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
|
||||
/** The title of a report without address words, at its (renamed) place. */
|
||||
export const eventTitle = (words, place) =>
|
||||
place ? `${words} in ${place}` : words;
|
||||
|
||||
/**
|
||||
* One export's table, or null while GDELT has not published it (GDELT
|
||||
* lists the newest in lastupdate.txt; see fetchLatestExport).
|
||||
*/
|
||||
async function fetchTable(url, { fetchImpl, signal, maxBytes, tableBytes }) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
} catch {
|
||||
throw new UpstreamError('GDELT exports unreachable');
|
||||
}
|
||||
if (response.status === 404) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw new UpstreamError(`GDELT exports answered ${response.status}`, {
|
||||
status: response.status,
|
||||
});
|
||||
}
|
||||
let bytes;
|
||||
try {
|
||||
bytes = await readResponseBytesCapped(response, maxBytes, signal);
|
||||
} catch (error) {
|
||||
// Over the cap stays over it; a read cut short may succeed again.
|
||||
throw new UpstreamError('GDELT export unreadable', {
|
||||
unreadable: error?.code === 'RESPONSE_TOO_LARGE',
|
||||
});
|
||||
}
|
||||
return unzipExport(bytes, tableBytes);
|
||||
}
|
||||
|
||||
/** One events export's events, or null while it is not published yet. */
|
||||
export async function fetchExport(url, { fetchImpl, signal } = {}) {
|
||||
const table = await fetchTable(url, {
|
||||
fetchImpl,
|
||||
signal,
|
||||
maxBytes: MAX_ZIP_BYTES,
|
||||
tableBytes: MAX_TABLE_BYTES,
|
||||
});
|
||||
return table === null ? null : parseEvents(table);
|
||||
}
|
||||
|
||||
/**
|
||||
* The page titles a GKG export holds for the addresses in `wanted`, or null
|
||||
* while it is not published yet.
|
||||
*/
|
||||
export async function fetchTitles(url, wanted, { fetchImpl, signal } = {}) {
|
||||
const table = await fetchTable(url, {
|
||||
fetchImpl,
|
||||
signal,
|
||||
maxBytes: MAX_GKG_ZIP_BYTES,
|
||||
tableBytes: MAX_GKG_TABLE_BYTES,
|
||||
});
|
||||
return table === null ? null : parseTitles(table, wanted);
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
addressWords,
|
||||
cleanTitle,
|
||||
gkgUrl,
|
||||
parseTitles,
|
||||
withNames,
|
||||
eventReports,
|
||||
exportsOfDay,
|
||||
exportUrl,
|
||||
fetchExport,
|
||||
parseEvents,
|
||||
unzipExport,
|
||||
} from './gdeltEvents.js';
|
||||
import { eventRow, gkgRow, zipTable } from './gdeltFixtures.mjs';
|
||||
|
||||
const ARTICLE =
|
||||
'https://www.wire.example/rubric-ato/3912-russian-drones-attack-kharkiv-overnight-three-killed.html';
|
||||
|
||||
test('the day is 96 exports, one each quarter hour, newest first', () => {
|
||||
const now = Date.parse('2026-11-05T15:07:30Z');
|
||||
assert.equal(
|
||||
exportUrl(now),
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105150000.export.CSV.zip',
|
||||
);
|
||||
const day = exportsOfDay(now);
|
||||
assert.equal(day.length, 96);
|
||||
assert.equal(
|
||||
day[1],
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105144500.export.CSV.zip',
|
||||
);
|
||||
assert.equal(
|
||||
day.at(-1),
|
||||
'https://data.gdeltproject.org/gdeltv2/20261104151500.export.CSV.zip',
|
||||
);
|
||||
});
|
||||
|
||||
test('a story with physical violence is kept with all its placed events', () => {
|
||||
const rows = [
|
||||
eventRow({ id: 1, code: '195', url: ARTICLE }),
|
||||
eventRow({ id: 2, code: '183', url: ARTICLE }),
|
||||
eventRow({ id: 3, code: '2001', url: ARTICLE }),
|
||||
// Not physical, but of the same story: kept to place it by.
|
||||
eventRow({ id: 4, code: '150', url: ARTICLE }),
|
||||
eventRow({ id: 7, code: '190', quad: '3', url: ARTICLE }),
|
||||
// A story with no violence in it is left out.
|
||||
eventRow({
|
||||
id: 5,
|
||||
code: '163',
|
||||
url: 'https://x.example/talks-resume-in-geneva-today-officials',
|
||||
}),
|
||||
eventRow({
|
||||
id: 6,
|
||||
code: '173',
|
||||
url: 'https://x.example/talks-resume-in-geneva-today-officials',
|
||||
}),
|
||||
// Unplaced, at 0,0, or without a link.
|
||||
eventRow({ id: 8, geoType: '0', url: ARTICLE }),
|
||||
eventRow({ id: 9, lat: 0, lon: 0, url: ARTICLE }),
|
||||
eventRow({ id: 10, url: 'javascript:alert(1)' }),
|
||||
'short\trow',
|
||||
];
|
||||
const events = parseEvents(rows.join('\n'));
|
||||
assert.deepEqual(
|
||||
events.map((e) => [e.id, e.violent]),
|
||||
[
|
||||
['1', true],
|
||||
['2', true],
|
||||
['3', true],
|
||||
['4', false],
|
||||
['7', false],
|
||||
],
|
||||
);
|
||||
assert.equal(events[0].precision, 'city');
|
||||
assert.equal(events[0].addedAt, '2026-11-05T11:45:00.000Z');
|
||||
});
|
||||
|
||||
test("an article is one report, at its most-mentioned event, in its address's words", () => {
|
||||
const events = parseEvents(
|
||||
[
|
||||
eventRow({ id: 11, code: '195', mentions: 2, url: ARTICLE }),
|
||||
eventRow({
|
||||
id: 12,
|
||||
code: '190',
|
||||
mentions: 9,
|
||||
url: ARTICLE,
|
||||
place: "Kupiansk, Kharkivs'ka Oblast', Ukraine",
|
||||
lat: 49.71,
|
||||
lon: 37.61,
|
||||
}),
|
||||
eventRow({
|
||||
id: 13,
|
||||
code: '190',
|
||||
geoType: '1',
|
||||
place: 'Ukraine',
|
||||
lat: 49,
|
||||
lon: 32,
|
||||
url: 'https://example.com/a/b/12345',
|
||||
}),
|
||||
].join('\n'),
|
||||
);
|
||||
const [drones, bare] = eventReports(events);
|
||||
// The story's first event (by time, then id) names it.
|
||||
assert.equal(drones.id, 'gdelt-11');
|
||||
// Its address words, with the place it is at written as a name.
|
||||
assert.equal(
|
||||
drones.title,
|
||||
'Russian drones attack Kharkiv overnight three killed',
|
||||
);
|
||||
// Kupiansk has more mentions; the article's own words name Kharkiv.
|
||||
assert.equal(drones.geo.place, "Kharkiv, Kharkivs'ka Oblast', Ukraine");
|
||||
assert.equal(drones.geo.precision, 'city');
|
||||
assert.equal(drones.locations.length, 0, 'Kupiansk is not named');
|
||||
assert.equal(drones.category, 'strike');
|
||||
assert.equal(drones.relevance, 'core');
|
||||
assert.equal(drones.severity, 2);
|
||||
assert.equal(drones.publisher, 'wire.example');
|
||||
assert.equal(drones.source.attribution, 'GDELT Project');
|
||||
assert.equal(drones.side, 'unknown', 'no actor is said');
|
||||
// No words in the address: GDELT's coding alone, so minor (not listed
|
||||
// under Conflicts), at least severity 1.
|
||||
assert.equal(bare.title, 'Fighting reported in Ukraine');
|
||||
assert.equal(bare.category, 'ground');
|
||||
assert.equal(bare.relevance, 'minor');
|
||||
assert.equal(bare.severity, 1);
|
||||
assert.equal(bare.geo.precision, 'country');
|
||||
});
|
||||
|
||||
test('an article GDELT codes as fighting whose words say otherwise stays out of Conflicts', () => {
|
||||
const [report] = eventReports(
|
||||
parseEvents(
|
||||
eventRow({
|
||||
url: 'https://news.example.com/2026/11/05/city-council-approves-new-budget-for-schools',
|
||||
id: 20,
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert.equal(report.relevance, 'minor');
|
||||
});
|
||||
|
||||
test('address words need four lettered words, most of the address', () => {
|
||||
assert.equal(addressWords('https://x.example/news/1234567'), null);
|
||||
assert.equal(addressWords('https://x.example/a/lueop1pzfi'), null);
|
||||
assert.equal(
|
||||
addressWords('https://x.example/2026/11/05/a-b-1-2-3-4-5'),
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
addressWords(
|
||||
'https://x.example/story/2638789/three-children-die-in-peshawar.html',
|
||||
),
|
||||
'Three children die in peshawar',
|
||||
);
|
||||
});
|
||||
|
||||
test('an export is unzipped and read; one not published yet is null', async () => {
|
||||
const zip = zipTable(eventRow({ id: 30, code: '193', url: ARTICLE }));
|
||||
assert.deepEqual(
|
||||
parseEvents(await unzipExport(zip)).map((e) => e.id),
|
||||
['30'],
|
||||
);
|
||||
const served = await fetchExport('https://data.gdeltproject.org/x.zip', {
|
||||
fetchImpl: async () => new Response(zip, { status: 200 }),
|
||||
});
|
||||
assert.deepEqual(
|
||||
served.map((e) => e.id),
|
||||
['30'],
|
||||
);
|
||||
const later = await fetchExport('https://data.gdeltproject.org/y.zip', {
|
||||
fetchImpl: async () => new Response('not found', { status: 404 }),
|
||||
});
|
||||
assert.equal(later, null);
|
||||
await assert.rejects(
|
||||
fetchExport('https://data.gdeltproject.org/z.zip', {
|
||||
fetchImpl: async () => new Response('busy', { status: 503 }),
|
||||
}),
|
||||
{ name: 'UpstreamError' },
|
||||
);
|
||||
await assert.rejects(unzipExport(new Uint8Array([1, 2, 3])), {
|
||||
name: 'UpstreamError',
|
||||
});
|
||||
});
|
||||
|
||||
test("a nationality is no place: a town the article's words name wins over a bare country", () => {
|
||||
const url =
|
||||
'https://www.sun.example/world/2026/11/05/security-officer-attacked-and-killed-near-sri-lankan-embassy-in-jordans-capital';
|
||||
const amman = { place: 'Amman, (JO11), Jordan', lat: 31.95, lon: 35.9333 };
|
||||
const cairo = { place: 'Cairo, Al Qahirah, Egypt', lat: 30.05, lon: 31.25 };
|
||||
const [report] = eventReports(
|
||||
parseEvents(
|
||||
[
|
||||
// "Sri Lankan embassy": GDELT's most-mentioned place, a bare country.
|
||||
eventRow({
|
||||
id: 1,
|
||||
mentions: 12,
|
||||
geoType: '1',
|
||||
place: 'Sri Lanka',
|
||||
lat: 7,
|
||||
lon: 81,
|
||||
url,
|
||||
}),
|
||||
eventRow({ id: 2, mentions: 6, ...cairo, url }),
|
||||
eventRow({ id: 3, mentions: 4, ...amman, url }),
|
||||
eventRow({ id: 4, mentions: 2, ...amman, url }),
|
||||
].join('\n'),
|
||||
),
|
||||
);
|
||||
assert.equal(report.geo.place, 'Amman, (JO11), Jordan');
|
||||
assert.equal(report.geo.precision, 'city');
|
||||
});
|
||||
|
||||
test('an agency story run by several outlets is one report, placed by all their events', () => {
|
||||
const words = 'jordan-security-officer-killed-near-sri-lanka-embassy';
|
||||
const times = `https://www.times.example/story/9386807/${words}/`;
|
||||
const chronicle = `https://www.chronicle.example/story/9386807/${words}/`;
|
||||
const reports = eventReports(
|
||||
parseEvents(
|
||||
[
|
||||
// The outlet's own country, at country level, on its violent events.
|
||||
eventRow({
|
||||
id: 1,
|
||||
mentions: 6,
|
||||
geoType: '1',
|
||||
place: 'Australia',
|
||||
lat: -25,
|
||||
lon: 135,
|
||||
url: times,
|
||||
added: '20261105141500',
|
||||
}),
|
||||
// Its other events are placed in Jordan.
|
||||
eventRow({
|
||||
id: 2,
|
||||
code: '120',
|
||||
quad: '3',
|
||||
mentions: 8,
|
||||
place: 'Karak, Jordan (general), Jordan',
|
||||
lat: 31.18,
|
||||
lon: 35.7,
|
||||
url: times,
|
||||
added: '20261105141500',
|
||||
}),
|
||||
eventRow({
|
||||
id: 3,
|
||||
mentions: 4,
|
||||
geoType: '1',
|
||||
place: 'Syria',
|
||||
lat: 35,
|
||||
lon: 38,
|
||||
url: chronicle,
|
||||
added: '20261105151500',
|
||||
}),
|
||||
].join('\n'),
|
||||
),
|
||||
);
|
||||
assert.equal(reports.length, 1);
|
||||
const [story] = reports;
|
||||
assert.equal(story.copies, 2);
|
||||
assert.equal(story.url, times, 'the first copy');
|
||||
assert.equal(story.publishedAt, '2026-11-05T14:15:00.000Z');
|
||||
assert.equal(story.geo.place, 'Karak, Jordan (general), Jordan');
|
||||
});
|
||||
|
||||
test("a story's title is a copy's page title from the GKG export", () => {
|
||||
const titles = parseTitles(
|
||||
[
|
||||
gkgRow(
|
||||
ARTICLE,
|
||||
'Russian drones attack Kharkiv overnight, three killed | Wire',
|
||||
),
|
||||
gkgRow('https://elsewhere.example/a', 'Not one of ours, not kept here'),
|
||||
].join('\n'),
|
||||
new Set([ARTICLE]),
|
||||
);
|
||||
assert.deepEqual(
|
||||
[...titles],
|
||||
[[ARTICLE, 'Russian drones attack Kharkiv overnight, three killed']],
|
||||
);
|
||||
const [report] = eventReports(
|
||||
parseEvents(eventRow({ id: 1, url: ARTICLE })),
|
||||
{
|
||||
titleOf: (url) => titles.get(url) || null,
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
report.title,
|
||||
'Russian drones attack Kharkiv overnight, three killed',
|
||||
);
|
||||
assert.equal(report.titled, 'page');
|
||||
assert.equal(
|
||||
gkgUrl(
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105141500.export.CSV.zip',
|
||||
),
|
||||
'https://data.gdeltproject.org/gdeltv2/20261105141500.gkg.csv.zip',
|
||||
);
|
||||
});
|
||||
|
||||
test('a page title loses its site name, keeps its words, and must be a headline', () => {
|
||||
assert.equal(
|
||||
cleanTitle(
|
||||
'Yemen's president sacks ministers | Yemen | The Daily Example',
|
||||
),
|
||||
"Yemen's president sacks ministers",
|
||||
);
|
||||
assert.equal(
|
||||
cleanTitle('Strike in Kharkiv - live updates - Wire Service'),
|
||||
'Strike in Kharkiv - live updates',
|
||||
);
|
||||
// Too little before the dash: the dash is part of the headline.
|
||||
assert.equal(cleanTitle('Iran - Wire Service'), 'Iran - Wire Service');
|
||||
assert.equal(cleanTitle('Home'), null);
|
||||
assert.equal(
|
||||
cleanTitle('Jordanian security officer killed in stabbing attack-Newswire'),
|
||||
'Jordanian security officer killed in stabbing attack',
|
||||
);
|
||||
assert.equal(
|
||||
cleanTitle('Five killed in Gaza, sources say-X'),
|
||||
'Five killed in Gaza, sources say',
|
||||
);
|
||||
// A hyphenated name is no tag.
|
||||
assert.equal(
|
||||
cleanTitle('Talks resume with Jean-Pierre'),
|
||||
'Talks resume with Jean-Pierre',
|
||||
);
|
||||
});
|
||||
|
||||
test('address words keep their numbers and name the places the story is at', () => {
|
||||
const words = addressWords(
|
||||
'https://x.example/2026/11/05/jordan-says-u-s-envoy-met-officials-in-amman-after-500-killed',
|
||||
);
|
||||
assert.equal(
|
||||
words,
|
||||
'Jordan says u s envoy met officials in amman after 500 killed',
|
||||
);
|
||||
assert.equal(
|
||||
withNames(words, [{ place: 'Amman, (JO11), Jordan' }]),
|
||||
'Jordan says US envoy met officials in Amman after 500 killed',
|
||||
);
|
||||
});
|
||||
|
||||
test("a story's id is its first event's: a later, more-mentioned event keeps it", () => {
|
||||
const first = eventRow({ id: 40, mentions: 1, url: ARTICLE });
|
||||
const [before] = eventReports(parseEvents(first));
|
||||
const [after] = eventReports(
|
||||
parseEvents(
|
||||
[
|
||||
first,
|
||||
eventRow({
|
||||
id: 41,
|
||||
code: '195',
|
||||
mentions: 30,
|
||||
url: ARTICLE,
|
||||
added: '20261105120000',
|
||||
}),
|
||||
].join('\n'),
|
||||
),
|
||||
);
|
||||
assert.equal(before.id, 'gdelt-40');
|
||||
assert.equal(after.id, 'gdelt-40');
|
||||
assert.equal(after.mentions, 31);
|
||||
});
|
||||
|
||||
test('words that read as no conflict leave GDELT no kind to name', () => {
|
||||
const [report] = eventReports(
|
||||
parseEvents(
|
||||
eventRow({
|
||||
id: 42,
|
||||
code: '195',
|
||||
url: 'https://x.example/2026/11/05/local-bakery-celebrates-fifty-years-of-baking',
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert.equal(report.relevance, 'minor');
|
||||
assert.equal(report.category, 'other');
|
||||
});
|
||||
|
||||
test('a GKG address written with http is the https one the events carry', () => {
|
||||
const plain = ARTICLE.replace('https:', 'http:');
|
||||
const [event] = parseEvents(eventRow({ id: 43, url: plain }));
|
||||
assert.equal(event.url, ARTICLE);
|
||||
const titles = parseTitles(
|
||||
gkgRow(plain, 'Russian drones attack Kharkiv overnight, three killed'),
|
||||
new Set([event.url]),
|
||||
);
|
||||
assert.equal(
|
||||
titles.get(ARTICLE),
|
||||
'Russian drones attack Kharkiv overnight, three killed',
|
||||
);
|
||||
});
|
||||
|
||||
test('a corrupt or oversized export is an UpstreamError, never a RangeError', async () => {
|
||||
const zip = zipTable(eventRow({ id: 44, url: ARTICLE }));
|
||||
const view = new DataView(zip.buffer);
|
||||
// The central directory's local header offset, pointing past the end.
|
||||
const dir = view.getUint32(zip.length - 22 + 16, true);
|
||||
const far = zip.slice();
|
||||
new DataView(far.buffer).setUint32(dir + 42, zip.length - 10, true);
|
||||
await assert.rejects(unzipExport(far), { name: 'UpstreamError' });
|
||||
// The end record cut short.
|
||||
await assert.rejects(unzipExport(zip.subarray(0, zip.length - 30)), {
|
||||
name: 'UpstreamError',
|
||||
});
|
||||
// Unzipped past the table's cap.
|
||||
await assert.rejects(unzipExport(zip, 16), { name: 'UpstreamError' });
|
||||
assert.equal((await unzipExport(zip)).length > 16, true);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
// Test fixtures: GDELT 2.0 export rows and their zip, as the export files
|
||||
// carry them (61 tab-separated columns, one zipped table).
|
||||
import { deflateRawSync } from 'node:zlib';
|
||||
|
||||
/** One events row; only the columns the reader looks at are filled. */
|
||||
export function eventRow({
|
||||
id,
|
||||
code = '190',
|
||||
quad = '4',
|
||||
mentions = 1,
|
||||
geoType = '4',
|
||||
place = "Kharkiv, Kharkivs'ka Oblast', Ukraine",
|
||||
lat = 49.9935,
|
||||
lon = 36.2304,
|
||||
added = '20261105114500',
|
||||
url,
|
||||
}) {
|
||||
const f = new Array(61).fill('');
|
||||
f[0] = String(id);
|
||||
f[26] = code;
|
||||
f[28] = code.slice(0, 2);
|
||||
f[29] = quad;
|
||||
f[31] = String(mentions);
|
||||
f[51] = geoType;
|
||||
f[52] = place;
|
||||
f[56] = String(lat);
|
||||
f[57] = String(lon);
|
||||
f[59] = added;
|
||||
f[60] = url;
|
||||
return f.join('\t');
|
||||
}
|
||||
|
||||
/** One GKG 2.1 row: an article's address and its page title (27 columns). */
|
||||
export function gkgRow(url, title) {
|
||||
const f = new Array(27).fill('');
|
||||
f[4] = url;
|
||||
f[26] = `<PAGE_PRECISEPUBTIMESTAMP>20261105114500</PAGE_PRECISEPUBTIMESTAMP><PAGE_TITLE>${title}</PAGE_TITLE>`;
|
||||
return f.join('\t');
|
||||
}
|
||||
|
||||
/** A single-entry zip of `text`, deflated, sizes in the central directory. */
|
||||
export function zipTable(text, name = 'table.export.CSV') {
|
||||
const data = Buffer.from(text, 'utf8');
|
||||
const packed = deflateRawSync(data);
|
||||
const nameBytes = Buffer.from(name);
|
||||
const local = Buffer.alloc(30);
|
||||
local.writeUInt32LE(0x04034b50, 0);
|
||||
local.writeUInt16LE(20, 4);
|
||||
local.writeUInt16LE(0x08, 6); // sizes follow the data (left 0 here)
|
||||
local.writeUInt16LE(8, 8);
|
||||
local.writeUInt16LE(nameBytes.length, 26);
|
||||
const dir = Buffer.alloc(46);
|
||||
dir.writeUInt32LE(0x02014b50, 0);
|
||||
dir.writeUInt16LE(20, 4);
|
||||
dir.writeUInt16LE(20, 6);
|
||||
dir.writeUInt16LE(0x08, 8);
|
||||
dir.writeUInt16LE(8, 10);
|
||||
dir.writeUInt32LE(packed.length, 20);
|
||||
dir.writeUInt32LE(data.length, 24);
|
||||
dir.writeUInt16LE(nameBytes.length, 28);
|
||||
dir.writeUInt32LE(0, 42);
|
||||
const dirAt = local.length + nameBytes.length + packed.length;
|
||||
const end = Buffer.alloc(22);
|
||||
end.writeUInt32LE(0x06054b50, 0);
|
||||
end.writeUInt16LE(1, 8);
|
||||
end.writeUInt16LE(1, 10);
|
||||
end.writeUInt32LE(dir.length + nameBytes.length, 12);
|
||||
end.writeUInt32LE(dirAt, 16);
|
||||
return new Uint8Array(
|
||||
Buffer.concat([local, nameBytes, packed, dir, nameBytes, end]),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,916 @@
|
||||
// The News layer's routes answered by Core itself, when no news server is
|
||||
// configured (GEV_NEWS_ORIGIN). Conflicts: the day's GDELT events (no key;
|
||||
// gdeltEvents.js), one report per article at GDELT's place for it, and with
|
||||
// LIVEUAMAP_API_KEY Liveuamap's mapped items too. Local: World News API
|
||||
// (WORLDNEWSAPI_KEY) by the country at the view centre; without its key the
|
||||
// section answers 503 {error: 'no_key', key} and reads nothing, and past
|
||||
// its day's reads (WORLDNEWSAPI_DAILY_READS) 503 {error: 'key_quota', key}.
|
||||
// Conflicts' categories, relevance and severity are the shared rules
|
||||
// (categorize.js); a country's own news keeps World News API's category.
|
||||
// A news server may replace these readings with its own.
|
||||
//
|
||||
// A read that has something to serve is not waited for: what was read is
|
||||
// served while it is read again. A section that cannot be read answers 502,
|
||||
// never an empty list.
|
||||
//
|
||||
// The answers are the same /api/news contract a news server serves
|
||||
// (src/layers/news/records.js reads both).
|
||||
|
||||
import {
|
||||
CATEGORY_ORDER,
|
||||
categorizeText,
|
||||
relevanceOf,
|
||||
severityByRules,
|
||||
} from './categorize.js';
|
||||
import {
|
||||
areaAt,
|
||||
countryAnchor,
|
||||
countryByCode,
|
||||
countryPolygons,
|
||||
} from './areas.js';
|
||||
import {
|
||||
eventReports,
|
||||
fetchLatestExport,
|
||||
eventTitle,
|
||||
exportAt,
|
||||
exportsOfDay,
|
||||
fetchExport,
|
||||
fetchTitles,
|
||||
gkgUrl,
|
||||
storyKey,
|
||||
} from './gdeltEvents.js';
|
||||
import { fetchLiveuamap, fetchWorldNews, UpstreamError } from './upstream.js';
|
||||
|
||||
/** The keys each section reads with, by the key registry's id. */
|
||||
export const NEWS_KEYS = Object.freeze({
|
||||
liveuamap: 'LIVEUAMAP_API_KEY',
|
||||
worldnewsapi: 'WORLDNEWSAPI_KEY',
|
||||
});
|
||||
|
||||
/** The one hotspot: the world, its reports drawn as markers at every zoom. */
|
||||
export const WORLDWIDE_HOTSPOT = 'worldwide';
|
||||
const HOUR_MS = 3_600_000;
|
||||
const DAY_MS = 24 * HOUR_MS;
|
||||
// GDELT publishes an export every 15 minutes and lists the newest in
|
||||
// lastupdate.txt: the day's missing ones up to it are read every 5 minutes,
|
||||
// newest first, a dozen a read (the first day's 96 fill in over the next
|
||||
// reads, a few seconds apart, while each reads something new). One listed
|
||||
// but missing (404) an hour and a half after its time is taken as never
|
||||
// published; one that fails GDELT_EXPORT_TRIES times (a try a round), or
|
||||
// cannot be read at all, is given up on, so one bad file never holds the
|
||||
// rest back. Only while GDELT itself answers is a file given up on. The first
|
||||
// answer waits at most FIRST_ANSWER_MS for the first exports.
|
||||
const GDELT_FRESH_MS = 5 * 60_000;
|
||||
const GDELT_EXPORTS_PER_READ = 12;
|
||||
const GDELT_PARALLEL = 4;
|
||||
const GDELT_NEXT_MS = 2_000;
|
||||
const GDELT_GONE_MS = 90 * 60_000;
|
||||
const GDELT_EXPORT_TRIES = 3;
|
||||
const MAX_GDELT_REPORTS = 5000;
|
||||
// Events kept of one story, to place it by.
|
||||
const MAX_STORY_EVENTS = 200;
|
||||
// Page titles are read from the GKG exports of the newest three hours: one
|
||||
// runs to 6 MB, so the day's 96 are not fetched. A file that fails is tried
|
||||
// with the next reads, a few times.
|
||||
const GDELT_TITLED_EXPORTS = 12;
|
||||
const GKG_TRIES = 3;
|
||||
const LIVE_FRESH_MS = 5 * 60_000;
|
||||
const COUNTRY_FRESH_MS = 30 * 60_000;
|
||||
// A failed read is not tried again for a minute; what was read before is
|
||||
// served meanwhile, up to these ages (GDELT's: since an export last came in).
|
||||
const RETRY_MS = 60_000;
|
||||
// A read with nothing to serve yet is waited for at most this long, well under
|
||||
// the browser's own 15 s: it goes on, and the next ask is served from it.
|
||||
const FIRST_ANSWER_MS = 10_000;
|
||||
const GDELT_STALE_MS = 2 * HOUR_MS;
|
||||
const LIVE_STALE_MS = HOUR_MS;
|
||||
// World News API's terms allow a cache of at most an hour, deleted after:
|
||||
// a country's news is kept no longer, served or not (dropExpiredCountries).
|
||||
const COUNTRY_STALE_MS = HOUR_MS;
|
||||
const MAX_COUNTRIES = 64;
|
||||
// World News API reads a UTC day, across all countries: each is about 1.5
|
||||
// of the free plan's 50 daily points.
|
||||
const DEFAULT_DAILY_READS = 30;
|
||||
const DEFAULT_LIMIT = 50;
|
||||
const EMPTY_OUTLINE = Object.freeze({
|
||||
polygons: [],
|
||||
borders: [],
|
||||
disputed: [],
|
||||
area: null,
|
||||
});
|
||||
// A country's own news keeps World News API's category: the rules read
|
||||
// fighting and crises, and in everyday news their words mislead ("advances
|
||||
// in the rankings" is no ground fighting). Fighting in the country reaches
|
||||
// Local as the Conflicts reports placed there.
|
||||
const WORLD_NEWS_CATEGORY = { politics: 'political', business: 'economy' };
|
||||
|
||||
const list = (value) =>
|
||||
value ? new Set(value.split(',').filter(Boolean)) : null;
|
||||
|
||||
/** The rules' reading of a Conflicts headline: category, relevance, severity. */
|
||||
function readTitle(title) {
|
||||
const read = categorizeText({ title });
|
||||
const { category } = read;
|
||||
return {
|
||||
category,
|
||||
relevance: relevanceOf({
|
||||
category,
|
||||
tags: read.tags,
|
||||
title,
|
||||
dedicated: true,
|
||||
}),
|
||||
severity: severityByRules({ category, tags: read.tags, title }),
|
||||
};
|
||||
}
|
||||
|
||||
/** A one-report event carrying the report's picture, for its card. */
|
||||
function pictureEvent(id, report, image, video, publisher) {
|
||||
return {
|
||||
id,
|
||||
status: 'single',
|
||||
size: 1,
|
||||
reports: 1,
|
||||
severity: report.severity,
|
||||
firstAt: report.publishedAt,
|
||||
lastAt: report.publishedAt,
|
||||
sides: [],
|
||||
image: { url: image, publisher, ...(video ? { video } : {}) },
|
||||
};
|
||||
}
|
||||
|
||||
const media = ({ image, video }) =>
|
||||
video ? ['video'] : image ? ['photo'] : [];
|
||||
|
||||
/** A Liveuamap item as a report of the Conflicts section. */
|
||||
export function liveuamapReport(item) {
|
||||
const id = `lum-${item.id}`;
|
||||
const report = {
|
||||
id,
|
||||
hotspotId: WORLDWIDE_HOTSPOT,
|
||||
hotspotIds: [WORLDWIDE_HOTSPOT],
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
publisher: 'Liveuamap',
|
||||
source: { name: 'Liveuamap' },
|
||||
kind: 'liveuamap',
|
||||
side: 'unknown',
|
||||
// A conflict map: everything it posts is at least related.
|
||||
...readTitle(item.title),
|
||||
severityBy: 'rules',
|
||||
publishedAt: item.publishedAt,
|
||||
geo: {
|
||||
lat: item.lat,
|
||||
lon: item.lon,
|
||||
place: item.place,
|
||||
precision: 'city',
|
||||
confidence: 1,
|
||||
mappable: true,
|
||||
method: 'source',
|
||||
},
|
||||
media: media(item),
|
||||
};
|
||||
if (item.image)
|
||||
report.event = pictureEvent(
|
||||
id,
|
||||
report,
|
||||
item.image,
|
||||
item.video,
|
||||
'Liveuamap',
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* A GDELT place named by its town and the area Natural Earth has there (a
|
||||
* disputed area as itself, else the country): GDELT's own names follow old
|
||||
* FIPS regions ("Bureij, Yerushalayim, Israel" for a camp in Gaza, "Abuja,
|
||||
* …, Niger" for Nigeria's capital). Its point stays. A town a disputed
|
||||
* area is named after ("Jerusalem" in East Jerusalem) lies on both sides
|
||||
* of its line: GDELT puts the whole city at one point, so it is named as
|
||||
* the town alone.
|
||||
*/
|
||||
export function gdeltPlace(geo) {
|
||||
const area = areaAt(geo.lat, geo.lon);
|
||||
const town = String(geo.place ?? '')
|
||||
.split(',')[0]
|
||||
.trim();
|
||||
const straddles =
|
||||
area?.kind === 'disputed' &&
|
||||
town &&
|
||||
area.name.toLowerCase().includes(town.toLowerCase());
|
||||
const place =
|
||||
geo.precision === 'country' && area?.kind === 'country'
|
||||
? area.name
|
||||
: straddles
|
||||
? town
|
||||
: [town, area?.name].filter(Boolean).join(', ');
|
||||
return { ...geo, place: place || null };
|
||||
}
|
||||
|
||||
/**
|
||||
* A World News API article as a report of the Local section: placed in its
|
||||
* publisher's country alone, at a point inside it.
|
||||
*/
|
||||
export function worldNewsReport(article, country) {
|
||||
const id = `wna-${article.id}`;
|
||||
const anchor = countryAnchor(country);
|
||||
const report = {
|
||||
id,
|
||||
hotspotId: null,
|
||||
hotspotIds: [],
|
||||
title: article.title,
|
||||
url: article.url,
|
||||
publisher: article.publisher,
|
||||
// World News API's free plan asks to be credited where its news shows.
|
||||
source: { name: article.publisher, attribution: 'World News API' },
|
||||
kind: 'worldnewsapi',
|
||||
side: 'unknown',
|
||||
category: WORLD_NEWS_CATEGORY[article.category] || 'other',
|
||||
// Local news: what the country's own outlets publish, with no severity
|
||||
// of its own.
|
||||
relevance: 'minor',
|
||||
publishedAt: article.publishedAt,
|
||||
geo: {
|
||||
lat: anchor.lat,
|
||||
lon: anchor.lon,
|
||||
place: country.name,
|
||||
precision: 'country',
|
||||
confidence: 1,
|
||||
mappable: true,
|
||||
method: 'source',
|
||||
},
|
||||
media: media(article),
|
||||
};
|
||||
if (article.image)
|
||||
report.event = pictureEvent(
|
||||
id,
|
||||
report,
|
||||
article.image,
|
||||
article.video,
|
||||
article.publisher,
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
const AT = /^(-?\d{1,2}(?:\.\d{1,2})?),(-?\d{1,3}(?:\.\d{1,2})?)$/;
|
||||
function point(value) {
|
||||
const match = AT.exec(String(value ?? ''));
|
||||
if (!match) return null;
|
||||
const [lat, lon] = [Number(match[1]), Number(match[2])];
|
||||
return Math.abs(lat) <= 90 && Math.abs(lon) <= 180 ? { lat, lon } : null;
|
||||
}
|
||||
|
||||
const noKey = (key) => ({ status: 503, body: { error: 'no_key', key } });
|
||||
const rejectedKey = (key) => ({
|
||||
status: 503,
|
||||
body: { error: 'key_rejected', key },
|
||||
});
|
||||
const spentKey = (key) => ({ status: 503, body: { error: 'key_quota', key } });
|
||||
|
||||
/**
|
||||
* The keyed news routes. `answer(path, params)` resolves to `{status, body}`
|
||||
* for `/hotspots`, `/feed` and `/outline`, whose queries the proxy has
|
||||
* already checked against the client's allowlist. `later(run, ms)` runs
|
||||
* the next GDELT read of a day still filling in (an unref'd timer).
|
||||
* `firstAnswerMs` bounds the wait for a read with nothing to serve yet.
|
||||
* @param {{env?: () => object, fetchImpl?: typeof fetch, now?: () => number, log?: (message: string) => void, later?: (run: () => void, ms: number) => void, firstAnswerMs?: number}} [options]
|
||||
*/
|
||||
export function createKeyedNews({
|
||||
env = () => process.env,
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
log = (message) => console.warn(message),
|
||||
later = (run, ms) => setTimeout(run, ms).unref(),
|
||||
firstAnswerMs = FIRST_ANSWER_MS,
|
||||
} = {}) {
|
||||
const keyOf = (id) => String(env()[NEWS_KEYS[id]] ?? '').trim();
|
||||
const inFlight = new Map();
|
||||
// GDELT's stories of the last day (their events and report, by
|
||||
// storyKey), and the exports read.
|
||||
const gdelt = {
|
||||
stories: new Map(),
|
||||
reports: new Map(),
|
||||
// Page titles by article address, of the stories kept.
|
||||
titles: new Map(),
|
||||
read: new Set(),
|
||||
// Exports still missing well after their time: not looked for again.
|
||||
gone: new Set(),
|
||||
// Failed reads of each export not read yet: `{tries, at}`.
|
||||
failures: new Map(),
|
||||
// Exports read among the newest whose GKG titles are not in yet, with
|
||||
// the tries made: `{tries, triedAt}`.
|
||||
untitled: new Map(),
|
||||
titling: null,
|
||||
// When a read last brought an export in.
|
||||
readAt: null,
|
||||
checkedAt: null,
|
||||
failedAt: null,
|
||||
};
|
||||
const live = new Map();
|
||||
const countries = new Map();
|
||||
// The keys refused so far, by section: logged once each.
|
||||
const refused = new Set();
|
||||
// World News API reads of the UTC day; `spent` once it answered 402.
|
||||
const budget = { day: null, reads: 0, spent: false, logged: false };
|
||||
// The areas (disputed area or country) of a placed Conflicts report's
|
||||
// pins, its first and the others, looked up once.
|
||||
const reportAreas = new WeakMap();
|
||||
const areasOf = (report) => {
|
||||
if (!reportAreas.has(report))
|
||||
reportAreas.set(
|
||||
report,
|
||||
new Set(
|
||||
[report.geo, ...(report.locations || [])]
|
||||
.filter(Boolean)
|
||||
.map((pin) => areaAt(pin.lat, pin.lon)?.key),
|
||||
),
|
||||
);
|
||||
return reportAreas.get(report);
|
||||
};
|
||||
|
||||
/** A failed read in the log: a refused key once, never the key itself. */
|
||||
function logFailure(section, what, key, error) {
|
||||
// The day's reads spent: said once, by spendRead.
|
||||
if (error.quota && error.status === null) return;
|
||||
if (error.rejected) {
|
||||
const id = `${section}\0${key}`;
|
||||
if (refused.has(id)) return;
|
||||
refused.add(id);
|
||||
log(
|
||||
`[news] ${NEWS_KEYS[section]} was refused (${error.status})${section === 'liveuamap' ? "; Conflicts are GDELT's alone" : ''}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
log(`[news] ${what} read failed: ${error.message}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* A cached read (`{key, value, readAt, failedAt, error}`, kept in `store`
|
||||
* under `id`): served as it is within `freshMs`; past that read again,
|
||||
* and served meanwhile up to `staleMs`, so only a read with nothing to
|
||||
* serve is waited for. One read at a time per key; a failed read is tried
|
||||
* again after RETRY_MS. A new key drops what the old one read.
|
||||
*/
|
||||
async function cached(
|
||||
store,
|
||||
id,
|
||||
{ section, key, freshMs, staleMs, read, what },
|
||||
) {
|
||||
const at = now();
|
||||
const stored = store.get(id);
|
||||
const prior = stored?.key === key ? stored : { key };
|
||||
if (prior.value && at - prior.readAt < freshMs) return prior;
|
||||
if (prior.failedAt && at - prior.failedAt < RETRY_MS) return prior;
|
||||
const flight = `${id}\0${key}`;
|
||||
if (!inFlight.has(flight))
|
||||
inFlight.set(
|
||||
flight,
|
||||
read(key)
|
||||
.then(
|
||||
(value) => ({ key, value, readAt: now() }),
|
||||
(error) => {
|
||||
logFailure(section, what, key, error);
|
||||
// What is held now: news deleted meanwhile stays deleted.
|
||||
const held = store.get(id);
|
||||
return {
|
||||
...(held?.key === key ? held : { key }),
|
||||
failedAt: now(),
|
||||
error,
|
||||
};
|
||||
},
|
||||
)
|
||||
.then((entry) => {
|
||||
// A read with a key replaced meanwhile is dropped.
|
||||
if (keyOf(section) === key) keep(store, id, entry);
|
||||
return entry;
|
||||
})
|
||||
.finally(() => inFlight.delete(flight)),
|
||||
);
|
||||
if (prior.value && at - prior.readAt < staleMs) return prior;
|
||||
// Meanwhile a refused key stays refused; an older failure is not this one.
|
||||
return firstAnswer(inFlight.get(flight), {
|
||||
...prior,
|
||||
error: prior.error?.rejected ? prior.error : null,
|
||||
});
|
||||
}
|
||||
|
||||
/** An entry stored as the most recently used (the countries are capped). */
|
||||
function keep(store, id, entry) {
|
||||
store.delete(id);
|
||||
store.set(id, entry);
|
||||
while (store.size > MAX_COUNTRIES) store.delete(store.keys().next().value);
|
||||
}
|
||||
|
||||
/** `read`, waited for at most firstAnswerMs; then `fallback`, as it goes on. */
|
||||
async function firstAnswer(read, fallback) {
|
||||
let timer;
|
||||
try {
|
||||
return await Promise.race([
|
||||
read,
|
||||
new Promise((resolve) => {
|
||||
timer = setTimeout(() => resolve(fallback), firstAnswerMs);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/** An entry's reports while they may be served, else why there are none. */
|
||||
function served(entry, staleMs, id) {
|
||||
if (entry.value && now() - entry.readAt < staleMs)
|
||||
return { reports: entry.value, readAt: entry.readAt };
|
||||
if (entry.error?.rejected) return { rejected: id };
|
||||
if (entry.error?.quota) return { quota: id };
|
||||
return { failed: true };
|
||||
}
|
||||
|
||||
async function liveuamapReports() {
|
||||
const key = keyOf('liveuamap');
|
||||
if (!key) return { missing: 'liveuamap' };
|
||||
const entry = await cached(live, 'liveuamap', {
|
||||
section: 'liveuamap',
|
||||
key,
|
||||
freshMs: LIVE_FRESH_MS,
|
||||
staleMs: LIVE_STALE_MS,
|
||||
what: 'liveuamap',
|
||||
read: async (k) =>
|
||||
(await fetchLiveuamap({ key: k, fetchImpl, now: now() }))
|
||||
.map(liveuamapReport)
|
||||
.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)),
|
||||
});
|
||||
return served(entry, LIVE_STALE_MS, 'liveuamap');
|
||||
}
|
||||
|
||||
/**
|
||||
* A story placed (again) from all its events and the titles in: an
|
||||
* agency story run by several outlets is one report.
|
||||
*/
|
||||
function placeStory(key) {
|
||||
const [report] = eventReports(gdelt.stories.get(key), {
|
||||
titleOf: (url) => gdelt.titles.get(url) || null,
|
||||
});
|
||||
if (!report) return;
|
||||
const { eventWords, ...rest } = report;
|
||||
const geo = gdeltPlace(report.geo);
|
||||
gdelt.reports.set(key, {
|
||||
...rest,
|
||||
...(eventWords ? { title: eventTitle(eventWords, geo.place) } : {}),
|
||||
hotspotId: WORLDWIDE_HOTSPOT,
|
||||
hotspotIds: [WORLDWIDE_HOTSPOT],
|
||||
geo,
|
||||
locations: report.locations.map(gdeltPlace),
|
||||
});
|
||||
}
|
||||
|
||||
/** An export's events into their stories, each placed again. */
|
||||
function addEvents(events) {
|
||||
const changed = new Set();
|
||||
for (const event of events) {
|
||||
const key = storyKey(event);
|
||||
const list = gdelt.stories.get(key) || [];
|
||||
if (list.length < MAX_STORY_EVENTS) list.push(event);
|
||||
gdelt.stories.set(key, list);
|
||||
changed.add(key);
|
||||
}
|
||||
for (const key of changed) placeStory(key);
|
||||
}
|
||||
|
||||
/** The stories a GKG export titled, re-titled. */
|
||||
function addTitles(titles) {
|
||||
for (const [key, list] of gdelt.stories) {
|
||||
const titled = list.filter((event) => titles.has(event.url));
|
||||
if (!titled.length) continue;
|
||||
for (const { url } of titled) gdelt.titles.set(url, titles.get(url));
|
||||
placeStory(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The page titles of the newest exports read, from their GKG exports one
|
||||
* at a time (each runs to 18 MB unzipped), in the background: no answer
|
||||
* waits for them, and a story is re-titled as its title comes in. A file
|
||||
* that fails or is not up yet only leaves address words: it is tried
|
||||
* again with a later read while among the newest, up to GKG_TRIES times.
|
||||
*/
|
||||
function readTitles() {
|
||||
gdelt.titling ??= (async () => {
|
||||
let failed = null;
|
||||
for (const [url, tried] of [...gdelt.untitled]) {
|
||||
if (!gdelt.untitled.has(url)) continue;
|
||||
if (tried.triedAt !== null && now() - tried.triedAt < GDELT_FRESH_MS)
|
||||
continue;
|
||||
tried.tries += 1;
|
||||
tried.triedAt = now();
|
||||
const wanted = new Set();
|
||||
for (const list of gdelt.stories.values())
|
||||
for (const event of list) wanted.add(event.url);
|
||||
let titles = null;
|
||||
try {
|
||||
titles = await fetchTitles(gkgUrl(url), wanted, { fetchImpl });
|
||||
} catch (error) {
|
||||
failed ??= error;
|
||||
}
|
||||
if (titles || tried.tries >= GKG_TRIES) gdelt.untitled.delete(url);
|
||||
if (titles?.size) addTitles(titles);
|
||||
}
|
||||
if (failed) log(`[news] gdelt titles read failed: ${failed.message}`);
|
||||
})()
|
||||
.catch((error) => log(`[news] gdelt titles failed: ${error.message}`))
|
||||
.finally(() => {
|
||||
gdelt.titling = null;
|
||||
});
|
||||
return gdelt.titling;
|
||||
}
|
||||
|
||||
/**
|
||||
* Up to a dozen of the day's exports not read yet, newest first; whether
|
||||
* to carry on at once (the round brought one in, and more are missing).
|
||||
* One not up yet is looked for again; one still missing GDELT_GONE_MS
|
||||
* after its time is not.
|
||||
*/
|
||||
async function readExports() {
|
||||
const at = now();
|
||||
const day = exportsOfDay(at);
|
||||
// Up to the newest GDELT lists; all of the day when its list fails.
|
||||
const latest = await fetchLatestExport({ fetchImpl });
|
||||
const listed = (url) => latest === null || exportAt(url) <= latest;
|
||||
const newest = new Set(day.slice(0, GDELT_TITLED_EXPORTS));
|
||||
const wanted = (url) =>
|
||||
listed(url) && !gdelt.read.has(url) && !gdelt.gone.has(url);
|
||||
const missing = day.filter(wanted).slice(0, GDELT_EXPORTS_PER_READ);
|
||||
let failed = null;
|
||||
let added = 0;
|
||||
const failing = [];
|
||||
const unreadable = [];
|
||||
const unpublished = [];
|
||||
for (let i = 0; i < missing.length; i += GDELT_PARALLEL)
|
||||
await Promise.all(
|
||||
missing.slice(i, i + GDELT_PARALLEL).map(async (url) => {
|
||||
let events;
|
||||
try {
|
||||
events = await fetchExport(url, { fetchImpl });
|
||||
} catch (error) {
|
||||
failed = error;
|
||||
(error.unreadable ? unreadable : failing).push(url);
|
||||
return;
|
||||
}
|
||||
if (!events) {
|
||||
if (at - exportAt(url) >= GDELT_GONE_MS) unpublished.push(url);
|
||||
return;
|
||||
}
|
||||
// The events count now; the titles follow (readTitles).
|
||||
gdelt.read.add(url);
|
||||
gdelt.failures.delete(url);
|
||||
added += 1;
|
||||
gdelt.readAt = now();
|
||||
addEvents(events);
|
||||
if (newest.has(url))
|
||||
gdelt.untitled.set(url, { tries: 0, triedAt: null });
|
||||
}),
|
||||
);
|
||||
if (failed) log(`[news] gdelt read failed: ${failed.message}`);
|
||||
// A file is blamed only while GDELT itself answers (its list read, or
|
||||
// another export of the round read): then a 404 long after its time is
|
||||
// never published, an unreadable file stays so, and a failing one gets a
|
||||
// few tries, one a GDELT_FRESH_MS. With GDELT down, no file is blamed.
|
||||
const answering = added > 0 || latest !== null;
|
||||
if (answering)
|
||||
for (const url of [...unpublished, ...unreadable]) gdelt.gone.add(url);
|
||||
if (added)
|
||||
for (const url of failing) {
|
||||
const prior = gdelt.failures.get(url);
|
||||
if (prior && now() - prior.at < GDELT_FRESH_MS) continue;
|
||||
const tries = (prior?.tries || 0) + 1;
|
||||
gdelt.failures.set(url, { tries, at: now() });
|
||||
if (tries >= GDELT_EXPORT_TRIES) gdelt.gone.add(url);
|
||||
}
|
||||
// Nothing read and a failure: tried again after RETRY_MS, not after
|
||||
// GDELT_FRESH_MS. Something read: the round counts.
|
||||
if (failed && !added) gdelt.failedAt = now();
|
||||
else {
|
||||
gdelt.checkedAt = now();
|
||||
gdelt.failedAt = null;
|
||||
}
|
||||
// Only the day is kept, and titled while among the newest.
|
||||
const kept = new Set(day);
|
||||
for (const set of [gdelt.read, gdelt.gone])
|
||||
for (const url of set) if (!kept.has(url)) set.delete(url);
|
||||
for (const url of gdelt.failures.keys())
|
||||
if (!kept.has(url)) gdelt.failures.delete(url);
|
||||
for (const url of gdelt.untitled.keys())
|
||||
if (!newest.has(url)) gdelt.untitled.delete(url);
|
||||
return (
|
||||
added > 0 && missing.length === GDELT_EXPORTS_PER_READ && day.some(wanted)
|
||||
);
|
||||
}
|
||||
|
||||
/** GDELT's reports of the last day, newest first, or `{failed}`. */
|
||||
async function gdeltReports() {
|
||||
const at = now();
|
||||
const fresh = gdelt.checkedAt && at - gdelt.checkedAt < GDELT_FRESH_MS;
|
||||
const resting = gdelt.failedAt && at - gdelt.failedAt < RETRY_MS;
|
||||
if (!fresh && !resting && !inFlight.has('gdelt')) {
|
||||
const round = readExports()
|
||||
.then(
|
||||
(more) => {
|
||||
// More of the day to fill in: carry on shortly, without holding
|
||||
// the answer.
|
||||
if (!more) return;
|
||||
gdelt.checkedAt = null;
|
||||
later(() => void gdeltReports(), GDELT_NEXT_MS);
|
||||
},
|
||||
(error) => {
|
||||
log(`[news] gdelt read failed: ${error.message}`);
|
||||
gdelt.failedAt = now();
|
||||
},
|
||||
)
|
||||
.finally(() => {
|
||||
inFlight.delete('gdelt');
|
||||
void readTitles();
|
||||
});
|
||||
inFlight.set('gdelt', round);
|
||||
}
|
||||
// Only a read with nothing (recent) to serve is waited for: the
|
||||
// newest exports' events, not their titles.
|
||||
const current = () =>
|
||||
gdelt.readAt !== null && now() - gdelt.readAt < GDELT_STALE_MS;
|
||||
// A slow GDELT must not outlast the browser's own timeout: what has come
|
||||
// in by then is served.
|
||||
if (!current() && inFlight.has('gdelt'))
|
||||
await firstAnswer(inFlight.get('gdelt'));
|
||||
if (!current()) return { failed: true };
|
||||
const dayAgo = new Date(now() - DAY_MS).toISOString();
|
||||
// Exports are read in parallel, so reports of one time arrive in any
|
||||
// order: the most mentioned comes first, then by id.
|
||||
const reports = [...gdelt.reports.values()]
|
||||
.filter((r) => r.publishedAt >= dayAgo)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
b.publishedAt.localeCompare(a.publishedAt) ||
|
||||
b.mentions - a.mentions ||
|
||||
a.id.localeCompare(b.id),
|
||||
)
|
||||
.slice(0, MAX_GDELT_REPORTS);
|
||||
if (reports.length !== gdelt.reports.size) {
|
||||
const kept = new Set(reports);
|
||||
for (const [key, report] of gdelt.reports)
|
||||
if (!kept.has(report)) {
|
||||
for (const event of gdelt.stories.get(key) || [])
|
||||
gdelt.titles.delete(event.url);
|
||||
gdelt.reports.delete(key);
|
||||
gdelt.stories.delete(key);
|
||||
}
|
||||
}
|
||||
return { reports, readAt: gdelt.readAt };
|
||||
}
|
||||
|
||||
/**
|
||||
* The Conflicts section's reports: GDELT's, with Liveuamap's when its key
|
||||
* is set. Liveuamap failing (or refusing its key) leaves GDELT's.
|
||||
*/
|
||||
async function conflictReports() {
|
||||
const [articles, items] = await Promise.all([
|
||||
gdeltReports(),
|
||||
keyOf('liveuamap') ? liveuamapReports() : {},
|
||||
]);
|
||||
if (!articles.reports && !items.reports) return { failed: true };
|
||||
return {
|
||||
reports: [...(articles.reports || []), ...(items.reports || [])].sort(
|
||||
(a, b) => b.publishedAt.localeCompare(a.publishedAt),
|
||||
),
|
||||
readAt: Math.max(articles.readAt || 0, items.readAt || 0),
|
||||
};
|
||||
}
|
||||
|
||||
const dailyReads = () => {
|
||||
const raw = String(env().WORLDNEWSAPI_DAILY_READS ?? '').trim();
|
||||
const reads = raw ? Number(raw) : NaN;
|
||||
return Number.isInteger(reads) && reads >= 0 ? reads : DEFAULT_DAILY_READS;
|
||||
};
|
||||
|
||||
/** Whether a World News API read is left today (UTC); counts it if so. */
|
||||
function spendRead() {
|
||||
const day = new Date(now()).toISOString().slice(0, 10);
|
||||
if (budget.day !== day)
|
||||
Object.assign(budget, { day, reads: 0, spent: false, logged: false });
|
||||
if (!budget.spent && budget.reads < dailyReads()) {
|
||||
budget.reads += 1;
|
||||
return true;
|
||||
}
|
||||
if (!budget.logged)
|
||||
log(
|
||||
"[news] World News API's reads for today are spent; Local reads again after 00:00 UTC",
|
||||
);
|
||||
budget.logged = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** World News API's news an hour old is deleted, as its terms ask. */
|
||||
function dropExpiredCountries() {
|
||||
for (const [code, entry] of countries)
|
||||
if (entry.value && now() - entry.readAt >= COUNTRY_STALE_MS) {
|
||||
const { value, readAt, ...rest } = entry;
|
||||
countries.set(code, rest);
|
||||
}
|
||||
}
|
||||
|
||||
async function countryReports(country) {
|
||||
const key = keyOf('worldnewsapi');
|
||||
if (!key) return { missing: 'worldnewsapi' };
|
||||
const stored = countries.get(country.code);
|
||||
if (stored) keep(countries, country.code, stored);
|
||||
const entry = await cached(countries, country.code, {
|
||||
section: 'worldnewsapi',
|
||||
key,
|
||||
freshMs: COUNTRY_FRESH_MS,
|
||||
staleMs: COUNTRY_STALE_MS,
|
||||
what: `worldnewsapi:${country.code}`,
|
||||
read: async (k) => {
|
||||
// Not asked: no status, unlike an answer.
|
||||
if (!spendRead())
|
||||
throw new UpstreamError("World News API's reads for today spent", {
|
||||
quota: true,
|
||||
});
|
||||
const { day } = budget;
|
||||
try {
|
||||
const reports = (
|
||||
await fetchWorldNews({
|
||||
key: k,
|
||||
country: country.code,
|
||||
fetchImpl,
|
||||
now: now(),
|
||||
})
|
||||
).map((article) => worldNewsReport(article, country));
|
||||
// Deleted an hour on, whether or not anything is asked then.
|
||||
setTimeout(dropExpiredCountries, COUNTRY_STALE_MS + 1000).unref?.();
|
||||
return reports;
|
||||
} catch (error) {
|
||||
// Out of points: nothing more is asked today.
|
||||
if (error.status === 402) budget.spent = true;
|
||||
// Never reached (offline): no read of the day it was counted in.
|
||||
if (error.unreached && budget.day === day && budget.reads > 0)
|
||||
budget.reads -= 1;
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
return served(entry, COUNTRY_STALE_MS, 'worldnewsapi');
|
||||
}
|
||||
|
||||
const refusal = (read) =>
|
||||
read.missing
|
||||
? noKey(read.missing)
|
||||
: read.rejected
|
||||
? rejectedKey(read.rejected)
|
||||
: read.quota
|
||||
? spentKey(read.quota)
|
||||
: read.failed
|
||||
? { status: 502, body: { error: 'news_upstream_error' } }
|
||||
: null;
|
||||
|
||||
async function hotspots(params) {
|
||||
const read = await conflictReports();
|
||||
const refused = refusal(read);
|
||||
if (refused) return refused;
|
||||
const hours = Number(params.get('hours')) || 24;
|
||||
const since = new Date(now() - hours * HOUR_MS).toISOString();
|
||||
// The reports Conflicts lists: a minor one is in no count.
|
||||
const shown = read.reports.filter(
|
||||
(r) => r.publishedAt >= since && r.relevance !== 'minor',
|
||||
);
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
hotspots: [
|
||||
{
|
||||
id: WORLDWIDE_HOTSPOT,
|
||||
name: 'Worldwide',
|
||||
kind: 'conflict',
|
||||
catchAll: true,
|
||||
bbox: [-180, -90, 180, 90],
|
||||
parties: [],
|
||||
developing: [],
|
||||
counts: {
|
||||
last24h: shown.length,
|
||||
mapped24h: shown.filter((r) => r.geo).length,
|
||||
prev24h: null,
|
||||
},
|
||||
latestAt: shown[0]?.publishedAt ?? null,
|
||||
},
|
||||
],
|
||||
categories: CATEGORY_ORDER,
|
||||
ingest: { lastRunAt: new Date(read.readAt).toISOString() },
|
||||
windowHours: hours,
|
||||
maxWindowHours: 24,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Reports filtered, newest first, and paged by `before`. */
|
||||
function page(reports, params) {
|
||||
const since = params.get('since');
|
||||
const before = params.get('before');
|
||||
const dayAgo = new Date(now() - DAY_MS).toISOString();
|
||||
const categories = list(params.get('category'));
|
||||
const relevance = list(params.get('relevance'));
|
||||
const hotspot = params.get('hotspot');
|
||||
const limit = Number(params.get('limit')) || DEFAULT_LIMIT;
|
||||
const kept = reports.filter(
|
||||
(r) =>
|
||||
(!hotspot || hotspot === 'all' || r.hotspotIds.includes(hotspot)) &&
|
||||
(!since || r.publishedAt >= since) &&
|
||||
(!before || r.publishedAt < before) &&
|
||||
(!params.has('today') || r.publishedAt >= dayAgo) &&
|
||||
(!params.has('geo') || r.geo) &&
|
||||
(!categories || categories.has(r.category)) &&
|
||||
(!relevance || relevance.has(r.relevance)),
|
||||
);
|
||||
const shown = kept.slice(0, limit);
|
||||
// A page ends between two times: the next asks for those `before` the
|
||||
// last, and a GDELT export's reports all share its quarter hour.
|
||||
while (
|
||||
shown.length < kept.length &&
|
||||
kept[shown.length].publishedAt === shown.at(-1).publishedAt
|
||||
)
|
||||
shown.push(kept[shown.length]);
|
||||
const events = params.has('events');
|
||||
return {
|
||||
reports: events ? shown : shown.map(({ event, ...report }) => report),
|
||||
nextBefore: kept.length > shown.length ? shown.at(-1).publishedAt : null,
|
||||
};
|
||||
}
|
||||
|
||||
async function feed(params) {
|
||||
const at = params.has('at') ? point(params.get('at')) : null;
|
||||
const area = at ? areaAt(at.lat, at.lon) : null;
|
||||
if (!area) {
|
||||
// Conflicts, or Local at sea (by distance, in the browser).
|
||||
const read = await conflictReports();
|
||||
return refusal(read) || { status: 200, body: page(read.reports, params) };
|
||||
}
|
||||
// Local in a country: its own outlets, with the Conflicts reports
|
||||
// placed inside it (those failing leave the outlets'). In a disputed
|
||||
// area no country's outlets speak for it: the Conflicts reports there
|
||||
// alone, so their failing is the answer's.
|
||||
const country = area.kind === 'country';
|
||||
const [national, conflicts] = await Promise.all([
|
||||
country ? countryReports(area.country) : { reports: [] },
|
||||
conflictReports(),
|
||||
]);
|
||||
const refused = refusal(national) || (!country && refusal(conflicts));
|
||||
if (refused) return refused;
|
||||
const inside = (conflicts.reports || []).filter((r) =>
|
||||
areasOf(r).has(area.key),
|
||||
);
|
||||
const reports = [...national.reports, ...inside].sort((a, b) =>
|
||||
b.publishedAt.localeCompare(a.publishedAt),
|
||||
);
|
||||
return { status: 200, body: page(reports, params) };
|
||||
}
|
||||
|
||||
function outline(params) {
|
||||
if (params.has('at')) {
|
||||
const at = point(params.get('at'));
|
||||
const area = at ? areaAt(at.lat, at.lon) : null;
|
||||
if (!area) return { status: 200, body: EMPTY_OUTLINE };
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
...EMPTY_OUTLINE,
|
||||
polygons:
|
||||
area.kind === 'country'
|
||||
? countryPolygons(area.country)
|
||||
: area.polygons,
|
||||
area: {
|
||||
key: area.key,
|
||||
name: area.name,
|
||||
kind: area.kind,
|
||||
country: area.code,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
// A developing situation's country; the world has no outline.
|
||||
const country = countryByCode(
|
||||
String(params.get('country') ?? '').slice(0, 2),
|
||||
);
|
||||
return {
|
||||
status: 200,
|
||||
body: country
|
||||
? { ...EMPTY_OUTLINE, polygons: countryPolygons(country) }
|
||||
: EMPTY_OUTLINE,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
async answer(path, params) {
|
||||
dropExpiredCountries();
|
||||
if (path === '/hotspots') return hotspots(params);
|
||||
if (path === '/feed') return feed(params);
|
||||
if (path === '/outline') return outline(params);
|
||||
return { status: 404, body: { error: 'not_found' } };
|
||||
},
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,283 @@
|
||||
// The keyed news APIs Core reads itself when no news server is configured:
|
||||
// Liveuamap's worldwide feed (`a=mpts`) beside GDELT's events in the
|
||||
// Conflicts section (gdeltEvents.js), World News API's search by source
|
||||
// country for Local. Each key stays on the server: Liveuamap's rides in the query, so
|
||||
// neither a URL nor an error body is ever logged or relayed.
|
||||
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
const LIVEUAMAP_URL = 'https://a.liveuamap.com/api';
|
||||
const WORLD_NEWS_URL = 'https://api.worldnewsapi.com/search-news';
|
||||
// 100 Liveuamap items run to about 500 KB.
|
||||
const LIVEUAMAP_COUNT = 500;
|
||||
const LIVEUAMAP_MAX_BYTES = 16 * 1024 * 1024;
|
||||
// One search per country: a point, and a hundredth of one per article, so
|
||||
// about 1.5 points a read (the free plan has 50 points a day).
|
||||
const WORLD_NEWS_ARTICLES = 50;
|
||||
const WORLD_NEWS_MAX_BYTES = 8 * 1024 * 1024;
|
||||
// World News API's categories that are not the news of the day. Its
|
||||
// `categories` filter is not asked for: it classifies few articles outside
|
||||
// English, so it would drop most of a country's own outlets; an article it
|
||||
// left unclassified is kept.
|
||||
export const WORLD_NEWS_LEFT_OUT = Object.freeze(
|
||||
new Set(['sports', 'entertainment', 'lifestyle', 'travel']),
|
||||
);
|
||||
const DAY_MS = 86_400_000;
|
||||
const TIMEOUT_MS = 20_000;
|
||||
|
||||
const ENTITIES = {
|
||||
amp: '&',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
nbsp: ' ',
|
||||
};
|
||||
|
||||
/** Text with HTML entities decoded and whitespace collapsed, or ''. */
|
||||
export function unescapeHtml(value) {
|
||||
return String(value ?? '')
|
||||
.replace(
|
||||
/&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]{2,6}));/gi,
|
||||
(all, dec, hex, name) => {
|
||||
if (name) return ENTITIES[name.toLowerCase()] ?? all;
|
||||
const code = dec ? Number(dec) : parseInt(hex, 16);
|
||||
return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : all;
|
||||
},
|
||||
)
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** An https URL without credentials, or null. */
|
||||
export function webLink(value) {
|
||||
try {
|
||||
const url = new URL(String(value ?? '').trim());
|
||||
return url.protocol === 'https:' && !url.username && !url.password
|
||||
? url.href
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** A clip a browser plays itself (an https MP4 or WebM file), or null. */
|
||||
export function playableVideo(value) {
|
||||
const url = webLink(value);
|
||||
return url && /\.(?:mp4|m4v|webm)$/i.test(new URL(url).pathname) ? url : null;
|
||||
}
|
||||
|
||||
const finiteNumber = (value) => {
|
||||
const number = typeof value === 'string' ? Number(value.trim()) : value;
|
||||
return typeof number === 'number' && Number.isFinite(number) ? number : null;
|
||||
};
|
||||
|
||||
/** Why an upstream could not be read; never carries its URL or body. */
|
||||
export class UpstreamError extends Error {
|
||||
constructor(
|
||||
message,
|
||||
{
|
||||
status = null,
|
||||
rejected = false,
|
||||
quota = false,
|
||||
unreadable = false,
|
||||
unreached = false,
|
||||
} = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'UpstreamError';
|
||||
this.status = status;
|
||||
// The key was refused (401 or 403): the operator's to fix.
|
||||
this.rejected = rejected;
|
||||
// The key's allowance is spent (402, or the day's reads): it answers
|
||||
// again later. A 429 is a per-second limit, not this.
|
||||
this.quota = quota;
|
||||
// The answer itself cannot be read (corrupt, or over a size cap): asking
|
||||
// again gets the same.
|
||||
this.unreadable = unreadable;
|
||||
// The request got no answer at all (offline, timed out).
|
||||
this.unreached = unreached;
|
||||
}
|
||||
}
|
||||
|
||||
async function getText(
|
||||
fetchImpl,
|
||||
url,
|
||||
{ headers = {}, maxBytes, signal, what },
|
||||
) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
headers: { Accept: 'application/json', ...headers },
|
||||
redirect: 'error',
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
} catch (cause) {
|
||||
// A request that timed out may have been read; one refused was not.
|
||||
const timedOut = cause?.name === 'TimeoutError';
|
||||
throw new UpstreamError(
|
||||
timedOut ? `${what} timed out` : `${what} unreachable`,
|
||||
{ unreached: !timedOut },
|
||||
);
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw new UpstreamError(`${what} answered ${response.status}`, {
|
||||
status: response.status,
|
||||
rejected: response.status === 401 || response.status === 403,
|
||||
quota: response.status === 402,
|
||||
});
|
||||
}
|
||||
try {
|
||||
return await readResponseTextCapped(response, maxBytes, signal);
|
||||
} catch {
|
||||
throw new UpstreamError(`${what} answer unreadable`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseJson(text, what) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
// Not the parser's message: an error answer may echo the request.
|
||||
throw new UpstreamError(`${what} answer unreadable`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Liveuamap's items with a headline, a point and a page:
|
||||
* `{id, title, lat, lon, place, publishedAt, url, image, video}`.
|
||||
*/
|
||||
export function parseLiveuamap(payload) {
|
||||
if (!Array.isArray(payload?.venues))
|
||||
throw new UpstreamError('Liveuamap answer unreadable');
|
||||
const items = [];
|
||||
for (const venue of payload.venues) {
|
||||
const id = finiteNumber(venue?.id);
|
||||
const title = unescapeHtml(venue?.name);
|
||||
const lat = finiteNumber(venue?.lat);
|
||||
const lon = finiteNumber(venue?.lng);
|
||||
const seconds = finiteNumber(venue?.timestamp);
|
||||
const url = webLink(venue?.link);
|
||||
if (
|
||||
id === null ||
|
||||
!title ||
|
||||
lat === null ||
|
||||
lon === null ||
|
||||
Math.abs(lat) > 90 ||
|
||||
Math.abs(lon) > 180 ||
|
||||
(lat === 0 && lon === 0) ||
|
||||
seconds === null ||
|
||||
!url
|
||||
)
|
||||
continue;
|
||||
items.push({
|
||||
id: Math.trunc(id),
|
||||
title,
|
||||
lat,
|
||||
lon,
|
||||
place: unescapeHtml(venue.location) || null,
|
||||
publishedAt: new Date(seconds * 1000).toISOString(),
|
||||
url,
|
||||
image: webLink(venue.img_share),
|
||||
video: playableVideo(venue.video),
|
||||
});
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
/** The worldwide feed of the last day, as of `now`. */
|
||||
export async function fetchLiveuamap({ key, fetchImpl, now, signal }) {
|
||||
const url = new URL(LIVEUAMAP_URL);
|
||||
url.search = new URLSearchParams({
|
||||
a: 'mpts',
|
||||
resid: '1',
|
||||
time: String(Math.floor((now - DAY_MS) / 1000)),
|
||||
count: String(LIVEUAMAP_COUNT),
|
||||
key,
|
||||
}).toString();
|
||||
const text = await getText(fetchImpl, url, {
|
||||
maxBytes: LIVEUAMAP_MAX_BYTES,
|
||||
signal,
|
||||
what: 'Liveuamap',
|
||||
});
|
||||
return parseLiveuamap(parseJson(text, 'Liveuamap'));
|
||||
}
|
||||
|
||||
// World News API writes times as "2026-10-30 01:31:00", in UTC.
|
||||
const worldNewsTime = (ms) =>
|
||||
new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
|
||||
function worldNewsDate(value) {
|
||||
const match = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)$/.exec(
|
||||
String(value ?? ''),
|
||||
);
|
||||
const ms = match ? Date.parse(`${match[1]}T${match[2]}Z`) : NaN;
|
||||
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A search's articles with a title, a link and a time, one per title (the
|
||||
* same story from several outlets is one), sport and entertainment left
|
||||
* out, newest first:
|
||||
* `{id, title, url, publisher, language, category, publishedAt, image, video}`.
|
||||
*/
|
||||
export function parseWorldNews(payload) {
|
||||
if (!Array.isArray(payload?.news))
|
||||
throw new UpstreamError('World News API answer unreadable');
|
||||
const seen = new Set();
|
||||
const articles = [];
|
||||
for (const article of payload.news) {
|
||||
const id = finiteNumber(article?.id);
|
||||
const title = unescapeHtml(article?.title);
|
||||
const url = webLink(article?.url);
|
||||
const publishedAt = worldNewsDate(article?.publish_date);
|
||||
const category =
|
||||
typeof article?.category === 'string'
|
||||
? article.category.toLowerCase()
|
||||
: null;
|
||||
if (id === null || !title || !url || !publishedAt) continue;
|
||||
if (WORLD_NEWS_LEFT_OUT.has(category)) continue;
|
||||
const same = title.toLowerCase();
|
||||
if (seen.has(same)) continue;
|
||||
seen.add(same);
|
||||
articles.push({
|
||||
id: Math.trunc(id),
|
||||
title,
|
||||
url,
|
||||
publisher: new URL(url).hostname.replace(/^www\./, ''),
|
||||
language:
|
||||
typeof article.language === 'string' &&
|
||||
/^[a-z]{2}$/i.test(article.language)
|
||||
? article.language.toLowerCase()
|
||||
: null,
|
||||
category,
|
||||
publishedAt,
|
||||
image: webLink(article.image),
|
||||
video: playableVideo(article.video),
|
||||
});
|
||||
}
|
||||
return articles.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt));
|
||||
}
|
||||
|
||||
/** A country's everyday news of the last day, as of `now`. */
|
||||
export async function fetchWorldNews({ key, country, fetchImpl, now, signal }) {
|
||||
const url = new URL(WORLD_NEWS_URL);
|
||||
url.search = new URLSearchParams({
|
||||
'source-country': country.toLowerCase(),
|
||||
// Both ends: sorted by time with neither, a search is cut to 31 days.
|
||||
'earliest-publish-date': worldNewsTime(now - DAY_MS),
|
||||
'latest-publish-date': worldNewsTime(now),
|
||||
sort: 'publish-time',
|
||||
'sort-direction': 'DESC',
|
||||
number: String(WORLD_NEWS_ARTICLES),
|
||||
}).toString();
|
||||
const text = await getText(fetchImpl, url, {
|
||||
headers: { 'x-api-key': key },
|
||||
maxBytes: WORLD_NEWS_MAX_BYTES,
|
||||
signal,
|
||||
what: 'World News API',
|
||||
});
|
||||
return parseWorldNews(parseJson(text, 'World News API'));
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createOpenAiChatHandler } from './openai/chat.js';
|
||||
import { defaultSourceRoot } from './common/source-root.js';
|
||||
import { handleHudSummary } from './openai/hud-summary.js';
|
||||
import { createDebugLogHandler } from './openai/debug-log.js';
|
||||
@@ -22,6 +23,10 @@ function openAiRealtimeProxy({
|
||||
} = {}) {
|
||||
const oauthLogin = createCodexOAuthLogin(realtime);
|
||||
function install(middlewares) {
|
||||
middlewares.use(
|
||||
'/api/openai/chat',
|
||||
createOpenAiChatHandler({ ...realtime, annotationGuidance }),
|
||||
);
|
||||
// Cost-bearing and log endpoints refuse cross-site browser requests
|
||||
// (see server/providers/common/same-site.js and SECURITY.md).
|
||||
middlewares.use('/api/openai/hud-summary', sameSiteGated(handleHudSummary));
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
import { createTextChatHandler } from '../common/textChat.js';
|
||||
import { preferCodexOAuth } from './codex-auth.js';
|
||||
import { OPENAI_DEFAULT_PER_MIN } from './rate-limit.js';
|
||||
import { invalidCalls } from '../common/textTools.js';
|
||||
import { textInstructions } from '../common/textInstructions.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
|
||||
const object = (value) =>
|
||||
value && typeof value === 'object' && !Array.isArray(value);
|
||||
const keys = (value, allowed) =>
|
||||
Object.keys(value).every((key) => allowed.includes(key));
|
||||
const callShape = (call) =>
|
||||
object(call) &&
|
||||
keys(call, ['id', 'type', 'function']) &&
|
||||
typeof call.id === 'string' &&
|
||||
call.id.length > 0 &&
|
||||
call.id.length <= 1024 &&
|
||||
call.type === 'function' &&
|
||||
object(call.function) &&
|
||||
keys(call.function, ['name', 'arguments']) &&
|
||||
typeof call.function.name === 'string' &&
|
||||
call.function.name.length <= 128 &&
|
||||
typeof call.function.arguments === 'string';
|
||||
function assistantShape(message) {
|
||||
return (
|
||||
object(message) &&
|
||||
keys(message, ['role', 'content', 'tool_calls']) &&
|
||||
message.role === 'assistant' &&
|
||||
(message.content == null || typeof message.content === 'string') &&
|
||||
(message.tool_calls === undefined ||
|
||||
(Array.isArray(message.tool_calls) &&
|
||||
message.tool_calls.length > 0 &&
|
||||
message.tool_calls.length <= 32 &&
|
||||
message.tool_calls.every(callShape) &&
|
||||
new Set(message.tool_calls.map((call) => call.id)).size ===
|
||||
message.tool_calls.length))
|
||||
);
|
||||
}
|
||||
function validMessages(messages) {
|
||||
if (!Array.isArray(messages) || !messages.length || messages.length > 96)
|
||||
return false;
|
||||
let pending = [];
|
||||
for (const message of messages) {
|
||||
if (!object(message)) return false;
|
||||
if (pending.length) {
|
||||
if (
|
||||
!keys(message, ['role', 'content', 'tool_call_id']) ||
|
||||
message.role !== 'tool' ||
|
||||
typeof message.content !== 'string' ||
|
||||
message.tool_call_id !== pending.shift()
|
||||
)
|
||||
return false;
|
||||
} else if (message.role === 'user') {
|
||||
if (
|
||||
!keys(message, ['role', 'content']) ||
|
||||
typeof message.content !== 'string'
|
||||
)
|
||||
return false;
|
||||
} else if (assistantShape(message)) {
|
||||
pending = (message.tool_calls || []).map((call) => call.id);
|
||||
} else return false;
|
||||
}
|
||||
return !pending.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* A tool schema without the `pattern`s Chat Completions refuses: it answers
|
||||
* 400 for a whole request whose tool uses Unicode property classes (`\p{L}`),
|
||||
* as `resolve_area`'s area ids do. Every argument is still checked against
|
||||
* the action's own contract before the tool runs, so the model loses a hint,
|
||||
* not a rule. Realtime voice refuses them too now (#1618) and uses this as well.
|
||||
*/
|
||||
export function chatSchema(schema) {
|
||||
if (Array.isArray(schema)) return schema.map(chatSchema);
|
||||
if (!object(schema)) return schema;
|
||||
return Object.fromEntries(
|
||||
Object.entries(schema)
|
||||
.filter(
|
||||
([key, value]) =>
|
||||
!(
|
||||
key === 'pattern' &&
|
||||
typeof value === 'string' &&
|
||||
/\\[pP]\{/.test(value)
|
||||
),
|
||||
)
|
||||
.map(([key, value]) => [key, chatSchema(value)]),
|
||||
);
|
||||
}
|
||||
|
||||
export function createOpenAiChatHandler({
|
||||
annotationGuidance,
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
// osm_query is offered, and the prompt names it, only with Overpass, as
|
||||
// for voice.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
resolveApiKey = () => process.env.OPENAI_API_KEY,
|
||||
resolveChatModel = () => process.env.OPENAI_CHAT_MODEL || 'gpt-4.1-mini',
|
||||
resolveProviderRateLimit = () => process.env.GEV_RATELIMIT_OPENAI_PER_MIN,
|
||||
environment = process.env,
|
||||
...options
|
||||
} = {}) {
|
||||
return createTextChatHandler({
|
||||
...options,
|
||||
provider: 'openai',
|
||||
resolveApiKey,
|
||||
resolveProviderRateLimit,
|
||||
providerPerMin: OPENAI_DEFAULT_PER_MIN,
|
||||
// Chat can only bill the API key, which this operator lock forbids.
|
||||
refuse() {
|
||||
let error =
|
||||
'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH, and typed OpenAI chat needs an API key.';
|
||||
try {
|
||||
if (!preferCodexOAuth(environment)) return null;
|
||||
} catch (invalid) {
|
||||
error = invalid.message;
|
||||
}
|
||||
return { error, code: 'CODEX_OAUTH_REQUIRED' };
|
||||
},
|
||||
prepare(input) {
|
||||
const model = String(resolveChatModel()).trim();
|
||||
if (
|
||||
!/^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,119}$/.test(model) ||
|
||||
/realtime|audio|tts/i.test(model)
|
||||
)
|
||||
return { errorStatus: 503 };
|
||||
if (
|
||||
!object(input) ||
|
||||
!keys(input, ['messages']) ||
|
||||
!validMessages(input.messages)
|
||||
)
|
||||
return null;
|
||||
const overpass = overpassConfigured();
|
||||
return {
|
||||
endpoint: 'https://api.openai.com/v1/chat/completions',
|
||||
headers: (key) => ({ Authorization: `Bearer ${key}` }),
|
||||
body: {
|
||||
model,
|
||||
n: 1,
|
||||
stream: false,
|
||||
store: false,
|
||||
max_completion_tokens: 4096,
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: textInstructions(annotationGuidance, { overpass }),
|
||||
},
|
||||
...input.messages,
|
||||
],
|
||||
tools: offeredTools(tools, { overpass }).map(
|
||||
({ name, description, parameters }) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name,
|
||||
description,
|
||||
parameters: chatSchema(parameters),
|
||||
strict: false,
|
||||
},
|
||||
}),
|
||||
),
|
||||
},
|
||||
};
|
||||
},
|
||||
parse(data) {
|
||||
const choice = data?.choices?.[0];
|
||||
const raw = choice?.message;
|
||||
if (!raw) return null;
|
||||
const content = {
|
||||
role: raw.role,
|
||||
content: raw.content ?? null,
|
||||
...(raw.tool_calls?.length ? { tool_calls: raw.tool_calls } : {}),
|
||||
};
|
||||
if (
|
||||
!assistantShape(content) ||
|
||||
(!content.content && !content.tool_calls?.length)
|
||||
)
|
||||
return null;
|
||||
const calls = (content.tool_calls || []).map((call) => {
|
||||
try {
|
||||
return {
|
||||
name: call.function.name,
|
||||
args: JSON.parse(call.function.arguments),
|
||||
};
|
||||
} catch {
|
||||
return { name: call.function.name, parseError: true };
|
||||
}
|
||||
});
|
||||
return {
|
||||
content,
|
||||
finishReason: choice.finish_reason,
|
||||
invalidCalls: invalidCalls(
|
||||
calls,
|
||||
offeredTools(tools, { overpass: overpassConfigured() }),
|
||||
),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -145,7 +145,7 @@ const CODEX_LOGIN_ENV =
|
||||
/^(?:PATH|PATHEXT|HOME|USER|USERNAME|LOGNAME|SHELL|LANG|LANGUAGE|LC_[A-Z_]+|TERM|TZ|TMPDIR|TEMP|TMP|XDG_[A-Z_]+|DISPLAY|WAYLAND_DISPLAY|DBUS_SESSION_BUS_ADDRESS|BROWSER|WSL_[A-Z_]+|WSLENV|(?:HTTPS?|ALL|NO)_PROXY|SSL_CERT_(?:FILE|DIR)|NODE_EXTRA_CA_CERTS|SYSTEMROOT|WINDIR|COMSPEC|USERPROFILE|APPDATA|LOCALAPPDATA|PROGRAMDATA|HOMEDRIVE|HOMEPATH|CODEX_[A-Z_]+)$/i;
|
||||
|
||||
/** The environment `codex login` runs with: the allowlist, then `CODEX_HOME`. */
|
||||
export function codexLoginEnvironment(environment, codexHome) {
|
||||
function codexLoginEnvironment(environment, codexHome) {
|
||||
const env = {};
|
||||
for (const [name, value] of Object.entries(environment || {}))
|
||||
if (CODEX_LOGIN_ENV.test(name) && value !== undefined) env[name] = value;
|
||||
|
||||
@@ -157,4 +157,4 @@ function createDebugLogHandler({
|
||||
};
|
||||
}
|
||||
|
||||
export { createDebugLogHandler, omitVoiceContent };
|
||||
export { createDebugLogHandler };
|
||||
@@ -1,5 +1,12 @@
|
||||
import { SNAPSHOT_FEED_STATES } from '../../../src/data/layerSnapshot.js';
|
||||
import {
|
||||
HUD_SUMMARY_INSTRUCTIONS,
|
||||
HUD_SUMMARY_MAX_CHARS,
|
||||
HUD_SUMMARY_MAX_UNSPACED_CHARS,
|
||||
HUD_SUMMARY_RESPONSE_FORMAT,
|
||||
hudSummaryInput,
|
||||
hudSummaryInstructions,
|
||||
hudSummaryIsUnspaced,
|
||||
hudSummaryLocale,
|
||||
keylessHudSummaryResponse,
|
||||
} from '../../../src/hudSummaryResponse.js';
|
||||
import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
|
||||
@@ -18,17 +25,74 @@ function extractOpenAiResponseText(data) {
|
||||
.trim();
|
||||
}
|
||||
|
||||
function toFiveWordHudSummary(value) {
|
||||
return String(value || '')
|
||||
.replace(/[^\p{L}\p{N}\s-]/gu, ' ')
|
||||
.trim()
|
||||
/** Parse the summary and normalize the canonical feed state. */
|
||||
function parseHudSummaryReply(text, locale) {
|
||||
let reply = null;
|
||||
try {
|
||||
reply = JSON.parse(text);
|
||||
} catch {
|
||||
/* not JSON: no summary */
|
||||
}
|
||||
const feedState = SNAPSHOT_FEED_STATES.includes(reply?.feedState)
|
||||
? reply.feedState
|
||||
: null;
|
||||
return {
|
||||
summary: toHudSummary(reply?.summary, locale),
|
||||
feedState,
|
||||
};
|
||||
}
|
||||
|
||||
function toHudSummary(value, requested) {
|
||||
const locale = hudSummaryLocale(requested);
|
||||
const words = String(value || '')
|
||||
.normalize('NFC')
|
||||
// Marks, and apostrophes and dots inside a word, belong to it: ar vowel
|
||||
// signs, fr d'Orléans, ja サンフランシスコ・ベイエリア.
|
||||
.replace(/[^\p{L}\p{M}\p{N}\s'’·・-]/gu, ' ')
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.slice(0, 5)
|
||||
.join(' ');
|
||||
.map((word) => word.replace(/^['’·・-]+|['’·・-]+$/gu, ''))
|
||||
.filter(Boolean);
|
||||
// Characters, not words: the model counts Rue de Rivoli as one word.
|
||||
const max = hudSummaryIsUnspaced(locale)
|
||||
? HUD_SUMMARY_MAX_UNSPACED_CHARS
|
||||
: HUD_SUMMARY_MAX_CHARS;
|
||||
return capAtWord(words.join(' '), max, locale);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cut `text` to `max` characters at a word boundary; a first word longer
|
||||
* than that is cut where the cap falls.
|
||||
*/
|
||||
function capAtWord(text, max, locale) {
|
||||
const graphemes = [
|
||||
...new Intl.Segmenter(locale, { granularity: 'grapheme' }).segment(text),
|
||||
];
|
||||
if (graphemes.length <= max) return text;
|
||||
const cut = graphemes[max].index;
|
||||
let end = 0;
|
||||
for (const { index, segment } of new Intl.Segmenter(locale, {
|
||||
granularity: 'word',
|
||||
}).segment(text)) {
|
||||
if (index + segment.length > cut) break;
|
||||
end = index + segment.length;
|
||||
}
|
||||
return text.slice(0, end).trim() || text.slice(0, cut);
|
||||
}
|
||||
|
||||
async function handleHudSummary(req, res) {
|
||||
// Capability probe: lets the HUD skip its paid place lookups when there is
|
||||
// no summary to feed them to.
|
||||
if (req.method === 'GET' && req.url?.split('?')[0] === '/status') {
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
configured: !keylessHudSummaryResponse(process.env.OPENAI_API_KEY),
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (req.method !== 'POST') {
|
||||
res.statusCode = 405;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
@@ -64,14 +128,18 @@ async function handleHudSummary(req, res) {
|
||||
model:
|
||||
process.env.OPENAI_HUD_SUMMARY_MODEL ||
|
||||
OPENAI_HUD_SUMMARY_MODEL_DEFAULT,
|
||||
instructions: HUD_SUMMARY_INSTRUCTIONS,
|
||||
input: JSON.stringify(context),
|
||||
instructions: hudSummaryInstructions(context?.locale),
|
||||
input: hudSummaryInput(context),
|
||||
text: { format: HUD_SUMMARY_RESPONSE_FORMAT },
|
||||
reasoning: { effort: 'minimal' },
|
||||
max_output_tokens: 100,
|
||||
max_output_tokens: 200,
|
||||
}),
|
||||
});
|
||||
const data = await response.json().catch(() => ({}));
|
||||
const summary = toFiveWordHudSummary(extractOpenAiResponseText(data));
|
||||
const { summary, feedState } = parseHudSummaryReply(
|
||||
extractOpenAiResponseText(data),
|
||||
context?.locale,
|
||||
);
|
||||
res.statusCode = response.ok && summary ? 200 : response.status || 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
@@ -80,6 +148,7 @@ async function handleHudSummary(req, res) {
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
summary: summary || null,
|
||||
feedState,
|
||||
// Never relay `data.error.message`: that is OpenAI's own wording, and
|
||||
// it carries request ids, organization hints and quota phrasing.
|
||||
error: response.ok ? null : 'OpenAI HUD summary request failed',
|
||||
@@ -97,4 +166,4 @@ async function handleHudSummary(req, res) {
|
||||
}
|
||||
}
|
||||
|
||||
export { handleHudSummary };
|
||||
export { handleHudSummary, parseHudSummaryReply };
|
||||
@@ -1,8 +1,14 @@
|
||||
const DEFAULT_ANNOTATION_GUIDANCE =
|
||||
'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
|
||||
|
||||
function realtimeInstructions(
|
||||
/**
|
||||
* The Realtime session directives, one per line. `overpass` says an operator
|
||||
* configured Overpass, which is what offers osm_query (see tools.js); without
|
||||
* it the prompt never mentions the tool.
|
||||
*/
|
||||
function realtimeDirectives(
|
||||
annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
|
||||
{ overpass = false } = {},
|
||||
) {
|
||||
return [
|
||||
"You are GEV Voice Control, a concise voice controller for a Cesium geospatial app called God's Eye View.",
|
||||
@@ -10,27 +16,31 @@ function realtimeInstructions(
|
||||
'Do not require a wake phrase. Treat direct commands like "zoom into London" or "open datacenters" as GEV control requests.',
|
||||
'Only control the app by calling the provided tools. Never invent tool names or arguments.',
|
||||
'Call tools only for clear GEV control, navigation, visual-style, layer, or app-state requests. For ordinary conversation, answer normally without tools.',
|
||||
'For requests to open, show, reveal, or focus a menu/panel, call set_panel_open or show_data_layers_menu. "Open Context" means only set_panel_open{panelId:"global-context-panel",open:true}; it does not activate a Context sub-mode. "Open Contacts" means set_context_mode{mode:"contacts"}; that action expands the parent Context panel before activating Contacts.',
|
||||
'Panels: set_panel_open or show_data_layers_menu; "Open Context" is only set_panel_open{panelId:"global-context-panel",open:true}; "Open Contacts" is set_context_mode contacts (expands Context itself). Unnamed settings: ask "Voice settings or Provider settings?" first; named panels apply directly. Contacts, Space Missions and Cockpit need explicit naming; opening Context or selecting/viewing an aircraft never enters them. In Cockpit, next/previous uses control_cockpit; track_entity/fly_to_location are refused: never retry, exit Cockpit first.',
|
||||
'For requests like "show me the datacenter layers", open the data layers menu and focus the matching layer row; do not enable the layer unless the user asks to turn it on.',
|
||||
'For questions like "what am I looking at?", "what is in view?", "what is this?", "that selected thing", nearby datacenter, dam, cable, ship, or current view contents, call get_entity_context first, then answer from the returned scene/entity context.',
|
||||
'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
|
||||
'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking stops the tracking.',
|
||||
'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
|
||||
'Globe spin: plain Rotate, including a repeat after a refusal, calls only move_camera motion rotate_globe; the app checks fit. For rotation, call zoom_to_globe only for this turn\'s explicit user Reset/Show Globe View request, never from your own plan, an earlier refusal or a bare yes/ok. On globe-view-required say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" Do not ask a question or call more tools for that request. Explicit reset-and-rotate: await reset success before rotate_globe; never both together. After a failed reset, do not retry for that request. Local turn/look left/right: move_camera motion rotate in that direction, not pan.',
|
||||
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
|
||||
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
|
||||
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
|
||||
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
|
||||
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
|
||||
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
|
||||
'WORLD QUERIES: query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and other get_* and find_* tools) look up any place without changing the map or enabling layers. Use them for places out of view or data whose layer is off. For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over layers instead; enable the layer first if asked. Use analyst_query and get_entity_context for loaded data or what is on screen. Answer from the result; name its source if supplied and never claim the map changed. If asked to show it, also use the matching app action (fly_to_location, set_layer_visibility).',
|
||||
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names, or an areaId you already have; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
|
||||
'AREAS: resolve_area turns a named boundary (country, state, province, county, city, district, natural region) into an areaId; to outline it pass draw=true; annotate_map is for marks, not boundaries. Count inside it → analyst_query kind=area with the areaId (or kind=region); "the area I drew" → kind=drawn; "the area you outlined" → kind=annotation. Imagery of a place → resolve_area, then find_imagery. Ambiguous name: call resolve_area; on needsClarification ask which, never guess.' +
|
||||
(overpass
|
||||
? ' "All hospitals/schools/… in <place>" → resolve_area, then osm_query; in view → osm_query alone; follow-ups → analyst_query layers [osm-places], scope anywhere.'
|
||||
: ''),
|
||||
'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
|
||||
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
|
||||
'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
|
||||
'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
|
||||
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
|
||||
'Do not mention disabled layers or stale selections.',
|
||||
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
|
||||
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
|
||||
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target, find_imagery area. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
|
||||
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, resolve_area, find_imagery, a searched fly_to_location, select_nearest_aircraft, show_street_level, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
|
||||
'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
|
||||
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
|
||||
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers, approximate outlines and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
|
||||
'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
|
||||
// INSTRUCTION-ONLY mapping for the two globe-scale named views.
|
||||
//
|
||||
@@ -46,25 +56,30 @@ function realtimeInstructions(
|
||||
// string is the whole rollback.
|
||||
'NAMED VIEWS are shorthand for tool calls you already have — there is no "mode" tool for them. Treat ONLY these as the shorthand: "infrastructure mode" / "the infrastructure view" / "show me global infrastructure" means three set_layer_visibility calls (local-datacenters, local-dams, telegeography-submarine-cables) plus zoom_to_globe; "environmental mode" / "earth watch" / "active events", said as the name of a view, means set_layer_visibility for local-firms and earthquakes plus zoom_to_globe. Anything vaguer is NOT this shorthand — an open-ended question about the world or the news is an ordinary question: answer it, or use analyst_query over the layers already on. Never switch a whole view on to answer a question nobody asked to see. When you do run one, make every call before speaking, then give one confirmation naming the resulting state; if the fires layer comes back unavailable because no FIRMS key is configured, say so plainly — the earthquakes still loaded. "Live contacts" and "space missions" are NOT this pattern: they stay set_context_mode{mode:"contacts"} and set_context_mode{mode:"space-missions"}.',
|
||||
'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
|
||||
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, snow, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
|
||||
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
|
||||
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any other mention of "satellite" or "satellites" means the satellites DATA LAYER via set_layer_visibility, never a basemap — except "recent/latest satellite imagery" (the recent-imagery layer) and "weather satellite"/clouds (weather-satellite). "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
|
||||
'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
|
||||
'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
|
||||
'HUD requests ("hud on/off", "switch to minimal/tactical layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
|
||||
'Sonar is a visual style over any HUD layout. Sonar requests (including "cyber mode", which was replaced by sonar) use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch layouts, reveal the HUD or enable sonar when changing a slider. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
|
||||
'Bloom/sharpen requests use set_post_processing. Scene requests ("play orbital watch", "stop the scene", "what scenes are there") use control_scene. CCTV camera requests ("next camera", "nearest camera", "select the Congress camera", "show coverage") use control_cctv — the CCTV layer must be enabled first.',
|
||||
'Radio playback requests use control_radio. "Turn on/start the radio" means action=play; action=enable only reveals Radio markers and must be reserved for explicit "show/enable the Radio layer/markers" requests. After a prepared playback result, briefly confirm any other completed actions and say "Turning on the radio"—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
|
||||
'"Track/follow <something specific>" (a callsign, ship name, satellite name) uses track_entity. "Take me to the biggest fire" uses track_entity with query "biggest fire" (the fires layer must be enabled). Bare "orbit" means camera orbit of the current landmark. "Stop following/tracking" uses stop_tracking.',
|
||||
'"Show me which planes are overhead"/"frame the ships"/"show me the satellites above" use frame_overhead with the matching target.',
|
||||
'Confirmations echo the RESULTING state, never the request: "HUD operator layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
|
||||
'Confirmations echo the RESULTING state, never the request: "HUD minimal layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
|
||||
'STREET LEVEL: "X on Street View/Mapillary/KartaView/Panoramax", "street view of X" → show_street_level once, not fly_to_location; no place = view centre. Street View is never a basemap.',
|
||||
'For destination requests such as "take me to Italy", "go to NYC", or "show me the Eiffel Tower", call fly_to_location. Prefer known city IDs when available; otherwise pass the plain place query.',
|
||||
'Navigation-only requests ("take me to X", "go to X", "fly to X") are NOT descriptions: call fly_to_location alone and do NOT also call annotate_map, unless the user explicitly asks to mark the place or you go on to explain specific places there. Never drop a point pin on a region-scale natural feature (a mountain range, desert, sea, or forest) — a single point in the middle of the Rockies is meaningless. If the user explicitly asks to mark such a region, prefer type=area.',
|
||||
'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
|
||||
'Only supply rangeM when the user asks for a particular numeric height, distance, closer view, or wider view.',
|
||||
'For relative requests such as "zoom out a little", "pull back", "zoom in more", or "get closer", always call adjust_camera_zoom. But "globe view", "whole earth", "the whole planet", or "zoom all the way out" is an ABSOLUTE framing: call zoom_to_globe once instead — repeated adjust_camera_zoom calls can never reach the globe. Never claim the camera moved without the tool returning ok=true.',
|
||||
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus, district or boundary, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park, district or boundary outline; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
|
||||
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus or district, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park or district footprint; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
|
||||
annotationGuidance,
|
||||
'annotate_map results: work any say line in naturally (places not found, a straight-line fallback with no travel time, or a full map — then ask before clearing). outlinePending means the boundary is still being traced: never claim it is drawn. A later map_annotation_outline system item reports each outcome; use it to quietly confirm or correct yourself.',
|
||||
'ROUTES vs DISTANCES: "walking/driving route from A to B" → type=route with the ordered points and the mode; the app draws the street path and reports distance and time you can read aloud. "How far is X from Y" or "X is next to Y" → type=arrow (straight-line distance), never route.',
|
||||
].join('\n');
|
||||
];
|
||||
}
|
||||
|
||||
export { realtimeInstructions };
|
||||
function realtimeInstructions(annotationGuidance, options) {
|
||||
return realtimeDirectives(annotationGuidance, options).join('\n');
|
||||
}
|
||||
|
||||
export { realtimeInstructions, realtimeDirectives };
|
||||
@@ -13,7 +13,9 @@ import {
|
||||
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
|
||||
} from './constants.js';
|
||||
import { realtimeInstructions } from './instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from './tools.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
|
||||
import { chatSchema } from './chat.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
import {
|
||||
preferCodexOAuth,
|
||||
readCodexOAuthAccessToken,
|
||||
@@ -127,6 +129,19 @@ function createRealtimeOAuthLoginHandler({
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The tools one Realtime session is offered, as OpenAI accepts them: it now
|
||||
* refuses a whole session whose tool schema uses `\p{…}` patterns (#1618), as
|
||||
* Chat Completions did (#1616), so voice drops the same pattern hints chat does.
|
||||
*/
|
||||
function realtimeSessionTools(tools, { overpass = false } = {}) {
|
||||
return offeredTools(tools, { overpass }).map((tool) =>
|
||||
tool.parameters
|
||||
? { ...tool, parameters: chatSchema(tool.parameters) }
|
||||
: tool,
|
||||
);
|
||||
}
|
||||
|
||||
function createRealtimeTokenHandler({
|
||||
annotationGuidance,
|
||||
endpoint = 'https://api.openai.com/v1/realtime/client_secrets',
|
||||
@@ -136,6 +151,8 @@ function createRealtimeTokenHandler({
|
||||
environment = process.env,
|
||||
models = {},
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
// osm_query is offered only when an operator configured Overpass.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
} = {}) {
|
||||
return async (req, res) => {
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
@@ -221,7 +238,12 @@ function createRealtimeTokenHandler({
|
||||
if (!bearer) {
|
||||
res.statusCode = 503;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'OPENAI_API_KEY is not set',
|
||||
code: 'API_KEY_MISSING',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -273,6 +295,7 @@ function createRealtimeTokenHandler({
|
||||
transcribeModel.toLowerCase() === 'off'
|
||||
? {}
|
||||
: { transcription: { model: transcribeModel } };
|
||||
const overpass = overpassConfigured();
|
||||
const sessionConfig = {
|
||||
session: {
|
||||
type: 'realtime',
|
||||
@@ -298,8 +321,8 @@ function createRealtimeTokenHandler({
|
||||
},
|
||||
output: { voice },
|
||||
},
|
||||
instructions: realtimeInstructions(annotationGuidance),
|
||||
tools,
|
||||
instructions: realtimeInstructions(annotationGuidance, { overpass }),
|
||||
tools: realtimeSessionTools(tools, { overpass }),
|
||||
tool_choice: 'auto',
|
||||
},
|
||||
};
|
||||
@@ -366,4 +389,5 @@ export {
|
||||
createRealtimeOAuthLoginHandler,
|
||||
createRealtimeOAuthStatusHandler,
|
||||
createRealtimeTokenHandler,
|
||||
realtimeSessionTools,
|
||||
};
|
||||
@@ -44,7 +44,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
select_nearest_aircraft: {
|
||||
description:
|
||||
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result. This does not open Contacts or Cockpit.',
|
||||
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -65,6 +65,28 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
show_street_level: {
|
||||
description:
|
||||
'Fly to a place and open its nearest street-level photo; Street View switches to Google 3D (billed). No query: view centre.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
query: {
|
||||
description: 'Place or address; "pointer" = under the cursor.',
|
||||
$position: 1,
|
||||
},
|
||||
provider: {
|
||||
description:
|
||||
'google = Street View; others by name; omit: the lit source.',
|
||||
$position: 1,
|
||||
},
|
||||
referent: {
|
||||
description: 'Numbered item n from the last result list.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
adjust_camera_zoom: {
|
||||
description:
|
||||
'Move the current Cesium camera closer to or farther from what it is presently looking at. Use for relative zoom requests without changing location.',
|
||||
@@ -111,12 +133,18 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
set_panel_open: {
|
||||
description: 'Open or close a GEV UI panel/dropdown.',
|
||||
description:
|
||||
'Visual Presets/styles/filters=control-panel; Scenes/saved scenes=scene-panel; Voice Settings=voice-settings; voice control/Preview=voice-preview; Provider Settings/API keys=provider-settings. Voice panels exit Cockpit directly.',
|
||||
$position: 1,
|
||||
},
|
||||
set_voice_inactivity_timeout: {
|
||||
description:
|
||||
'Set the requested exact inactivity duration directly: integer minutes 1–60, 0=None. Report applied/persisted truthfully; an unsaved live change is not saved.',
|
||||
$position: 1,
|
||||
},
|
||||
set_context_mode: {
|
||||
description:
|
||||
'Enter or exit the Global Context sub-mode used by Contacts and Space Missions. Use Contacts only when the user explicitly requests Contacts, and Space Missions only when explicitly requested. A request to open the parent Context panel alone uses set_panel_open and must not activate either sub-mode. Selecting an aircraft does not imply Context.',
|
||||
'Enter/exit Contacts or Space Missions only when explicitly requested; opening Context alone uses set_panel_open.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -129,23 +157,23 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
control_cockpit: {
|
||||
description:
|
||||
'Read or control Cockpit when the user explicitly requests Cockpit: establish Contacts and enter from a selected or tracked aircraft; exit; or navigate nearby Contacts with optional filters. Selecting or viewing an aircraft alone must not enter Cockpit.',
|
||||
'Explicit Cockpit requests only: enter from a selected/tracked aircraft (needs only its own Flights/Military layer on, not Contacts; opens Contacts itself, no set_context_mode), exit, status, or next/previous. Explicit exit-and-show voice: voicePanel preview/settings (Voice Settings), no confirmation.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
action: {
|
||||
description:
|
||||
'previous/next (or prev) navigates through nearby contacts in Cockpit context.',
|
||||
'previous/next (or prev) cycles nearby Cockpit contacts.',
|
||||
$position: 2,
|
||||
},
|
||||
targetLayer: {
|
||||
description:
|
||||
'Optional contact layer filter for next/previous (for example military for a military-only cycle).',
|
||||
'Optional next/previous layer filter (e.g. military-only).',
|
||||
$position: 2,
|
||||
},
|
||||
aircraftClass: {
|
||||
description:
|
||||
'Optional aircraft class filter (for example helicopter) when using next/previous navigation.',
|
||||
'Optional next/previous aircraft class filter (e.g. helicopter).',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
@@ -197,7 +225,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
set_cyber_sonar: {
|
||||
description:
|
||||
'Adjust Cyber-only sonar controls. Requires the Cyber HUD layout; never switches layout or HUD visibility. Omitted fields stay unchanged. Returns actual settings and separate map/contact sweep activity. A saved setting does not mean the effect is active.',
|
||||
'Adjust the Sonar visual style, which works over any HUD layout. Never switches layout or HUD visibility. Omitted fields stay unchanged. Returns actual settings and separate map/contact sweep activity. A saved setting does not mean the effect is active.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -281,6 +309,11 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
open_stories: {
|
||||
description:
|
||||
'Open the Stories Explore list on request ("open Stories", "show the story list"). Turns off voice.',
|
||||
$position: 1,
|
||||
},
|
||||
control_scene: {
|
||||
description:
|
||||
'Cinematic scene playback: list scenes, play one scene by name, stop, advance, or read status. Play starts a single named scene and returns immediately.',
|
||||
@@ -399,7 +432,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
annotate_map: {
|
||||
description:
|
||||
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints and boundaries, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
|
||||
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -511,20 +544,24 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
move_camera: {
|
||||
description:
|
||||
'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly").',
|
||||
'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly"). Globe rotation (rotate_globe) spins Earth right continuously by default; explicit direction, mode and speed take precedence. It checks fit and may refuse.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
direction: {
|
||||
description:
|
||||
'Required except for orbit (defaults right/clockwise) and stop.',
|
||||
'Pan/rotate/tilt steps; orbit/rotate_globe default right.',
|
||||
$position: 2,
|
||||
},
|
||||
mode: {
|
||||
description:
|
||||
'once = bounded eased nudge (default); continuous = until stop/manual input.',
|
||||
'once = bounded eased nudge; continuous = until stop/manual input. rotate_globe defaults continuous; other motions default once.',
|
||||
$position: 2,
|
||||
},
|
||||
pitchDeg: {
|
||||
description: 'Absolute; -45 default, -89 top-down.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -541,9 +578,48 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
resolve_area: {
|
||||
description:
|
||||
'Named place → reusable areaId for analyst_query scope {kind:area} and other area tools. draw=true outlines it ("outline Bagmati province"); around=true is "an area around <landmark>": its enclosing mapped area, else a buffer marked approximate. needsClarification: ask which, then call again with its candidateId. AREA_TIMEOUT: retry once.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
query: {
|
||||
description: 'The place as said, with any kind word.',
|
||||
$position: 1,
|
||||
},
|
||||
level: {
|
||||
description: 'Only when the user names the kind.',
|
||||
$position: 1,
|
||||
},
|
||||
within: {
|
||||
description: 'Country or state the user added.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
find_imagery: {
|
||||
description:
|
||||
'Recent satellite imagery (Sentinel-2/Landsat 30 m, VIIRS daily; last 30 days): pins the best acquisition in the Recent Imagery panel. areaId from resolve_area; area=pointer for "this spot"; omit for the view. BOX_REFUSED: offer to zoom in. DISPLACED: leave it.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
maxCloudPct: {
|
||||
description: '"cloud-free" → 10.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
osm_query: {
|
||||
description:
|
||||
'OpenStreetMap places of one kind ("hospitals in Kathmandu"). areaId from resolve_area; omit for the view. Shows them in OSM Places with numbered referents; they become the last answer. countScope bounding-box means an approximate count. OSM_TIMEOUT: retry once.',
|
||||
$position: 1,
|
||||
},
|
||||
analyst_query: {
|
||||
description:
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), NEEDS_CLARIFICATION (ask which), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -554,7 +630,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
scope: {
|
||||
description:
|
||||
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"). Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates. There is no drawn-area scope yet: say so rather than answering for the view.',
|
||||
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"); area + areaId from resolve_area; drawn = "the area I drew"; annotation = "the area you outlined". Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates.',
|
||||
$position: 2,
|
||||
properties: {
|
||||
name: {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createActionTools } from '../../../src/voice/actionSchemas.js';
|
||||
import { ACTION_DESCRIPTIONS } from './toolDescriptions.js';
|
||||
|
||||
/** Every voice tool, as the Realtime session describes them. */
|
||||
export const GEV_REALTIME_TOOLS = createActionTools(ACTION_DESCRIPTIONS);
|
||||
|
||||
/**
|
||||
@@ -14,3 +15,27 @@ export function realtimeSessionTools(additional = []) {
|
||||
...additional.filter((tool) => !names.has(tool.name)),
|
||||
];
|
||||
}
|
||||
|
||||
/** Tools that need an operator-configured Overpass (OVERPASS_UPSTREAMS). */
|
||||
export const OVERPASS_ONLY_TOOLS = Object.freeze(['osm_query']);
|
||||
|
||||
/**
|
||||
* The tools one session is offered. Bulk OpenStreetMap place search runs
|
||||
* only on an operator's Overpass, so without one `osm_query` is left out
|
||||
* rather than offered and refused.
|
||||
* @param {Array<{name: string}>} tools
|
||||
* @param {{overpass?: boolean}} [options]
|
||||
*/
|
||||
export function offeredTools(tools, { overpass = false } = {}) {
|
||||
return overpass
|
||||
? tools
|
||||
: tools.filter((tool) => !OVERPASS_ONLY_TOOLS.includes(tool.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* The app actions one session is offered.
|
||||
* @param {{overpass?: boolean}} [options]
|
||||
*/
|
||||
export function realtimeTools({ overpass = false } = {}) {
|
||||
return offeredTools(GEV_REALTIME_TOOLS, { overpass });
|
||||
}
|
||||
+105
-61
@@ -1,5 +1,12 @@
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import {
|
||||
makeRateLimiter,
|
||||
clientKey,
|
||||
clientNetworkKey,
|
||||
isLoopbackPeer,
|
||||
} from './common/rate-limit.js';
|
||||
import { hasProxySignals } from '../../src/localRequestGate.mjs';
|
||||
import { readRequestBodyCapped } from './common/request.js';
|
||||
import { admitSameSiteKeyless } from './common/same-site.js';
|
||||
import {
|
||||
OVERPASS_MAX_BODY_BYTES,
|
||||
OVERPASS_MAX_CONCURRENT,
|
||||
@@ -21,18 +28,65 @@ import {
|
||||
overpassNotConfigured,
|
||||
} from './overpass/transport.js';
|
||||
import { installRouteMiddleware } from './places/routes.js';
|
||||
import { installOsmFeaturesRoute } from './overpass/osmFeaturesRoute.js';
|
||||
|
||||
/** @type {Map<string,Promise>} In-flight Overpass requests keyed by normalized query body. */
|
||||
const _overpassInFlight = new Map();
|
||||
|
||||
let _overpassConcurrent = 0;
|
||||
|
||||
// Bound body-reading and validation work, including malformed requests and
|
||||
// cache hits, before a byte is read. Remote peers share one global window so
|
||||
// address rotation cannot mint admission without bound (IPv6 peers are keyed
|
||||
// by /64 for the same reason). A direct loopback request — the owner's own
|
||||
// browser and the repo QA harnesses, with no forwarding headers — has its own
|
||||
// per-peer window outside that global, so busy LAN peers cannot lock the owner
|
||||
// out of the map. A request relayed by a reverse proxy on this machine also
|
||||
// arrives from loopback, but it is remote traffic: it is admitted from the
|
||||
// remote pool under its own socket-keyed bucket, never from the owner's. The
|
||||
// forwarded client address is not trusted as identity. Only admission is
|
||||
// isolated: the upstream budget below (90/min per key, 300/min globally) is
|
||||
// shared, so an owner request that misses cache can still be refused there.
|
||||
const OVERPASS_ADMISSION_WINDOW = { windowMs: 60_000, max: 300 };
|
||||
const _overpassLoopbackAdmission = makeRateLimiter(OVERPASS_ADMISSION_WINDOW);
|
||||
const _overpassPeerAdmission = makeRateLimiter({
|
||||
...OVERPASS_ADMISSION_WINDOW,
|
||||
globalMax: 900,
|
||||
});
|
||||
|
||||
const _overpassRateLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 90,
|
||||
globalMax: 300,
|
||||
});
|
||||
|
||||
/**
|
||||
* Classify a request for admission. Direct loopback — a loopback socket with
|
||||
* no forwarding header — is the owner's own pool; everything else, including a
|
||||
* loopback socket that carries forwarding headers (a local reverse proxy), is
|
||||
* remote. A proxied request keeps its socket identity under a `proxied:`
|
||||
* prefix, so it never shares a bucket with the owner's direct requests.
|
||||
* @param {import('http').IncomingMessage} req
|
||||
* @returns {{direct: boolean, key: string}}
|
||||
*/
|
||||
function overpassAdmissionClass(req) {
|
||||
const peer = clientNetworkKey(req);
|
||||
if (!isLoopbackPeer(clientKey(req))) return { direct: false, key: peer };
|
||||
if (hasProxySignals(req.headers || {}))
|
||||
return { direct: false, key: `proxied:${peer}` };
|
||||
return { direct: true, key: peer };
|
||||
}
|
||||
|
||||
/**
|
||||
* Spend one admission slot for this request; false when its bucket is over its
|
||||
* window (or, for remote traffic, when the shared global window is full).
|
||||
* @param {{direct: boolean, key: string}} admission - `overpassAdmissionClass(req)`.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function admitOverpassRequest({ direct, key }) {
|
||||
return (direct ? _overpassLoopbackAdmission : _overpassPeerAdmission)(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a completed Overpass payload to the HTTP response.
|
||||
*
|
||||
@@ -94,6 +148,22 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Keyless gate: refuses cross-site browser requests but, unlike the
|
||||
// cost-bearing routes, admits the forwarding headers a reverse proxy
|
||||
// adds — the layers that depend on Overpass must keep working for an
|
||||
// instance deployed the way the README documents.
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
const admission = overpassAdmissionClass(req);
|
||||
if (!admitOverpassRequest(admission)) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
'Retry-After': '60',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
|
||||
// Collect POST body with a hard byte cap (Overpass QL queries are small)
|
||||
let body;
|
||||
try {
|
||||
@@ -143,7 +213,31 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
// requests; boundary-class queries keep their month-long TTL.
|
||||
readDisk: () =>
|
||||
readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
|
||||
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
|
||||
allowUpstream: () => _overpassRateLimiter(admission.key),
|
||||
loadUpstream: async () => {
|
||||
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT)
|
||||
return {
|
||||
status: 503,
|
||||
contentType: 'application/json',
|
||||
retryAfterMs: 2000,
|
||||
body: JSON.stringify({
|
||||
error: 'Overpass proxy busy — try again shortly',
|
||||
}),
|
||||
};
|
||||
_overpassConcurrent++;
|
||||
try {
|
||||
const payload = await fetchOverpassPayload(safeBody);
|
||||
if (overpassPayloadIsData(payload)) {
|
||||
const entry = { ...payload, cachedAt: Date.now() };
|
||||
_overpassCache.set(cacheKey, entry);
|
||||
trimOverpassCache();
|
||||
writeOverpassDisk(cacheKey, entry);
|
||||
}
|
||||
return payload;
|
||||
} finally {
|
||||
_overpassConcurrent--;
|
||||
}
|
||||
},
|
||||
});
|
||||
if (preflight.source === 'RATE_LIMITED') {
|
||||
res.writeHead(429, {
|
||||
@@ -153,71 +247,20 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
if (preflight.source !== 'UPSTREAM') {
|
||||
// A coalesced caller sees the same failure as the original request
|
||||
// and must get the same last-good fallback, not the raw refusal.
|
||||
if (!overpassPayloadIsData(preflight.payload)) {
|
||||
const stale = await readStaleOverpass(cacheKey);
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (preflight.source === 'DISK') {
|
||||
_overpassCache.set(cacheKey, preflight.payload);
|
||||
trimOverpassCache();
|
||||
}
|
||||
sendOverpassResponse(res, preflight.payload, preflight.source);
|
||||
return;
|
||||
}
|
||||
|
||||
// From here onward the request is genuinely upstream-bound and has
|
||||
// consumed one local limiter slot. Cache and dedupe hits above do not.
|
||||
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT) {
|
||||
res.writeHead(503, {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '2',
|
||||
});
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'Overpass proxy busy — try again shortly',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
_overpassConcurrent += 1;
|
||||
const requestPromise = fetchOverpassPayload(safeBody)
|
||||
.then((payload) => {
|
||||
// Only a 2xx is data. `< 500` cached every 4xx, so one mirror's
|
||||
// refusal was written to memory AND disk — and boundary-class
|
||||
// queries hold a month-long TTL, so a single 406 outlived the
|
||||
// outage that caused it.
|
||||
if (overpassPayloadIsData(payload)) {
|
||||
const entry = { ...payload, cachedAt: Date.now() };
|
||||
_overpassCache.set(cacheKey, entry);
|
||||
trimOverpassCache();
|
||||
writeOverpassDisk(cacheKey, entry);
|
||||
}
|
||||
return payload;
|
||||
})
|
||||
.finally(() => {
|
||||
_overpassConcurrent -= 1;
|
||||
_overpassInFlight.delete(cacheKey);
|
||||
});
|
||||
|
||||
_overpassInFlight.set(cacheKey, requestPromise);
|
||||
const payload = await requestPromise;
|
||||
// Degraded upstream (rate-limited on every mirror / 5xx / runtime
|
||||
// error): last-good roads beat an empty layer — serve stale from
|
||||
// memory or disk at ANY age before surfacing the failure.
|
||||
if (!overpassPayloadIsData(payload)) {
|
||||
if (!overpassPayloadIsData(preflight.payload)) {
|
||||
const stale = await readStaleOverpass(cacheKey);
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
}
|
||||
}
|
||||
sendOverpassResponse(res, payload, 'MISS');
|
||||
// Preflight already published the disk hit to memory; only trim.
|
||||
if (preflight.source === 'DISK') trimOverpassCache();
|
||||
sendOverpassResponse(
|
||||
res,
|
||||
preflight.payload,
|
||||
preflight.source === 'UPSTREAM' ? 'MISS' : preflight.source,
|
||||
);
|
||||
} catch (e) {
|
||||
// Every mirror threw (network-level). Same serve-stale rule.
|
||||
const stale = cacheKey ? await readStaleOverpass(cacheKey) : null;
|
||||
@@ -232,6 +275,7 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
});
|
||||
|
||||
installRouteMiddleware(server.middlewares, routing);
|
||||
installOsmFeaturesRoute(server.middlewares);
|
||||
};
|
||||
return {
|
||||
name: 'overpass-proxy',
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { coalesceProxyRequest } from '../common/http.js';
|
||||
import { isOverpassBoundaryQuery } from './query.js';
|
||||
import {
|
||||
OVERPASS_BOUNDARY_DISK_TTL_MS,
|
||||
@@ -78,7 +79,9 @@ function writeOverpassDisk(cacheKey, payload) {
|
||||
/**
|
||||
* Resolve every cache/coalescing layer before admitting a request to the local
|
||||
* upstream rate limiter. The injected limiter callback is invoked exactly once
|
||||
* for a complete cache miss and never for memory, in-flight, or disk hits.
|
||||
* for a complete cache miss and never for memory, in-flight, or disk hits. A
|
||||
* caller that joined a reservation rejected by another client's limiter
|
||||
* retries with its own limiter instead of inheriting that 429.
|
||||
* Exported so the admission ordering can be tested without a Vite server.
|
||||
*
|
||||
* @param {object} options
|
||||
@@ -87,6 +90,7 @@ function writeOverpassDisk(cacheKey, payload) {
|
||||
* @param {Map<string, Promise<object>>} options.inFlight
|
||||
* @param {()=>Promise<object|null>} options.readDisk
|
||||
* @param {()=>boolean} options.allowUpstream
|
||||
* @param {()=>Promise<object>} options.loadUpstream Fetch and publish the admitted result.
|
||||
* @param {number} [options.now]
|
||||
* @param {number} [options.cacheMs]
|
||||
* @returns {Promise<{source:'HIT'|'INFLIGHT'|'DISK'|'UPSTREAM'|'RATE_LIMITED', payload:object|null}>}
|
||||
@@ -97,22 +101,34 @@ async function resolveOverpassPreflight({
|
||||
inFlight,
|
||||
readDisk,
|
||||
allowUpstream,
|
||||
loadUpstream = async () => null,
|
||||
now = Date.now(),
|
||||
cacheMs = OVERPASS_CACHE_MS,
|
||||
}) {
|
||||
const cached = memoryCache.get(cacheKey);
|
||||
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs)
|
||||
return { source: 'HIT', payload: cached };
|
||||
for (;;) {
|
||||
const cached = memoryCache.get(cacheKey);
|
||||
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs)
|
||||
return { source: 'HIT', payload: cached };
|
||||
|
||||
const pending = inFlight.get(cacheKey);
|
||||
if (pending) return { source: 'INFLIGHT', payload: await pending };
|
||||
|
||||
const disk = await readDisk();
|
||||
if (overpassPayloadIsData(disk)) return { source: 'DISK', payload: disk };
|
||||
|
||||
return allowUpstream()
|
||||
? { source: 'UPSTREAM', payload: null }
|
||||
: { source: 'RATE_LIMITED', payload: null };
|
||||
// Reserve synchronously before disk I/O. The owner retains the reservation
|
||||
// through its own upstream admission, fetch and cache publication.
|
||||
const request = coalesceProxyRequest(inFlight, cacheKey, async () => {
|
||||
const disk = await readDisk();
|
||||
if (overpassPayloadIsData(disk)) {
|
||||
memoryCache.set(cacheKey, disk);
|
||||
return { source: 'DISK', payload: disk };
|
||||
}
|
||||
if (!allowUpstream()) return { source: 'RATE_LIMITED', payload: null };
|
||||
return { source: 'UPSTREAM', payload: await loadUpstream() };
|
||||
});
|
||||
const result = await request.promise;
|
||||
if (!request.shared) return result;
|
||||
// The owner's quota rejection belongs to the owner's client. The rejected
|
||||
// reservation is released before joiners resume, so retry and let this
|
||||
// caller own the next reservation (or join an admitted one) on its quota.
|
||||
if (result.source === 'RATE_LIMITED') continue;
|
||||
return { ...result, source: 'INFLIGHT' };
|
||||
}
|
||||
}
|
||||
|
||||
/** Return only last-good Overpass data, regardless of its age. */
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { OSM_PRESETS } from '../../../src/data/osmPresets.js';
|
||||
|
||||
/**
|
||||
* Bounded, typed OpenStreetMap feature search: one preset from the curated
|
||||
* table inside a bounding box.
|
||||
*
|
||||
* The request carries a preset id, a box, a mode and a limit — never query
|
||||
* text. The server writes the Overpass query itself and bounds it: box size
|
||||
* per mode (a count may cover more ground than a fetch), result count,
|
||||
* Overpass timeout and memory, response bytes, and one query in flight. It
|
||||
* runs only against the operator's `OVERPASS_UPSTREAMS`; the generic
|
||||
* `/api/overpass` guard is untouched.
|
||||
*
|
||||
* There is deliberately no boundary (`area`) filter: Overpass servers differ
|
||||
* in whether they hold area data, and one without it answers zero rather than
|
||||
* failing. The browser filters the box's features by the exact area geometry
|
||||
* instead, so a count inside a province is exact and a box count is labelled
|
||||
* as a box count.
|
||||
*/
|
||||
export const OSM_FEATURE_LIMITS = Object.freeze({
|
||||
/** Largest box a fetch may cover, in square degrees (~200 × 200 km at the equator). */
|
||||
maxFetchDeg2: 4,
|
||||
/** Largest box a count may cover. */
|
||||
maxCountDeg2: 36,
|
||||
/** Most features one fetch returns. */
|
||||
maxLimit: 1000,
|
||||
/** Overpass server-side timeout, seconds: it gives up before we stop waiting. */
|
||||
timeoutS: 20,
|
||||
/** Overpass server-side memory cap, bytes. */
|
||||
maxsize: 64 * 1024 * 1024,
|
||||
/** Response bytes read from the upstream. */
|
||||
maxResponseBytes: 16 * 1024 * 1024,
|
||||
/** Per-upstream wait: the Overpass timeout plus transfer time. */
|
||||
upstreamTimeoutMs: 22_000,
|
||||
/**
|
||||
* The whole search, across every configured endpoint in turn. It ends
|
||||
* before the voice answer's own deadline (`OSM_QUERY_LIMITS.budgetMs` in
|
||||
* `src/voice/osmActions.js`, 25 s): a browser that gives up first closes
|
||||
* the request, which cancels the upstream query, so nothing would be
|
||||
* cached for the retry.
|
||||
*/
|
||||
upstreamBudgetMs: 22_000,
|
||||
});
|
||||
|
||||
/** Tags a returned feature keeps, besides the preset's own. */
|
||||
const KEPT_TAGS = [
|
||||
'name',
|
||||
'name:en',
|
||||
'operator',
|
||||
'brand',
|
||||
'opening_hours',
|
||||
'wheelchair',
|
||||
'emergency',
|
||||
'healthcare',
|
||||
'addr:street',
|
||||
'addr:city',
|
||||
];
|
||||
|
||||
const quote = (text) =>
|
||||
`"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
||||
|
||||
/**
|
||||
* Parse and bound a request. Returns the compiled query or a refusal.
|
||||
* @param {{preset: string, bbox: string|number[], mode?: string, limit?: string|number}} request
|
||||
* @returns {{ok: true, ql: string, mode: string, preset: object, limit: number, boxes: number[][]}|{ok: false, code: string, error: string}}
|
||||
*/
|
||||
export function compileOsmFeatureQuery(request = {}) {
|
||||
const refuse = (code, error) => ({ ok: false, code, error });
|
||||
const presetId = String(request.preset || '');
|
||||
const preset = Object.hasOwn(OSM_PRESETS, presetId)
|
||||
? OSM_PRESETS[presetId]
|
||||
: null;
|
||||
if (!preset) return refuse('UNKNOWN_PRESET', 'Unknown feature kind.');
|
||||
const mode = request.mode === 'count' ? 'count' : 'features';
|
||||
const parts = (
|
||||
Array.isArray(request.bbox)
|
||||
? request.bbox
|
||||
: String(request.bbox || '').split(',')
|
||||
).map(Number);
|
||||
if (parts.length !== 4 || !parts.every(Number.isFinite))
|
||||
return refuse('BAD_BOX', 'bbox needs west,south,east,north.');
|
||||
const [west, south, east, north] = parts;
|
||||
if (
|
||||
Math.abs(west) > 180 ||
|
||||
Math.abs(east) > 180 ||
|
||||
south < -90 ||
|
||||
north > 90 ||
|
||||
south >= north ||
|
||||
west === east
|
||||
)
|
||||
return refuse('BAD_BOX', 'bbox edges are out of range.');
|
||||
// A box crossing the antimeridian is searched as its two halves.
|
||||
const boxes =
|
||||
west < east
|
||||
? [[south, west, north, east]]
|
||||
: [
|
||||
[south, west, north, 180],
|
||||
[south, -180, north, east],
|
||||
];
|
||||
const deg2 = boxes.reduce((sum, [s, w, n, e]) => sum + (n - s) * (e - w), 0);
|
||||
const cap =
|
||||
mode === 'count'
|
||||
? OSM_FEATURE_LIMITS.maxCountDeg2
|
||||
: OSM_FEATURE_LIMITS.maxFetchDeg2;
|
||||
if (deg2 > cap)
|
||||
return refuse(
|
||||
'AREA_TOO_LARGE',
|
||||
mode === 'count'
|
||||
? 'That area is too large to search; narrow it down.'
|
||||
: 'That area is too large to list; count first or narrow it down.',
|
||||
);
|
||||
// An absent or empty limit is the default, not Number(null) = 0.
|
||||
const requested =
|
||||
request.limit === null ||
|
||||
request.limit === undefined ||
|
||||
String(request.limit).trim() === ''
|
||||
? NaN
|
||||
: Number(request.limit);
|
||||
const limit = Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
OSM_FEATURE_LIMITS.maxLimit,
|
||||
Number.isFinite(requested) ? Math.trunc(requested) : 500,
|
||||
),
|
||||
);
|
||||
const selectors = [];
|
||||
for (const [key, value] of preset.tags)
|
||||
for (const [s, w, n, e] of boxes)
|
||||
selectors.push(
|
||||
`nwr[${quote(key)}=${quote(value)}](${s},${w},${n},${e});`,
|
||||
);
|
||||
const ql =
|
||||
`[out:json][timeout:${OSM_FEATURE_LIMITS.timeoutS}][maxsize:${OSM_FEATURE_LIMITS.maxsize}];` +
|
||||
`(${selectors.join('')});` +
|
||||
(mode === 'count' ? 'out count;' : `out center tags ${limit};`);
|
||||
return { ok: true, ql, mode, preset, limit, boxes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Overpass JSON → the compact answer the browser gets.
|
||||
* @param {object} payload Parsed Overpass JSON.
|
||||
* @param {{mode: string, preset: object, limit: number}} compiled
|
||||
*/
|
||||
export function summarizeOsmFeatures(payload, compiled) {
|
||||
if (!Array.isArray(payload?.elements))
|
||||
throw new TypeError('Malformed Overpass feature response');
|
||||
const elements = payload.elements;
|
||||
if (compiled.mode === 'count') {
|
||||
const tags = elements.find((el) => el?.type === 'count')?.tags;
|
||||
const rawTotal = tags?.total;
|
||||
const total = Number(rawTotal);
|
||||
if (
|
||||
!tags ||
|
||||
!Object.hasOwn(tags, 'total') ||
|
||||
!['string', 'number'].includes(typeof rawTotal) ||
|
||||
String(rawTotal).trim() === '' ||
|
||||
!Number.isSafeInteger(total) ||
|
||||
total < 0
|
||||
)
|
||||
throw new TypeError('Malformed Overpass count response');
|
||||
return {
|
||||
mode: 'count',
|
||||
preset: compiled.preset.id,
|
||||
count: total,
|
||||
nodes: Number(tags.nodes) || 0,
|
||||
ways: Number(tags.ways) || 0,
|
||||
relations: Number(tags.relations) || 0,
|
||||
};
|
||||
}
|
||||
const features = [];
|
||||
let unplaced = 0;
|
||||
for (const el of elements) {
|
||||
if (
|
||||
!['node', 'way', 'relation'].includes(el?.type) ||
|
||||
(!Number.isFinite(Number(el?.id)) && typeof el?.id !== 'string')
|
||||
) {
|
||||
unplaced += 1;
|
||||
continue;
|
||||
}
|
||||
const lat = Number.isFinite(el?.lat) ? el.lat : el?.center?.lat;
|
||||
const lon = Number.isFinite(el?.lon) ? el.lon : el?.center?.lon;
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) {
|
||||
unplaced += 1;
|
||||
continue;
|
||||
}
|
||||
const tags = {};
|
||||
for (const key of [...KEPT_TAGS, ...compiled.preset.tags.map(([k]) => k)])
|
||||
if (typeof el.tags?.[key] === 'string')
|
||||
tags[key] = el.tags[key].slice(0, 120);
|
||||
features.push({ id: `${el.type}/${el.id}`, lat, lon, tags });
|
||||
}
|
||||
// A genuinely empty `elements` array is authoritative. A non-empty answer
|
||||
// whose every row is unusable is a schema failure, not "zero places"; do
|
||||
// not let the route cache that as current state for a day.
|
||||
if (elements.length && features.length === 0)
|
||||
throw new TypeError('Overpass feature response has no usable rows');
|
||||
return {
|
||||
mode: 'features',
|
||||
preset: compiled.preset.id,
|
||||
count: features.length,
|
||||
features,
|
||||
truncated: elements.length >= compiled.limit,
|
||||
...(unplaced ? { unplaced } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
import path from 'node:path';
|
||||
import { makeRateLimiter, clientNetworkKey } from '../common/rate-limit.js';
|
||||
import { admitSameSiteKeyless } from '../common/same-site.js';
|
||||
import { createBoundedCache } from '../common/boundedCache.js';
|
||||
import {
|
||||
compileOsmFeatureQuery,
|
||||
OSM_FEATURE_LIMITS,
|
||||
summarizeOsmFeatures,
|
||||
} from './osmFeatures.js';
|
||||
import {
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
} from './constants.js';
|
||||
import { fetchOverpassPayload, overpassPayloadIsData } from './transport.js';
|
||||
|
||||
/** Places of interest change slowly: answers are kept a day. */
|
||||
const OSM_FEATURES_TTL_MS = 86_400_000;
|
||||
/** Upstream searches at once, across all clients (one per client). */
|
||||
const OSM_MAX_RUNNING = 2;
|
||||
/** Cache-key version; nothing cached under another version is reused. */
|
||||
const CACHE_VERSION = 'v2';
|
||||
|
||||
/** Never simplify: these answers are points and counts, not drawings. */
|
||||
const keepBody = (body) => body;
|
||||
|
||||
const jsonError = (status, error, extra = {}) => ({
|
||||
status,
|
||||
body: JSON.stringify({ error, ...extra }),
|
||||
});
|
||||
|
||||
/**
|
||||
* One upstream Overpass query at a time, with identical requests sharing it
|
||||
* and answers in a bounded memory + disk cache.
|
||||
*/
|
||||
function createOverpassAnswerer({
|
||||
fetchPayload,
|
||||
cache,
|
||||
maxResponseBytes,
|
||||
upstreamTimeoutMs,
|
||||
upstreamBudgetMs,
|
||||
maxRunning = OSM_MAX_RUNNING,
|
||||
}) {
|
||||
const inFlight = new Map();
|
||||
// One search per client, a few in all: one client can't hold the route.
|
||||
const runningBy = new Map();
|
||||
let running = 0;
|
||||
return async function answer(key, ql, summarize, signal = null, owner = '') {
|
||||
signal?.throwIfAborted?.();
|
||||
const held = await cache.get(key);
|
||||
signal?.throwIfAborted?.();
|
||||
if (typeof held === 'string')
|
||||
return { status: 200, body: held, cache: 'HIT' };
|
||||
let operation = inFlight.get(key);
|
||||
if (operation?.controller.signal.aborted) {
|
||||
inFlight.delete(key);
|
||||
operation = null;
|
||||
}
|
||||
if (!operation) {
|
||||
if (runningBy.get(owner) || running >= maxRunning)
|
||||
return {
|
||||
...jsonError(
|
||||
503,
|
||||
runningBy.get(owner)
|
||||
? 'Your previous map search is still running — try again shortly.'
|
||||
: 'Another map search is running — try again shortly.',
|
||||
),
|
||||
retryAfter: '3',
|
||||
};
|
||||
running += 1;
|
||||
runningBy.set(owner, (runningBy.get(owner) || 0) + 1);
|
||||
const controller = new AbortController();
|
||||
// The whole search, every configured endpoint in turn, gets one budget.
|
||||
const budget = AbortSignal.timeout(upstreamBudgetMs);
|
||||
operation = { controller, waiters: 0, settled: false, promise: null };
|
||||
const owned = operation;
|
||||
owned.promise = (async () => {
|
||||
try {
|
||||
const payload = await fetchPayload(
|
||||
`data=${encodeURIComponent(ql)}`,
|
||||
maxResponseBytes,
|
||||
{
|
||||
simplify: keepBody,
|
||||
timeoutMs: upstreamTimeoutMs,
|
||||
signal: AbortSignal.any([controller.signal, budget]),
|
||||
},
|
||||
);
|
||||
if (!overpassPayloadIsData(payload))
|
||||
return {
|
||||
...jsonError(
|
||||
payload?.rateLimited ? 429 : 503,
|
||||
'OpenStreetMap search is unavailable right now.',
|
||||
),
|
||||
retryAfter: '10',
|
||||
};
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(payload.body);
|
||||
} catch {
|
||||
return jsonError(
|
||||
502,
|
||||
'OpenStreetMap returned an unreadable answer.',
|
||||
);
|
||||
}
|
||||
const body = JSON.stringify(summarize(parsed));
|
||||
controller.signal.throwIfAborted();
|
||||
await cache.set(key, body);
|
||||
controller.signal.throwIfAborted();
|
||||
return { status: 200, body, cache: 'MISS' };
|
||||
} catch {
|
||||
return budget.aborted && !controller.signal.aborted
|
||||
? jsonError(504, 'OpenStreetMap took too long to answer.')
|
||||
: jsonError(502, 'OpenStreetMap search failed.');
|
||||
} finally {
|
||||
owned.settled = true;
|
||||
running -= 1;
|
||||
const left = (runningBy.get(owner) || 1) - 1;
|
||||
if (left > 0) runningBy.set(owner, left);
|
||||
else runningBy.delete(owner);
|
||||
if (inFlight.get(key) === owned) inFlight.delete(key);
|
||||
}
|
||||
})();
|
||||
inFlight.set(key, owned);
|
||||
}
|
||||
|
||||
operation.waiters += 1;
|
||||
let abort = null;
|
||||
const abandoned = signal
|
||||
? new Promise((_, reject) => {
|
||||
abort = () => {
|
||||
operation.waiters -= 1;
|
||||
if (operation.waiters === 0 && !operation.settled)
|
||||
operation.controller.abort();
|
||||
reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
|
||||
};
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
if (signal.aborted) abort();
|
||||
})
|
||||
: null;
|
||||
try {
|
||||
return await (abandoned
|
||||
? Promise.race([operation.promise, abandoned])
|
||||
: operation.promise);
|
||||
} finally {
|
||||
if (signal && abort) signal.removeEventListener('abort', abort);
|
||||
if (!signal?.aborted) operation.waiters -= 1;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function sender(res) {
|
||||
return ({ status, body, cache, retryAfter }) => {
|
||||
if (res.writableEnded) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(cache ? { 'X-OSM-Cache': cache } : {}),
|
||||
...(retryAfter ? { 'Retry-After': retryAfter } : {}),
|
||||
});
|
||||
res.end(body);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Install `GET /api/osm/features?preset=&bbox=w,s,e,n[&mode=count][&limit=]`:
|
||||
* one curated preset inside a box, answered by the operator's Overpass
|
||||
* (`OVERPASS_UPSTREAMS`). With none configured the route answers
|
||||
* `OVERPASS_NOT_CONFIGURED` and fetches nothing; public Nominatim is never a
|
||||
* substitute for bulk place search.
|
||||
*
|
||||
* @param {{use: Function}} middlewares
|
||||
* @param {{fetchPayload?: Function, featuresCache?: object, cacheRoot?: string|null, configured?: () => boolean, upstreamTag?: () => string}} [deps]
|
||||
*/
|
||||
export function installOsmFeaturesRoute(
|
||||
middlewares,
|
||||
{
|
||||
fetchPayload = fetchOverpassPayload,
|
||||
configured = () => resolveOverpassUpstreams().length > 0,
|
||||
// Keys each answer to the instance that gave it, as the other Overpass
|
||||
// caches do, so repointing OVERPASS_UPSTREAMS never serves the old one's.
|
||||
upstreamTag = resolveOverpassUpstreamTag,
|
||||
upstreamBudgetMs = OSM_FEATURE_LIMITS.upstreamBudgetMs,
|
||||
cacheRoot = path.join(process.cwd(), '.gev-cache'),
|
||||
featuresCache = createBoundedCache({
|
||||
dir: cacheRoot ? path.join(cacheRoot, 'osm-features') : null,
|
||||
ttlMs: OSM_FEATURES_TTL_MS,
|
||||
maxMemoryEntries: 60,
|
||||
maxDiskEntries: 400,
|
||||
maxDiskBytes: 256 * 1024 * 1024,
|
||||
}),
|
||||
} = {},
|
||||
) {
|
||||
const limiter = makeRateLimiter({ windowMs: 60_000, max: 12, globalMax: 30 });
|
||||
const features = createOverpassAnswerer({
|
||||
fetchPayload,
|
||||
cache: featuresCache,
|
||||
maxResponseBytes: OSM_FEATURE_LIMITS.maxResponseBytes,
|
||||
upstreamTimeoutMs: OSM_FEATURE_LIMITS.upstreamTimeoutMs,
|
||||
upstreamBudgetMs,
|
||||
});
|
||||
|
||||
middlewares.use('/api/osm/features', async (req, res) => {
|
||||
const send = sender(res);
|
||||
if (req.method !== 'GET') return send(jsonError(405, 'Method Not Allowed'));
|
||||
// A page elsewhere must not spend the operator's Overpass. The keyless
|
||||
// gate, as on /api/overpass: voice works behind a reverse proxy too.
|
||||
if (admitSameSiteKeyless(req, res)) return undefined;
|
||||
if (!configured())
|
||||
return send(
|
||||
jsonError(503, 'Place search needs a configured Overpass server.', {
|
||||
code: 'OVERPASS_NOT_CONFIGURED',
|
||||
retryable: false,
|
||||
}),
|
||||
);
|
||||
const url = new URL(req.url || '', 'http://localhost');
|
||||
const compiled = compileOsmFeatureQuery({
|
||||
preset: url.searchParams.get('preset'),
|
||||
bbox: url.searchParams.get('bbox'),
|
||||
mode: url.searchParams.get('mode'),
|
||||
limit: url.searchParams.get('limit'),
|
||||
});
|
||||
if (!compiled.ok)
|
||||
return send(jsonError(400, compiled.error, { code: compiled.code }));
|
||||
// IPv6 peers by /64, as /api/overpass does: rotating addresses inside
|
||||
// one prefix earns neither a second rate bucket nor a second slot.
|
||||
const client = clientNetworkKey(req);
|
||||
if (!limiter(client))
|
||||
return send({
|
||||
...jsonError(429, 'Rate limit exceeded'),
|
||||
retryAfter: '10',
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort();
|
||||
req.once?.('aborted', abort);
|
||||
res.once?.('close', abort);
|
||||
try {
|
||||
const answer = await features(
|
||||
`${CACHE_VERSION}|features|${upstreamTag()}|${compiled.ql}`,
|
||||
compiled.ql,
|
||||
(parsed) => summarizeOsmFeatures(parsed, compiled),
|
||||
controller.signal,
|
||||
client,
|
||||
);
|
||||
if (!controller.signal.aborted) send(answer);
|
||||
} catch {
|
||||
if (!controller.signal.aborted)
|
||||
send(jsonError(502, 'OpenStreetMap search failed.'));
|
||||
} finally {
|
||||
req.removeListener?.('aborted', abort);
|
||||
res.removeListener?.('close', abort);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -18,6 +18,29 @@ function isOverpassBoundaryQuery(cacheKey) {
|
||||
return /is_in\s*\(|\bpivot\b/i.test(String(cacheKey || ''));
|
||||
}
|
||||
|
||||
// Input has already had quoted literals and comments removed. Consume each
|
||||
// character once: a regex that searches for a closing bracket from every `[` is
|
||||
// quadratic on an unterminated filter. Nested or unmatched brackets are invalid.
|
||||
function stripTagFilters(text) {
|
||||
const parts = [];
|
||||
let start = 0;
|
||||
let inFilter = false;
|
||||
for (let i = 0; i < text.length; i += 1) {
|
||||
if (text[i] === '[') {
|
||||
if (inFilter) return null;
|
||||
parts.push(text.slice(start, i), ' ');
|
||||
inFilter = true;
|
||||
} else if (text[i] === ']') {
|
||||
if (!inFilter) return null;
|
||||
inFilter = false;
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
if (inFilter) return null;
|
||||
parts.push(text.slice(start));
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate + clamp an Overpass form body. Defends the generic proxy against
|
||||
* planet-scale abuse: requires exactly one `data` query in which EVERY element
|
||||
@@ -50,6 +73,10 @@ function sanitizeOverpassBody(rawBody) {
|
||||
// Blank quoted literals + strip comments in one lexer pass so a fake bound or a
|
||||
// `//` inside a string can't hide an unbounded selector (or satisfy a bound).
|
||||
const stripped = stripOverpassNoise(data);
|
||||
const withoutTags = stripTagFilters(stripped);
|
||||
if (withoutTags === null) {
|
||||
return { ok: false, error: 'Malformed Overpass tag filter' };
|
||||
}
|
||||
|
||||
// Reject oversized radii in EVERY around form — point `around:r,lat,lon` AND the
|
||||
// input-set form `around.set:r` — and parse the full numeric token so scientific
|
||||
@@ -93,20 +120,21 @@ function sanitizeOverpassBody(rawBody) {
|
||||
// statement. So `way[...]->.a` (global assigned to a set) is rejected, while the
|
||||
// app's `is_in(...)->.a; area.a[...]` and `area(id)->.x; rel(pivot.x)` validate.
|
||||
const boundedSets = new Set();
|
||||
for (let stmt of stripped.split(';')) {
|
||||
for (let stmt of withoutTags.split(';')) {
|
||||
stmt = stmt.trim();
|
||||
if (!stmt || stmt.startsWith('[') || /^out\b/.test(stmt)) continue;
|
||||
// The settings block (`[out:json][timeout:25]`) was blanked together with
|
||||
// the tag filters, so it arrives here as an empty statement.
|
||||
if (!stmt || /^out\b/.test(stmt)) continue;
|
||||
|
||||
// Strip output-set assignments (NOT input bounds), then strip bracket tag
|
||||
// filters so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) can never be
|
||||
// misread as a spatial bound. Bounds live in (...) / function calls / set
|
||||
// refs, never inside [...], so the probe loses nothing real.
|
||||
// Strip output-set assignments (NOT input bounds). With the tag filters
|
||||
// already gone, a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot
|
||||
// be misread as a spatial bound; bounds live in (...) / function calls /
|
||||
// set refs, never inside [...], so nothing real was lost.
|
||||
const outSets = [];
|
||||
const body = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
|
||||
const probe = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
|
||||
outSets.push(name);
|
||||
return ' ';
|
||||
});
|
||||
const probe = body.replace(/\[[^\]]*\]/g, ' ');
|
||||
|
||||
// Reject element-in-area scans on the TAG-STRIPPED probe, so a tag filter
|
||||
// between the selector and the area filter (way["highway"](area.a)) can't hide
|
||||
@@ -140,8 +168,10 @@ function sanitizeOverpassBody(rawBody) {
|
||||
if (bounded) for (const name of outSets) boundedSets.add(name);
|
||||
}
|
||||
|
||||
// Tolerate whitespace around the directive so `[ timeout : N ]` is clamped
|
||||
// rather than passed through as written.
|
||||
const clamped = data.replace(
|
||||
/\[timeout:\s*(\d+)\s*\]/gi,
|
||||
/\[\s*timeout\s*:\s*(\d+)\s*\]/gi,
|
||||
(_, n) =>
|
||||
`[timeout:${Math.min(Number(n) || OVERPASS_MAX_QL_TIMEOUT, OVERPASS_MAX_QL_TIMEOUT)}]`,
|
||||
);
|
||||
|
||||
@@ -0,0 +1,428 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { Readable } from 'node:stream';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { overpassProxy } from '../overpass.js';
|
||||
import { _overpassCache } from './cache.js';
|
||||
import { sanitizeOverpassBody } from './query.js';
|
||||
|
||||
const form = (query) => `data=${encodeURIComponent(query)}`;
|
||||
|
||||
// The limiters are module-level, so every test that spends admission mocks a
|
||||
// clock a full day past the previous one in execution order: no test can see
|
||||
// another's window, whatever order (or randomization) they run in. A new
|
||||
// test that reaches admission must call it too: earlier tests leave
|
||||
// timestamps days ahead of the real clock, and those still count.
|
||||
let clockBase = Date.now() + 86_400_000;
|
||||
function freshClock(t) {
|
||||
clockBase += 86_400_000;
|
||||
const clock = { now: clockBase };
|
||||
t.mock.method(Date, 'now', () => clock.now);
|
||||
return clock;
|
||||
}
|
||||
|
||||
test('bracket validation rejects malformed filters without trusting their contents as bounds', () => {
|
||||
for (const query of [
|
||||
'node[',
|
||||
'node]',
|
||||
'node[[name]](around:20,1,1);out;',
|
||||
'node[name=around:20];out;',
|
||||
'node[is_in];out;',
|
||||
'node[name="(around:20,1,1)"];out;',
|
||||
'node(around:20,1,1);way[name];out;',
|
||||
'way[highway](area.a);out;',
|
||||
'node[->.a](around:20,1,1);way.a;out;',
|
||||
]) {
|
||||
assert.equal(sanitizeOverpassBody(form(query)).ok, false, query);
|
||||
}
|
||||
});
|
||||
|
||||
test('bounded queries preserve quoted brackets, comments, multiple tags and timeout clamping', () => {
|
||||
for (const query of [
|
||||
'[out:json][timeout:999];node[name="[Town]; ]"](around:20,1,1);out;',
|
||||
'/* [ */way[highway][name](30.1,-97.9,30.5,-97.5);out geom;',
|
||||
'is_in(30.2,-97.7)->.a;area.a[boundary="administrative"];out tags;',
|
||||
'area(3600039152)->.x;rel(pivot.x);out geom;',
|
||||
]) {
|
||||
const result = sanitizeOverpassBody(form(query));
|
||||
assert.equal(result.ok, true, query);
|
||||
assert.equal(
|
||||
new URLSearchParams(result.body).get('data'),
|
||||
query.replace('[timeout:999]', '[timeout:30]'),
|
||||
);
|
||||
}
|
||||
// Whitespace inside the directive is clamped too, not passed through.
|
||||
const spaced = sanitizeOverpassBody(
|
||||
form('[out:json][ timeout : 999 ];node(around:20,1,1);out;'),
|
||||
);
|
||||
assert.equal(spaced.ok, true);
|
||||
assert.equal(
|
||||
new URLSearchParams(spaced.body).get('data'),
|
||||
'[out:json][timeout:30];node(around:20,1,1);out;',
|
||||
);
|
||||
});
|
||||
|
||||
test('a burst of maximum-size unterminated filters completes within a bounded process deadline', () => {
|
||||
// A child deadline can stop synchronous regex backtracking; a test timeout in
|
||||
// the same event loop cannot. The old parser takes several seconds for this.
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
'--input-type=module',
|
||||
'-e',
|
||||
`import { sanitizeOverpassBody } from ${JSON.stringify(new URL('./query.js', import.meta.url).href)};
|
||||
for (let i = 0; i < 20; i++) {
|
||||
if (sanitizeOverpassBody('data=node' + '['.repeat(24000)).ok) process.exit(1);
|
||||
}`,
|
||||
],
|
||||
{ timeout: 2000, encoding: 'utf8' },
|
||||
);
|
||||
assert.ifError(result.error);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
});
|
||||
|
||||
function handler(hook = 'configureServer') {
|
||||
let route;
|
||||
overpassProxy()[hook]({
|
||||
middlewares: {
|
||||
use(path, fn) {
|
||||
if (path === '/api/overpass') route = fn;
|
||||
},
|
||||
},
|
||||
});
|
||||
return route;
|
||||
}
|
||||
|
||||
async function call(
|
||||
route,
|
||||
{
|
||||
headers = {},
|
||||
peer = '127.0.0.1',
|
||||
body = 'data=node[',
|
||||
forbidRead = false,
|
||||
method = 'POST',
|
||||
url = '/',
|
||||
} = {},
|
||||
) {
|
||||
let reads = 0;
|
||||
const req = new Readable({
|
||||
read() {
|
||||
reads++;
|
||||
assert.equal(forbidRead, false, 'refused request must not read its body');
|
||||
this.push(Buffer.from(body));
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
Object.assign(req, {
|
||||
method,
|
||||
url,
|
||||
headers: { host: 'localhost:4173', ...headers },
|
||||
socket: { remoteAddress: peer },
|
||||
});
|
||||
const res = {
|
||||
statusCode: 200,
|
||||
headers: {},
|
||||
setHeader(key, value) {
|
||||
this.headers[key.toLowerCase()] = value;
|
||||
},
|
||||
writeHead(status, values) {
|
||||
this.statusCode = status;
|
||||
for (const [key, value] of Object.entries(values))
|
||||
this.setHeader(key, value);
|
||||
},
|
||||
end(value) {
|
||||
this.body = value;
|
||||
},
|
||||
};
|
||||
await route(req, res);
|
||||
return { ...res, reads };
|
||||
}
|
||||
|
||||
for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
test(`${hook}: cross-site requests are rejected before consuming a body or fetching`, async (t) => {
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler(hook);
|
||||
for (const headers of [
|
||||
{ origin: 'https://foreign.example', 'content-type': 'text/plain' },
|
||||
{ 'sec-fetch-site': 'cross-site' },
|
||||
{ 'sec-fetch-site': 'same-site' },
|
||||
{ origin: 'null' },
|
||||
// Forwarding headers do not launder a cross-site request.
|
||||
{ 'x-forwarded-for': '10.0.0.9', 'sec-fetch-site': 'cross-site' },
|
||||
{
|
||||
'x-forwarded-proto': 'https',
|
||||
origin: 'http://localhost:4173',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
},
|
||||
]) {
|
||||
const response = await call(route, { headers, forbidRead: true });
|
||||
assert.equal(response.statusCode, 403);
|
||||
assert.equal(response.reads, 0);
|
||||
assert.equal(response.headers['cache-control'], 'no-store');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('same-origin, LAN, proxied and non-browser requests still reach validation; GET status stays available', async (t) => {
|
||||
freshClock(t);
|
||||
const route = handler();
|
||||
for (const headers of [
|
||||
{},
|
||||
{ origin: 'http://localhost:4173', 'sec-fetch-site': 'same-origin' },
|
||||
{ host: '192.168.1.10:4173', origin: 'http://192.168.1.10:4173' },
|
||||
// The README's reverse-proxy deployment: Overpass is keyless, so the
|
||||
// forwarding headers the proxy adds are admitted (as for terrain heights).
|
||||
{
|
||||
host: 'globe.example.lan',
|
||||
origin: 'https://globe.example.lan',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'x-forwarded-for': '203.0.113.7',
|
||||
'x-forwarded-proto': 'https',
|
||||
'x-forwarded-host': 'globe.example.lan',
|
||||
},
|
||||
{ 'x-forwarded-for': '127.0.0.1' },
|
||||
]) {
|
||||
const response = await call(route, { headers });
|
||||
assert.equal(response.statusCode, 400);
|
||||
assert.equal(response.reads, 1);
|
||||
assert.match(response.body, /Malformed Overpass tag filter/);
|
||||
}
|
||||
const response = await call(route, {
|
||||
method: 'GET',
|
||||
url: '/status',
|
||||
forbidRead: true,
|
||||
});
|
||||
assert.equal(response.statusCode, 200);
|
||||
assert.equal(typeof JSON.parse(response.body).configured, 'boolean');
|
||||
});
|
||||
|
||||
test('invalid requests spend admission quota before parsing, including on an unconfigured server', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
const peer = 'admission-limit-test';
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, { peer })).statusCode, 400);
|
||||
}
|
||||
const denied = await call(route, { peer, forbidRead: true });
|
||||
assert.equal(denied.statusCode, 429);
|
||||
assert.equal(denied.reads, 0);
|
||||
assert.equal(denied.headers['retry-after'], '60');
|
||||
assert.equal((await call(route, { peer: 'other-peer' })).statusCode, 400);
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('the 900/min global window spans non-loopback peers only; loopback keeps its own window', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (const peer of ['10.0.0.1', '10.0.0.2', '10.0.0.3']) {
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, { peer })).statusCode, 400, peer);
|
||||
}
|
||||
}
|
||||
const starved = await call(route, { peer: '10.0.0.4', forbidRead: true });
|
||||
assert.equal(starved.statusCode, 429, 'global window is full for LAN peers');
|
||||
assert.equal(starved.reads, 0);
|
||||
for (const peer of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
|
||||
const owner = await call(route, { peer });
|
||||
assert.equal(owner.statusCode, 400, `${peer} is admitted past the global`);
|
||||
assert.equal(owner.reads, 1);
|
||||
}
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer: '10.0.0.4' })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('IPv6 peers share a /64 admission bucket and IPv4-mapped peers fold to IPv4', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer: `2001:db8:1:2::${(i % 200) + 1}` }))
|
||||
.statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const rotated = await call(route, {
|
||||
peer: '2001:db8:1:2:ffff:ffff:ffff:9',
|
||||
forbidRead: true,
|
||||
});
|
||||
assert.equal(rotated.statusCode, 429, 'a rotating host stays in one bucket');
|
||||
assert.equal(
|
||||
(await call(route, { peer: '2001:db8:1:3::1' })).statusCode,
|
||||
400,
|
||||
);
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer: '::ffff:10.1.1.1' })).statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const unmapped = await call(route, { peer: '10.1.1.1', forbidRead: true });
|
||||
assert.equal(unmapped.statusCode, 429, 'mapped and bare IPv4 share a bucket');
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer: '10.1.1.1' })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('cache hits spend admission budget but not the 90/min upstream budget', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
_overpassCache.delete(hitKey);
|
||||
});
|
||||
let fetches = 0;
|
||||
t.mock.method(globalThis, 'fetch', async () => {
|
||||
fetches++;
|
||||
// Not data: nothing is cached or written to disk by this test.
|
||||
return new Response('upstream unavailable', { status: 503 });
|
||||
});
|
||||
const hit = 'node(around:20,1,1);out;';
|
||||
const hitKey = `data=${encodeURIComponent(hit)}`;
|
||||
_overpassCache.set(hitKey, {
|
||||
status: 200,
|
||||
body: '{"elements":[]}',
|
||||
contentType: 'application/json',
|
||||
endpoint: 'https://overpass.example/api/interpreter',
|
||||
cachedAt: clock.now,
|
||||
});
|
||||
const route = handler();
|
||||
const peer = '10.7.7.7';
|
||||
// More cache hits than the upstream limiter would allow per peer.
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const response = await call(route, { peer, body: form(hit) });
|
||||
assert.equal(response.statusCode, 200);
|
||||
assert.equal(response.headers['x-overpass-cache'], 'HIT');
|
||||
}
|
||||
assert.equal(fetches, 0);
|
||||
const miss = await call(route, {
|
||||
peer,
|
||||
body: form('node(around:20,2,2);out;'),
|
||||
});
|
||||
assert.equal(fetches, 1, 'the first miss still reaches upstream');
|
||||
assert.notEqual(miss.statusCode, 429);
|
||||
for (let i = 0; i < 199; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer, body: form(hit) })).statusCode,
|
||||
200,
|
||||
);
|
||||
}
|
||||
const exhausted = await call(route, {
|
||||
peer,
|
||||
body: form(hit),
|
||||
forbidRead: true,
|
||||
});
|
||||
assert.equal(exhausted.statusCode, 429, 'hits spent the admission window');
|
||||
assert.equal(exhausted.reads, 0);
|
||||
});
|
||||
|
||||
test('a local reverse proxy cannot spend the direct-loopback reserve; owner cache hits survive it', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
const hit = 'node(around:20,3,3);out;';
|
||||
const hitKey = `data=${encodeURIComponent(hit)}`;
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
_overpassCache.delete(hitKey);
|
||||
});
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
_overpassCache.set(hitKey, {
|
||||
status: 200,
|
||||
body: '{"elements":[]}',
|
||||
contentType: 'application/json',
|
||||
endpoint: 'https://overpass.example/api/interpreter',
|
||||
cachedAt: clock.now,
|
||||
});
|
||||
const route = handler();
|
||||
// The proxy connects over loopback; a rotating forwarded client address
|
||||
// must not mint fresh buckets, so all 300 land in one socket-keyed bucket.
|
||||
for (const proxyPeer of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
|
||||
const proxied = (i) => ({
|
||||
peer: proxyPeer,
|
||||
headers: { 'x-forwarded-for': `203.0.113.${i % 250}` },
|
||||
body: form(hit),
|
||||
});
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, proxied(i))).statusCode, 200, proxyPeer);
|
||||
}
|
||||
const refused = await call(route, { ...proxied(300), forbidRead: true });
|
||||
assert.equal(refused.statusCode, 429, `${proxyPeer} proxy bucket is full`);
|
||||
assert.equal(refused.reads, 0);
|
||||
for (const peer of ['127.0.0.1', '::1']) {
|
||||
const owner = await call(route, { peer, body: form(hit) });
|
||||
assert.equal(owner.statusCode, 200, `direct ${peer} is still admitted`);
|
||||
assert.equal(owner.headers['x-overpass-cache'], 'HIT');
|
||||
}
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, proxied(0))).statusCode, 200);
|
||||
}
|
||||
});
|
||||
|
||||
test('proxied loopback traffic draws on the shared remote global window', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (const peer of ['10.0.0.1', '10.0.0.2']) {
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, { peer })).statusCode, 400, peer);
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { headers: { via: '1.1 proxy' } })).statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const starved = await call(route, { peer: '10.0.0.3', forbidRead: true });
|
||||
assert.equal(starved.statusCode, 429, 'proxied requests counted globally');
|
||||
assert.equal((await call(route, {})).statusCode, 400, 'owner unaffected');
|
||||
});
|
||||
|
||||
test("a local proxy has its own upstream bucket, so its misses cannot spend the owner's", async (t) => {
|
||||
freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
});
|
||||
// Not data: nothing is cached, so every request below is a miss.
|
||||
t.mock.method(
|
||||
globalThis,
|
||||
'fetch',
|
||||
async () => new Response('upstream unavailable', { status: 503 }),
|
||||
);
|
||||
const route = handler();
|
||||
const miss = form('node(around:20,4,4);out;');
|
||||
// The upstream limiter's own refusal is the only miss without this header.
|
||||
const passedLimiter = (response) =>
|
||||
response.headers['x-overpass-cache'] !== undefined;
|
||||
const proxied = { peer: '127.0.0.1', headers: { via: '1.1 proxy' } };
|
||||
for (let i = 0; i < 90; i++) {
|
||||
const response = await call(route, { ...proxied, body: miss });
|
||||
assert.equal(passedLimiter(response), true, `proxied miss ${i}`);
|
||||
}
|
||||
const spent = await call(route, { ...proxied, body: miss });
|
||||
assert.equal(spent.statusCode, 429);
|
||||
assert.equal(passedLimiter(spent), false, 'the proxy bucket is spent');
|
||||
const owner = await call(route, { peer: '127.0.0.1', body: miss });
|
||||
assert.equal(passedLimiter(owner), true, 'the owner keeps its own bucket');
|
||||
});
|
||||
@@ -124,6 +124,8 @@ async function fetchOverpassPayload(
|
||||
readBody = readResponseTextCapped,
|
||||
simplify = simplifyOverpassPayloadBody,
|
||||
now = Date.now,
|
||||
timeoutMs = OVERPASS_TIMEOUT_MS,
|
||||
signal = null,
|
||||
} = {},
|
||||
) {
|
||||
if (!endpoints.length) return overpassNotConfigured();
|
||||
@@ -135,7 +137,10 @@ async function fetchOverpassPayload(
|
||||
continue;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
const requestSignal = signal
|
||||
? AbortSignal.any([signal, controller.signal])
|
||||
: controller.signal;
|
||||
try {
|
||||
const requestUrl = new URL(endpoint);
|
||||
const authorization =
|
||||
@@ -156,7 +161,7 @@ async function fetchOverpassPayload(
|
||||
...(authorization ? { Authorization: authorization } : {}),
|
||||
},
|
||||
body,
|
||||
signal: controller.signal,
|
||||
signal: requestSignal,
|
||||
});
|
||||
const responseBody = await readBody(upstream, maxResponseBytes);
|
||||
const parsed = parseJson(responseBody);
|
||||
@@ -207,7 +212,12 @@ async function fetchOverpassPayload(
|
||||
upstreams: resolveOverpassUpstreamTag(),
|
||||
rateLimited: false,
|
||||
};
|
||||
} catch {
|
||||
} catch (error) {
|
||||
// A route whose last requester disconnected owns this cancellation. It
|
||||
// is not evidence that the configured upstream is unhealthy, so do not
|
||||
// put the endpoint on cooldown or translate the abort into a response
|
||||
// that a now-gone client could cache.
|
||||
if (signal?.aborted) throw signal.reason ?? error;
|
||||
const failures = (previous?.failures || 0) + 1;
|
||||
const delay = retryDelay(null, failures - 1, now());
|
||||
cooldowns.set(endpoint, { until: now() + delay, failures, status: 502 });
|
||||
|
||||
@@ -72,6 +72,14 @@ class UpstreamRateLimitError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** Thrown when the routing service failed or sent no JSON; answered as 502. */
|
||||
class UpstreamUnavailableError extends Error {
|
||||
constructor() {
|
||||
super('routing service unavailable');
|
||||
this.name = 'UpstreamUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one outbound request no sooner than the shared gate allows.
|
||||
* @template T
|
||||
@@ -167,18 +175,26 @@ async function fetchRoute({
|
||||
`${base.replace(/\/$/, '')}/route/v1/${osrmProfile}/${coords}` +
|
||||
`?overview=full&geometries=geojson&alternatives=false&steps=${withSteps ? 'true' : 'false'}`;
|
||||
const controller = new AbortController();
|
||||
// The 12 s budget includes queue wait, so Directions (15 s) hears back.
|
||||
const timer = setTimeout(() => controller.abort(), 12000);
|
||||
let osrm;
|
||||
try {
|
||||
const upstreamRes = await throughUpstreamGate(() =>
|
||||
fetchImpl(upstream, {
|
||||
signal: controller.signal,
|
||||
// The endpoint is configured above; a redirect is the one way out of
|
||||
// it, so it is refused rather than followed.
|
||||
redirect: 'error',
|
||||
headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
|
||||
}),
|
||||
);
|
||||
const upstreamRes = await throughUpstreamGate(async () => {
|
||||
// Expiry before dispatch reflects our backlog, not an upstream outage.
|
||||
if (controller.signal.aborted)
|
||||
throw new RouteBusyError(_upstreamIntervalMs);
|
||||
try {
|
||||
return await fetchImpl(upstream, {
|
||||
signal: controller.signal,
|
||||
// The endpoint is configured above; a redirect is the one way out of
|
||||
// it, so it is refused rather than followed.
|
||||
redirect: 'error',
|
||||
headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
|
||||
});
|
||||
} catch {
|
||||
throw new UpstreamUnavailableError();
|
||||
}
|
||||
});
|
||||
if (upstreamRes.status === 429) {
|
||||
await cancelBody(upstreamRes);
|
||||
const retryAfter = Number(upstreamRes.headers.get('retry-after'));
|
||||
@@ -186,20 +202,28 @@ async function fetchRoute({
|
||||
Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : 5,
|
||||
);
|
||||
}
|
||||
if (!upstreamRes.ok) {
|
||||
// OSRM answers 400 for a route it cannot find; other failures are outages.
|
||||
if (upstreamRes.status === 400) {
|
||||
await cancelBody(upstreamRes);
|
||||
return { payload: null, error: 'no route found' };
|
||||
}
|
||||
if (!upstreamRes.ok) {
|
||||
await cancelBody(upstreamRes);
|
||||
throw new UpstreamUnavailableError();
|
||||
}
|
||||
const ctype = upstreamRes.headers.get('content-type') || '';
|
||||
if (!ctype.includes('json')) {
|
||||
await cancelBody(upstreamRes);
|
||||
return { payload: null, error: 'no route found' };
|
||||
throw new UpstreamUnavailableError();
|
||||
}
|
||||
// Read and parse failures are outages; later ones stay "route proxy error".
|
||||
try {
|
||||
osrm = JSON.parse(
|
||||
await readResponseTextCapped(upstreamRes, ROUTE_MAX_RESPONSE_BYTES),
|
||||
);
|
||||
} catch {
|
||||
throw new UpstreamUnavailableError();
|
||||
}
|
||||
const text = await readResponseTextCapped(
|
||||
upstreamRes,
|
||||
ROUTE_MAX_RESPONSE_BYTES,
|
||||
);
|
||||
osrm = JSON.parse(text);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
@@ -227,6 +251,11 @@ export function installRouteMiddleware(
|
||||
// only shape that asks the upstream for them. A response WITHOUT steps is
|
||||
// byte for byte what this endpoint has always returned.
|
||||
middlewares.use('/api/route', async (req, res) => {
|
||||
if (req.method !== 'GET') {
|
||||
res.writeHead(405, { 'Content-Type': 'application/json', Allow: 'GET' });
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
const fail = (msg) => {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ ok: false, error: msg }));
|
||||
@@ -350,6 +379,11 @@ export function installRouteMiddleware(
|
||||
upstreamError.retryAfterSec,
|
||||
);
|
||||
}
|
||||
if (upstreamError instanceof UpstreamUnavailableError) {
|
||||
res.writeHead(502, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ ok: false, error: upstreamError.message }));
|
||||
return;
|
||||
}
|
||||
throw upstreamError;
|
||||
}
|
||||
if (error || !payload) return fail(error || 'no route found');
|
||||
|
||||
@@ -7,9 +7,13 @@ export {
|
||||
publicRadioHttpsUrl,
|
||||
} from './radio/stations.js';
|
||||
export function radioBrowserProxy() {
|
||||
const middleware = createRadioProxyMiddleware();
|
||||
let middleware;
|
||||
const initialize = () =>
|
||||
(middleware ||= createRadioProxyMiddleware({
|
||||
priorityCountries: process.env.RADIO_PRIORITY_COUNTRIES || '',
|
||||
}));
|
||||
const install = (server) => {
|
||||
server.middlewares.use('/api/radio', middleware);
|
||||
server.middlewares.use('/api/radio', initialize());
|
||||
};
|
||||
return {
|
||||
name: 'radio-browser-proxy',
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js';
|
||||
import { lookup as lookupDns } from 'node:dns/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
@@ -5,7 +6,7 @@ import { readResponseTextCapped } from '../common/http.js';
|
||||
import {
|
||||
normalizeRadioBrowserStation,
|
||||
publicRadioStation,
|
||||
cleanRadioText,
|
||||
normalizeRadioTag,
|
||||
} from './stations.js';
|
||||
import {
|
||||
radioMirrorOrigin,
|
||||
@@ -25,8 +26,37 @@ import {
|
||||
RADIO_USER_AGENT,
|
||||
RADIO_UUID_RE,
|
||||
RADIO_FALLBACK_MIRRORS,
|
||||
RADIO_REJECTED_STATION_COUNT_MAX,
|
||||
} from './constants.js';
|
||||
export async function mapRadioConcurrent(values, concurrency, mapper) {
|
||||
const PRIORITY_COUNTRY_LIMIT = 8;
|
||||
const COUNTRY_QUERY_LIMIT = 200;
|
||||
const COUNTRY_RESERVED_STATIONS = 40;
|
||||
const REFRESH_RETRY_MS = 60_000;
|
||||
const MAX_MIRROR_ATTEMPTS = 3;
|
||||
|
||||
/** Validate an operator-owned list; malformed configuration must not silently widen queries. */
|
||||
export function parseRadioPriorityCountries(value = '') {
|
||||
const entries = Array.isArray(value) ? value : String(value).split(',');
|
||||
const countries = [];
|
||||
for (const entry of entries) {
|
||||
const input = String(entry).trim().toUpperCase();
|
||||
if (!input) continue;
|
||||
const normalized = normalizeRadioCountryInput(input);
|
||||
if (!/^[A-Z]{2}$/.test(input) || !normalized.valid)
|
||||
throw new TypeError(
|
||||
'RADIO_PRIORITY_COUNTRIES requires ISO alpha-2 country codes',
|
||||
);
|
||||
const code = normalized.code;
|
||||
if (!countries.includes(code)) countries.push(code);
|
||||
}
|
||||
if (countries.length > PRIORITY_COUNTRY_LIMIT)
|
||||
throw new RangeError(
|
||||
'RADIO_PRIORITY_COUNTRIES supports at most eight countries',
|
||||
);
|
||||
return countries;
|
||||
}
|
||||
|
||||
async function mapRadioConcurrent(values, concurrency, mapper) {
|
||||
const results = new Array(values.length);
|
||||
let cursor = 0;
|
||||
const workers = Array.from(
|
||||
@@ -48,7 +78,9 @@ export function createRadioProxyMiddleware({
|
||||
fetchImpl = null,
|
||||
lookupImpl = lookupDns,
|
||||
now = Date.now,
|
||||
priorityCountries = [],
|
||||
} = {}) {
|
||||
const countries = parseRadioPriorityCountries(priorityCountries);
|
||||
let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 };
|
||||
let mirrorPromise = null;
|
||||
let catalogCache = null;
|
||||
@@ -60,6 +92,9 @@ export function createRadioProxyMiddleware({
|
||||
const catalogInstance = randomUUID();
|
||||
let servedStationIds = new Set();
|
||||
let refreshPromise = null;
|
||||
let retryAfter = 0;
|
||||
let partialCatalog = null;
|
||||
let refreshError = null;
|
||||
|
||||
async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) {
|
||||
const destination = radioProxyDestination(url);
|
||||
@@ -138,7 +173,7 @@ export function createRadioProxyMiddleware({
|
||||
|
||||
async function fetchPath(pathname) {
|
||||
let lastError = null;
|
||||
for (const origin of await mirrors()) {
|
||||
for (const origin of (await mirrors()).slice(0, MAX_MIRROR_ATTEMPTS)) {
|
||||
try {
|
||||
return await fetchJson(`${origin}${pathname}`);
|
||||
} catch (error) {
|
||||
@@ -159,47 +194,55 @@ export function createRadioProxyMiddleware({
|
||||
'aviation',
|
||||
'marine',
|
||||
'traffic',
|
||||
];
|
||||
].map((tag) => ({ tag }));
|
||||
const baseQueryCount = queries.length;
|
||||
queries.push(...countries.map((country) => ({ country })));
|
||||
const outcomes = await mapRadioConcurrent(
|
||||
queries,
|
||||
3,
|
||||
async (tag, index) => {
|
||||
async ({ tag, country }, index) => {
|
||||
const params = new URLSearchParams({
|
||||
has_geo_info: 'true',
|
||||
is_https: 'true',
|
||||
hidebroken: 'true',
|
||||
order: 'clickcount',
|
||||
reverse: 'true',
|
||||
limit: index === 0 ? '1800' : '220',
|
||||
limit: country
|
||||
? String(COUNTRY_QUERY_LIMIT)
|
||||
: index === 0
|
||||
? '1800'
|
||||
: '220',
|
||||
});
|
||||
if (tag) params.set('tag', tag);
|
||||
if (country) params.set('countrycode', country);
|
||||
try {
|
||||
const rows = await fetchPath(`/json/stations/search?${params}`);
|
||||
if (!Array.isArray(rows))
|
||||
throw new Error('Radio Browser catalog payload was not an array');
|
||||
if (
|
||||
!rows.every(
|
||||
(row) =>
|
||||
row &&
|
||||
typeof row === 'object' &&
|
||||
!Array.isArray(row) &&
|
||||
typeof row.stationuuid === 'string' &&
|
||||
typeof row.name === 'string' &&
|
||||
(typeof row.url_resolved === 'string' ||
|
||||
typeof row.url === 'string'),
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
'Radio Browser catalog contained a malformed station row',
|
||||
);
|
||||
const stations = rows
|
||||
const structuralRows = rows.filter(
|
||||
(row) =>
|
||||
row &&
|
||||
typeof row === 'object' &&
|
||||
!Array.isArray(row) &&
|
||||
typeof row.stationuuid === 'string' &&
|
||||
typeof row.name === 'string' &&
|
||||
(typeof row.url_resolved === 'string' ||
|
||||
typeof row.url === 'string'),
|
||||
);
|
||||
// Malformed rows are skipped individually, but counted so the
|
||||
// catalog keeps a diagnostic for upstream schema drift. Policy
|
||||
// rejections in normalization (offline, non-HTTPS, codec) are normal
|
||||
// filtering and are not counted.
|
||||
const rejectedRows = rows.length - structuralRows.length;
|
||||
const stations = structuralRows
|
||||
.map(normalizeRadioBrowserStation)
|
||||
.filter(Boolean);
|
||||
const requestedTag = cleanRadioText(tag, 80)
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[_-]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
.filter(
|
||||
(station) =>
|
||||
station && (!country || station.countryCode === country),
|
||||
);
|
||||
if (country && rows.length > COUNTRY_QUERY_LIMIT)
|
||||
throw new Error('Radio Browser exceeded the country query limit');
|
||||
const requestedTag = normalizeRadioTag(tag);
|
||||
const requestedTagCovered =
|
||||
!requestedTag ||
|
||||
stations.some((station) =>
|
||||
@@ -210,14 +253,17 @@ export function createRadioProxyMiddleware({
|
||||
),
|
||||
);
|
||||
return {
|
||||
// Query coverage is based on accepted rows, not merely a payload that
|
||||
// happens to match the upstream schema. Specialist responses must
|
||||
// also contain an accepted station tagged for the requested category.
|
||||
succeeded: stations.length > 0 && requestedTagCovered,
|
||||
// A valid country response can contain no playable stations. Country
|
||||
// coverage is advisory; only broad/specialist queries gate health.
|
||||
// Specialist responses need an accepted station with matching tags.
|
||||
succeeded: country
|
||||
? rows.length === 0 || structuralRows.length > 0
|
||||
: stations.length > 0 && requestedTagCovered,
|
||||
stations,
|
||||
rejectedRows,
|
||||
};
|
||||
} catch {
|
||||
return { succeeded: false, stations: [] };
|
||||
return { succeeded: false, stations: [], rejectedRows: 0 };
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -235,9 +281,24 @@ export function createRadioProxyMiddleware({
|
||||
seen.add(station.id);
|
||||
selected.push(station);
|
||||
};
|
||||
// Reserve country slots first; low-click regional stations cannot lose all
|
||||
// representation to the global charts. Keep room for all eight specialist
|
||||
// seeds (8 * 45) and global fill even at the eight-country limit (8 * 40).
|
||||
for (const rows of resultSets.slice(baseQueryCount)) {
|
||||
const ranked = [...rows].sort(
|
||||
(a, b) => b.clickCount - a.clickCount || a.id.localeCompare(b.id),
|
||||
);
|
||||
let added = 0;
|
||||
for (const station of ranked) {
|
||||
if (seen.has(station.id)) continue;
|
||||
take(station);
|
||||
if (++added >= COUNTRY_RESERVED_STATIONS) break;
|
||||
}
|
||||
}
|
||||
// Seed specialist station-tag queries before popularity fill so operational
|
||||
// categories remain represented even when global click charts skew musical.
|
||||
for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take);
|
||||
for (const rows of resultSets.slice(1, baseQueryCount))
|
||||
rows.slice(0, 45).forEach(take);
|
||||
resultSets
|
||||
.flat()
|
||||
.sort(
|
||||
@@ -245,9 +306,9 @@ export function createRadioProxyMiddleware({
|
||||
)
|
||||
.forEach(take);
|
||||
const timestamp = now();
|
||||
const successfulQueries = outcomes.filter(
|
||||
(outcome) => outcome.succeeded,
|
||||
).length;
|
||||
const successfulQueries = outcomes
|
||||
.slice(0, baseQueryCount)
|
||||
.filter((outcome) => outcome.succeeded).length;
|
||||
const broadQueryHealthy =
|
||||
outcomes[0].succeeded && outcomes[0].stations.length > 0;
|
||||
const healthReasons = [];
|
||||
@@ -257,9 +318,24 @@ export function createRadioProxyMiddleware({
|
||||
if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS)
|
||||
healthReasons.push('station-coverage-below-policy');
|
||||
const degraded = healthReasons.length > 0;
|
||||
const rejectedStationCount = Math.min(
|
||||
RADIO_REJECTED_STATION_COUNT_MAX,
|
||||
outcomes.reduce((total, outcome) => total + outcome.rejectedRows, 0),
|
||||
);
|
||||
const coverage = {
|
||||
successfulQueries,
|
||||
totalQueries: queries.length,
|
||||
totalQueries: baseQueryCount,
|
||||
...(countries.length
|
||||
? {
|
||||
priorityCountries: countries.map((country, index) => ({
|
||||
countryCode: country,
|
||||
succeeded: outcomes[baseQueryCount + index].succeeded,
|
||||
stationCount: selected.filter(
|
||||
(station) => station.countryCode === country,
|
||||
).length,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
stationCount: selected.length,
|
||||
healthyStationMinimum: RADIO_CATALOG_HEALTHY_MIN_STATIONS,
|
||||
};
|
||||
@@ -271,6 +347,9 @@ export function createRadioProxyMiddleware({
|
||||
degraded,
|
||||
degradedReason: degraded ? healthReasons.join(',') : null,
|
||||
coverage,
|
||||
// Diagnostic only: it never fails health policy, so a usable generation
|
||||
// with a few skipped rows is still admitted. The client reports it.
|
||||
rejectedStationCount,
|
||||
};
|
||||
if (degraded && catalogCache) {
|
||||
const error = new Error(
|
||||
@@ -310,7 +389,24 @@ export function createRadioProxyMiddleware({
|
||||
return { ...catalogCache, stale: false };
|
||||
}
|
||||
if (!refreshPromise) {
|
||||
refreshPromise = refreshCatalog().finally(() => {
|
||||
refreshPromise = (async () => {
|
||||
if (now() < retryAfter) {
|
||||
if (refreshError) throw refreshError;
|
||||
return partialCatalog;
|
||||
}
|
||||
try {
|
||||
const catalog = await refreshCatalog();
|
||||
refreshError = null;
|
||||
partialCatalog = catalog.degraded ? catalog : null;
|
||||
retryAfter = catalog.degraded ? now() + REFRESH_RETRY_MS : 0;
|
||||
return catalog;
|
||||
} catch (error) {
|
||||
refreshError = error;
|
||||
partialCatalog = null;
|
||||
retryAfter = now() + REFRESH_RETRY_MS;
|
||||
throw error;
|
||||
}
|
||||
})().finally(() => {
|
||||
refreshPromise = null;
|
||||
});
|
||||
}
|
||||
@@ -360,6 +456,7 @@ export function createRadioProxyMiddleware({
|
||||
coverage: catalog.coverage || null,
|
||||
acceptedGeneration: catalog.acceptedGeneration ?? null,
|
||||
catalogInstance,
|
||||
rejectedStationCount: catalog.rejectedStationCount ?? 0,
|
||||
});
|
||||
} catch (error) {
|
||||
sendJson(res, 503, {
|
||||
|
||||
@@ -11,6 +11,9 @@ export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(
|
||||
export const RADIO_USER_AGENT =
|
||||
'GodsEyeView/1.0 (Radio Browser directory client)';
|
||||
export { RADIO_UUID_RE } from '../../../src/sources/radioBrowser.js';
|
||||
// Upper bound for the structurally malformed upstream rows reported in the
|
||||
// /stations envelope. The client layer validates against the same bound.
|
||||
export const RADIO_REJECTED_STATION_COUNT_MAX = 10_000;
|
||||
export const RADIO_FALLBACK_MIRRORS = Object.freeze([
|
||||
'https://de1.api.radio-browser.info',
|
||||
'https://de2.api.radio-browser.info',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export {
|
||||
cleanRadioText,
|
||||
normalizeRadioTag,
|
||||
isNonGlobalIpv4,
|
||||
publicRadioHttpsUrl,
|
||||
normalizeRadioBrowserStation,
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
import { open, stat } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createRailService } from '../../src/sources/railService.js';
|
||||
import { RAIL_FEED_ID_PATTERN } from '../../src/data/railFeeds.js';
|
||||
|
||||
/**
|
||||
* Where development and preview look for path and schedule packs when
|
||||
* RAIL_PATHS_DIR is unset: the default output of scripts/build-rail-paths.mjs.
|
||||
*/
|
||||
export const DEFAULT_RAIL_PATHS_DIR = path.resolve(
|
||||
path.dirname(fileURLToPath(import.meta.url)),
|
||||
'../../.cache/rail-paths',
|
||||
);
|
||||
|
||||
/**
|
||||
* Pack bytes the development server keeps in memory, across path and
|
||||
* schedule packs: a few of the largest national packs.
|
||||
*/
|
||||
export const RAIL_PACK_CACHE_MAX_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* A bounded cache of pack files' bytes, by path, least recently read first
|
||||
* out. An entry is used only while the file's modification time and size
|
||||
* are the ones it was read with, so a rebuilt pack is read again; a file
|
||||
* larger than the whole byte budget is never kept. Reads of the same file at
|
||||
* the same time share one read.
|
||||
* @param {number} [maxBytes]
|
||||
* @returns {{load: (file: string, info: {mtimeMs: number, size: number},
|
||||
* read: () => Promise<Buffer>) => Promise<Buffer>, bytes: () => number}}
|
||||
*/
|
||||
export function createRailPackCache(maxBytes = RAIL_PACK_CACHE_MAX_BYTES) {
|
||||
/** @type {Map<string, {mtimeMs: number, size: number, bytes: Buffer}>} */
|
||||
const entries = new Map();
|
||||
/** @type {Map<string, Promise<Buffer>>} */
|
||||
const pending = new Map();
|
||||
let total = 0;
|
||||
const drop = (file) => {
|
||||
const entry = entries.get(file);
|
||||
if (!entry) return;
|
||||
entries.delete(file);
|
||||
total -= entry.bytes.byteLength;
|
||||
};
|
||||
return {
|
||||
async load(file, info, read) {
|
||||
const entry = entries.get(file);
|
||||
if (entry && entry.mtimeMs === info.mtimeMs && entry.size === info.size) {
|
||||
// Most recently read last: the first entry is the next one out.
|
||||
entries.delete(file);
|
||||
entries.set(file, entry);
|
||||
return entry.bytes;
|
||||
}
|
||||
const version = `${file}|${info.mtimeMs}|${info.size}`;
|
||||
const reading = pending.get(version);
|
||||
if (reading) return reading;
|
||||
const promise = (async () => {
|
||||
const bytes = await read();
|
||||
drop(file);
|
||||
if (bytes.byteLength <= maxBytes) {
|
||||
entries.set(file, { mtimeMs: info.mtimeMs, size: info.size, bytes });
|
||||
total += bytes.byteLength;
|
||||
for (const [oldest] of entries) {
|
||||
if (total <= maxBytes) break;
|
||||
drop(oldest);
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
})().finally(() => pending.delete(version));
|
||||
pending.set(version, promise);
|
||||
return promise;
|
||||
},
|
||||
bytes: () => total,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A pack reader for the rail service that reads `<dir>/<feedId><suffix>`.
|
||||
* Ids are re-checked against the registry's id pattern, so a name can never
|
||||
* leave the directory; a missing file is "no pack", and a file over the cap is
|
||||
* refused without being read (an error with `code: 'PACK_TOO_LARGE'`). The
|
||||
* pack is returned as the file's bytes, never decoded to text; with a
|
||||
* `cache` (createRailPackCache), an unchanged file is read once.
|
||||
* @param {string} dir
|
||||
* @param {string} suffix `.json` (path packs) or `.schedule.json`.
|
||||
* @param {ReturnType<typeof createRailPackCache>|null} [cache]
|
||||
* @returns {(feedId: string, options?: {maxBytes?: number}) =>
|
||||
* Promise<Buffer|null>}
|
||||
*/
|
||||
function createFsPackReader(dir, suffix, cache = null) {
|
||||
const root = path.resolve(dir);
|
||||
return async (feedId, { maxBytes = Infinity } = {}) => {
|
||||
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
|
||||
return null;
|
||||
const file = path.join(root, `${feedId}${suffix}`);
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(file, 'r');
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const info = await handle.stat();
|
||||
if (!info.isFile()) return null;
|
||||
if (info.size > maxBytes) {
|
||||
const error = new Error(`pack is ${info.size} bytes`);
|
||||
error.code = 'PACK_TOO_LARGE';
|
||||
throw error;
|
||||
}
|
||||
return cache
|
||||
? await cache.load(file, info, () => handle.readFile())
|
||||
: await handle.readFile();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A `readPathPack` that reads `<dir>/<feedId>.json`.
|
||||
* @param {string} dir
|
||||
* @param {{cache?: ReturnType<typeof createRailPackCache>|null}} [options]
|
||||
*/
|
||||
export function createFsPathPackReader(dir, { cache = null } = {}) {
|
||||
return createFsPackReader(dir, '.json', cache);
|
||||
}
|
||||
|
||||
/**
|
||||
* A `readSchedulePack` that reads `<dir>/<feedId>.schedule.json`.
|
||||
* @param {string} dir
|
||||
* @param {{cache?: ReturnType<typeof createRailPackCache>|null}} [options]
|
||||
*/
|
||||
export function createFsSchedulePackReader(dir, { cache = null } = {}) {
|
||||
return createFsPackReader(dir, '.schedule.json', cache);
|
||||
}
|
||||
|
||||
/**
|
||||
* A `hasSchedulePack` for the rail service: whether
|
||||
* `<dir>/<feedId>.schedule.json` is a file. Only the directory entry is
|
||||
* read, never the pack; a missing file is false, any other error is thrown
|
||||
* (the service then lists the feed, so the read shows the failure).
|
||||
* @param {string} dir
|
||||
* @returns {(feedId: string) => Promise<boolean>}
|
||||
*/
|
||||
export function createFsSchedulePackProbe(dir) {
|
||||
const root = path.resolve(dir);
|
||||
return async (feedId) => {
|
||||
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
|
||||
return false;
|
||||
try {
|
||||
return (await stat(path.join(root, `${feedId}.schedule.json`))).isFile();
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return false;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A pack version for the rail service: the modification time and size of
|
||||
* `<dir>/<feedId><suffix>`, or null when there is no such file. Only the
|
||||
* directory entry is read; a rebuilt pack has a new version.
|
||||
* @param {string} dir
|
||||
* @param {string} suffix
|
||||
* @returns {(feedId: string) => Promise<string|null>}
|
||||
*/
|
||||
function createFsPackVersion(dir, suffix) {
|
||||
const root = path.resolve(dir);
|
||||
return async (feedId) => {
|
||||
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
|
||||
return null;
|
||||
try {
|
||||
const info = await stat(path.join(root, `${feedId}${suffix}`));
|
||||
return info.isFile() ? `${info.mtimeMs}:${info.size}` : null;
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A `schedulePackVersion` for the rail service: the modification time and
|
||||
* size of `<dir>/<feedId>.schedule.json`, or null when there is no such
|
||||
* file.
|
||||
* @param {string} dir
|
||||
* @returns {(feedId: string) => Promise<string|null>}
|
||||
*/
|
||||
export function createFsSchedulePackVersion(dir) {
|
||||
return createFsPackVersion(dir, '.schedule.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* A `pathPackVersion` for the rail service: the same for
|
||||
* `<dir>/<feedId>.json`.
|
||||
* @param {string} dir
|
||||
* @returns {(feedId: string) => Promise<string|null>}
|
||||
*/
|
||||
export function createFsPathPackVersion(dir) {
|
||||
return createFsPackVersion(dir, '.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a service Response to a Node response, streaming its body: a pack's
|
||||
* bytes go to the socket as they are, with no text decode and no copy.
|
||||
* @param {Response} response
|
||||
* @param {import('node:http').ServerResponse} res
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function sendRailResponse(response, res) {
|
||||
res.writeHead(response.status, Object.fromEntries(response.headers));
|
||||
if (!response.body) {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
await pipeline(Readable.fromWeb(response.body), res);
|
||||
}
|
||||
|
||||
/** Connect the reusable rail request service to development and preview. */
|
||||
export function railProxy(options = {}) {
|
||||
const pathsDir =
|
||||
options.pathsDir || process.env.RAIL_PATHS_DIR || DEFAULT_RAIL_PATHS_DIR;
|
||||
// One byte budget for both kinds of pack.
|
||||
const cache = createRailPackCache();
|
||||
const service = createRailService({
|
||||
...options,
|
||||
readPathPack:
|
||||
options.readPathPack || createFsPathPackReader(pathsDir, { cache }),
|
||||
readSchedulePack:
|
||||
options.readSchedulePack ||
|
||||
createFsSchedulePackReader(pathsDir, { cache }),
|
||||
// Probe the same directory only when its fs reader is the one in use.
|
||||
hasSchedulePack:
|
||||
options.hasSchedulePack ||
|
||||
(options.readSchedulePack ? null : createFsSchedulePackProbe(pathsDir)),
|
||||
schedulePackVersion:
|
||||
options.schedulePackVersion ||
|
||||
(options.readSchedulePack ? null : createFsSchedulePackVersion(pathsDir)),
|
||||
pathPackVersion:
|
||||
options.pathPackVersion ||
|
||||
(options.readPathPack ? null : createFsPathPackVersion(pathsDir)),
|
||||
});
|
||||
// A throw here would escape into the HTTP server's event loop and end the
|
||||
// whole dev server (Node exits on an unhandled rejection or a throwing
|
||||
// 'close' listener), so failures stay inside the rail routes.
|
||||
function closeService() {
|
||||
try {
|
||||
service.close();
|
||||
} catch (error) {
|
||||
console.warn('[rail] provider close failed', error);
|
||||
}
|
||||
}
|
||||
function install(server) {
|
||||
server.middlewares.use('/api/rail', async (req, res) => {
|
||||
try {
|
||||
const response = await service.handle({
|
||||
url: `http://localhost/api/rail${req.url || '/'}`,
|
||||
method: req.method,
|
||||
// Read for If-None-Match on a pack.
|
||||
headers: req.headers,
|
||||
});
|
||||
await sendRailResponse(response, res);
|
||||
} catch (error) {
|
||||
// An answer already under way (a viewer that left mid-pack) has
|
||||
// nothing to add: the stream is closed and that is all.
|
||||
if (res.headersSent) {
|
||||
res.destroy();
|
||||
return;
|
||||
}
|
||||
console.warn('[rail] request failed', error);
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Rail provider error' }));
|
||||
}
|
||||
});
|
||||
server.httpServer?.once('close', closeService);
|
||||
}
|
||||
return {
|
||||
name: 'rail-proxy',
|
||||
closeBundle: closeService,
|
||||
configureServer: install,
|
||||
configurePreviewServer: install,
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { fetchRegionalPlace } from './place.js';
|
||||
import { fetchRegionalWeather } from './weather.js';
|
||||
import { fetchRegionalNews } from './news.js';
|
||||
import { validRegionalPoint } from './query.js';
|
||||
import { REGIONAL_NEWS_WINDOW_HOURS } from '../../../src/data/regionalModel.js';
|
||||
import { coalesceProxyRequest } from '../common/http.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -50,10 +51,11 @@ function regionalBriefProxy({ placeProvider = fetchRegionalPlace } = {}) {
|
||||
if (!regionalBriefHasAnySource({ place, weather, news })) {
|
||||
throw new Error('All regional briefing sources unavailable');
|
||||
}
|
||||
const retrievedAt = new Date().toISOString();
|
||||
const payload = {
|
||||
status:
|
||||
place && weather && news.status !== 'unavailable' ? 'ready' : 'partial',
|
||||
retrievedAt: new Date().toISOString(),
|
||||
retrievedAt,
|
||||
coordinates: point,
|
||||
place,
|
||||
placeStatus: place ? 'ready' : 'unavailable',
|
||||
@@ -62,6 +64,9 @@ function regionalBriefProxy({ placeProvider = fetchRegionalPlace } = {}) {
|
||||
newsStatus: news.status,
|
||||
newsQuery: news.query,
|
||||
newsSource: news.source,
|
||||
newsWindowHours: REGIONAL_NEWS_WINDOW_HOURS,
|
||||
// A cached or stale answer keeps this, so the client can show its age.
|
||||
newsFetchedAt: retrievedAt,
|
||||
articles: news.articles,
|
||||
};
|
||||
_regionalBriefCache.set(key, { payload, cachedAt: Date.now() });
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { fetchRegionalText, fetchRegionalJson } from './http.js';
|
||||
import { normalizeRegionalArticles } from '../../../src/data/regionalModel.js';
|
||||
import {
|
||||
REGIONAL_NEWS_WINDOW_HOURS,
|
||||
googleNewsWindowOperator,
|
||||
normalizeRegionalArticles,
|
||||
recentRegionalArticles,
|
||||
} from '../../../src/data/regionalModel.js';
|
||||
|
||||
function decodeRssText(value) {
|
||||
return String(value || '')
|
||||
@@ -22,7 +27,10 @@ function rssTag(block, tag) {
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeRssArticles(xml, limit = 5) {
|
||||
// Feed items read before the window filter; Google returns about 100.
|
||||
const RSS_ITEM_CAP = 100;
|
||||
|
||||
function normalizeRssArticles(xml, limit = RSS_ITEM_CAP) {
|
||||
const seen = new Set();
|
||||
const articles = [];
|
||||
for (const match of String(xml || '').matchAll(
|
||||
@@ -57,12 +65,27 @@ function normalizeRssArticles(xml, limit = 5) {
|
||||
return articles;
|
||||
}
|
||||
|
||||
async function fetchRegionalNews(place) {
|
||||
/**
|
||||
* Headlines for a place from the last REGIONAL_NEWS_WINDOW_HOURS, newest
|
||||
* first: Google News RSS, then GDELT when RSS fails or has nothing recent.
|
||||
* `empty` means a source answered with nothing in the window; `unavailable`
|
||||
* means there was no place or no source answered.
|
||||
* @param {{locality?: string, region?: string, country?: string}|null} place
|
||||
* @param {{now?: () => number}} [options]
|
||||
*/
|
||||
async function fetchRegionalNews(place, { now = Date.now } = {}) {
|
||||
const query = place?.locality || place?.region || place?.country;
|
||||
if (!query)
|
||||
return { status: 'unavailable', query: null, articles: [], source: null };
|
||||
const recent = (articles) =>
|
||||
recentRegionalArticles(articles, {
|
||||
now: now(),
|
||||
windowHours: REGIONAL_NEWS_WINDOW_HOURS,
|
||||
limit: 5,
|
||||
});
|
||||
let answered = false;
|
||||
const rssParams = new URLSearchParams({
|
||||
q: String(query).replace(/["\\]/g, ' ').trim(),
|
||||
q: `${String(query).replace(/["\\]/g, ' ').trim()} ${googleNewsWindowOperator()}`,
|
||||
hl: 'en-US',
|
||||
gl: 'US',
|
||||
ceid: 'US:en',
|
||||
@@ -75,7 +98,8 @@ async function fetchRegionalNews(place) {
|
||||
timeoutMs: 12_000,
|
||||
},
|
||||
);
|
||||
const articles = normalizeRssArticles(xml, 5);
|
||||
answered = true;
|
||||
const articles = recent(normalizeRssArticles(xml));
|
||||
if (articles.length)
|
||||
return { status: 'ready', query, articles, source: 'Google News RSS' };
|
||||
} catch {
|
||||
@@ -87,7 +111,7 @@ async function fetchRegionalNews(place) {
|
||||
format: 'json',
|
||||
maxrecords: '5',
|
||||
sort: 'datedesc',
|
||||
timespan: '48h',
|
||||
timespan: `${REGIONAL_NEWS_WINDOW_HOURS}h`,
|
||||
});
|
||||
try {
|
||||
const payload = await fetchRegionalJson(
|
||||
@@ -97,7 +121,7 @@ async function fetchRegionalNews(place) {
|
||||
timeoutMs: 12_000,
|
||||
},
|
||||
);
|
||||
const articles = normalizeRegionalArticles(payload, 5);
|
||||
const articles = recent(normalizeRegionalArticles(payload, 5));
|
||||
return {
|
||||
status: articles.length ? 'ready' : 'empty',
|
||||
query,
|
||||
@@ -105,7 +129,10 @@ async function fetchRegionalNews(place) {
|
||||
source: 'GDELT fallback',
|
||||
};
|
||||
} catch {
|
||||
return { status: 'unavailable', query, articles: [], source: null };
|
||||
// An RSS answer with nothing recent is still a true empty result.
|
||||
return answered
|
||||
? { status: 'empty', query, articles: [], source: 'Google News RSS' }
|
||||
: { status: 'unavailable', query, articles: [], source: null };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { normalizePlaceName as foldName } from '../../../src/data/placeNames.js';
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/** Entry files this cache writes: a 40-hex-digit key hash. */
|
||||
const CACHE_FILE_RE = /^[0-9a-f]{40}\.json$/;
|
||||
|
||||
/** How long an answer is kept, by what it was. */
|
||||
export const NOMINATIM_CACHE_TTL_MS = Object.freeze({
|
||||
outline: 90 * DAY_MS,
|
||||
place: 30 * DAY_MS,
|
||||
notFound: DAY_MS,
|
||||
});
|
||||
|
||||
/**
|
||||
* Comparable form of a place name for matching results to the ask: the
|
||||
* shared place-name key (see `normalizePlaceName` in placeNames.js, so
|
||||
* "St. Paul" and "Saint Paul" agree), bounded in length.
|
||||
*
|
||||
* @param {string} value
|
||||
* @returns {string}
|
||||
*/
|
||||
export function normalizePlaceName(value) {
|
||||
return foldName(value).slice(0, 200);
|
||||
}
|
||||
|
||||
/**
|
||||
* Memory + disk cache for Nominatim answers.
|
||||
*
|
||||
* Every entry carries its own expiry. Memory is an LRU bounded by entries;
|
||||
* disk is bounded by entries and bytes, with expired files removed during a
|
||||
* periodic sweep. A null `dir` keeps everything in memory.
|
||||
*/
|
||||
export function createNominatimCache({
|
||||
dir = null,
|
||||
maxMemoryEntries = 256,
|
||||
maxDiskEntries = 4000,
|
||||
maxDiskBytes = 64 * 1024 * 1024,
|
||||
maxEntryBytes = 2 * 1024 * 1024,
|
||||
sweepEvery = 50,
|
||||
now = Date.now,
|
||||
} = {}) {
|
||||
const memory = new Map();
|
||||
let writesSinceSweep = 0;
|
||||
let sweeping = null;
|
||||
|
||||
const digestFor = (key) =>
|
||||
crypto.createHash('sha256').update(key).digest('hex');
|
||||
|
||||
const fileFor = (key) =>
|
||||
path.join(dir, `${digestFor(key).slice(0, 40)}.json`);
|
||||
|
||||
function remember(key, entry) {
|
||||
memory.delete(key);
|
||||
memory.set(key, entry);
|
||||
while (memory.size > maxMemoryEntries)
|
||||
memory.delete(memory.keys().next().value);
|
||||
}
|
||||
|
||||
async function get(key) {
|
||||
const hit = memory.get(key);
|
||||
if (hit) {
|
||||
if (hit.expiresAt > now()) {
|
||||
remember(key, hit);
|
||||
return hit;
|
||||
}
|
||||
memory.delete(key);
|
||||
}
|
||||
if (!dir) return null;
|
||||
try {
|
||||
const entry = JSON.parse(await fs.readFile(fileFor(key), 'utf8'));
|
||||
if (entry?.keyDigest !== digestFor(key) || !(entry.expiresAt > now()))
|
||||
return null;
|
||||
remember(key, entry);
|
||||
return entry;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function set(key, payload, ttlMs) {
|
||||
const entry = {
|
||||
keyDigest: digestFor(key),
|
||||
payload,
|
||||
storedAt: now(),
|
||||
expiresAt: now() + ttlMs,
|
||||
};
|
||||
remember(key, entry);
|
||||
if (!dir) return;
|
||||
const body = JSON.stringify(entry);
|
||||
if (Buffer.byteLength(body) > maxEntryBytes) return;
|
||||
let temp = null;
|
||||
try {
|
||||
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
|
||||
await fs.chmod(dir, 0o700);
|
||||
const file = fileFor(key);
|
||||
temp = `${file}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`;
|
||||
await fs.writeFile(temp, body, { mode: 0o600, flag: 'wx' });
|
||||
await fs.rename(temp, file);
|
||||
temp = null;
|
||||
} catch {
|
||||
return;
|
||||
} finally {
|
||||
if (temp) await fs.rm(temp, { force: true }).catch(() => {});
|
||||
}
|
||||
writesSinceSweep += 1;
|
||||
if (writesSinceSweep >= sweepEvery) {
|
||||
writesSinceSweep = 0;
|
||||
await sweep();
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove expired files, then the oldest until both disk bounds hold. */
|
||||
async function sweep() {
|
||||
if (!dir) return;
|
||||
if (sweeping) return sweeping;
|
||||
sweeping = (async () => {
|
||||
let names;
|
||||
try {
|
||||
names = await fs.readdir(dir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const files = [];
|
||||
for (const name of names) {
|
||||
// Only this cache's own entry files; anything else in the directory
|
||||
// is left alone.
|
||||
if (!CACHE_FILE_RE.test(name)) continue;
|
||||
const file = path.join(dir, name);
|
||||
let stat;
|
||||
let text;
|
||||
try {
|
||||
[stat, text] = await Promise.all([
|
||||
fs.stat(file),
|
||||
fs.readFile(file, 'utf8'),
|
||||
]);
|
||||
} catch {
|
||||
// Unreadable right now (gone, busy, too many open files): an error
|
||||
// reading is not evidence the entry is bad, so it stays.
|
||||
continue;
|
||||
}
|
||||
let expiresAt;
|
||||
try {
|
||||
expiresAt = JSON.parse(text)?.expiresAt;
|
||||
} catch {
|
||||
expiresAt = null; // corrupt: removed below
|
||||
}
|
||||
if (!(expiresAt > now())) {
|
||||
await fs.rm(file, { force: true }).catch(() => {});
|
||||
continue;
|
||||
}
|
||||
files.push({ file, size: stat.size, at: stat.mtimeMs });
|
||||
}
|
||||
files.sort((a, b) => a.at - b.at);
|
||||
let bytes = files.reduce((sum, entry) => sum + entry.size, 0);
|
||||
while (
|
||||
files.length &&
|
||||
(files.length > maxDiskEntries || bytes > maxDiskBytes)
|
||||
) {
|
||||
const oldest = files.shift();
|
||||
bytes -= oldest.size;
|
||||
await fs.rm(oldest.file, { force: true }).catch(() => {});
|
||||
}
|
||||
})().finally(() => {
|
||||
sweeping = null;
|
||||
});
|
||||
return sweeping;
|
||||
}
|
||||
|
||||
return { get, set, sweep };
|
||||
}
|
||||
@@ -0,0 +1,955 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* One gate for every request this server sends to a Nominatim instance.
|
||||
*
|
||||
* Place search and voice outlines share it. The service is off until an operator
|
||||
* configures an endpoint. Explicit public requests coordinate fetch invocation
|
||||
* across processes sharing this installation's state file, at least 1.1 s apart.
|
||||
* Independent installations do not share an app-wide limiter; these local guards
|
||||
* do not establish compliance for an application's combined public traffic.
|
||||
*/
|
||||
|
||||
/** The public instance. Only this host is subject to the public-use limits. */
|
||||
export const PUBLIC_NOMINATIM_SEARCH =
|
||||
'https://nominatim.openstreetmap.org/search';
|
||||
|
||||
const PUBLIC_NOMINATIM_HOST = 'nominatim.openstreetmap.org';
|
||||
|
||||
/**
|
||||
* The public usage policy asks for a User-Agent or Referer that identifies the
|
||||
* application. The identity is stable: it is never rotated or varied.
|
||||
*/
|
||||
const NOMINATIM_HEADERS = Object.freeze({
|
||||
'User-Agent':
|
||||
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
||||
Referer: 'https://github.com/bilawalsidhu/gods-eye-view',
|
||||
});
|
||||
|
||||
/** Absolute policy maximum is one request per second; 1.1 s absorbs jitter. */
|
||||
const NOMINATIM_MIN_SPACING_MS = 1100;
|
||||
|
||||
/** Requests allowed to wait for their turn; past this, refuse at once. */
|
||||
export const NOMINATIM_MAX_PENDING = 4;
|
||||
|
||||
/** A queued request older than this has no reader left; drop it unsent. */
|
||||
const NOMINATIM_MAX_WAIT_MS = 10_000;
|
||||
|
||||
/** Default public requests per install per UTC day, outlines and search together. */
|
||||
const NOMINATIM_DEFAULT_DAILY_CAP = 50;
|
||||
|
||||
/**
|
||||
* Pause after a refusal without Retry-After. A Retry-After from the server is
|
||||
* honoured in full, however long: no local maximum may cause an earlier retry.
|
||||
*/
|
||||
const DEFAULT_BACKOFF_MS = 60_000;
|
||||
/** A 403 from the public instance means this identity is blocked: stop longer. */
|
||||
const BLOCKED_BACKOFF_MS = 3_600_000;
|
||||
/** Pause after a server error or a failed retry. */
|
||||
const OUTAGE_BACKOFF_MS = 30_000;
|
||||
/** How old an unfinished dispatch must be before the next lock owner reclaims it. */
|
||||
const DISPATCH_RECLAIM_MS = 10 * 60_000;
|
||||
|
||||
const DISABLED_VALUES = new Set(['', 'off', 'none', 'disabled', '0']);
|
||||
|
||||
/**
|
||||
* node:sqlite, loaded on first public-instance use. Node 24 still flags it
|
||||
* experimental and prints that once per process; only that one warning is
|
||||
* dropped, and only while the module loads.
|
||||
*/
|
||||
let sqlite = null;
|
||||
function sqliteModule() {
|
||||
if (sqlite) return sqlite;
|
||||
const emitWarning = process.emitWarning;
|
||||
process.emitWarning = function (warning, ...rest) {
|
||||
const type = typeof rest[0] === 'string' ? rest[0] : rest[0]?.type;
|
||||
if (type === 'ExperimentalWarning' && /SQLite/.test(String(warning)))
|
||||
return;
|
||||
return emitWarning.call(this, warning, ...rest);
|
||||
};
|
||||
try {
|
||||
sqlite = process.getBuiltinModule('node:sqlite');
|
||||
} finally {
|
||||
process.emitWarning = emitWarning;
|
||||
}
|
||||
return sqlite;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the search endpoint setting.
|
||||
*
|
||||
* Unset, empty or `off` disables every Nominatim request. An explicit endpoint
|
||||
* must be an http(s) URL
|
||||
* without credentials, query or fragment; a bare host gets `/search`. A value
|
||||
* that is not usable disables the service rather than falling back to the
|
||||
* public instance.
|
||||
*
|
||||
* @param {Record<string, string|undefined>} [env]
|
||||
* @returns {{endpoint: string|null, isPublic: boolean, dailyCap: number, invalid?: boolean}}
|
||||
*/
|
||||
export function resolveNominatimSettings(env = process.env) {
|
||||
const dailyCap = parseDailyCap(env.NOMINATIM_DAILY_CAP);
|
||||
const raw = env.NOMINATIM_URL;
|
||||
if (raw === undefined) return { endpoint: null, isPublic: false, dailyCap };
|
||||
const value = String(raw).trim();
|
||||
if (DISABLED_VALUES.has(value.toLowerCase()))
|
||||
return { endpoint: null, isPublic: false, dailyCap };
|
||||
let url;
|
||||
try {
|
||||
url = new URL(value);
|
||||
} catch {
|
||||
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
|
||||
}
|
||||
if (
|
||||
!['http:', 'https:'].includes(url.protocol) ||
|
||||
!url.hostname ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.search ||
|
||||
url.hash
|
||||
)
|
||||
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
|
||||
const publicHost =
|
||||
url.hostname.toLowerCase().replace(/\.$/, '') === PUBLIC_NOMINATIM_HOST;
|
||||
if (publicHost && url.protocol !== 'https:')
|
||||
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
|
||||
if (url.pathname === '/' || url.pathname === '') url.pathname = '/search';
|
||||
return {
|
||||
endpoint: url.href.replace(/\/$/, ''),
|
||||
// A trailing DNS dot names the same host.
|
||||
isPublic: publicHost,
|
||||
dailyCap,
|
||||
};
|
||||
}
|
||||
|
||||
function parseDailyCap(raw) {
|
||||
if (raw === undefined || String(raw).trim() === '')
|
||||
return NOMINATIM_DEFAULT_DAILY_CAP;
|
||||
const value = Number(raw);
|
||||
return Number.isFinite(value) && value >= 0
|
||||
? Math.floor(value)
|
||||
: NOMINATIM_DEFAULT_DAILY_CAP;
|
||||
}
|
||||
|
||||
function gateError(code, message, extra = {}) {
|
||||
return Object.assign(new Error(message), { code, ...extra });
|
||||
}
|
||||
|
||||
const HTTP_MONTHS = Object.freeze([
|
||||
'Jan',
|
||||
'Feb',
|
||||
'Mar',
|
||||
'Apr',
|
||||
'May',
|
||||
'Jun',
|
||||
'Jul',
|
||||
'Aug',
|
||||
'Sep',
|
||||
'Oct',
|
||||
'Nov',
|
||||
'Dec',
|
||||
]);
|
||||
const HTTP_WEEKDAYS = Object.freeze([
|
||||
'Sun',
|
||||
'Mon',
|
||||
'Tue',
|
||||
'Wed',
|
||||
'Thu',
|
||||
'Fri',
|
||||
'Sat',
|
||||
]);
|
||||
const HTTP_WEEKDAYS_LONG = Object.freeze([
|
||||
'Sunday',
|
||||
'Monday',
|
||||
'Tuesday',
|
||||
'Wednesday',
|
||||
'Thursday',
|
||||
'Friday',
|
||||
'Saturday',
|
||||
]);
|
||||
|
||||
/** Strict RFC 9110 HTTP-date, including the two obsolete recipient formats. */
|
||||
function parseHttpDate(raw, now) {
|
||||
let match = raw.match(
|
||||
/^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/,
|
||||
);
|
||||
let weekday, day, month, year, hour, minute, second;
|
||||
if (match) {
|
||||
[, weekday, day, month, year, hour, minute, second] = match;
|
||||
weekday = HTTP_WEEKDAYS.indexOf(weekday);
|
||||
} else {
|
||||
match = raw.match(
|
||||
/^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), (\d{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{2}) (\d{2}):(\d{2}):(\d{2}) GMT$/,
|
||||
);
|
||||
if (match) {
|
||||
[, weekday, day, month, year, hour, minute, second] = match;
|
||||
weekday = HTTP_WEEKDAYS_LONG.indexOf(weekday);
|
||||
const currentYear = new Date(now).getUTCFullYear();
|
||||
if (!Number.isFinite(currentYear)) return null;
|
||||
year = Math.floor(currentYear / 100) * 100 + Number(year);
|
||||
if (year > currentYear + 50) year -= 100;
|
||||
} else {
|
||||
match = raw.match(
|
||||
/^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ([ \d]\d) (\d{2}):(\d{2}):(\d{2}) (\d{4})$/,
|
||||
);
|
||||
if (!match) return null;
|
||||
[, weekday, month, day, hour, minute, second, year] = match;
|
||||
weekday = HTTP_WEEKDAYS.indexOf(weekday);
|
||||
}
|
||||
}
|
||||
day = Number(day);
|
||||
month = HTTP_MONTHS.indexOf(month);
|
||||
year = Number(year);
|
||||
hour = Number(hour);
|
||||
minute = Number(minute);
|
||||
second = Number(second);
|
||||
if (
|
||||
weekday < 0 ||
|
||||
month < 0 ||
|
||||
year < 1601 ||
|
||||
hour > 23 ||
|
||||
minute > 59 ||
|
||||
second > 59
|
||||
)
|
||||
return null;
|
||||
const at = Date.UTC(year, month, day, hour, minute, second);
|
||||
const date = new Date(at);
|
||||
return date.getUTCFullYear() === year &&
|
||||
date.getUTCMonth() === month &&
|
||||
date.getUTCDate() === day &&
|
||||
date.getUTCHours() === hour &&
|
||||
date.getUTCMinutes() === minute &&
|
||||
date.getUTCSeconds() === second &&
|
||||
date.getUTCDay() === weekday
|
||||
? at
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Seconds or an HTTP date → milliseconds from now, or null. */
|
||||
export function parseRetryAfter(value, now = Date.now()) {
|
||||
if (value == null) return null;
|
||||
const raw = String(value).trim();
|
||||
if (!raw) return null;
|
||||
if (/^\d+$/.test(raw)) {
|
||||
const seconds = Number(raw);
|
||||
const milliseconds = seconds * 1000;
|
||||
return Number.isSafeInteger(milliseconds) ? milliseconds : null;
|
||||
}
|
||||
// Numeric-looking but invalid delay-seconds must not be reinterpreted as a date.
|
||||
if (/^[+-]?(?:\d|\.\d)/.test(raw)) return null;
|
||||
const at = parseHttpDate(raw, now);
|
||||
return at === null ? null : Math.max(0, at - now);
|
||||
}
|
||||
|
||||
/** UTC calendar day, the unit of the daily cap. */
|
||||
function utcDay(ms) {
|
||||
return new Date(ms).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Public-service state shared by every process using the same file: today's
|
||||
* request count, the next reserved start, and any pause the server asked for.
|
||||
*
|
||||
* Every change is a read-modify-write under an OS-owned SQLite transaction,
|
||||
* and the JSON file is replaced atomically, so two dev servers sharing a
|
||||
* checkout add to one count instead of overwriting each other. A process exit
|
||||
* releases only that process's transaction; there is no stale path to delete
|
||||
* and therefore no stale reclaimer that can remove a successor's live lock.
|
||||
* Memory keeps the highest values this process has seen, so a failed or
|
||||
* unreadable file never lowers the count or shortens a pause; a persistence
|
||||
* failure is reported once and in `status()`.
|
||||
*/
|
||||
export function createGateStateStore({
|
||||
file = null,
|
||||
lockWaitMs = 500,
|
||||
onError = (error) =>
|
||||
console.warn(
|
||||
`[Nominatim] could not persist usage state: ${error?.message || error}`,
|
||||
),
|
||||
} = {}) {
|
||||
let memory = { day: '', count: 0, pausedUntil: 0, nextStartAt: 0 };
|
||||
let lastError = null;
|
||||
|
||||
const report = (error) => {
|
||||
if (!lastError) onError(error);
|
||||
lastError = error;
|
||||
};
|
||||
|
||||
const pauseMarkerPrefix = file ? `${path.basename(file)}.pause.` : null;
|
||||
const dispatchMarkerPrefix = file ? `${path.basename(file)}.dispatch.` : null;
|
||||
|
||||
// A crash or persistence failure during dispatch must never reopen admission
|
||||
// using an older timestamp. A dispatch removes its own marker after
|
||||
// persisting the post-invocation deadline. A marker is only ever seen under
|
||||
// the lock after its owner left the lock, so one that names a start older
|
||||
// than DISPATCH_RECLAIM_MS (far beyond any request's timeout) is reclaimed
|
||||
// by the next lock owner, which restarts spacing from its own clock.
|
||||
function pendingDispatches({ strict = false } = {}) {
|
||||
if (!file) return [];
|
||||
try {
|
||||
return fs
|
||||
.readdirSync(path.dirname(file))
|
||||
.filter((name) => name.startsWith(dispatchMarkerPrefix));
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return [];
|
||||
report(error);
|
||||
if (strict) throw error;
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Under the lock: remove dispatch markers whose start is older than
|
||||
* DISPATCH_RECLAIM_MS and report whether any were. Any other marker (newer,
|
||||
* or naming no start) still refuses admission.
|
||||
*/
|
||||
function reclaimDispatches(at) {
|
||||
const names = pendingDispatches({ strict: true });
|
||||
for (const name of names) {
|
||||
const since = /^since-(\d+)$/.exec(
|
||||
name.slice(dispatchMarkerPrefix.length).split('.', 1)[0],
|
||||
);
|
||||
const started = since ? Number(since[1]) : NaN;
|
||||
if (!Number.isSafeInteger(started) || at - started < DISPATCH_RECLAIM_MS)
|
||||
throw new Error('unfinished durable Nominatim dispatch');
|
||||
}
|
||||
for (const name of names)
|
||||
fs.unlinkSync(path.join(path.dirname(file), name));
|
||||
return names.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* A refusal must remain visible even while another process owns the SQLite
|
||||
* transaction. Immutable marker names make that write atomic and monotonic:
|
||||
* concurrent writers cannot shorten one another's pause, and the next lock
|
||||
* owner folds every marker into the normal JSON state.
|
||||
*/
|
||||
function pendingPauses({ strict = false } = {}) {
|
||||
if (!file) return [];
|
||||
let names;
|
||||
try {
|
||||
names = fs.readdirSync(path.dirname(file));
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return [];
|
||||
report(error);
|
||||
if (strict) throw error;
|
||||
return [];
|
||||
}
|
||||
const pauses = [];
|
||||
for (const name of names) {
|
||||
if (!name.startsWith(pauseMarkerPrefix)) continue;
|
||||
const encoded = name.slice(pauseMarkerPrefix.length).split('.', 1)[0];
|
||||
if (!/^\d+$/.test(encoded)) continue;
|
||||
const pausedUntil = Number(encoded);
|
||||
if (!Number.isSafeInteger(pausedUntil)) {
|
||||
const error = new Error('invalid durable Nominatim pause marker');
|
||||
report(error);
|
||||
if (strict) throw error;
|
||||
continue;
|
||||
}
|
||||
pauses.push({
|
||||
path: path.join(path.dirname(file), name),
|
||||
pausedUntil,
|
||||
});
|
||||
}
|
||||
return pauses;
|
||||
}
|
||||
|
||||
function markPause(at) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
const pausedUntil = Math.ceil(at);
|
||||
const marker = `${file}.pause.${pausedUntil}.${process.pid}.${process.hrtime.bigint()}`;
|
||||
fs.writeFileSync(marker, '', { flag: 'wx' });
|
||||
return pausedUntil;
|
||||
}
|
||||
|
||||
function clearPersistedPauses(pausedUntil) {
|
||||
for (const marker of pendingPauses()) {
|
||||
if (marker.pausedUntil > pausedUntil) continue;
|
||||
try {
|
||||
fs.unlinkSync(marker.path);
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') report(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const merge = (a, b) => ({
|
||||
day: a.day >= b.day ? a.day : b.day,
|
||||
count:
|
||||
a.day === b.day
|
||||
? Math.max(a.count, b.count)
|
||||
: a.day > b.day
|
||||
? a.count
|
||||
: b.count,
|
||||
pausedUntil: Math.max(a.pausedUntil || 0, b.pausedUntil || 0),
|
||||
nextStartAt: Math.max(a.nextStartAt || 0, b.nextStartAt || 0),
|
||||
});
|
||||
|
||||
const parseState = (parsed) => {
|
||||
const validDay =
|
||||
parsed?.day === '' ||
|
||||
(typeof parsed?.day === 'string' &&
|
||||
/^\d{4}-\d{2}-\d{2}$/.test(parsed.day) &&
|
||||
new Date(`${parsed.day}T00:00:00.000Z`)
|
||||
.toISOString()
|
||||
.startsWith(parsed.day));
|
||||
const validCount =
|
||||
Number.isInteger(parsed?.count) &&
|
||||
parsed.count >= 0 &&
|
||||
(parsed.day !== '' || parsed.count === 0);
|
||||
const validPausedUntil =
|
||||
Number.isFinite(parsed?.pausedUntil) && parsed.pausedUntil >= 0;
|
||||
// nextStartAt was added after the original state format. Its absence is
|
||||
// the one supported migration; a present value must still be trustworthy.
|
||||
const validNextStartAt =
|
||||
parsed?.nextStartAt === undefined ||
|
||||
(Number.isFinite(parsed.nextStartAt) && parsed.nextStartAt >= 0);
|
||||
if (
|
||||
!parsed ||
|
||||
Array.isArray(parsed) ||
|
||||
typeof parsed !== 'object' ||
|
||||
!validDay ||
|
||||
!validCount ||
|
||||
!validPausedUntil ||
|
||||
!validNextStartAt
|
||||
)
|
||||
throw new Error('invalid durable Nominatim usage state');
|
||||
return {
|
||||
day: parsed.day,
|
||||
count: parsed.count,
|
||||
pausedUntil: parsed.pausedUntil,
|
||||
nextStartAt: parsed.nextStartAt ?? 0,
|
||||
};
|
||||
};
|
||||
|
||||
function readDisk({ strict = false } = {}) {
|
||||
if (!file) {
|
||||
if (strict)
|
||||
throw new Error('no durable Nominatim usage state is configured');
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return parseState(JSON.parse(fs.readFileSync(file, 'utf8')));
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return null;
|
||||
report(error);
|
||||
if (strict) throw error;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function current() {
|
||||
const disk = readDisk();
|
||||
if (disk) memory = merge(memory, disk);
|
||||
const pending = pendingPauses();
|
||||
if (pending.length)
|
||||
memory = merge(memory, {
|
||||
...memory,
|
||||
pausedUntil: Math.max(...pending.map((item) => item.pausedUntil)),
|
||||
});
|
||||
return memory;
|
||||
}
|
||||
|
||||
function withLock(work) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
const waitMs = Number.isFinite(lockWaitMs)
|
||||
? Math.max(0, Math.floor(lockWaitMs))
|
||||
: 500;
|
||||
const database = new (sqliteModule().DatabaseSync)(`${file}.lock.sqlite`, {
|
||||
timeout: waitMs,
|
||||
});
|
||||
let transactionOpen = false;
|
||||
try {
|
||||
database.exec('BEGIN IMMEDIATE');
|
||||
transactionOpen = true;
|
||||
const result = work();
|
||||
database.exec('COMMIT');
|
||||
transactionOpen = false;
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (transactionOpen) {
|
||||
try {
|
||||
database.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Closing the connection below still releases this owner's lock.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
database.close();
|
||||
} catch {
|
||||
// The transaction has already been committed or rolled back. Closing
|
||||
// is best effort during exceptional teardown and never removes a lock.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function writeDisk(next) {
|
||||
const temp = `${file}.${process.pid}.${process.hrtime.bigint()}.tmp`;
|
||||
try {
|
||||
fs.writeFileSync(temp, JSON.stringify(next));
|
||||
fs.renameSync(temp, file);
|
||||
} finally {
|
||||
try {
|
||||
fs.unlinkSync(temp);
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') report(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Apply `change` to the freshest state and persist it atomically. */
|
||||
function update(change) {
|
||||
if (!file) {
|
||||
memory = merge(memory, change(memory));
|
||||
return memory;
|
||||
}
|
||||
try {
|
||||
withLock(() => {
|
||||
const next = merge(memory, change(current()));
|
||||
writeDisk(next);
|
||||
memory = next;
|
||||
clearPersistedPauses(next.pausedUntil);
|
||||
lastError = null;
|
||||
});
|
||||
} catch (error) {
|
||||
report(error);
|
||||
memory = merge(memory, change(memory));
|
||||
}
|
||||
return memory;
|
||||
}
|
||||
|
||||
return {
|
||||
count(day) {
|
||||
const state = current();
|
||||
return state.day === day ? state.count : 0;
|
||||
},
|
||||
increment(day) {
|
||||
return update((state) => ({
|
||||
day,
|
||||
count: state.day === day ? state.count + 1 : 1,
|
||||
pausedUntil: state.pausedUntil,
|
||||
nextStartAt: state.nextStartAt,
|
||||
})).count;
|
||||
},
|
||||
dispatch(
|
||||
day,
|
||||
{ clock, dailyCap, minSpacingMs, latestStartAt = Infinity },
|
||||
start,
|
||||
) {
|
||||
if (!file) {
|
||||
const error = new Error(
|
||||
'no durable Nominatim usage state is configured',
|
||||
);
|
||||
report(error);
|
||||
return { status: 'unavailable', error };
|
||||
}
|
||||
let value;
|
||||
try {
|
||||
let result;
|
||||
withLock(() => {
|
||||
const reclaimed = reclaimDispatches(clock());
|
||||
const disk = readDisk({ strict: true });
|
||||
if (disk) memory = merge(memory, disk);
|
||||
const pauses = pendingPauses({ strict: true });
|
||||
const state = merge(memory, {
|
||||
...memory,
|
||||
pausedUntil: Math.max(
|
||||
memory.pausedUntil,
|
||||
...pauses.map((p) => p.pausedUntil),
|
||||
),
|
||||
});
|
||||
const at = clock();
|
||||
if (reclaimed) {
|
||||
state.nextStartAt = Math.max(state.nextStartAt, at + minSpacingMs);
|
||||
writeDisk(state);
|
||||
memory = state;
|
||||
}
|
||||
if (at > latestStartAt) {
|
||||
result = { status: 'abandoned' };
|
||||
return;
|
||||
}
|
||||
if (state.pausedUntil > at) {
|
||||
result = { status: 'paused', retryAfterMs: state.pausedUntil - at };
|
||||
return;
|
||||
}
|
||||
const count = state.day === day ? state.count : 0;
|
||||
if (count >= dailyCap) {
|
||||
result = { status: 'cap' };
|
||||
return;
|
||||
}
|
||||
if (state.nextStartAt > at) {
|
||||
result =
|
||||
state.nextStartAt > latestStartAt
|
||||
? { status: 'abandoned' }
|
||||
: { status: 'wait', waitMs: state.nextStartAt - at };
|
||||
return;
|
||||
}
|
||||
const marker = `${file}.dispatch.since-${at}.${process.pid}.${process.hrtime.bigint()}`;
|
||||
fs.writeFileSync(marker, '', { flag: 'wx' });
|
||||
const next = {
|
||||
day,
|
||||
count: count + 1,
|
||||
pausedUntil: state.pausedUntil,
|
||||
nextStartAt: at + minSpacingMs,
|
||||
};
|
||||
writeDisk(next);
|
||||
memory = next;
|
||||
const latePause = Math.max(
|
||||
next.pausedUntil,
|
||||
...pendingPauses({ strict: true }).map((p) => p.pausedUntil),
|
||||
);
|
||||
if (clock() > latestStartAt) result = { status: 'abandoned' };
|
||||
else if (latePause > clock())
|
||||
result = { status: 'paused', retryAfterMs: latePause - clock() };
|
||||
else {
|
||||
// Invoke, but never await, the transport while still owning the lock.
|
||||
try {
|
||||
value = start();
|
||||
} catch (error) {
|
||||
value = Promise.reject(error);
|
||||
}
|
||||
result = { status: 'started', value };
|
||||
}
|
||||
const after = {
|
||||
...next,
|
||||
pausedUntil: latePause,
|
||||
nextStartAt: Math.max(next.nextStartAt, clock() + minSpacingMs),
|
||||
};
|
||||
writeDisk(after);
|
||||
memory = after;
|
||||
fs.unlinkSync(marker);
|
||||
clearPersistedPauses(after.pausedUntil);
|
||||
lastError = null;
|
||||
});
|
||||
return result;
|
||||
} catch (error) {
|
||||
// The caller aborts its transport on admission failure. Observe a started
|
||||
// promise even when persistence failed, without masking the store error.
|
||||
if (value?.then) Promise.resolve(value).catch(() => {});
|
||||
report(error);
|
||||
return { status: 'unavailable', error };
|
||||
}
|
||||
},
|
||||
pausedUntil() {
|
||||
return current().pausedUntil || 0;
|
||||
},
|
||||
pauseUntil(at) {
|
||||
if (file) {
|
||||
try {
|
||||
at = markPause(at);
|
||||
} catch (error) {
|
||||
report(error);
|
||||
}
|
||||
}
|
||||
update((state) => ({ ...state, pausedUntil: at }));
|
||||
},
|
||||
status: () => {
|
||||
const dispatchPending = pendingDispatches().length > 0;
|
||||
return {
|
||||
file,
|
||||
durable: Boolean(file),
|
||||
persisted: Boolean(file) && !lastError && !dispatchPending,
|
||||
dispatchPending,
|
||||
error: lastError
|
||||
? String(lastError.message || lastError)
|
||||
: dispatchPending
|
||||
? 'unfinished durable Nominatim dispatch'
|
||||
: null,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a gate. `settings` may be a function so the environment is read
|
||||
* after the dev server loads `.env`.
|
||||
*
|
||||
* @param {object} [options]
|
||||
* @param {(() => object)|object} [options.settings]
|
||||
* @param {typeof fetch} [options.fetchImpl]
|
||||
* @param {() => number} [options.now]
|
||||
* @param {(ms: number) => Promise<void>} [options.sleep]
|
||||
* @param {ReturnType<typeof createGateStateStore>} [options.usage]
|
||||
*/
|
||||
export function createNominatimGate({
|
||||
settings = () => resolveNominatimSettings(),
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = Date.now,
|
||||
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
usage = createGateStateStore(),
|
||||
minSpacingMs = NOMINATIM_MIN_SPACING_MS,
|
||||
maxPending = NOMINATIM_MAX_PENDING,
|
||||
maxWaitMs = NOMINATIM_MAX_WAIT_MS,
|
||||
timeoutMs = 9000,
|
||||
} = {}) {
|
||||
let queue = Promise.resolve();
|
||||
let pending = 0;
|
||||
const pausedUntilByEndpoint = new Map();
|
||||
let upstreamRequests = 0;
|
||||
let usageStore = usage;
|
||||
|
||||
const current = () =>
|
||||
typeof settings === 'function' ? settings() : settings;
|
||||
|
||||
/**
|
||||
* Pause until `ms` from now. The public service's pause is also written to
|
||||
* the shared state so a restart or a second process honours it.
|
||||
*/
|
||||
function pause(ms, config) {
|
||||
const until = now() + Math.max(0, ms);
|
||||
const key = String(config?.endpoint || '');
|
||||
if (until > (pausedUntilByEndpoint.get(key) || 0))
|
||||
pausedUntilByEndpoint.set(key, until);
|
||||
if (config?.isPublic) usageStore.pauseUntil(until);
|
||||
}
|
||||
|
||||
function pauseEnd(config) {
|
||||
const endpointPause =
|
||||
pausedUntilByEndpoint.get(String(config?.endpoint || '')) || 0;
|
||||
return config?.isPublic
|
||||
? Math.max(endpointPause, usageStore.pausedUntil())
|
||||
: endpointPause;
|
||||
}
|
||||
|
||||
function pausedError(config) {
|
||||
return gateError('NOMINATIM_PAUSED', 'Place service is pausing', {
|
||||
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
||||
});
|
||||
}
|
||||
|
||||
function admissionError(config) {
|
||||
if (!config.endpoint)
|
||||
return gateError('NOMINATIM_DISABLED', 'Place service is not configured');
|
||||
if (now() < pauseEnd(config)) return pausedError(config);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** One upstream request; classifies refusals and pauses the gate. */
|
||||
async function send(url, config, maxBytes, signal, queuedAt) {
|
||||
const controller = new AbortController();
|
||||
const onAbort = () => controller.abort();
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
// One deadline covers the headers AND the body: a server that answers its
|
||||
// headers and then stalls must not hold the shared queue.
|
||||
let timer;
|
||||
const start = () => {
|
||||
upstreamRequests += 1;
|
||||
timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
return fetchImpl(url, {
|
||||
headers: NOMINATIM_HEADERS,
|
||||
redirect: 'error',
|
||||
signal: controller.signal,
|
||||
});
|
||||
};
|
||||
try {
|
||||
return await exchange(
|
||||
config,
|
||||
maxBytes,
|
||||
signal,
|
||||
controller.signal,
|
||||
queuedAt,
|
||||
start,
|
||||
);
|
||||
} catch (error) {
|
||||
controller.abort();
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
async function exchange(
|
||||
config,
|
||||
maxBytes,
|
||||
signal,
|
||||
requestSignal,
|
||||
queuedAt,
|
||||
start,
|
||||
) {
|
||||
const turn = await takeTurn(config, queuedAt, signal, start);
|
||||
let response;
|
||||
try {
|
||||
response = await turn.value;
|
||||
} catch (cause) {
|
||||
if (signal?.aborted) throw gateError('NOMINATIM_ABANDONED', 'Abandoned');
|
||||
throw gateError('NOMINATIM_TRANSPORT', 'Place service unreachable', {
|
||||
cause,
|
||||
transport: true,
|
||||
});
|
||||
}
|
||||
if (!response.ok) {
|
||||
void response.body?.cancel?.().catch(() => {});
|
||||
const status = response.status;
|
||||
const retryAfter = parseRetryAfter(
|
||||
response.headers?.get?.('retry-after'),
|
||||
now(),
|
||||
);
|
||||
if (status === 429 || status === 503) {
|
||||
pause(retryAfter ?? DEFAULT_BACKOFF_MS, config);
|
||||
throw gateError('NOMINATIM_REFUSED', 'Place service asked to wait', {
|
||||
status,
|
||||
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
||||
});
|
||||
}
|
||||
if (status === 403 || status === 418) {
|
||||
pause(retryAfter ?? BLOCKED_BACKOFF_MS, config);
|
||||
throw gateError('NOMINATIM_REFUSED', 'Place service refused', {
|
||||
status,
|
||||
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
||||
});
|
||||
}
|
||||
if (status >= 500) pause(retryAfter ?? OUTAGE_BACKOFF_MS, config);
|
||||
throw gateError('NOMINATIM_UPSTREAM', `Upstream returned ${status}`, {
|
||||
status,
|
||||
});
|
||||
}
|
||||
try {
|
||||
return await readResponseJsonCapped(response, maxBytes, requestSignal);
|
||||
} catch (cause) {
|
||||
if (cause?.code === 'RESPONSE_TOO_LARGE')
|
||||
throw gateError('NOMINATIM_TOO_LARGE', 'Place answer too large');
|
||||
if (signal?.aborted) throw gateError('NOMINATIM_ABANDONED', 'Abandoned');
|
||||
throw gateError('NOMINATIM_TRANSPORT', 'Place answer unreadable', {
|
||||
cause,
|
||||
transport: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Revalidate public dispatch under shared ownership after every wake. */
|
||||
async function takeTurn(config, queuedAt, signal, start) {
|
||||
while (true) {
|
||||
if (signal?.aborted || now() - queuedAt > maxWaitMs)
|
||||
throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned');
|
||||
const refused = admissionError(config);
|
||||
if (refused) throw refused;
|
||||
// An operator-selected endpoint is not governed by the public instance's
|
||||
// shared budget, pacing, or durable-state requirement.
|
||||
if (!config.isPublic) {
|
||||
try {
|
||||
return { value: start() };
|
||||
} catch (error) {
|
||||
return { value: Promise.reject(error) };
|
||||
}
|
||||
}
|
||||
const reserved = usageStore.dispatch(
|
||||
utcDay(now()),
|
||||
{
|
||||
clock: now,
|
||||
dailyCap: config.dailyCap,
|
||||
minSpacingMs,
|
||||
latestStartAt: queuedAt + maxWaitMs,
|
||||
},
|
||||
start,
|
||||
);
|
||||
if (reserved.status === 'unavailable')
|
||||
throw gateError(
|
||||
'NOMINATIM_STATE_UNAVAILABLE',
|
||||
'Public place service usage state is unavailable',
|
||||
{ cause: reserved.error },
|
||||
);
|
||||
if (reserved.status === 'cap')
|
||||
throw gateError(
|
||||
'NOMINATIM_DAILY_CAP',
|
||||
'Daily place lookup allowance is used',
|
||||
);
|
||||
if (reserved.status === 'paused')
|
||||
throw gateError('NOMINATIM_PAUSED', 'Place service is pausing', {
|
||||
retryAfterMs: reserved.retryAfterMs,
|
||||
});
|
||||
if (reserved.status === 'abandoned')
|
||||
throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned');
|
||||
if (reserved.status === 'started') return reserved;
|
||||
if (reserved.status === 'wait') await sleep(reserved.waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue one JSON request for `buildUrl(endpoint)`. At most one retry, and
|
||||
* only after a transport failure; a failed retry pauses the gate.
|
||||
*
|
||||
* @param {(endpoint: string) => string} buildUrl
|
||||
* @param {{signal?: AbortSignal, maxBytes?: number}} [options]
|
||||
*/
|
||||
function requestJson(buildUrl, { signal, maxBytes = 2 * 1024 * 1024 } = {}) {
|
||||
const config = current();
|
||||
const refused = admissionError(config);
|
||||
if (refused) return Promise.reject(refused);
|
||||
if (signal?.aborted)
|
||||
return Promise.reject(
|
||||
gateError('NOMINATIM_ABANDONED', 'Place search was abandoned'),
|
||||
);
|
||||
if (pending >= maxPending)
|
||||
return Promise.reject(
|
||||
gateError('NOMINATIM_QUEUE_FULL', 'Place search queue is full'),
|
||||
);
|
||||
pending += 1;
|
||||
const queuedAt = now();
|
||||
const url = buildUrl(config.endpoint);
|
||||
const task = queue.then(async () => {
|
||||
try {
|
||||
try {
|
||||
return await send(url, config, maxBytes, signal, queuedAt);
|
||||
} catch (error) {
|
||||
if (!error?.transport) throw error;
|
||||
try {
|
||||
return await send(url, config, maxBytes, signal, now());
|
||||
} catch (retryError) {
|
||||
if (retryError?.transport) pause(OUTAGE_BACKOFF_MS, config);
|
||||
throw retryError;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
pending -= 1;
|
||||
}
|
||||
});
|
||||
queue = task.catch(() => null);
|
||||
return task;
|
||||
}
|
||||
|
||||
return {
|
||||
requestJson,
|
||||
settings: current,
|
||||
/** Share public admission, daily count and pauses through one install's file. */
|
||||
persistUsage(file) {
|
||||
usageStore = createGateStateStore({ file });
|
||||
},
|
||||
stats: () => ({
|
||||
upstreamRequests,
|
||||
pending,
|
||||
pausedForMs: Math.max(0, pauseEnd(current()) - now()),
|
||||
usedToday: usageStore.count(utcDay(now())),
|
||||
persistence: usageStore.status?.() || null,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
let sharedGate = createNominatimGate();
|
||||
|
||||
/** The process-wide gate: search and outlines share one budget and pacer. */
|
||||
export function sharedNominatimGate() {
|
||||
return sharedGate;
|
||||
}
|
||||
|
||||
/** Replace the process-wide gate (tests, embedders with their own transport). */
|
||||
export function setSharedNominatimGate(gate) {
|
||||
sharedGate = gate || createNominatimGate();
|
||||
return sharedGate;
|
||||
}
|
||||
|
||||
/** Codes that mean "not now" rather than "no such place". */
|
||||
export function isNominatimBusy(error) {
|
||||
return [
|
||||
'NOMINATIM_QUEUE_FULL',
|
||||
'NOMINATIM_ABANDONED',
|
||||
'NOMINATIM_PAUSED',
|
||||
'NOMINATIM_REFUSED',
|
||||
'NOMINATIM_DAILY_CAP',
|
||||
].includes(error?.code);
|
||||
}
|
||||
@@ -0,0 +1,746 @@
|
||||
import { normalizePlaceName } from './nominatimCache.js';
|
||||
import {
|
||||
containsAnchor,
|
||||
pointInRing,
|
||||
} from '../../../src/data/adminBoundaries.js';
|
||||
import {
|
||||
douglasPeucker,
|
||||
ringAreaM2,
|
||||
} from '../../../src/sources/featureGeometry.js';
|
||||
|
||||
/**
|
||||
* Which Nominatim results may answer which voice ask, and how their polygons
|
||||
* are bounded. Pure: no I/O.
|
||||
*
|
||||
* A search returns whatever matches the words best. "Mission District" can
|
||||
* come back as a car-charging station and "Bandra" as a railway station, so a
|
||||
* result is accepted only when its OSM class (and, for broad classes, its
|
||||
* type) suits the kind of thing that was asked for, it carries a polygon, and
|
||||
* its name actually answers the words.
|
||||
*/
|
||||
|
||||
/** Ask kinds the outline route answers. */
|
||||
export const OUTLINE_KINDS = Object.freeze([
|
||||
'city',
|
||||
'admin',
|
||||
'neighborhood',
|
||||
'landmark',
|
||||
'building',
|
||||
]);
|
||||
|
||||
/** Simplification tolerance (degrees) sent upstream, per ask kind. */
|
||||
export const OUTLINE_POLYGON_THRESHOLD = Object.freeze({
|
||||
city: 0.0005,
|
||||
admin: 0.0005,
|
||||
neighborhood: 0.0005,
|
||||
landmark: 0.00005,
|
||||
building: 0.00005,
|
||||
});
|
||||
|
||||
/**
|
||||
* Size limits for one outline. Vertices beyond the budget are simplified away;
|
||||
* more parts, or more holes in one part, than these limits refuse the outline
|
||||
* whole, because a truncated multipart place would misrepresent it.
|
||||
*/
|
||||
const OUTLINE_MAX_VERTICES = 6000;
|
||||
const OUTLINE_MAX_PARTS = 24;
|
||||
const OUTLINE_MAX_HOLES = 16;
|
||||
|
||||
const ANY = '*';
|
||||
|
||||
const SETTLEMENT_PLACES = [
|
||||
'city',
|
||||
'town',
|
||||
'village',
|
||||
'municipality',
|
||||
'borough',
|
||||
'hamlet',
|
||||
];
|
||||
|
||||
const CIVIC_AMENITIES = [
|
||||
'university',
|
||||
'college',
|
||||
'school',
|
||||
'kindergarten',
|
||||
'hospital',
|
||||
'clinic',
|
||||
'place_of_worship',
|
||||
'monastery',
|
||||
'townhall',
|
||||
'courthouse',
|
||||
'library',
|
||||
'theatre',
|
||||
'cinema',
|
||||
'arts_centre',
|
||||
'community_centre',
|
||||
'conference_centre',
|
||||
'events_venue',
|
||||
'exhibition_centre',
|
||||
'music_venue',
|
||||
'planetarium',
|
||||
'marketplace',
|
||||
'prison',
|
||||
'grave_yard',
|
||||
'fire_station',
|
||||
'police',
|
||||
'embassy',
|
||||
'public_building',
|
||||
];
|
||||
|
||||
const LANDMARK_TOURISM = [
|
||||
'attraction',
|
||||
'theme_park',
|
||||
'zoo',
|
||||
'aquarium',
|
||||
'museum',
|
||||
'gallery',
|
||||
'viewpoint',
|
||||
'camp_site',
|
||||
'hotel',
|
||||
'resort',
|
||||
];
|
||||
|
||||
const MAN_MADE_STRUCTURES = [
|
||||
'tower',
|
||||
'lighthouse',
|
||||
'pier',
|
||||
'bridge',
|
||||
'works',
|
||||
'dam',
|
||||
'observatory',
|
||||
'wastewater_plant',
|
||||
'water_works',
|
||||
'breakwater',
|
||||
];
|
||||
|
||||
/**
|
||||
* Accepted `class → types` per ask kind. `*` accepts every type of that
|
||||
* class. Anything absent is rejected: roads, railways, stations, shops,
|
||||
* guideposts and single facilities never answer an area ask.
|
||||
*/
|
||||
const OUTLINE_ACCEPT = Object.freeze({
|
||||
city: {
|
||||
boundary: ['administrative'],
|
||||
place: SETTLEMENT_PLACES,
|
||||
},
|
||||
admin: {
|
||||
boundary: ['administrative'],
|
||||
place: ['country', 'state', 'province', 'region', 'county', 'district'],
|
||||
},
|
||||
neighborhood: {
|
||||
boundary: ['administrative'],
|
||||
place: ['suburb', 'quarter', 'neighbourhood', 'borough', 'city_block'],
|
||||
},
|
||||
landmark: {
|
||||
leisure: ANY,
|
||||
landuse: ANY,
|
||||
amenity: CIVIC_AMENITIES,
|
||||
tourism: LANDMARK_TOURISM,
|
||||
building: ANY,
|
||||
natural: ANY,
|
||||
water: ANY,
|
||||
waterway: ['riverbank', 'dock'],
|
||||
historic: ANY,
|
||||
man_made: MAN_MADE_STRUCTURES,
|
||||
office: ['government', 'diplomatic'],
|
||||
aeroway: ['aerodrome', 'heliport', 'terminal'],
|
||||
military: ANY,
|
||||
boundary: ['national_park', 'protected_area'],
|
||||
place: ['island', 'islet', 'square'],
|
||||
},
|
||||
// Single structures only: no campuses, grounds, parks or camp sites.
|
||||
building: {
|
||||
building: ANY,
|
||||
amenity: [
|
||||
'place_of_worship',
|
||||
'townhall',
|
||||
'courthouse',
|
||||
'library',
|
||||
'theatre',
|
||||
'cinema',
|
||||
'arts_centre',
|
||||
'community_centre',
|
||||
'conference_centre',
|
||||
'exhibition_centre',
|
||||
'music_venue',
|
||||
'planetarium',
|
||||
'fire_station',
|
||||
'police',
|
||||
'embassy',
|
||||
'public_building',
|
||||
],
|
||||
tourism: ['museum', 'gallery', 'hotel', 'attraction'],
|
||||
historic: ['building', 'castle', 'church', 'fort', 'palace', 'monastery'],
|
||||
man_made: ['tower', 'lighthouse', 'observatory'],
|
||||
office: ANY,
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Administrative evidence per ask kind: a recognized address type is required.
|
||||
* A supplied numeric place_rank must also fit the ask's broad band (country 4,
|
||||
* state 8, county 10-12, city 12-16, suburb 18-20, neighbourhood 20-22).
|
||||
*/
|
||||
const ADMIN_EVIDENCE = Object.freeze({
|
||||
city: {
|
||||
types: ['city', 'town', 'village', 'municipality', 'borough', 'hamlet'],
|
||||
ranks: [12, 18],
|
||||
},
|
||||
admin: {
|
||||
types: [
|
||||
'country',
|
||||
'state',
|
||||
'province',
|
||||
'region',
|
||||
'county',
|
||||
'district',
|
||||
'state_district',
|
||||
],
|
||||
ranks: [2, 14],
|
||||
},
|
||||
neighborhood: {
|
||||
types: ['suburb', 'quarter', 'neighbourhood', 'city_district', 'borough'],
|
||||
ranks: [17, 24],
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Nominatim address types whose meaning is stable enough for the area consumer's
|
||||
* country/admin1/admin2 contract. `region`, `district` and numeric admin levels
|
||||
* vary by country, so they remain unverified instead of being guessed.
|
||||
*/
|
||||
const VERIFIED_ADMIN_LEVEL = Object.freeze({
|
||||
country: 'country',
|
||||
state: 'admin1',
|
||||
province: 'admin1',
|
||||
county: 'admin2',
|
||||
state_district: 'admin2',
|
||||
});
|
||||
|
||||
const EXPLICIT_NAME_LEVEL = Object.freeze({
|
||||
country: 'country',
|
||||
nation: 'country',
|
||||
province: 'admin1',
|
||||
state: 'admin1',
|
||||
region: 'admin1',
|
||||
oblast: 'admin1',
|
||||
prefecture: 'admin1',
|
||||
canton: 'admin1',
|
||||
governorate: 'admin1',
|
||||
county: 'admin2',
|
||||
district: 'admin2',
|
||||
department: 'admin2',
|
||||
zone: 'retired',
|
||||
});
|
||||
|
||||
function explicitNameEvidence(value) {
|
||||
const normalized = String(value || '')
|
||||
.normalize('NFKC')
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ')
|
||||
.trim();
|
||||
const prefix = normalized.match(
|
||||
/^(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)\s+of\b/,
|
||||
)?.[1];
|
||||
const suffix = normalized.match(
|
||||
/\b(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)$/,
|
||||
)?.[1];
|
||||
const levels = new Set(
|
||||
[prefix, suffix]
|
||||
.filter(Boolean)
|
||||
.map((type) => EXPLICIT_NAME_LEVEL[type])
|
||||
.filter(Boolean),
|
||||
);
|
||||
if (levels.size > 1) return { contradictory: true, level: null };
|
||||
return { contradictory: false, level: levels.values().next().value || null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verified administrative identity carried by the selected result itself.
|
||||
* Never infer it from the request, desired kind, rank, or a separate geocoder
|
||||
* anchor. A retired name is preserved without a level so downstream policy can
|
||||
* reject it explicitly; every other contradiction stays wholly unverified.
|
||||
*/
|
||||
function selectedAdminIdentity(kind, row) {
|
||||
if (kind !== 'admin') return null;
|
||||
const adminLevel =
|
||||
VERIFIED_ADMIN_LEVEL[String(row?.addresstype || '').toLowerCase()];
|
||||
const authoritativeAdminArea = String(row?.name || '').trim();
|
||||
if (!adminLevel || !authoritativeAdminArea) return null;
|
||||
// Validate the complete authoritative name. `adminArea` is truncated only
|
||||
// for display after every prefix/suffix signal has been considered.
|
||||
const { contradictory, level: nameLevel } = explicitNameEvidence(
|
||||
authoritativeAdminArea,
|
||||
);
|
||||
if (contradictory) return null;
|
||||
const adminArea = authoritativeAdminArea.slice(0, 160);
|
||||
if (nameLevel === 'retired') {
|
||||
// Nepal replaced its zones in 2015, but zones are current units elsewhere
|
||||
// (Ethiopia). Without a country the name stays retired.
|
||||
if (!zoneIsCurrentIn(row)) return { adminArea };
|
||||
return { adminArea, adminLevel };
|
||||
}
|
||||
if (nameLevel && nameLevel !== adminLevel) return null;
|
||||
return { adminArea, adminLevel };
|
||||
}
|
||||
|
||||
/** Whether the row names a country (last display_name part) other than Nepal. */
|
||||
function zoneIsCurrentIn(row) {
|
||||
const country = String(row?.display_name || '')
|
||||
.split(',')
|
||||
.slice(1)
|
||||
.pop()
|
||||
?.trim()
|
||||
.toLocaleLowerCase();
|
||||
return Boolean(country) && country !== 'nepal';
|
||||
}
|
||||
|
||||
/** Whether supplied administrative evidence contradicts the accepted identity. */
|
||||
function adminLevelFits(kind, row) {
|
||||
const evidence = ADMIN_EVIDENCE[kind];
|
||||
if (!evidence) return true;
|
||||
const place = String(row?.category || row?.class).toLowerCase() === 'place';
|
||||
// A place row already has an accepted type. Missing address evidence must
|
||||
// not erase it, but explicit contradictory evidence must not bypass checks.
|
||||
if (!place || Object.hasOwn(row, 'addresstype')) {
|
||||
if (typeof row?.addresstype !== 'string') return false;
|
||||
const type = row.addresstype.toLowerCase();
|
||||
if (
|
||||
!evidence.types.includes(type) &&
|
||||
!(place && type === String(row.type).toLowerCase())
|
||||
)
|
||||
return false;
|
||||
}
|
||||
// Rank can corroborate or contradict an explicit identity, never invent it.
|
||||
// JSONv2 supplies a number; coercing arrays or strings hides malformed data.
|
||||
if (!Object.hasOwn(row, 'place_rank')) return true;
|
||||
const rank = row.place_rank;
|
||||
return (
|
||||
Number.isInteger(rank) &&
|
||||
// Boundary rank bands do not describe every accepted place type.
|
||||
(place || (rank >= evidence.ranks[0] && rank <= evidence.ranks[1]))
|
||||
);
|
||||
}
|
||||
|
||||
/** Largest footprint (m²) a non-`building` result may have for a building ask. */
|
||||
const BUILDING_MAX_AREA_M2 = 150_000;
|
||||
|
||||
/** Whether a result's class/type suits the ask kind. */
|
||||
export function acceptsOutlineClass(kind, osmClass, osmType) {
|
||||
const rules = OUTLINE_ACCEPT[kind];
|
||||
if (!rules) return false;
|
||||
const types = rules[String(osmClass || '').toLowerCase()];
|
||||
if (!types) return false;
|
||||
return types === ANY || types.includes(String(osmType || '').toLowerCase());
|
||||
}
|
||||
|
||||
const STOP_WORDS = new Set(['the', 'of', 'de', 'la', 'le', 'el', 'and']);
|
||||
|
||||
/** Scripts written without spaces between words. */
|
||||
const UNSPACED_SCRIPT_RE =
|
||||
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
|
||||
|
||||
/** Significant words of the part of a query before the first comma. */
|
||||
function queryWords(query) {
|
||||
const head = String(query ?? '').split(',')[0];
|
||||
return normalizePlaceName(head)
|
||||
.split(' ')
|
||||
.filter((word) => word && !STOP_WORDS.has(word));
|
||||
}
|
||||
|
||||
/**
|
||||
* Words naming what a building is, which OSM rarely carries in the name or
|
||||
* type ("Burj Khalifa", building=yes). Enclosures (zoo, garden, arch,
|
||||
* conservatory) are left out: those name a feature inside a park.
|
||||
*/
|
||||
const BUILDING_WORDS = [
|
||||
'building',
|
||||
'tower',
|
||||
'church',
|
||||
'cathedral',
|
||||
'basilica',
|
||||
'chapel',
|
||||
'temple',
|
||||
'mosque',
|
||||
'synagogue',
|
||||
'shrine',
|
||||
'museum',
|
||||
'palace',
|
||||
'castle',
|
||||
'hall',
|
||||
'library',
|
||||
];
|
||||
|
||||
/** How well a result's names answer the ask. */
|
||||
const NAME_MISS = 0;
|
||||
const NAME_OVERLAP = 1;
|
||||
const NAME_WHOLE = 2;
|
||||
const NAME_EXACT = 3;
|
||||
|
||||
/**
|
||||
* Settlements require one complete normalized name or explicit returned alias.
|
||||
* Landmark/building asks retain their word-overlap and unspaced-script
|
||||
* shorthand. An exact name ranks first, then one holding every asked word,
|
||||
* then an overlap.
|
||||
*/
|
||||
function outlineNameMatch(row, query, kind) {
|
||||
const names = new Set();
|
||||
const add = (value) => {
|
||||
const name = normalizePlaceName(value);
|
||||
if (name) names.add(name);
|
||||
};
|
||||
add(row?.name);
|
||||
add(String(row?.display_name || '').split(',')[0]);
|
||||
// OSM name tags may hold several names separated by ";" ("York;Eboracum").
|
||||
if (row?.namedetails && typeof row.namedetails === 'object')
|
||||
for (const value of Object.values(row.namedetails))
|
||||
for (const part of String(value ?? '').split(';')) add(part);
|
||||
if (kind === 'city' || kind === 'admin' || kind === 'neighborhood') {
|
||||
const head = normalizePlaceName(String(query ?? '').split(',')[0]);
|
||||
return head && names.has(head) ? NAME_EXACT : NAME_MISS;
|
||||
}
|
||||
const words = queryWords(query);
|
||||
if (!words.length) return NAME_MISS;
|
||||
const whole = words.join(' ');
|
||||
const unspaced = UNSPACED_SCRIPT_RE.test(whole);
|
||||
// The last word usually names the feature ("Zoo", "Arch"): a name without
|
||||
// it is the place around the feature ("Central Park"), unless the word is
|
||||
// where the result is ("Space Needle Seattle"), what it is ("… Bridge"), or
|
||||
// a building word on a single-structure result ("Burj Khalifa Tower").
|
||||
const context = new Set(
|
||||
normalizePlaceName(
|
||||
[
|
||||
String(row?.display_name || '')
|
||||
.split(',')
|
||||
.slice(1)
|
||||
.join(' '),
|
||||
row?.type,
|
||||
row?.category || row?.class,
|
||||
]
|
||||
.join(' ')
|
||||
.replace(/_/g, ' '),
|
||||
).split(' '),
|
||||
);
|
||||
// Only a single structure, as a building ask would accept: never a campus,
|
||||
// zoo or district whose name merely starts the ask ("Duke University").
|
||||
if (acceptsOutlineClass('building', row?.category || row?.class, row?.type))
|
||||
for (const word of BUILDING_WORDS) context.add(word);
|
||||
let match = NAME_MISS;
|
||||
for (const name of names) {
|
||||
if (
|
||||
name
|
||||
.split(' ')
|
||||
.filter((word) => !STOP_WORDS.has(word))
|
||||
.join(' ') === whole
|
||||
)
|
||||
return NAME_EXACT;
|
||||
// Scripts without spaces compare by characters instead.
|
||||
if (unspaced && name.replace(/ /g, '').includes(whole.replace(/ /g, '')))
|
||||
match = NAME_WHOLE;
|
||||
// Whole words only: "York" never matches "Yorkshire Park".
|
||||
const nameSet = new Set(name.split(' '));
|
||||
const hits = words.filter((word) => nameSet.has(word)).length;
|
||||
if (hits === words.length) match = NAME_WHOLE;
|
||||
const last = words.at(-1);
|
||||
if (
|
||||
words.length > 1 &&
|
||||
hits >= Math.ceil(words.length * 0.6) &&
|
||||
(unspaced || nameSet.has(last) || context.has(last))
|
||||
)
|
||||
match = Math.max(match, NAME_OVERLAP);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
const finiteLon = (value) =>
|
||||
Number.isFinite(value) && value >= -180 && value <= 180;
|
||||
const finiteLat = (value) =>
|
||||
Number.isFinite(value) && value >= -90 && value <= 90;
|
||||
|
||||
/** A valid closed ring of [lon, lat] numbers, or null. */
|
||||
function cleanRing(ring) {
|
||||
if (!Array.isArray(ring)) return null;
|
||||
const out = [];
|
||||
for (const point of ring) {
|
||||
if (!Array.isArray(point)) return null;
|
||||
const [lon, lat] = point;
|
||||
if (!finiteLon(lon) || !finiteLat(lat)) return null;
|
||||
out.push([lon, lat]);
|
||||
}
|
||||
if (out.length < 4) return null;
|
||||
const [first, last] = [out[0], out.at(-1)];
|
||||
if (first[0] !== last[0] || first[1] !== last[1]) out.push([...first]);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Polygon/MultiPolygon GeoJSON → `[[outer, ...holes], …]`, largest first. */
|
||||
function polygonsFromGeoJson(geometry) {
|
||||
const raw =
|
||||
geometry?.type === 'Polygon'
|
||||
? [geometry.coordinates]
|
||||
: geometry?.type === 'MultiPolygon'
|
||||
? geometry.coordinates
|
||||
: null;
|
||||
if (!Array.isArray(raw) || !raw.length || raw.length > OUTLINE_MAX_PARTS)
|
||||
return null;
|
||||
const polygons = [];
|
||||
for (const polygon of raw) {
|
||||
if (
|
||||
!Array.isArray(polygon) ||
|
||||
!polygon.length ||
|
||||
polygon.length - 1 > OUTLINE_MAX_HOLES
|
||||
)
|
||||
return null;
|
||||
const outer = cleanRing(polygon[0]);
|
||||
if (!outer) return null;
|
||||
const holes = polygon.slice(1).map(cleanRing);
|
||||
if (holes.some((hole) => !hole)) return null;
|
||||
polygons.push({ rings: [outer, ...holes], area: ringAreaM2(outer) });
|
||||
}
|
||||
polygons.sort((a, b) => b.area - a.area);
|
||||
return polygons.map((entry) => entry.rings);
|
||||
}
|
||||
|
||||
/** Iterative Douglas–Peucker on a closed ring (keeps it closed, ≥ 4 points). */
|
||||
function simplifyClosedRing(ring, tolerance) {
|
||||
if (ring.length <= 8 || tolerance <= 0) return ring;
|
||||
const out = douglasPeucker(ring, tolerance);
|
||||
return out.length >= 4 ? out : ring;
|
||||
}
|
||||
|
||||
const countVertices = (polygons) =>
|
||||
polygons.reduce(
|
||||
(sum, rings) => sum + rings.reduce((n, ring) => n + ring.length, 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const edgeSide = (p, q, r) =>
|
||||
(q[0] - p[0]) * (r[1] - p[1]) - (q[1] - p[1]) * (r[0] - p[0]);
|
||||
const onEdge = (p, q, r) =>
|
||||
r[0] >= Math.min(p[0], q[0]) &&
|
||||
r[0] <= Math.max(p[0], q[0]) &&
|
||||
r[1] >= Math.min(p[1], q[1]) &&
|
||||
r[1] <= Math.max(p[1], q[1]);
|
||||
|
||||
/** Intersection, including touching, of two planar closed-ring edges. */
|
||||
function edgesIntersect(a, b, c, d) {
|
||||
if (
|
||||
Math.max(a[0], b[0]) < Math.min(c[0], d[0]) ||
|
||||
Math.max(c[0], d[0]) < Math.min(a[0], b[0]) ||
|
||||
Math.max(a[1], b[1]) < Math.min(c[1], d[1]) ||
|
||||
Math.max(c[1], d[1]) < Math.min(a[1], b[1])
|
||||
)
|
||||
return false;
|
||||
const ac = edgeSide(a, b, c);
|
||||
const ad = edgeSide(a, b, d);
|
||||
const ca = edgeSide(c, d, a);
|
||||
const cb = edgeSide(c, d, b);
|
||||
return (
|
||||
(ac === 0 && onEdge(a, b, c)) ||
|
||||
(ad === 0 && onEdge(a, b, d)) ||
|
||||
(ca === 0 && onEdge(c, d, a)) ||
|
||||
(cb === 0 && onEdge(c, d, b)) ||
|
||||
(ac > 0 !== ad > 0 && ca > 0 !== cb > 0)
|
||||
);
|
||||
}
|
||||
|
||||
function ringsIntersect(a, b) {
|
||||
for (let i = 1; i < a.length; i++)
|
||||
for (let j = 1; j < b.length; j++)
|
||||
if (edgesIntersect(a[i - 1], a[i], b[j - 1], b[j])) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check only generated bounded output, not a new input-geometry policy.
|
||||
* Touching generated boundaries are conservatively unsupported here.
|
||||
*/
|
||||
function simplifiedTopologyFits(polygons) {
|
||||
for (const [outer, ...holes] of polygons) {
|
||||
for (const ring of [outer, ...holes]) {
|
||||
if (ringAreaM2(ring) === 0) return false;
|
||||
for (let i = 1; i < ring.length; i++)
|
||||
for (let j = i + 2; j < ring.length; j++) {
|
||||
if (i === 1 && j === ring.length - 1) continue;
|
||||
if (edgesIntersect(ring[i - 1], ring[i], ring[j - 1], ring[j]))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < holes.length; i++) {
|
||||
const hole = holes[i];
|
||||
if (
|
||||
!hole.every(([lon, lat]) => pointInRing(outer, lat, lon)) ||
|
||||
ringsIntersect(outer, hole)
|
||||
)
|
||||
return false;
|
||||
for (let j = 0; j < i; j++) {
|
||||
const other = holes[j];
|
||||
if (
|
||||
ringsIntersect(hole, other) ||
|
||||
pointInRing(other, hole[0][1], hole[0][0]) ||
|
||||
pointInRing(hole, other[0][1], other[0][0])
|
||||
)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Simplify until the whole outline fits the vertex cap. */
|
||||
export function capOutlineVertices(
|
||||
polygons,
|
||||
maxVertices = OUTLINE_MAX_VERTICES,
|
||||
) {
|
||||
let current = polygons;
|
||||
let tolerance = 0.00002;
|
||||
for (let round = 0; round < 12; round++) {
|
||||
if (countVertices(current) <= maxVertices)
|
||||
return current === polygons || simplifiedTopologyFits(current)
|
||||
? current
|
||||
: null;
|
||||
current = polygons.map((rings) =>
|
||||
rings.map((ring) => simplifyClosedRing(ring, tolerance)),
|
||||
);
|
||||
tolerance *= 2;
|
||||
}
|
||||
// Never turn a resource refusal into an apparently complete partial outline.
|
||||
return countVertices(current) <= maxVertices &&
|
||||
simplifiedTopologyFits(current)
|
||||
? current
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every result within the limit that suits the ask, names the place and
|
||||
* carries a polygon — best name match first, then upstream order — with the reason each other result was passed over. `outline` is
|
||||
* the first of them, or null.
|
||||
*
|
||||
* @param {object[]} rows Nominatim jsonv2 results, in upstream order.
|
||||
* @param {{kind: string, query: string}} ask
|
||||
*/
|
||||
export function selectOutlineResult(rows, { kind, query }) {
|
||||
const skipped = [];
|
||||
const found = [];
|
||||
for (const row of Array.isArray(rows) ? rows : []) {
|
||||
const osmClass = row?.category || row?.class;
|
||||
const osmType = row?.type;
|
||||
const label = `${osmClass}=${osmType}`;
|
||||
if (!acceptsOutlineClass(kind, osmClass, osmType)) {
|
||||
skipped.push(`${label}: wrong type`);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
(String(osmClass).toLowerCase() === 'place' ||
|
||||
(String(osmClass).toLowerCase() === 'boundary' &&
|
||||
String(osmType).toLowerCase() === 'administrative')) &&
|
||||
!adminLevelFits(kind, row)
|
||||
) {
|
||||
skipped.push(`${label}: level`);
|
||||
continue;
|
||||
}
|
||||
const nameMatch = outlineNameMatch(row, query, kind);
|
||||
if (nameMatch === NAME_MISS) {
|
||||
skipped.push(`${label}: name`);
|
||||
continue;
|
||||
}
|
||||
const parsed = polygonsFromGeoJson(row?.geojson);
|
||||
if (!parsed) {
|
||||
skipped.push(`${label}: no polygon`);
|
||||
continue;
|
||||
}
|
||||
// A building ask needs building evidence: the building class, or a
|
||||
// footprint no larger than a large building.
|
||||
if (
|
||||
kind === 'building' &&
|
||||
String(osmClass).toLowerCase() !== 'building' &&
|
||||
ringAreaM2(parsed[0][0]) > BUILDING_MAX_AREA_M2
|
||||
) {
|
||||
skipped.push(`${label}: not a building`);
|
||||
continue;
|
||||
}
|
||||
const polygons = capOutlineVertices(parsed);
|
||||
if (!polygons) {
|
||||
skipped.push(`${label}: too large`);
|
||||
continue;
|
||||
}
|
||||
found.push({
|
||||
nameMatch,
|
||||
outline: {
|
||||
name: String(row.name || '').slice(0, 160) || null,
|
||||
displayName: String(row.display_name || '').slice(0, 300) || null,
|
||||
class: String(osmClass),
|
||||
type: String(osmType),
|
||||
osm:
|
||||
row.osm_type && row.osm_id != null
|
||||
? `${row.osm_type}/${row.osm_id}`
|
||||
: null,
|
||||
...(selectedAdminIdentity(kind, row) || {}),
|
||||
polygons,
|
||||
},
|
||||
});
|
||||
}
|
||||
// Stable: equal matches keep upstream order.
|
||||
found.sort((a, b) => b.nameMatch - a.nameMatch);
|
||||
const outlines = found.map((entry) => entry.outline);
|
||||
const ranks = found.map((entry) => entry.nameMatch);
|
||||
return { outline: outlines[0] || null, outlines, ranks, skipped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Among the best-named outlines (`ranks` from selectOutlineResult; all of
|
||||
* them without), the one holding `lat, lon` (the resolved anchor), else the
|
||||
* first. A worse name never wins by containment alone: an exact building
|
||||
* that misses a pin by metres is not replaced by the plaza around it.
|
||||
* Without an anchor the first answers, as before.
|
||||
*/
|
||||
export function outlineAt(outlines, lat, lon, ranks = null) {
|
||||
const list = Array.isArray(outlines) ? outlines : [];
|
||||
if (!finiteLat(lat) || !finiteLon(lon)) return list[0] || null;
|
||||
const best = Array.isArray(ranks) ? ranks[0] : undefined;
|
||||
return (
|
||||
list.find(
|
||||
(outline, index) =>
|
||||
(best === undefined || ranks[index] === best) &&
|
||||
containsAnchor(outline.polygons, lat, lon),
|
||||
) ||
|
||||
list[0] ||
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Coarse view bias: the centre is snapped to a grid (1° for places, 0.05°
|
||||
* for small grounds) so small camera moves reuse the same cache entry, and
|
||||
* the viewbox sent upstream is built from the snapped centre.
|
||||
*/
|
||||
export function outlineBias(kind, lat, lon) {
|
||||
// Missing is missing: Number(null) and Number('') would read as (0, 0).
|
||||
const given = (value) =>
|
||||
value !== null &&
|
||||
value !== undefined &&
|
||||
(typeof value === 'number' || String(value).trim() !== '');
|
||||
if (!given(lat) || !given(lon)) return null;
|
||||
const latitude = Number(lat);
|
||||
const longitude = Number(lon);
|
||||
if (!finiteLat(latitude) || !finiteLon(longitude)) return null;
|
||||
const small = kind === 'landmark' || kind === 'building';
|
||||
const step = small ? 0.05 : 1;
|
||||
const half = small ? 0.25 : 2;
|
||||
const snap = (value) => Math.round(value / step) * step;
|
||||
const cLat = Number(snap(latitude).toFixed(2));
|
||||
const cLon = Number(snap(longitude).toFixed(2));
|
||||
const clampLat = (value) => Math.max(-90, Math.min(90, value));
|
||||
const clampLon = (value) => Math.max(-180, Math.min(180, value));
|
||||
return {
|
||||
key: `${cLat},${cLon}`,
|
||||
viewbox: [
|
||||
clampLon(cLon - half),
|
||||
clampLat(cLat + half),
|
||||
clampLon(cLon + half),
|
||||
clampLat(cLat - half),
|
||||
]
|
||||
.map((value) => Number(value.toFixed(4)))
|
||||
.join(','),
|
||||
};
|
||||
}
|
||||
+451
-196
@@ -1,114 +1,140 @@
|
||||
import path from 'node:path';
|
||||
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { coalesceProxyRequest } from '../common/http.js';
|
||||
import { fetchRegionalJson } from './http.js';
|
||||
import { admitSameSite } from '../common/same-site.js';
|
||||
import { defaultSourceRoot } from '../common/source-root.js';
|
||||
import { naturalRegionAtPoint } from '../../../src/data/naturalEarthRegions.js';
|
||||
import {
|
||||
nominatimToGeocodeResult,
|
||||
nominatimViewboxFromBounds,
|
||||
} from '../../../src/nominatimGeocode.js';
|
||||
|
||||
/**
|
||||
* The usage policy for the public Nominatim instance asks for a User-Agent or
|
||||
* Referer that identifies the application, and states that stock library
|
||||
* agents will not do. Both are sent.
|
||||
*/
|
||||
const NOMINATIM_HEADERS = Object.freeze({
|
||||
'User-Agent':
|
||||
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
||||
Referer: 'https://github.com/bilawalsidhu/gods-eye-view',
|
||||
});
|
||||
|
||||
/**
|
||||
* Minimum spacing between upstream calls. The policy states an absolute
|
||||
* maximum of one request per second; 1.1 s keeps clock jitter from crossing it.
|
||||
*/
|
||||
const NOMINATIM_MIN_SPACING_MS = 1100;
|
||||
|
||||
/**
|
||||
* How many searches may be waiting for their turn.
|
||||
*
|
||||
* One request per second and an unbounded queue are incompatible: a burst of
|
||||
* searches would keep the upstream busy long after everyone who asked has given
|
||||
* up, which is exactly the load the policy asks callers not to create. Past
|
||||
* this depth a search is refused at once instead of being promised a slot
|
||||
* minutes away.
|
||||
*/
|
||||
const NOMINATIM_MAX_PENDING = 4;
|
||||
|
||||
/**
|
||||
* How long a queued search may wait before it is not worth sending. The
|
||||
* browser gives up well before this, so anything reaching the front later than
|
||||
* this is answering nobody.
|
||||
*/
|
||||
const NOMINATIM_MAX_WAIT_MS = 10_000;
|
||||
|
||||
const NOMINATIM_SEARCH_CACHE_MS = 5 * 60_000;
|
||||
|
||||
const NOMINATIM_SEARCH_MAX_CACHE = 80;
|
||||
import {
|
||||
NOMINATIM_MAX_PENDING,
|
||||
isNominatimBusy,
|
||||
sharedNominatimGate,
|
||||
} from './nominatimGate.js';
|
||||
import {
|
||||
NOMINATIM_CACHE_TTL_MS,
|
||||
createNominatimCache,
|
||||
} from './nominatimCache.js';
|
||||
import {
|
||||
OUTLINE_KINDS,
|
||||
OUTLINE_POLYGON_THRESHOLD,
|
||||
outlineAt,
|
||||
outlineBias,
|
||||
selectOutlineResult,
|
||||
} from './outlineSelect.js';
|
||||
|
||||
const NOMINATIM_SEARCH_MAX_QUERY = 200;
|
||||
|
||||
// The pacer is deliberately module state while everything else here is
|
||||
// per-instance. One request per second is a budget for the whole application,
|
||||
// not for each provider object: two instances each pacing themselves would
|
||||
// send two requests a second between them. Only last-resort forward searches
|
||||
// use this queue.
|
||||
let _nominatimQueue = Promise.resolve();
|
||||
/** Results considered per outline ask: enough to skip a node or a wrong type. */
|
||||
const OUTLINE_RESULT_LIMIT = 3;
|
||||
|
||||
let _nominatimLastRequestAt = 0;
|
||||
/** Largest upstream answer read for an outline (polygons are simplified upstream). */
|
||||
const OUTLINE_MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
let _nominatimPending = 0;
|
||||
/** Bump when cached answers change shape or selection authority. */
|
||||
const CACHE_SCHEMA = 'v3';
|
||||
|
||||
/** Raised when the queue is already as deep as it is allowed to get. */
|
||||
function queueFullError() {
|
||||
return Object.assign(new Error('Place search queue is full'), {
|
||||
code: 'NOMINATIM_QUEUE_FULL',
|
||||
/**
|
||||
* The query text actually sent upstream: Unicode-composed, whitespace
|
||||
* collapsed. Nothing else is changed, so two queries share a cache entry only
|
||||
* when they are the same upstream request.
|
||||
*/
|
||||
export function upstreamQuery(query) {
|
||||
return String(query ?? '')
|
||||
.normalize('NFC')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** Cache key: endpoint, schema and the exact upstream parameters. */
|
||||
function requestKey(gate, kind, params) {
|
||||
const endpoint = gate.settings?.()?.endpoint || 'none';
|
||||
return `${CACHE_SCHEMA}|${endpoint}|${kind}|${params}`;
|
||||
}
|
||||
|
||||
/** A decimal coordinate query value, or NaN: never "" or " " as 0. */
|
||||
function coordinateParam(value) {
|
||||
return /^-?\d+(\.\d+)?$/.test(value ?? '') ? Number(value) : NaN;
|
||||
}
|
||||
|
||||
function malformed() {
|
||||
return Object.assign(new Error('Place answer was not understood'), {
|
||||
code: 'NOMINATIM_MALFORMED',
|
||||
});
|
||||
}
|
||||
|
||||
/** Raised when a queued search waited so long that nobody is left to answer. */
|
||||
function abandonedError() {
|
||||
function abandoned() {
|
||||
return Object.assign(new Error('Place search was abandoned'), {
|
||||
code: 'NOMINATIM_ABANDONED',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one piece of upstream work, never closer than the policy spacing to the
|
||||
* last one.
|
||||
*
|
||||
* `bounded` applies queue-depth and staleness limits to interactive forward searches.
|
||||
*
|
||||
* @param {() => Promise<unknown>} work
|
||||
* @param {{bounded?: boolean, signal?: AbortSignal}} [options]
|
||||
* Share one upstream call while keeping each caller's cancellation independent.
|
||||
* The shared controller is aborted only after the last subscriber leaves.
|
||||
*/
|
||||
function enqueueNominatim(work, { bounded = false, signal } = {}) {
|
||||
if (bounded && _nominatimPending >= NOMINATIM_MAX_PENDING)
|
||||
return Promise.reject(queueFullError());
|
||||
if (bounded) _nominatimPending += 1;
|
||||
const queuedAt = Date.now();
|
||||
const task = _nominatimQueue.then(async () => {
|
||||
try {
|
||||
const waitMs = Math.max(
|
||||
0,
|
||||
NOMINATIM_MIN_SPACING_MS - (Date.now() - _nominatimLastRequestAt),
|
||||
);
|
||||
if (waitMs) await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
// Nobody is waiting for this any more: sending it would spend the one
|
||||
// request per second the policy allows on an answer with no reader.
|
||||
if (bounded && Date.now() - queuedAt > NOMINATIM_MAX_WAIT_MS)
|
||||
throw abandonedError();
|
||||
if (signal?.aborted) throw abandonedError();
|
||||
_nominatimLastRequestAt = Date.now();
|
||||
return await work();
|
||||
} finally {
|
||||
if (bounded) _nominatimPending -= 1;
|
||||
}
|
||||
function subscribeSharedRequest(inFlight, key, signal, create) {
|
||||
if (signal?.aborted) return Promise.reject(abandoned());
|
||||
let flight = inFlight.get(key);
|
||||
if (!flight) {
|
||||
flight = {
|
||||
controller: new AbortController(),
|
||||
users: 0,
|
||||
settled: false,
|
||||
promise: null,
|
||||
};
|
||||
const owned = flight;
|
||||
owned.promise = Promise.resolve()
|
||||
.then(() => create(owned.controller.signal))
|
||||
.finally(() => {
|
||||
owned.settled = true;
|
||||
if (inFlight.get(key) === owned) inFlight.delete(key);
|
||||
});
|
||||
inFlight.set(key, owned);
|
||||
}
|
||||
flight.users += 1;
|
||||
return new Promise((resolve, reject) => {
|
||||
let left = false;
|
||||
const leave = () => {
|
||||
if (left) return;
|
||||
left = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
flight.users -= 1;
|
||||
if (
|
||||
flight.users === 0 &&
|
||||
!flight.settled &&
|
||||
inFlight.get(key) === flight
|
||||
) {
|
||||
inFlight.delete(key);
|
||||
flight.controller.abort();
|
||||
}
|
||||
};
|
||||
const onAbort = () => {
|
||||
leave();
|
||||
reject(abandoned());
|
||||
};
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
flight.promise.then(
|
||||
(value) => {
|
||||
leave();
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
leave();
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
_nominatimQueue = task.catch(() => null);
|
||||
return task;
|
||||
}
|
||||
|
||||
/** Where the dev server keeps Nominatim answers and the daily count. */
|
||||
export const NOMINATIM_CACHE_DIR = path.join(
|
||||
defaultSourceRoot,
|
||||
'.gev-cache',
|
||||
'nominatim',
|
||||
);
|
||||
|
||||
/** Construct the offline regional context provider from bundled Natural Earth polygons. */
|
||||
export function createRegionalPlaceProvider() {
|
||||
return (point) => naturalRegionAtPoint(point.latitude, point.longitude);
|
||||
@@ -117,137 +143,366 @@ export function createRegionalPlaceProvider() {
|
||||
export const fetchRegionalPlace = createRegionalPlaceProvider();
|
||||
|
||||
/**
|
||||
* Construct the forward search adapter with a trusted endpoint.
|
||||
* Snap a `"swLat,swLng|neLat,neLng"` view to a coarse grid so small camera
|
||||
* moves reuse one cache entry. The viewbox is a soft preference upstream.
|
||||
*/
|
||||
function coarseBounds(bounds) {
|
||||
// Validate the original hint before snapping or clamping can disguise a
|
||||
// wrapped, inverted or out-of-range box as an unrelated valid viewbox.
|
||||
if (!nominatimViewboxFromBounds(bounds)) return null;
|
||||
const match = String(bounds || '')
|
||||
.trim()
|
||||
.split('|')
|
||||
.map((corner) => corner.split(',').map(Number));
|
||||
if (match.length !== 2 || match.some((c) => c.length !== 2)) return null;
|
||||
const [[swLat, swLng], [neLat, neLng]] = match;
|
||||
if (![swLat, swLng, neLat, neLng].every(Number.isFinite)) return null;
|
||||
const span = Math.max(neLat - swLat, neLng - swLng, 0.01);
|
||||
const step = 2 ** Math.ceil(Math.log2(span));
|
||||
const cLat = Math.round((swLat + neLat) / 2 / (step / 4)) * (step / 4);
|
||||
const cLng = Math.round((swLng + neLng) / 2 / (step / 4)) * (step / 4);
|
||||
const clamp = (v, limit) => Math.max(-limit, Math.min(limit, v));
|
||||
const fix = (v) => Number(v.toFixed(4));
|
||||
return `${fix(clamp(cLat - step / 2, 90))},${fix(clamp(cLng - step / 2, 180))}|${fix(clamp(cLat + step / 2, 90))},${fix(clamp(cLng + step / 2, 180))}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct the forward search adapter over the shared gate.
|
||||
*
|
||||
* The policy asks that results be cached, and warns that a client repeating the
|
||||
* same query may be treated as faulty, so an answer is remembered and identical
|
||||
* searches already in flight share one upstream call rather than queueing
|
||||
* behind each other.
|
||||
* The policy asks that results be cached, and warns that a client repeating
|
||||
* the same query may be treated as faulty, so answers are kept (30 days for a
|
||||
* hit, 24 hours for a genuine no-result; a failure is never kept) and
|
||||
* identical searches already in flight share one upstream call.
|
||||
*/
|
||||
export function createNominatimSearchProvider({
|
||||
endpoint = 'https://nominatim.openstreetmap.org/search',
|
||||
requestJson = fetchRegionalJson,
|
||||
gate = () => sharedNominatimGate(),
|
||||
cache = createNominatimCache(),
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
|
||||
const trimCache = () => {
|
||||
while (cache.size > NOMINATIM_SEARCH_MAX_CACHE) {
|
||||
const oldest = cache.keys().next().value;
|
||||
if (oldest === undefined) break;
|
||||
cache.delete(oldest);
|
||||
}
|
||||
};
|
||||
const currentGate = () => (typeof gate === 'function' ? gate() : gate);
|
||||
|
||||
return async function fetchNominatimSearch(query, bounds, { signal } = {}) {
|
||||
const cacheKey = `${query.toLowerCase()}|${bounds || ''}`;
|
||||
const cached = cache.get(cacheKey);
|
||||
if (cached && Date.now() - cached.cachedAt <= NOMINATIM_SEARCH_CACHE_MS) {
|
||||
return { ...cached.payload, cached: true };
|
||||
}
|
||||
const { promise } = coalesceProxyRequest(inFlight, cacheKey, async () => {
|
||||
const params = new URLSearchParams({
|
||||
format: 'jsonv2',
|
||||
q: query,
|
||||
addressdetails: '1',
|
||||
limit: '1',
|
||||
'accept-language': 'en',
|
||||
});
|
||||
const viewbox = nominatimViewboxFromBounds(bounds);
|
||||
if (viewbox) params.set('viewbox', viewbox);
|
||||
const rows = await enqueueNominatim(
|
||||
() =>
|
||||
requestJson(`${endpoint}?${params}`, {
|
||||
headers: NOMINATIM_HEADERS,
|
||||
redirect: 'error',
|
||||
}),
|
||||
{ bounded: true, signal },
|
||||
);
|
||||
const result = nominatimToGeocodeResult(
|
||||
Array.isArray(rows) ? rows[0] : null,
|
||||
);
|
||||
const payload = result
|
||||
? { status: 'OK', results: [result] }
|
||||
: { status: 'ZERO_RESULTS', results: [] };
|
||||
cache.set(cacheKey, { payload, cachedAt: Date.now() });
|
||||
trimCache();
|
||||
return payload;
|
||||
const coarse = coarseBounds(bounds);
|
||||
const params = new URLSearchParams({
|
||||
format: 'jsonv2',
|
||||
q: upstreamQuery(query),
|
||||
addressdetails: '1',
|
||||
limit: '1',
|
||||
'accept-language': 'en',
|
||||
});
|
||||
return await promise;
|
||||
const viewbox = nominatimViewboxFromBounds(coarse);
|
||||
if (viewbox) params.set('viewbox', viewbox);
|
||||
const gateNow = currentGate();
|
||||
const cacheKey = requestKey(gateNow, 'search', params);
|
||||
const cached = await cache.get(cacheKey);
|
||||
if (cached) return { ...cached.payload, cached: true };
|
||||
return await subscribeSharedRequest(
|
||||
inFlight,
|
||||
cacheKey,
|
||||
signal,
|
||||
async (sharedSignal) => {
|
||||
const rows = await gateNow.requestJson(
|
||||
(endpoint) => `${endpoint}?${params}`,
|
||||
{ signal: sharedSignal },
|
||||
);
|
||||
// Only a well-formed empty list is "no such place"; anything else is a
|
||||
// failure and is never cached.
|
||||
if (!Array.isArray(rows)) throw malformed();
|
||||
const result = nominatimToGeocodeResult(rows[0] ?? null);
|
||||
if (rows.length && !result) throw malformed();
|
||||
const payload = result
|
||||
? { status: 'OK', results: [result] }
|
||||
: { status: 'ZERO_RESULTS', results: [] };
|
||||
await cache.set(
|
||||
cacheKey,
|
||||
payload,
|
||||
result
|
||||
? NOMINATIM_CACHE_TTL_MS.place
|
||||
: NOMINATIM_CACHE_TTL_MS.notFound,
|
||||
);
|
||||
return payload;
|
||||
},
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct the outline lookup: one guarded search with polygons for one
|
||||
* explicit ask, accepting only results whose class suits the ask kind.
|
||||
* Every accepted result is kept (90 days; genuine no-results 24 hours), and
|
||||
* each ask is answered with the one holding its anchor, so a same-name place
|
||||
* elsewhere neither answers nor repeats from the cache.
|
||||
*/
|
||||
export function createNominatimOutlineProvider({
|
||||
gate = () => sharedNominatimGate(),
|
||||
cache = createNominatimCache(),
|
||||
} = {}) {
|
||||
const inFlight = new Map();
|
||||
const currentGate = () => (typeof gate === 'function' ? gate() : gate);
|
||||
|
||||
const answer = (payload, anchorLat, anchorLon, cached) => ({
|
||||
status: payload.status,
|
||||
outline: outlineAt(payload.outlines, anchorLat, anchorLon, payload.ranks),
|
||||
...(payload.skipped ? { skipped: payload.skipped } : {}),
|
||||
...(cached ? { cached: true } : {}),
|
||||
});
|
||||
|
||||
return async function fetchNominatimOutline(
|
||||
{ query, kind, lat, lon, anchorLat = null, anchorLon = null },
|
||||
{ signal } = {},
|
||||
) {
|
||||
const bias = outlineBias(kind, lat, lon);
|
||||
const sent = upstreamQuery(query);
|
||||
const params = new URLSearchParams({
|
||||
format: 'jsonv2',
|
||||
q: sent,
|
||||
limit: String(OUTLINE_RESULT_LIMIT),
|
||||
polygon_geojson: '1',
|
||||
polygon_threshold: String(OUTLINE_POLYGON_THRESHOLD[kind]),
|
||||
namedetails: '1',
|
||||
'accept-language': 'en',
|
||||
});
|
||||
if (bias) params.set('viewbox', bias.viewbox);
|
||||
const gateNow = currentGate();
|
||||
const cacheKey = requestKey(gateNow, `outlines-${kind}`, params);
|
||||
const cached = await cache.get(cacheKey);
|
||||
if (cached) return answer(cached.payload, anchorLat, anchorLon, true);
|
||||
const payload = await subscribeSharedRequest(
|
||||
inFlight,
|
||||
cacheKey,
|
||||
signal,
|
||||
async (sharedSignal) => {
|
||||
const rows = await gateNow.requestJson(
|
||||
(endpoint) => `${endpoint}?${params}`,
|
||||
{ signal: sharedSignal, maxBytes: OUTLINE_MAX_RESPONSE_BYTES },
|
||||
);
|
||||
if (!Array.isArray(rows)) throw malformed();
|
||||
const { outlines, ranks, skipped } = selectOutlineResult(rows, {
|
||||
kind,
|
||||
query: sent,
|
||||
});
|
||||
const found = outlines.length
|
||||
? { status: 'OK', outlines, ranks }
|
||||
: { status: 'ZERO_RESULTS', outlines: [], skipped };
|
||||
await cache.set(
|
||||
cacheKey,
|
||||
found,
|
||||
outlines.length
|
||||
? NOMINATIM_CACHE_TTL_MS.outline
|
||||
: NOMINATIM_CACHE_TTL_MS.notFound,
|
||||
);
|
||||
return found;
|
||||
},
|
||||
);
|
||||
return answer(payload, anchorLat, anchorLon, false);
|
||||
};
|
||||
}
|
||||
|
||||
export const fetchNominatimSearch = createNominatimSearchProvider();
|
||||
|
||||
/** Vite plugin: last-resort place search over the public Nominatim instance. */
|
||||
export function geocodeProxy({ search = fetchNominatimSearch } = {}) {
|
||||
const limiter = makeRateLimiter({
|
||||
function sendJson(res, status, body, headers = {}) {
|
||||
if (res.writableEnded) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...headers,
|
||||
});
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
/** Map a gate refusal to the route's answer. Never "not found". */
|
||||
function sendFailure(res, error, what) {
|
||||
if (error?.code === 'NOMINATIM_DISABLED') {
|
||||
sendJson(res, 503, {
|
||||
error: `${what} is not configured`,
|
||||
code: 'NOMINATIM_DISABLED',
|
||||
retryable: false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (error?.code === 'NOMINATIM_DAILY_CAP') {
|
||||
sendJson(
|
||||
res,
|
||||
429,
|
||||
{
|
||||
error: `${what} daily allowance is used`,
|
||||
code: 'NOMINATIM_DAILY_CAP',
|
||||
retryable: false,
|
||||
},
|
||||
{ 'Retry-After': '3600' },
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (isNominatimBusy(error)) {
|
||||
const seconds = Math.max(5, Math.ceil((error.retryAfterMs || 0) / 1000));
|
||||
sendJson(
|
||||
res,
|
||||
429,
|
||||
{ error: `${what} is busy`, code: error.code },
|
||||
{ 'Retry-After': String(seconds) },
|
||||
);
|
||||
return;
|
||||
}
|
||||
sendJson(res, 503, {
|
||||
error: `${what} is temporarily unavailable`,
|
||||
code: 'NOMINATIM_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Vite plugin: last-resort place search (`/api/geocode`) and voice outlines
|
||||
* (`/api/geocode/outline`) over the configured Nominatim endpoint.
|
||||
*
|
||||
* `storageDir` turns on the disk cache and the persisted daily count; the
|
||||
* application composition supplies it, tests and embedders may omit it.
|
||||
*/
|
||||
export function geocodeProxy({
|
||||
search,
|
||||
outline,
|
||||
storageDir = null,
|
||||
gate = null,
|
||||
} = {}) {
|
||||
const currentGate = () => gate || sharedNominatimGate();
|
||||
if (storageDir) {
|
||||
// The shared count and pause live beside, never inside, the answer cache,
|
||||
// so no cache sweep can reset them.
|
||||
currentGate().persistUsage(path.join(storageDir, 'state.json'));
|
||||
const diskCache = createNominatimCache({
|
||||
dir: path.join(storageDir, 'answers'),
|
||||
});
|
||||
void diskCache.sweep();
|
||||
search ||= createNominatimSearchProvider({
|
||||
gate: currentGate,
|
||||
cache: diskCache,
|
||||
});
|
||||
outline ||= createNominatimOutlineProvider({
|
||||
gate: currentGate,
|
||||
cache: diskCache,
|
||||
});
|
||||
}
|
||||
search ||= gate
|
||||
? createNominatimSearchProvider({ gate })
|
||||
: fetchNominatimSearch;
|
||||
outline ||= createNominatimOutlineProvider({ gate: currentGate });
|
||||
|
||||
const searchLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 30,
|
||||
globalMax: 90,
|
||||
});
|
||||
// Outlines are explicit voice asks: a person cannot make many a minute.
|
||||
const outlineLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 12,
|
||||
globalMax: 30,
|
||||
});
|
||||
|
||||
/** Common front door: method, same-site, rate limit, abandonment signal. */
|
||||
function admit(req, res, limiter) {
|
||||
if (admitSameSite(req, res)) return null;
|
||||
if (req.method !== 'GET') {
|
||||
sendJson(res, 405, { error: 'Method Not Allowed' });
|
||||
return null;
|
||||
}
|
||||
if (!limiter(clientKey(req))) {
|
||||
sendJson(
|
||||
res,
|
||||
429,
|
||||
{ error: 'Rate limit exceeded' },
|
||||
{ 'Retry-After': '10' },
|
||||
);
|
||||
return null;
|
||||
}
|
||||
// A browser that gave up is no longer waiting; the queue reads this
|
||||
// before spending its slot.
|
||||
const abandoned = new AbortController();
|
||||
req.on?.('aborted', () => abandoned.abort());
|
||||
res.on?.('close', () => abandoned.abort());
|
||||
return abandoned.signal;
|
||||
}
|
||||
|
||||
async function handleOutline(req, res, url) {
|
||||
const signal = admit(req, res, outlineLimiter);
|
||||
if (!signal) return;
|
||||
const query = String(url.searchParams.get('q') || '').trim();
|
||||
const kind = String(url.searchParams.get('kind') || '');
|
||||
if (
|
||||
!query ||
|
||||
query.length > NOMINATIM_SEARCH_MAX_QUERY ||
|
||||
!OUTLINE_KINDS.includes(kind)
|
||||
) {
|
||||
sendJson(res, 400, {
|
||||
error:
|
||||
'A place query of 1-200 characters and a known kind are required',
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const payload = await outline(
|
||||
{
|
||||
query,
|
||||
kind,
|
||||
lat: url.searchParams.get('lat'),
|
||||
lon: url.searchParams.get('lon'),
|
||||
anchorLat: coordinateParam(url.searchParams.get('alat')),
|
||||
anchorLon: coordinateParam(url.searchParams.get('alon')),
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
sendJson(
|
||||
res,
|
||||
200,
|
||||
{ status: payload.status, outline: payload.outline || null },
|
||||
{
|
||||
'Cache-Control': payload.cached ? 'private, max-age=300' : 'no-store',
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
sendFailure(res, error, 'Outline lookup');
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSearch(req, res, url) {
|
||||
const signal = admit(req, res, searchLimiter);
|
||||
if (!signal) return;
|
||||
const query = String(url.searchParams.get('q') || '').trim();
|
||||
if (!query || query.length > NOMINATIM_SEARCH_MAX_QUERY) {
|
||||
sendJson(res, 400, {
|
||||
error: 'A place query of 1-200 characters is required',
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const payload = await search(query, url.searchParams.get('bounds'), {
|
||||
signal,
|
||||
});
|
||||
sendJson(
|
||||
res,
|
||||
200,
|
||||
{ status: payload.status, results: payload.results },
|
||||
{
|
||||
'Cache-Control': payload.cached ? 'private, max-age=60' : 'no-store',
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
sendFailure(res, error, 'Place search');
|
||||
}
|
||||
}
|
||||
|
||||
function install(middlewares) {
|
||||
middlewares.use('/api/geocode', async (req, res) => {
|
||||
if (req.method !== 'GET') {
|
||||
res.writeHead(405, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
if (!limiter(clientKey(req))) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '10',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
middlewares.use('/api/geocode', async (req, res, next) => {
|
||||
const url = new URL(req.url || '', 'http://localhost');
|
||||
const query = String(url.searchParams.get('q') || '').trim();
|
||||
if (!query || query.length > NOMINATIM_SEARCH_MAX_QUERY) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'A place query of 1-200 characters is required',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
// A browser that gave up is no longer waiting; the queue reads this
|
||||
// before spending its slot.
|
||||
const abandoned = new AbortController();
|
||||
req.on?.('aborted', () => abandoned.abort());
|
||||
res.on?.('close', () => abandoned.abort());
|
||||
try {
|
||||
const payload = await search(query, url.searchParams.get('bounds'), {
|
||||
signal: abandoned.signal,
|
||||
// Capability probe, like /api/overpass/status: clients skip lookups a
|
||||
// server without a Nominatim endpoint could only refuse.
|
||||
if (url.pathname === '/status' && req.method === 'GET')
|
||||
return sendJson(res, 200, {
|
||||
configured: Boolean(currentGate().settings()?.endpoint),
|
||||
});
|
||||
if (res.writableEnded) return;
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': payload.cached ? 'public, max-age=60' : 'no-store',
|
||||
});
|
||||
res.end(
|
||||
JSON.stringify({ status: payload.status, results: payload.results }),
|
||||
);
|
||||
} catch (error) {
|
||||
if (res.writableEnded) return;
|
||||
const busy =
|
||||
error?.code === 'NOMINATIM_QUEUE_FULL' ||
|
||||
error?.code === 'NOMINATIM_ABANDONED';
|
||||
res.writeHead(busy ? 429 : 503, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(busy ? { 'Retry-After': '5' } : {}),
|
||||
});
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: busy
|
||||
? 'Place search is busy'
|
||||
: 'Place search is temporarily unavailable',
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (url.pathname === '/outline') return handleOutline(req, res, url);
|
||||
if (url.pathname === '/' || url.pathname === '')
|
||||
return handleSearch(req, res, url);
|
||||
if (typeof next === 'function') return next();
|
||||
sendJson(res, 404, { error: 'Not Found' });
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import {
|
||||
EMODNET_WFS_URL,
|
||||
WIKIDATA_QUERY,
|
||||
mergeShipwreckRecords,
|
||||
normalizeEmodnetFeatures,
|
||||
normalizeWikidataResults,
|
||||
} from '../../src/layers/shipwrecks/records.js';
|
||||
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
|
||||
import { clientKey, makeRateLimiter } from './common/rate-limit.js';
|
||||
|
||||
const MIB = 1024 * 1024;
|
||||
/** EMODnet republishes the wreck layer about once a year; Wikidata daily at most. */
|
||||
const REFRESH_MS = 24 * 60 * 60 * 1000;
|
||||
/** A source's last good records keep being served this long, marked stale. */
|
||||
const STALE_MAX_MS = 7 * REFRESH_MS;
|
||||
/** After a source fails, it is not asked again for this long. */
|
||||
const RETRY_MS = 15 * 60 * 1000;
|
||||
/** A refresh with fewer than this share of the last good records is refused. */
|
||||
const MIN_KEEP_RATIO = 0.5;
|
||||
// Only the columns the card uses; ship_char/obj_desc histories can name people
|
||||
// and are not read beyond a short vessel type.
|
||||
const EMODNET_PROPERTIES = [
|
||||
'source_id',
|
||||
'country',
|
||||
'name',
|
||||
'least_depth',
|
||||
'depth_prec',
|
||||
'loc_prec',
|
||||
'sink_context',
|
||||
'sink_yr',
|
||||
'dating',
|
||||
'period',
|
||||
'statutory',
|
||||
'ship_char',
|
||||
'obj_type',
|
||||
'artefacts',
|
||||
'source_inf',
|
||||
'website1',
|
||||
'point_info',
|
||||
'the_geom',
|
||||
].join(',');
|
||||
const EMODNET_URL = `${EMODNET_WFS_URL}&propertyName=${EMODNET_PROPERTIES}`;
|
||||
const WIKIDATA_URL =
|
||||
'https://query.wikidata.org/sparql?format=json&query=' +
|
||||
encodeURIComponent(WIKIDATA_QUERY);
|
||||
// Wikimedia asks automated clients to name themselves.
|
||||
const USER_AGENT =
|
||||
"God's Eye View shipwrecks layer (https://github.com/bilawalsidhu/gods-eye-view)";
|
||||
|
||||
/** Fixed-origin, cached EMODnet + Wikidata shipwreck snapshot for dev and preview. */
|
||||
export function shipwrecksProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
} = {}) {
|
||||
// Each source keeps its last good records on its own, so one failing (or
|
||||
// answering with nothing usable) never removes the other's wrecks or its
|
||||
// own earlier ones. `retryAt` holds a failed source back for RETRY_MS.
|
||||
const sources = {
|
||||
emodnet: { load: fetchEmodnet, good: null, retryAt: 0 },
|
||||
wikidata: { load: fetchWikidata, good: null, retryAt: 0 },
|
||||
};
|
||||
let snapshot = null;
|
||||
const inFlight = new Map();
|
||||
const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 });
|
||||
|
||||
async function upstreamJson(url, maxBytes) {
|
||||
const signal = AbortSignal.timeout(90_000);
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' },
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('upstream_unavailable');
|
||||
}
|
||||
return readResponseJsonCapped(response, maxBytes, signal);
|
||||
}
|
||||
|
||||
async function fetchEmodnet() {
|
||||
const records = normalizeEmodnetFeatures(
|
||||
await upstreamJson(EMODNET_URL, 32 * MIB),
|
||||
);
|
||||
if (!records.length) throw new Error('invalid_emodnet_layer');
|
||||
return records;
|
||||
}
|
||||
|
||||
async function fetchWikidata() {
|
||||
const payload = await upstreamJson(WIKIDATA_URL, 16 * MIB);
|
||||
if (!Array.isArray(payload?.results?.bindings))
|
||||
throw new Error('invalid_wikidata_results');
|
||||
const records = normalizeWikidataResults(payload);
|
||||
// Wikidata lists over a thousand wrecks; none at all is a broken answer.
|
||||
if (!records.length) throw new Error('invalid_wikidata_results');
|
||||
return records;
|
||||
}
|
||||
|
||||
const fresh = (state, at) =>
|
||||
state.good !== null && at - state.good.fetchedAt < REFRESH_MS;
|
||||
const usable = (state, at) =>
|
||||
state.good !== null && at - state.good.fetchedAt <= STALE_MAX_MS;
|
||||
const due = (state, at) => !fresh(state, at) && at >= state.retryAt;
|
||||
|
||||
/** Refresh the sources that are due; failures keep their last good records. */
|
||||
async function refresh() {
|
||||
const at = now();
|
||||
const names = Object.keys(sources).filter((name) => due(sources[name], at));
|
||||
const results = await Promise.allSettled(
|
||||
names.map((name) => sources[name].load()),
|
||||
);
|
||||
const done = now();
|
||||
results.forEach((result, i) => {
|
||||
const state = sources[names[i]];
|
||||
// A valid answer far smaller than the last good set (a truncated or
|
||||
// half-built upstream) is a failure too, until the old set expires.
|
||||
const shrunk =
|
||||
result.status === 'fulfilled' &&
|
||||
usable(state, done) &&
|
||||
result.value.length < state.good.records.length * MIN_KEEP_RATIO;
|
||||
if (result.status === 'fulfilled' && !shrunk)
|
||||
state.good = { records: result.value, fetchedAt: done };
|
||||
else state.retryAt = done + RETRY_MS;
|
||||
if (shrunk)
|
||||
console.warn(
|
||||
`[shipwrecks] ${names[i]} refresh refused: ${result.value.length} ` +
|
||||
`records against ${state.good.records.length} last good`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** The served snapshot: every source with usable records, merged once. */
|
||||
function compose(at) {
|
||||
const sourceFlags = {};
|
||||
// How old each source's records are, on this clock, so the page can
|
||||
// expire them by its own clock without comparing the two.
|
||||
const ages = {};
|
||||
let stale = false;
|
||||
let fetchedAt = null;
|
||||
for (const [name, state] of Object.entries(sources)) {
|
||||
sourceFlags[name] = usable(state, at);
|
||||
ages[name] = sourceFlags[name] ? at - state.good.fetchedAt : null;
|
||||
if (!sourceFlags[name]) continue;
|
||||
if (!fresh(state, at)) stale = true;
|
||||
fetchedAt = Math.min(fetchedAt ?? Infinity, state.good.fetchedAt);
|
||||
}
|
||||
if (!sourceFlags.emodnet && !sourceFlags.wikidata) return null;
|
||||
const key = `${sourceFlags.emodnet && sources.emodnet.good.fetchedAt}:${
|
||||
sourceFlags.wikidata && sources.wikidata.good.fetchedAt
|
||||
}`;
|
||||
if (snapshot?.key !== key) {
|
||||
// The merge writes EMODnet facts into Wikidata records: work on copies
|
||||
// so a source's kept records stay as fetched.
|
||||
const copy = (name) =>
|
||||
sourceFlags[name]
|
||||
? sources[name].good.records.map((record) => ({ ...record }))
|
||||
: [];
|
||||
snapshot = {
|
||||
key,
|
||||
records: mergeShipwreckRecords(copy('emodnet'), copy('wikidata')),
|
||||
};
|
||||
}
|
||||
return {
|
||||
value: {
|
||||
fetchedAt,
|
||||
version: key,
|
||||
sources: sourceFlags,
|
||||
ages,
|
||||
maxAgeMs: STALE_MAX_MS,
|
||||
records: snapshot.records,
|
||||
},
|
||||
stale,
|
||||
};
|
||||
}
|
||||
|
||||
async function acquire() {
|
||||
const at = now();
|
||||
if (Object.values(sources).some((state) => due(state, at)))
|
||||
await coalesceProxyRequest(inFlight, 'refresh', refresh).promise;
|
||||
const composed = compose(now());
|
||||
if (!composed) throw new Error('shipwrecks_unavailable');
|
||||
return composed;
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (status, value, stale = false) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...(status === 429 ? { 'Retry-After': '60' } : {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
if (path !== '/' && path !== '')
|
||||
return json(404, { error: 'unknown_route' });
|
||||
if (!allow(clientKey(req))) return json(429, { error: 'rate_limited' });
|
||||
try {
|
||||
const { value, stale } = await acquire();
|
||||
json(200, stale ? { ...value, stale: true } : value, stale);
|
||||
} catch {
|
||||
json(502, { error: 'shipwrecks_unavailable' });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'shipwrecks',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/shipwrecks', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/shipwrecks', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
|
||||
import {
|
||||
celestrakSatcatUrl,
|
||||
celestrakTleUrl,
|
||||
} from '../../../src/data/spaceProviderRequests.js';
|
||||
import { providerCooldownMs } from './cooldown.js';
|
||||
import { admitSameSiteKeyless } from '../common/same-site.js';
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
@@ -20,79 +25,166 @@ import { readResponseTextCapped } from '../common/http.js';
|
||||
* 6 h; on upstream failure the freshest stale copy is served (a stale TLE
|
||||
* beats an empty satellites layer). Pattern mirrors openSkyProxy's
|
||||
* cache+serve-stale. Adapted from skylight's TleStore (MIT).
|
||||
*
|
||||
* The same plugin serves CelesTrak SATCAT records for the satellites layer's
|
||||
* groups at `/api/celestrak-satcat/<group>` (object type, operational status
|
||||
* and owner, which file each object under a class). The catalog changes
|
||||
* slowly, so SATCAT is cached for a day. CelesTrak throttles the client, not
|
||||
* the endpoint, so a 403 or 429 holds both routes off; any other failure
|
||||
* holds off only the route that failed, so a bad SATCAT answer never stops
|
||||
* TLE refreshes.
|
||||
*/
|
||||
export function celestrakProxy() {
|
||||
const TLE_TTL_MS = 6 * 3600_000;
|
||||
const SATCAT_TTL_MS = 24 * 3600_000;
|
||||
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
|
||||
const mem = new Map(); // group -> { at: epochMs, body: string }
|
||||
const inflight = new Map(); // group -> Promise<{at, body}|null>
|
||||
|
||||
const diskPath = (group) => path.join(CACHE_DIR, `celestrak-${group}.json`);
|
||||
|
||||
async function readDisk(group) {
|
||||
try {
|
||||
const parsed = JSON.parse(await fsp.readFile(diskPath(group), 'utf8'));
|
||||
if (typeof parsed?.body === 'string' && Number.isFinite(parsed?.at))
|
||||
return parsed;
|
||||
} catch {
|
||||
/* no disk cache yet */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function writeDisk(group, entry) {
|
||||
try {
|
||||
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
||||
await fsp.writeFile(diskPath(group), JSON.stringify(entry), 'utf8');
|
||||
} catch (err) {
|
||||
console.warn('[celestrak-proxy] cache write failed');
|
||||
}
|
||||
}
|
||||
/** The satellites layer's core groups (src/layers/satellites/policy.js). */
|
||||
const coreGroups = [
|
||||
'stations',
|
||||
'visual',
|
||||
'gps-ops',
|
||||
'glo-ops',
|
||||
'galileo',
|
||||
'geo',
|
||||
'resource',
|
||||
'weather',
|
||||
'planet',
|
||||
'sar',
|
||||
'science',
|
||||
];
|
||||
const feeds = [
|
||||
{
|
||||
route: '/api/celestrak',
|
||||
ttlMs: TLE_TTL_MS,
|
||||
cacheHeader: 'x-tle-cache',
|
||||
cachedAtHeader: 'X-TLE-Cached-At',
|
||||
diskName: (group) => `celestrak-${group}.json`,
|
||||
upstreamUrl: celestrakTleUrl,
|
||||
groups: new Set([
|
||||
...coreGroups,
|
||||
'starlink',
|
||||
// Space Missions (`getActiveTle`) reads the full active catalog.
|
||||
'active',
|
||||
]),
|
||||
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
|
||||
validate: (body) => /^1 /m.test(body),
|
||||
invalid: 'no TLE lines in response',
|
||||
},
|
||||
{
|
||||
route: '/api/celestrak-satcat',
|
||||
ttlMs: SATCAT_TTL_MS,
|
||||
cacheHeader: 'x-satcat-cache',
|
||||
cachedAtHeader: 'X-SATCAT-Cached-At',
|
||||
diskName: (group) => `celestrak-satcat-${group}.json`,
|
||||
upstreamUrl: celestrakSatcatUrl,
|
||||
groups: new Set(coreGroups),
|
||||
// A real answer, even for an empty group, starts with the CSV header.
|
||||
validate: (body) => /^OBJECT_NAME,/.test(body),
|
||||
invalid: 'no SATCAT header in response',
|
||||
},
|
||||
];
|
||||
// CelesTrak throttling (403/429) is per client, so it holds every feed
|
||||
// off; any other failure holds off only the feed that failed, so a bad
|
||||
// SATCAT answer never blocks the TLEs the layer cannot do without.
|
||||
let hostCooldownUntil = 0;
|
||||
|
||||
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
|
||||
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
async function fetchUpstream(group) {
|
||||
const url = celestrakTleUrl(group);
|
||||
const signal = AbortSignal.timeout(20000);
|
||||
const res = await fetch(url.toString(), {
|
||||
signal,
|
||||
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
|
||||
// descriptive User-Agent with a contact point.
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
||||
},
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const body = await readResponseTextCapped(
|
||||
res,
|
||||
CELESTRAK_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
|
||||
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
|
||||
return { at: Date.now(), body };
|
||||
}
|
||||
function createFeedHandler(feed) {
|
||||
let feedCooldownUntil = 0;
|
||||
const cooldownUntil = () => Math.max(hostCooldownUntil, feedCooldownUntil);
|
||||
const mem = new Map(); // group -> { at: epochMs, body: string }
|
||||
const inflight = new Map(); // group -> Promise<{at, body}|null>
|
||||
const diskPath = (group) => path.join(CACHE_DIR, feed.diskName(group));
|
||||
|
||||
const installMiddleware = (server) => {
|
||||
server.middlewares.use('/api/celestrak', async (req, res) => {
|
||||
async function readDisk(group) {
|
||||
try {
|
||||
const parsed = JSON.parse(await fsp.readFile(diskPath(group), 'utf8'));
|
||||
if (typeof parsed?.body === 'string' && Number.isFinite(parsed?.at))
|
||||
return parsed;
|
||||
} catch {
|
||||
/* no disk cache yet */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function writeDisk(group, entry) {
|
||||
try {
|
||||
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
||||
await fsp.writeFile(diskPath(group), JSON.stringify(entry), 'utf8');
|
||||
} catch (err) {
|
||||
console.warn('[celestrak-proxy] cache write failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchUpstream(group) {
|
||||
const url = feed.upstreamUrl(group);
|
||||
const signal = AbortSignal.timeout(20000);
|
||||
const res = await fetch(url.toString(), {
|
||||
signal,
|
||||
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
|
||||
// descriptive User-Agent with a contact point.
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
// Group refreshes run in parallel: a later failure without Retry-After
|
||||
// must never shorten a longer deadline an earlier one already set.
|
||||
const until = Date.now() + providerCooldownMs(res);
|
||||
if (res.status === 403 || res.status === 429)
|
||||
hostCooldownUntil = Math.max(hostCooldownUntil, until);
|
||||
else feedCooldownUntil = Math.max(feedCooldownUntil, until);
|
||||
void res.body?.cancel().catch(() => {});
|
||||
throw new Error(`HTTP ${res.status}`);
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
res,
|
||||
CELESTRAK_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
if (!feed.validate(body)) throw new Error(feed.invalid);
|
||||
return { at: Date.now(), body };
|
||||
}
|
||||
|
||||
return async (req, res) => {
|
||||
const group = String(req.url || '')
|
||||
.replace(/^\//, '')
|
||||
.split('?')[0];
|
||||
if (!/^[a-z0-9-]+$/i.test(group)) {
|
||||
// Keyless and needed to render the satellites layer: refuse cross-site
|
||||
// browsers but keep the documented reverse-proxy deployment working.
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
if (req.method !== 'GET') {
|
||||
res.writeHead(405);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (!feed.groups.has(group)) {
|
||||
res.writeHead(400, { 'Content-Type': 'text/plain' });
|
||||
res.end('invalid group');
|
||||
return;
|
||||
}
|
||||
const send = (status, body, cacheStatus) => {
|
||||
const send = (status, body, cacheStatus, entry) => {
|
||||
// Guard against a double-send (e.g. a throw AFTER a response already
|
||||
// went out routing into the catch's send): writeHead after headersSent
|
||||
// throws "Cannot set headers after they are sent".
|
||||
if (res.headersSent) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'text/plain',
|
||||
'x-tle-cache': cacheStatus,
|
||||
[feed.cacheHeader]: cacheStatus,
|
||||
'Cache-Control': 'no-store',
|
||||
...(entry
|
||||
? { [feed.cachedAtHeader]: new Date(entry.at).toISOString() }
|
||||
: {}),
|
||||
...(cooldownUntil() > 0 && Date.now() < cooldownUntil()
|
||||
? {
|
||||
'Retry-After': String(
|
||||
Math.ceil((cooldownUntil() - Date.now()) / 1000),
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
res.end(body);
|
||||
};
|
||||
@@ -103,8 +195,17 @@ export function celestrakProxy() {
|
||||
entry = await readDisk(group);
|
||||
if (entry) mem.set(group, entry);
|
||||
}
|
||||
if (entry && now - entry.at < TLE_TTL_MS) {
|
||||
send(200, entry.body, 'HIT');
|
||||
if (entry && now - entry.at < feed.ttlMs) {
|
||||
send(200, entry.body, 'HIT', entry);
|
||||
return;
|
||||
}
|
||||
if (Date.now() < cooldownUntil()) {
|
||||
send(
|
||||
entry ? 200 : 502,
|
||||
entry?.body || 'celestrak upstream cooling down',
|
||||
entry ? 'STALE-ERROR' : 'NONE',
|
||||
entry,
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Stale or missing → refresh, single-flight per group.
|
||||
@@ -118,6 +219,8 @@ export function celestrakProxy() {
|
||||
return fresh;
|
||||
})
|
||||
.catch((err) => {
|
||||
if (Date.now() >= feedCooldownUntil)
|
||||
feedCooldownUntil = Date.now() + 60000;
|
||||
console.warn(
|
||||
'[celestrak-proxy] refresh failed — serving cache if any',
|
||||
);
|
||||
@@ -128,9 +231,9 @@ export function celestrakProxy() {
|
||||
}
|
||||
const fresh = await inflight.get(group);
|
||||
if (fresh) {
|
||||
send(200, fresh.body, 'MISS');
|
||||
send(200, fresh.body, 'MISS', fresh);
|
||||
} else if (entry) {
|
||||
send(200, entry.body, 'STALE-ERROR'); // upstream down — stale beats empty
|
||||
send(200, entry.body, 'STALE-ERROR', entry); // upstream down — stale beats empty
|
||||
} else {
|
||||
send(502, 'celestrak fetch failed and no cache available', 'NONE');
|
||||
}
|
||||
@@ -138,7 +241,13 @@ export function celestrakProxy() {
|
||||
console.error('[celestrak-proxy] request failed');
|
||||
send(500, 'celestrak proxy error', 'ERROR');
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
const handlers = feeds.map((feed) => [feed.route, createFeedHandler(feed)]);
|
||||
const installMiddleware = (server) => {
|
||||
for (const [route, handler] of handlers)
|
||||
server.middlewares.use(route, handler);
|
||||
};
|
||||
return {
|
||||
name: 'celestrak-proxy',
|
||||
|
||||
Loaded 100 of 4343 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user