mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
Merge branch 'main' into feat/voice-geometry
Picks up #1308 (Overpass admission), #114 (Sea Marks and Ski Pistes), #1206 (radio country coverage) and #1435. Resolutions: - SECURITY.md: main's rewritten cross-site paragraph, with voice place search listed beside /api/overpass among the keyless map routes. - scripts/format-scope.json: both sides. A merge on main dropped 97 entries (being restored there); keeping them, this PR's 12 and main's 2 new ones keeps every file in format:check. All 1,222 paths exist. - Translation catalogs: main's two new keys and this PR's 47, no overlap. - Layer count 36 (33 + OSM Places + Sea Marks + Ski Pistes); both schema pins re-derived, the first-run length is exactly 30,919 + 292 + 107.
This commit is contained in:
commit
1e12d5ba26
56 files changed
+1834
-209
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { isIPv6 } from 'node:net';
|
||||
import { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
export { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
|
||||
@@ -60,3 +61,46 @@ export function makeCostRateLimiter(envValue, defaultPerMin) {
|
||||
export function clientKey(req) {
|
||||
return String(req.socket?.remoteAddress || 'local');
|
||||
}
|
||||
|
||||
const IPV4_MAPPED_RE = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/i;
|
||||
|
||||
/**
|
||||
* Whether a socket peer address is this machine: IPv4 loopback in any of its
|
||||
* forms (`127.0.0.0/8`, including the IPv4-mapped `::ffff:127.x` a dual-stack
|
||||
* listener reports) or the IPv6 loopback. Non-address keys are not loopback.
|
||||
* @param {string} address - A `clientKey` value.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isLoopbackPeer(address) {
|
||||
const raw = String(address || '');
|
||||
const v4 = IPV4_MAPPED_RE.exec(raw)?.[1] ?? raw;
|
||||
return v4.startsWith('127.') || raw === '::1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate-limit key that survives address rotation, for limiters whose global
|
||||
* backstop everyone shares. IPv6 privacy extensions hand one host a fresh
|
||||
* address at will inside its /64, so keying by the full address lets a single
|
||||
* host mint per-peer quota until the global window closes for everyone else.
|
||||
* IPv6 peers are keyed by their /64 prefix instead, and an IPv4-mapped
|
||||
* address folds to its IPv4 form so one host cannot hold two buckets. Any
|
||||
* other value (IPv4, or the `local` fallback) passes through unchanged.
|
||||
* @param {import('node:http').IncomingMessage} req
|
||||
* @returns {string}
|
||||
*/
|
||||
export function clientNetworkKey(req) {
|
||||
const address = clientKey(req);
|
||||
const mapped = IPV4_MAPPED_RE.exec(address);
|
||||
if (mapped) return mapped[1];
|
||||
const bare = address.replace(/%.*$/, ''); // link-local zone id
|
||||
if (!isIPv6(bare)) return address;
|
||||
const [head, tail = ''] = bare.split('::');
|
||||
const headParts = head ? head.split(':') : [];
|
||||
const tailParts = tail ? tail.split(':') : [];
|
||||
const missing = Math.max(0, 8 - headParts.length - tailParts.length);
|
||||
const hextets = [...headParts, ...Array(missing).fill('0'), ...tailParts];
|
||||
return `${hextets
|
||||
.slice(0, 4)
|
||||
.map((h) => h.toLowerCase().replace(/^0+(?=.)/, ''))
|
||||
.join(':')}::/64`;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { clientNetworkKey, isLoopbackPeer } from './rate-limit.js';
|
||||
|
||||
const req = (remoteAddress) => ({ socket: { remoteAddress } });
|
||||
|
||||
test('clientNetworkKey keys IPv6 peers by /64 and folds IPv4-mapped peers', () => {
|
||||
assert.equal(clientNetworkKey(req('2001:db8:1:2::1')), '2001:db8:1:2::/64');
|
||||
assert.equal(
|
||||
clientNetworkKey(req('2001:0db8:0001:0002:abcd:ef01:2345:6789')),
|
||||
'2001:db8:1:2::/64',
|
||||
);
|
||||
assert.equal(clientNetworkKey(req('2001:DB8::1')), '2001:db8:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('fe80::1%en0')), 'fe80:0:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('::1')), '0:0:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('::ffff:10.0.0.5')), '10.0.0.5');
|
||||
assert.equal(clientNetworkKey(req('10.0.0.5')), '10.0.0.5');
|
||||
assert.equal(clientNetworkKey(req(undefined)), 'local');
|
||||
assert.equal(clientNetworkKey(req('not-an-address')), 'not-an-address');
|
||||
});
|
||||
|
||||
test('isLoopbackPeer recognizes every loopback form and nothing else', () => {
|
||||
for (const address of ['127.0.0.1', '127.9.9.9', '::1', '::ffff:127.0.0.1']) {
|
||||
assert.equal(isLoopbackPeer(address), true, address);
|
||||
}
|
||||
for (const address of [
|
||||
'10.0.0.1',
|
||||
'::ffff:10.0.0.1',
|
||||
'2001:db8::1',
|
||||
'::2',
|
||||
'local',
|
||||
'',
|
||||
undefined,
|
||||
]) {
|
||||
assert.equal(isLoopbackPeer(address), false, String(address));
|
||||
}
|
||||
});
|
||||
@@ -58,7 +58,7 @@ function forwardedProtocol(req) {
|
||||
|
||||
/**
|
||||
* Cross-site gate for keyless routes the map needs in order to render
|
||||
* (`/api/terrain/heights`). It refuses the same cross-site browser shapes as
|
||||
* (`/api/terrain/heights`, `/api/overpass`). It refuses the same cross-site browser shapes as
|
||||
* `admitSameSite` (a `Sec-Fetch-Site` other than same-origin/none, a foreign
|
||||
* or opaque `Origin`) but does not refuse reverse-proxy forwarding headers:
|
||||
* those protect credentials and spend, which a keyless route has neither of,
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import {
|
||||
makeRateLimiter,
|
||||
clientKey,
|
||||
clientNetworkKey,
|
||||
isLoopbackPeer,
|
||||
} from './common/rate-limit.js';
|
||||
import { hasProxySignals } from '../../src/localRequestGate.mjs';
|
||||
import { readRequestBodyCapped } from './common/request.js';
|
||||
import { admitSameSite } from './common/same-site.js';
|
||||
import { admitSameSiteKeyless } from './common/same-site.js';
|
||||
import {
|
||||
OVERPASS_MAX_BODY_BYTES,
|
||||
OVERPASS_MAX_CONCURRENT,
|
||||
@@ -30,10 +36,21 @@ const _overpassInFlight = new Map();
|
||||
let _overpassConcurrent = 0;
|
||||
|
||||
// Bound body-reading and validation work, including malformed requests and
|
||||
// cache hits. Keep the tighter upstream budget below independent of this gate.
|
||||
const _overpassRequestLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 300,
|
||||
// cache hits, before a byte is read. Remote peers share one global window so
|
||||
// address rotation cannot mint admission without bound (IPv6 peers are keyed
|
||||
// by /64 for the same reason). A direct loopback request — the owner's own
|
||||
// browser and the repo QA harnesses, with no forwarding headers — has its own
|
||||
// per-peer window outside that global, so busy LAN peers cannot lock the owner
|
||||
// out of the map. A request relayed by a reverse proxy on this machine also
|
||||
// arrives from loopback, but it is remote traffic: it is admitted from the
|
||||
// remote pool under its own socket-keyed bucket, never from the owner's. The
|
||||
// forwarded client address is not trusted as identity. Only admission is
|
||||
// isolated: the upstream budget below (90/min per key, 300/min globally) is
|
||||
// shared, so an owner request that misses cache can still be refused there.
|
||||
const OVERPASS_ADMISSION_WINDOW = { windowMs: 60_000, max: 300 };
|
||||
const _overpassLoopbackAdmission = makeRateLimiter(OVERPASS_ADMISSION_WINDOW);
|
||||
const _overpassPeerAdmission = makeRateLimiter({
|
||||
...OVERPASS_ADMISSION_WINDOW,
|
||||
globalMax: 900,
|
||||
});
|
||||
|
||||
@@ -43,6 +60,33 @@ const _overpassRateLimiter = makeRateLimiter({
|
||||
globalMax: 300,
|
||||
});
|
||||
|
||||
/**
|
||||
* Classify a request for admission. Direct loopback — a loopback socket with
|
||||
* no forwarding header — is the owner's own pool; everything else, including a
|
||||
* loopback socket that carries forwarding headers (a local reverse proxy), is
|
||||
* remote. A proxied request keeps its socket identity under a `proxied:`
|
||||
* prefix, so it never shares a bucket with the owner's direct requests.
|
||||
* @param {import('http').IncomingMessage} req
|
||||
* @returns {{direct: boolean, key: string}}
|
||||
*/
|
||||
function overpassAdmissionClass(req) {
|
||||
const peer = clientNetworkKey(req);
|
||||
if (!isLoopbackPeer(clientKey(req))) return { direct: false, key: peer };
|
||||
if (hasProxySignals(req.headers || {}))
|
||||
return { direct: false, key: `proxied:${peer}` };
|
||||
return { direct: true, key: peer };
|
||||
}
|
||||
|
||||
/**
|
||||
* Spend one admission slot for this request; false when its bucket is over its
|
||||
* window (or, for remote traffic, when the shared global window is full).
|
||||
* @param {{direct: boolean, key: string}} admission - `overpassAdmissionClass(req)`.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function admitOverpassRequest({ direct, key }) {
|
||||
return (direct ? _overpassLoopbackAdmission : _overpassPeerAdmission)(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a completed Overpass payload to the HTTP response.
|
||||
*
|
||||
@@ -104,8 +148,13 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (admitSameSite(req, res)) return;
|
||||
if (!_overpassRequestLimiter(clientKey(req))) {
|
||||
// Keyless gate: refuses cross-site browser requests but, unlike the
|
||||
// cost-bearing routes, admits the forwarding headers a reverse proxy
|
||||
// adds — the layers that depend on Overpass must keep working for an
|
||||
// instance deployed the way the README documents.
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
const admission = overpassAdmissionClass(req);
|
||||
if (!admitOverpassRequest(admission)) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
@@ -164,7 +213,7 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
// requests; boundary-class queries keep their month-long TTL.
|
||||
readDisk: () =>
|
||||
readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
|
||||
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
|
||||
allowUpstream: () => _overpassRateLimiter(admission.key),
|
||||
loadUpstream: async () => {
|
||||
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT)
|
||||
return {
|
||||
|
||||
@@ -122,12 +122,14 @@ function sanitizeOverpassBody(rawBody) {
|
||||
const boundedSets = new Set();
|
||||
for (let stmt of withoutTags.split(';')) {
|
||||
stmt = stmt.trim();
|
||||
if (!stmt || stmt.startsWith('[') || /^out\b/.test(stmt)) continue;
|
||||
// The settings block (`[out:json][timeout:25]`) was blanked together with
|
||||
// the tag filters, so it arrives here as an empty statement.
|
||||
if (!stmt || /^out\b/.test(stmt)) continue;
|
||||
|
||||
// Strip output-set assignments (NOT input bounds). Bracket tag filters were
|
||||
// removed above so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot be
|
||||
// misread as a spatial bound. Bounds live in (...) / function calls / set
|
||||
// refs, never inside [...], so the probe loses nothing real.
|
||||
// Strip output-set assignments (NOT input bounds). With the tag filters
|
||||
// already gone, a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot
|
||||
// be misread as a spatial bound; bounds live in (...) / function calls /
|
||||
// set refs, never inside [...], so nothing real was lost.
|
||||
const outSets = [];
|
||||
const probe = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
|
||||
outSets.push(name);
|
||||
@@ -166,8 +168,10 @@ function sanitizeOverpassBody(rawBody) {
|
||||
if (bounded) for (const name of outSets) boundedSets.add(name);
|
||||
}
|
||||
|
||||
// Tolerate whitespace around the directive so `[ timeout : N ]` is clamped
|
||||
// rather than passed through as written.
|
||||
const clamped = data.replace(
|
||||
/\[timeout:\s*(\d+)\s*\]/gi,
|
||||
/\[\s*timeout\s*:\s*(\d+)\s*\]/gi,
|
||||
(_, n) =>
|
||||
`[timeout:${Math.min(Number(n) || OVERPASS_MAX_QL_TIMEOUT, OVERPASS_MAX_QL_TIMEOUT)}]`,
|
||||
);
|
||||
|
||||
@@ -3,10 +3,24 @@ import assert from 'node:assert/strict';
|
||||
import { Readable } from 'node:stream';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { overpassProxy } from '../overpass.js';
|
||||
import { _overpassCache } from './cache.js';
|
||||
import { sanitizeOverpassBody } from './query.js';
|
||||
|
||||
const form = (query) => `data=${encodeURIComponent(query)}`;
|
||||
|
||||
// The limiters are module-level, so every test that spends admission mocks a
|
||||
// clock a full day past the previous one in execution order: no test can see
|
||||
// another's window, whatever order (or randomization) they run in. A new
|
||||
// test that reaches admission must call it too: earlier tests leave
|
||||
// timestamps days ahead of the real clock, and those still count.
|
||||
let clockBase = Date.now() + 86_400_000;
|
||||
function freshClock(t) {
|
||||
clockBase += 86_400_000;
|
||||
const clock = { now: clockBase };
|
||||
t.mock.method(Date, 'now', () => clock.now);
|
||||
return clock;
|
||||
}
|
||||
|
||||
test('bracket validation rejects malformed filters without trusting their contents as bounds', () => {
|
||||
for (const query of [
|
||||
'node[',
|
||||
@@ -37,6 +51,15 @@ test('bounded queries preserve quoted brackets, comments, multiple tags and time
|
||||
query.replace('[timeout:999]', '[timeout:30]'),
|
||||
);
|
||||
}
|
||||
// Whitespace inside the directive is clamped too, not passed through.
|
||||
const spaced = sanitizeOverpassBody(
|
||||
form('[out:json][ timeout : 999 ];node(around:20,1,1);out;'),
|
||||
);
|
||||
assert.equal(spaced.ok, true);
|
||||
assert.equal(
|
||||
new URLSearchParams(spaced.body).get('data'),
|
||||
'[out:json][timeout:30];node(around:20,1,1);out;',
|
||||
);
|
||||
});
|
||||
|
||||
test('a burst of maximum-size unterminated filters completes within a bounded process deadline', () => {
|
||||
@@ -126,7 +149,13 @@ for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
{ 'sec-fetch-site': 'cross-site' },
|
||||
{ 'sec-fetch-site': 'same-site' },
|
||||
{ origin: 'null' },
|
||||
{ 'x-forwarded-for': '127.0.0.1' },
|
||||
// Forwarding headers do not launder a cross-site request.
|
||||
{ 'x-forwarded-for': '10.0.0.9', 'sec-fetch-site': 'cross-site' },
|
||||
{
|
||||
'x-forwarded-proto': 'https',
|
||||
origin: 'http://localhost:4173',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
},
|
||||
]) {
|
||||
const response = await call(route, { headers, forbidRead: true });
|
||||
assert.equal(response.statusCode, 403);
|
||||
@@ -136,12 +165,24 @@ for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
});
|
||||
}
|
||||
|
||||
test('same-origin, LAN and non-browser requests still reach validation; GET status stays available', async () => {
|
||||
test('same-origin, LAN, proxied and non-browser requests still reach validation; GET status stays available', async (t) => {
|
||||
freshClock(t);
|
||||
const route = handler();
|
||||
for (const headers of [
|
||||
{},
|
||||
{ origin: 'http://localhost:4173', 'sec-fetch-site': 'same-origin' },
|
||||
{ host: '192.168.1.10:4173', origin: 'http://192.168.1.10:4173' },
|
||||
// The README's reverse-proxy deployment: Overpass is keyless, so the
|
||||
// forwarding headers the proxy adds are admitted (as for terrain heights).
|
||||
{
|
||||
host: 'globe.example.lan',
|
||||
origin: 'https://globe.example.lan',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'x-forwarded-for': '203.0.113.7',
|
||||
'x-forwarded-proto': 'https',
|
||||
'x-forwarded-host': 'globe.example.lan',
|
||||
},
|
||||
{ 'x-forwarded-for': '127.0.0.1' },
|
||||
]) {
|
||||
const response = await call(route, { headers });
|
||||
assert.equal(response.statusCode, 400);
|
||||
@@ -158,8 +199,7 @@ test('same-origin, LAN and non-browser requests still reach validation; GET stat
|
||||
});
|
||||
|
||||
test('invalid requests spend admission quota before parsing, including on an unconfigured server', async (t) => {
|
||||
let now = Date.now() + 120000;
|
||||
t.mock.method(Date, 'now', () => now);
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
@@ -173,6 +213,216 @@ test('invalid requests spend admission quota before parsing, including on an unc
|
||||
assert.equal(denied.reads, 0);
|
||||
assert.equal(denied.headers['retry-after'], '60');
|
||||
assert.equal((await call(route, { peer: 'other-peer' })).statusCode, 400);
|
||||
now += 60001;
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('the 900/min global window spans non-loopback peers only; loopback keeps its own window', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (const peer of ['10.0.0.1', '10.0.0.2', '10.0.0.3']) {
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, { peer })).statusCode, 400, peer);
|
||||
}
|
||||
}
|
||||
const starved = await call(route, { peer: '10.0.0.4', forbidRead: true });
|
||||
assert.equal(starved.statusCode, 429, 'global window is full for LAN peers');
|
||||
assert.equal(starved.reads, 0);
|
||||
for (const peer of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
|
||||
const owner = await call(route, { peer });
|
||||
assert.equal(owner.statusCode, 400, `${peer} is admitted past the global`);
|
||||
assert.equal(owner.reads, 1);
|
||||
}
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer: '10.0.0.4' })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('IPv6 peers share a /64 admission bucket and IPv4-mapped peers fold to IPv4', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer: `2001:db8:1:2::${(i % 200) + 1}` }))
|
||||
.statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const rotated = await call(route, {
|
||||
peer: '2001:db8:1:2:ffff:ffff:ffff:9',
|
||||
forbidRead: true,
|
||||
});
|
||||
assert.equal(rotated.statusCode, 429, 'a rotating host stays in one bucket');
|
||||
assert.equal(
|
||||
(await call(route, { peer: '2001:db8:1:3::1' })).statusCode,
|
||||
400,
|
||||
);
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer: '::ffff:10.1.1.1' })).statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const unmapped = await call(route, { peer: '10.1.1.1', forbidRead: true });
|
||||
assert.equal(unmapped.statusCode, 429, 'mapped and bare IPv4 share a bucket');
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer: '10.1.1.1' })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('cache hits spend admission budget but not the 90/min upstream budget', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
_overpassCache.delete(hitKey);
|
||||
});
|
||||
let fetches = 0;
|
||||
t.mock.method(globalThis, 'fetch', async () => {
|
||||
fetches++;
|
||||
// Not data: nothing is cached or written to disk by this test.
|
||||
return new Response('upstream unavailable', { status: 503 });
|
||||
});
|
||||
const hit = 'node(around:20,1,1);out;';
|
||||
const hitKey = `data=${encodeURIComponent(hit)}`;
|
||||
_overpassCache.set(hitKey, {
|
||||
status: 200,
|
||||
body: '{"elements":[]}',
|
||||
contentType: 'application/json',
|
||||
endpoint: 'https://overpass.example/api/interpreter',
|
||||
cachedAt: clock.now,
|
||||
});
|
||||
const route = handler();
|
||||
const peer = '10.7.7.7';
|
||||
// More cache hits than the upstream limiter would allow per peer.
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const response = await call(route, { peer, body: form(hit) });
|
||||
assert.equal(response.statusCode, 200);
|
||||
assert.equal(response.headers['x-overpass-cache'], 'HIT');
|
||||
}
|
||||
assert.equal(fetches, 0);
|
||||
const miss = await call(route, {
|
||||
peer,
|
||||
body: form('node(around:20,2,2);out;'),
|
||||
});
|
||||
assert.equal(fetches, 1, 'the first miss still reaches upstream');
|
||||
assert.notEqual(miss.statusCode, 429);
|
||||
for (let i = 0; i < 199; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer, body: form(hit) })).statusCode,
|
||||
200,
|
||||
);
|
||||
}
|
||||
const exhausted = await call(route, {
|
||||
peer,
|
||||
body: form(hit),
|
||||
forbidRead: true,
|
||||
});
|
||||
assert.equal(exhausted.statusCode, 429, 'hits spent the admission window');
|
||||
assert.equal(exhausted.reads, 0);
|
||||
});
|
||||
|
||||
test('a local reverse proxy cannot spend the direct-loopback reserve; owner cache hits survive it', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
const hit = 'node(around:20,3,3);out;';
|
||||
const hitKey = `data=${encodeURIComponent(hit)}`;
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
_overpassCache.delete(hitKey);
|
||||
});
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
_overpassCache.set(hitKey, {
|
||||
status: 200,
|
||||
body: '{"elements":[]}',
|
||||
contentType: 'application/json',
|
||||
endpoint: 'https://overpass.example/api/interpreter',
|
||||
cachedAt: clock.now,
|
||||
});
|
||||
const route = handler();
|
||||
// The proxy connects over loopback; a rotating forwarded client address
|
||||
// must not mint fresh buckets, so all 300 land in one socket-keyed bucket.
|
||||
for (const proxyPeer of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
|
||||
const proxied = (i) => ({
|
||||
peer: proxyPeer,
|
||||
headers: { 'x-forwarded-for': `203.0.113.${i % 250}` },
|
||||
body: form(hit),
|
||||
});
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, proxied(i))).statusCode, 200, proxyPeer);
|
||||
}
|
||||
const refused = await call(route, { ...proxied(300), forbidRead: true });
|
||||
assert.equal(refused.statusCode, 429, `${proxyPeer} proxy bucket is full`);
|
||||
assert.equal(refused.reads, 0);
|
||||
for (const peer of ['127.0.0.1', '::1']) {
|
||||
const owner = await call(route, { peer, body: form(hit) });
|
||||
assert.equal(owner.statusCode, 200, `direct ${peer} is still admitted`);
|
||||
assert.equal(owner.headers['x-overpass-cache'], 'HIT');
|
||||
}
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, proxied(0))).statusCode, 200);
|
||||
}
|
||||
});
|
||||
|
||||
test('proxied loopback traffic draws on the shared remote global window', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (const peer of ['10.0.0.1', '10.0.0.2']) {
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, { peer })).statusCode, 400, peer);
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { headers: { via: '1.1 proxy' } })).statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const starved = await call(route, { peer: '10.0.0.3', forbidRead: true });
|
||||
assert.equal(starved.statusCode, 429, 'proxied requests counted globally');
|
||||
assert.equal((await call(route, {})).statusCode, 400, 'owner unaffected');
|
||||
});
|
||||
|
||||
test("a local proxy has its own upstream bucket, so its misses cannot spend the owner's", async (t) => {
|
||||
freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
});
|
||||
// Not data: nothing is cached, so every request below is a miss.
|
||||
t.mock.method(
|
||||
globalThis,
|
||||
'fetch',
|
||||
async () => new Response('upstream unavailable', { status: 503 }),
|
||||
);
|
||||
const route = handler();
|
||||
const miss = form('node(around:20,4,4);out;');
|
||||
// The upstream limiter's own refusal is the only miss without this header.
|
||||
const passedLimiter = (response) =>
|
||||
response.headers['x-overpass-cache'] !== undefined;
|
||||
const proxied = { peer: '127.0.0.1', headers: { via: '1.1 proxy' } };
|
||||
for (let i = 0; i < 90; i++) {
|
||||
const response = await call(route, { ...proxied, body: miss });
|
||||
assert.equal(passedLimiter(response), true, `proxied miss ${i}`);
|
||||
}
|
||||
const spent = await call(route, { ...proxied, body: miss });
|
||||
assert.equal(spent.statusCode, 429);
|
||||
assert.equal(passedLimiter(spent), false, 'the proxy bucket is spent');
|
||||
const owner = await call(route, { peer: '127.0.0.1', body: miss });
|
||||
assert.equal(passedLimiter(owner), true, 'the owner keeps its own bucket');
|
||||
});
|
||||
@@ -7,9 +7,13 @@ export {
|
||||
publicRadioHttpsUrl,
|
||||
} from './radio/stations.js';
|
||||
export function radioBrowserProxy() {
|
||||
const middleware = createRadioProxyMiddleware();
|
||||
let middleware;
|
||||
const initialize = () =>
|
||||
(middleware ||= createRadioProxyMiddleware({
|
||||
priorityCountries: process.env.RADIO_PRIORITY_COUNTRIES || '',
|
||||
}));
|
||||
const install = (server) => {
|
||||
server.middlewares.use('/api/radio', middleware);
|
||||
server.middlewares.use('/api/radio', initialize());
|
||||
};
|
||||
return {
|
||||
name: 'radio-browser-proxy',
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js';
|
||||
import { lookup as lookupDns } from 'node:dns/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
@@ -27,6 +28,34 @@ import {
|
||||
RADIO_FALLBACK_MIRRORS,
|
||||
RADIO_REJECTED_STATION_COUNT_MAX,
|
||||
} from './constants.js';
|
||||
const PRIORITY_COUNTRY_LIMIT = 8;
|
||||
const COUNTRY_QUERY_LIMIT = 200;
|
||||
const COUNTRY_RESERVED_STATIONS = 40;
|
||||
const REFRESH_RETRY_MS = 60_000;
|
||||
const MAX_MIRROR_ATTEMPTS = 3;
|
||||
|
||||
/** Validate an operator-owned list; malformed configuration must not silently widen queries. */
|
||||
export function parseRadioPriorityCountries(value = '') {
|
||||
const entries = Array.isArray(value) ? value : String(value).split(',');
|
||||
const countries = [];
|
||||
for (const entry of entries) {
|
||||
const input = String(entry).trim().toUpperCase();
|
||||
if (!input) continue;
|
||||
const normalized = normalizeRadioCountryInput(input);
|
||||
if (!/^[A-Z]{2}$/.test(input) || !normalized.valid)
|
||||
throw new TypeError(
|
||||
'RADIO_PRIORITY_COUNTRIES requires ISO alpha-2 country codes',
|
||||
);
|
||||
const code = normalized.code;
|
||||
if (!countries.includes(code)) countries.push(code);
|
||||
}
|
||||
if (countries.length > PRIORITY_COUNTRY_LIMIT)
|
||||
throw new RangeError(
|
||||
'RADIO_PRIORITY_COUNTRIES supports at most eight countries',
|
||||
);
|
||||
return countries;
|
||||
}
|
||||
|
||||
async function mapRadioConcurrent(values, concurrency, mapper) {
|
||||
const results = new Array(values.length);
|
||||
let cursor = 0;
|
||||
@@ -49,7 +78,9 @@ export function createRadioProxyMiddleware({
|
||||
fetchImpl = null,
|
||||
lookupImpl = lookupDns,
|
||||
now = Date.now,
|
||||
priorityCountries = [],
|
||||
} = {}) {
|
||||
const countries = parseRadioPriorityCountries(priorityCountries);
|
||||
let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 };
|
||||
let mirrorPromise = null;
|
||||
let catalogCache = null;
|
||||
@@ -61,6 +92,9 @@ export function createRadioProxyMiddleware({
|
||||
const catalogInstance = randomUUID();
|
||||
let servedStationIds = new Set();
|
||||
let refreshPromise = null;
|
||||
let retryAfter = 0;
|
||||
let partialCatalog = null;
|
||||
let refreshError = null;
|
||||
|
||||
async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) {
|
||||
const destination = radioProxyDestination(url);
|
||||
@@ -139,7 +173,7 @@ export function createRadioProxyMiddleware({
|
||||
|
||||
async function fetchPath(pathname) {
|
||||
let lastError = null;
|
||||
for (const origin of await mirrors()) {
|
||||
for (const origin of (await mirrors()).slice(0, MAX_MIRROR_ATTEMPTS)) {
|
||||
try {
|
||||
return await fetchJson(`${origin}${pathname}`);
|
||||
} catch (error) {
|
||||
@@ -160,20 +194,27 @@ export function createRadioProxyMiddleware({
|
||||
'aviation',
|
||||
'marine',
|
||||
'traffic',
|
||||
];
|
||||
].map((tag) => ({ tag }));
|
||||
const baseQueryCount = queries.length;
|
||||
queries.push(...countries.map((country) => ({ country })));
|
||||
const outcomes = await mapRadioConcurrent(
|
||||
queries,
|
||||
3,
|
||||
async (tag, index) => {
|
||||
async ({ tag, country }, index) => {
|
||||
const params = new URLSearchParams({
|
||||
has_geo_info: 'true',
|
||||
is_https: 'true',
|
||||
hidebroken: 'true',
|
||||
order: 'clickcount',
|
||||
reverse: 'true',
|
||||
limit: index === 0 ? '1800' : '220',
|
||||
limit: country
|
||||
? String(COUNTRY_QUERY_LIMIT)
|
||||
: index === 0
|
||||
? '1800'
|
||||
: '220',
|
||||
});
|
||||
if (tag) params.set('tag', tag);
|
||||
if (country) params.set('countrycode', country);
|
||||
try {
|
||||
const rows = await fetchPath(`/json/stations/search?${params}`);
|
||||
if (!Array.isArray(rows))
|
||||
@@ -195,7 +236,12 @@ export function createRadioProxyMiddleware({
|
||||
const rejectedRows = rows.length - structuralRows.length;
|
||||
const stations = structuralRows
|
||||
.map(normalizeRadioBrowserStation)
|
||||
.filter(Boolean);
|
||||
.filter(
|
||||
(station) =>
|
||||
station && (!country || station.countryCode === country),
|
||||
);
|
||||
if (country && rows.length > COUNTRY_QUERY_LIMIT)
|
||||
throw new Error('Radio Browser exceeded the country query limit');
|
||||
const requestedTag = normalizeRadioTag(tag);
|
||||
const requestedTagCovered =
|
||||
!requestedTag ||
|
||||
@@ -207,10 +253,12 @@ export function createRadioProxyMiddleware({
|
||||
),
|
||||
);
|
||||
return {
|
||||
// Query coverage is based on accepted rows, not merely a payload that
|
||||
// happens to match the upstream schema. Specialist responses must
|
||||
// also contain an accepted station tagged for the requested category.
|
||||
succeeded: stations.length > 0 && requestedTagCovered,
|
||||
// A valid country response can contain no playable stations. Country
|
||||
// coverage is advisory; only broad/specialist queries gate health.
|
||||
// Specialist responses need an accepted station with matching tags.
|
||||
succeeded: country
|
||||
? rows.length === 0 || structuralRows.length > 0
|
||||
: stations.length > 0 && requestedTagCovered,
|
||||
stations,
|
||||
rejectedRows,
|
||||
};
|
||||
@@ -233,9 +281,24 @@ export function createRadioProxyMiddleware({
|
||||
seen.add(station.id);
|
||||
selected.push(station);
|
||||
};
|
||||
// Reserve country slots first; low-click regional stations cannot lose all
|
||||
// representation to the global charts. Keep room for all eight specialist
|
||||
// seeds (8 * 45) and global fill even at the eight-country limit (8 * 40).
|
||||
for (const rows of resultSets.slice(baseQueryCount)) {
|
||||
const ranked = [...rows].sort(
|
||||
(a, b) => b.clickCount - a.clickCount || a.id.localeCompare(b.id),
|
||||
);
|
||||
let added = 0;
|
||||
for (const station of ranked) {
|
||||
if (seen.has(station.id)) continue;
|
||||
take(station);
|
||||
if (++added >= COUNTRY_RESERVED_STATIONS) break;
|
||||
}
|
||||
}
|
||||
// Seed specialist station-tag queries before popularity fill so operational
|
||||
// categories remain represented even when global click charts skew musical.
|
||||
for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take);
|
||||
for (const rows of resultSets.slice(1, baseQueryCount))
|
||||
rows.slice(0, 45).forEach(take);
|
||||
resultSets
|
||||
.flat()
|
||||
.sort(
|
||||
@@ -243,9 +306,9 @@ export function createRadioProxyMiddleware({
|
||||
)
|
||||
.forEach(take);
|
||||
const timestamp = now();
|
||||
const successfulQueries = outcomes.filter(
|
||||
(outcome) => outcome.succeeded,
|
||||
).length;
|
||||
const successfulQueries = outcomes
|
||||
.slice(0, baseQueryCount)
|
||||
.filter((outcome) => outcome.succeeded).length;
|
||||
const broadQueryHealthy =
|
||||
outcomes[0].succeeded && outcomes[0].stations.length > 0;
|
||||
const healthReasons = [];
|
||||
@@ -261,7 +324,18 @@ export function createRadioProxyMiddleware({
|
||||
);
|
||||
const coverage = {
|
||||
successfulQueries,
|
||||
totalQueries: queries.length,
|
||||
totalQueries: baseQueryCount,
|
||||
...(countries.length
|
||||
? {
|
||||
priorityCountries: countries.map((country, index) => ({
|
||||
countryCode: country,
|
||||
succeeded: outcomes[baseQueryCount + index].succeeded,
|
||||
stationCount: selected.filter(
|
||||
(station) => station.countryCode === country,
|
||||
).length,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
stationCount: selected.length,
|
||||
healthyStationMinimum: RADIO_CATALOG_HEALTHY_MIN_STATIONS,
|
||||
};
|
||||
@@ -315,7 +389,24 @@ export function createRadioProxyMiddleware({
|
||||
return { ...catalogCache, stale: false };
|
||||
}
|
||||
if (!refreshPromise) {
|
||||
refreshPromise = refreshCatalog().finally(() => {
|
||||
refreshPromise = (async () => {
|
||||
if (now() < retryAfter) {
|
||||
if (refreshError) throw refreshError;
|
||||
return partialCatalog;
|
||||
}
|
||||
try {
|
||||
const catalog = await refreshCatalog();
|
||||
refreshError = null;
|
||||
partialCatalog = catalog.degraded ? catalog : null;
|
||||
retryAfter = catalog.degraded ? now() + REFRESH_RETRY_MS : 0;
|
||||
return catalog;
|
||||
} catch (error) {
|
||||
refreshError = error;
|
||||
partialCatalog = null;
|
||||
retryAfter = now() + REFRESH_RETRY_MS;
|
||||
throw error;
|
||||
}
|
||||
})().finally(() => {
|
||||
refreshPromise = null;
|
||||
});
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user