mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 02:44:02 +00:00
Traffic roads come from OpenFreeMap vector tiles, or TomTom flow tiles with a key (Hybrid by default). Military areas use OpenFreeMap polygons with a bundled worldwide name index; wide views show named points. ALPR reads an hourly OpenStreetMap extract. Cockpit place context and country, state and US county outlines use bundled Natural Earth and US Census data. Overpass is used only when an operator configures their own endpoint. Also: camera arrivals no longer land below ground, keyless terrain retries on 429, bounded relation geometry for configured Overpass, one shared OpenStreetMap credit, and separate tile and API transports for embedders. Co-authored-by: theGaryLarson <[email protected]> Co-authored-by: Milan Khanal <[email protected]>
This commit is contained in:
200 files changed
+26665
-1977
No files matched your search
+63
-181
@@ -1,8 +1,8 @@
|
||||
// OVERPASS PROXY — which upstream answers count as an answer.
|
||||
//
|
||||
// One predicate governs cache reads, writes, and stale fallback. A mirror's
|
||||
// refusal must neither end the search for healthy alternatives nor persist as
|
||||
// data under the week/month-long cache TTLs. These cases use no live providers.
|
||||
// One predicate governs cache reads, writes, and stale fallback. A configured
|
||||
// upstream's refusal is never data, and only operator-supplied alternatives
|
||||
// may be tried. These cases use no live providers.
|
||||
//
|
||||
// Run with: npm test
|
||||
import { test } from 'node:test';
|
||||
@@ -13,30 +13,6 @@ import path from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import createViteConfig, { fetchOverpassPayload, overpassPayloadIsData, readOverpassDisk } from '../vite.config.js';
|
||||
|
||||
const ENDPOINTS = ['https://a.example/api', 'https://b.example/api', 'https://c.example/api'];
|
||||
|
||||
/** Answer each endpoint from a map of url → {status, body}; record the order. */
|
||||
function mirrors(byUrl) {
|
||||
const tried = [];
|
||||
const fetchImpl = async (url) => {
|
||||
tried.push(url);
|
||||
const answer = byUrl[url];
|
||||
if (answer instanceof Error) throw answer;
|
||||
return { status: answer.status, headers: { get: () => answer.contentType || 'application/json' } };
|
||||
};
|
||||
return { fetchImpl, tried, readBody: async (_, __) => byUrl[tried[tried.length - 1]]?.body ?? '' };
|
||||
}
|
||||
|
||||
const run = (byUrl) => {
|
||||
const m = mirrors(byUrl);
|
||||
return fetchOverpassPayload('data=x', 1e6, {
|
||||
endpoints: ENDPOINTS,
|
||||
fetchImpl: m.fetchImpl,
|
||||
readBody: m.readBody,
|
||||
simplify: (body) => body,
|
||||
}).then((payload) => ({ payload, tried: m.tried }), (error) => ({ error, tried: m.tried }));
|
||||
};
|
||||
|
||||
const DATA = { status: 200, body: '{"elements":[]}' };
|
||||
|
||||
test('disk cache rejects old refusals for fresh and stale reads but preserves last-good data', async () => {
|
||||
@@ -82,167 +58,70 @@ test('only a 2xx that is neither rate-limited nor a runtime error is data', () =
|
||||
assert.equal(overpassPayloadIsData(null), false);
|
||||
});
|
||||
|
||||
// ── The fan-out ──────────────────────────────────────────────────────────────
|
||||
|
||||
test('a refusal moves to the next mirror instead of ending the fan-out', async () => {
|
||||
// The exact shape measured against the live mirrors.
|
||||
const { payload, tried } = await run({
|
||||
[ENDPOINTS[0]]: { status: 406, contentType: 'text/html', body: '<!DOCTYPE HTML><title>406</title>' },
|
||||
[ENDPOINTS[1]]: DATA,
|
||||
[ENDPOINTS[2]]: DATA,
|
||||
// Configured endpoints replace defaults; refusals are sanitized and cooled down.
|
||||
test('configured failover accepts empty data and keeps the application identity', async () => {
|
||||
const endpoints = ['https://first.example/query?secret=one', 'http://localhost:12345/api'];
|
||||
const seen = [];
|
||||
const payload = await fetchOverpassPayload('data=x', 1024, {
|
||||
endpoints,
|
||||
fetchImpl: async (url, options) => {
|
||||
seen.push(url);
|
||||
assert.equal(options.headers['User-Agent'], 'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)');
|
||||
assert.equal(options.redirect, 'error');
|
||||
return new Response(seen.length === 1 ? url : DATA.body, { status: seen.length === 1 ? 406 : 200 });
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(payload.status, 200);
|
||||
assert.equal(payload.endpoint, ENDPOINTS[1]);
|
||||
assert.deepEqual(tried, ENDPOINTS.slice(0, 2), 'the healthy mirror must be reached, and no further');
|
||||
assert.deepEqual(seen, endpoints);
|
||||
assert.equal(payload.body, DATA.body);
|
||||
assert.equal(payload.endpoint, 'configured');
|
||||
});
|
||||
|
||||
test('every mirror is asked with a User-Agent that identifies the application', async () => {
|
||||
// The OSM API usage policy asks for a "Valid User-Agent identifying
|
||||
// application and version". Every outbound request must carry it, not just
|
||||
// the first: a mirror further down the list refusing an unidentified client
|
||||
// is exactly the case the fan-out exists to survive.
|
||||
const seen = [];
|
||||
const fetchImpl = async (url, options) => {
|
||||
seen.push({ url, agent: options?.headers?.['User-Agent'] });
|
||||
// Refuse everywhere, so the loop is forced through the whole list.
|
||||
return { status: 503, headers: { get: () => 'text/html' } };
|
||||
};
|
||||
await fetchOverpassPayload('data=x', 1e6, {
|
||||
endpoints: ENDPOINTS,
|
||||
fetchImpl,
|
||||
readBody: async () => 'upstream down',
|
||||
simplify: (body) => body,
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
seen.map((request) => request.url),
|
||||
ENDPOINTS,
|
||||
'the fan-out must reach every mirror',
|
||||
);
|
||||
for (const request of seen) {
|
||||
const agent = String(request.agent || '');
|
||||
assert.match(
|
||||
agent,
|
||||
/^gods-eye-view\/\d/,
|
||||
`${request.url} must name the application and its version`,
|
||||
);
|
||||
assert.ok(
|
||||
!/proxy\/1\.0$/.test(agent),
|
||||
`${request.url} must not fall back to the unidentified label`,
|
||||
);
|
||||
assert.match(
|
||||
agent,
|
||||
/github\.com\/bilawalsidhu\/gods-eye-view/,
|
||||
`${request.url} must carry a route back to the project`,
|
||||
);
|
||||
test('406 and 429 honor Retry-After across different queries without more egress', async () => {
|
||||
for (const status of [406, 429]) {
|
||||
const endpoint = `https://cooldown-${status}.example/private?token=secret`;
|
||||
let count = 0, now = 1_000_000;
|
||||
const options = { endpoints: [endpoint], now: () => now, fetchImpl: async () => {
|
||||
count++;
|
||||
return new Response(`Refused ${endpoint}`, { status, headers: { 'Retry-After': '90' } });
|
||||
} };
|
||||
const first = await fetchOverpassPayload('first', 1024, options);
|
||||
assert.equal(first.status, status);
|
||||
assert.equal(first.retryAfterMs, 90_000);
|
||||
assert.doesNotMatch(JSON.stringify(first), /secret|https:/);
|
||||
now += 30_000;
|
||||
const next = await fetchOverpassPayload('different', 1024, options);
|
||||
assert.equal(count, 1);
|
||||
assert.equal(next.retryAfterMs, 60_000);
|
||||
now += 60_001;
|
||||
await fetchOverpassPayload('third', 1024, options);
|
||||
assert.equal(count, 2);
|
||||
}
|
||||
});
|
||||
|
||||
test('a mirror that refuses the old label serves the same query under the identifying one', async () => {
|
||||
// Recorded from the canonical instance on 2026-09-14: the unidentified label
|
||||
// is answered with a 406 Not Acceptable HTML page before the query is read,
|
||||
// and the identifying one is served. This stub replays that shape so the
|
||||
// behaviour the header change buys is pinned without a live mirror.
|
||||
const REFUSED = 'gods-eye-view-overpass-proxy/1.0';
|
||||
const answer = (agent) =>
|
||||
String(agent || '').startsWith(REFUSED)
|
||||
? {
|
||||
status: 406,
|
||||
body: '<!DOCTYPE HTML><title>406 Not Acceptable</title>',
|
||||
contentType: 'text/html',
|
||||
}
|
||||
: { status: 200, body: DATA.body, contentType: 'application/json' };
|
||||
|
||||
let last = null;
|
||||
const run = (agentOverride) =>
|
||||
fetchOverpassPayload('data=x', 1e6, {
|
||||
endpoints: [ENDPOINTS[0]],
|
||||
fetchImpl: async (url, options) => {
|
||||
last = answer(agentOverride ?? options?.headers?.['User-Agent']);
|
||||
return { status: last.status, headers: { get: () => last.contentType } };
|
||||
},
|
||||
readBody: async () => last.body,
|
||||
simplify: (body) => body,
|
||||
});
|
||||
|
||||
const refused = await run(REFUSED);
|
||||
assert.equal(refused.status, 406, 'the old label is refused by the mirror');
|
||||
assert.equal(
|
||||
overpassPayloadIsData(refused),
|
||||
false,
|
||||
'a refusal is never treated as data',
|
||||
);
|
||||
|
||||
const served = await run(undefined);
|
||||
assert.equal(served.status, 200, 'the header the proxy now sends is served');
|
||||
assert.equal(overpassPayloadIsData(served), true);
|
||||
assert.equal(served.body, DATA.body);
|
||||
test('HTTP-date Retry-After is respected, and absent delays use bounded backoff', async () => {
|
||||
let now = Date.parse('2026-09-23T12:00:00Z');
|
||||
const endpoint = 'https://dated.example/api';
|
||||
const payload = await fetchOverpassPayload('x', 1024, { endpoints: [endpoint], now: () => now,
|
||||
fetchImpl: async () => new Response('busy', { status: 429, headers: { 'Retry-After': 'Wed, 23 Sep 2026 12:02:00 GMT' } }) });
|
||||
assert.equal(payload.retryAfterMs, 120_000);
|
||||
const options = { endpoints: ['https://bounded.example'], now: () => now,
|
||||
fetchImpl: async () => { throw new Error('https://bounded.example/secret'); } };
|
||||
for (let i = 0; i < 8; i++) {
|
||||
const reply = await fetchOverpassPayload('x', 1024, options);
|
||||
assert.ok(reply.retryAfterMs <= 300_000);
|
||||
assert.doesNotMatch(reply.body, /secret|https:/);
|
||||
now += reply.retryAfterMs + 1;
|
||||
}
|
||||
});
|
||||
|
||||
test('the first mirror to answer wins, and the rest are left alone', async () => {
|
||||
const { payload, tried } = await run({
|
||||
[ENDPOINTS[0]]: DATA, [ENDPOINTS[1]]: DATA, [ENDPOINTS[2]]: DATA,
|
||||
});
|
||||
|
||||
assert.equal(payload.endpoint, ENDPOINTS[0]);
|
||||
assert.deepEqual(tried, [ENDPOINTS[0]]);
|
||||
});
|
||||
|
||||
test('a refusal every mirror agrees on is reported, not swallowed', async () => {
|
||||
// A genuinely bad query must still say what upstream said — but only after
|
||||
// every mirror has had its chance to answer it.
|
||||
const refusal = { status: 400, body: 'line 1: parse error' };
|
||||
const { payload, tried } = await run({
|
||||
[ENDPOINTS[0]]: refusal, [ENDPOINTS[1]]: refusal, [ENDPOINTS[2]]: refusal,
|
||||
});
|
||||
|
||||
assert.equal(payload.status, 400);
|
||||
assert.equal(payload.endpoint, ENDPOINTS[0], 'the FIRST refusal is the one reported');
|
||||
assert.deepEqual(tried, ENDPOINTS);
|
||||
assert.equal(overpassPayloadIsData(payload), false, 'so it is neither cached nor served as data');
|
||||
});
|
||||
|
||||
test('a mirror that throws is no different from one that refuses', async () => {
|
||||
const { payload, tried } = await run({
|
||||
[ENDPOINTS[0]]: new Error('ECONNRESET'),
|
||||
[ENDPOINTS[1]]: { status: 503, body: 'busy' },
|
||||
[ENDPOINTS[2]]: DATA,
|
||||
});
|
||||
|
||||
assert.equal(payload.endpoint, ENDPOINTS[2]);
|
||||
assert.deepEqual(tried, ENDPOINTS);
|
||||
});
|
||||
|
||||
test('when every mirror is unreachable the caller gets a throw, not a payload', async () => {
|
||||
const { error, payload } = await run({
|
||||
[ENDPOINTS[0]]: new Error('ECONNRESET'),
|
||||
[ENDPOINTS[1]]: new Error('ETIMEDOUT'),
|
||||
[ENDPOINTS[2]]: new Error('ENOTFOUND'),
|
||||
});
|
||||
|
||||
assert.equal(payload, undefined);
|
||||
assert.match(error.message, /ENOTFOUND/);
|
||||
});
|
||||
|
||||
test('production reader rotates past oversized, runtime-error and rate-limited bodies', async () => {
|
||||
for (const [status, body] of [
|
||||
[200, 'x'.repeat(200)], [200, '{"remark":"runtime error: timed out","elements":[]}'],
|
||||
[200, 'rate_limited'], [429, 'busy'], [403, 'forbidden'],
|
||||
]) {
|
||||
const tried = [];
|
||||
const payload = await fetchOverpassPayload('data=x', 100, {
|
||||
endpoints: ENDPOINTS,
|
||||
fetchImpl: async (url) => {
|
||||
tried.push(url);
|
||||
return new Response(tried.length === 1 ? body : DATA.body, {
|
||||
status: tried.length === 1 ? status : 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
},
|
||||
});
|
||||
assert.equal(payload.body, DATA.body);
|
||||
assert.deepEqual(tried, ENDPOINTS.slice(0, 2));
|
||||
test('production reader rejects oversized, malformed and runtime-failure bodies', async () => {
|
||||
for (const [i, body] of ['x'.repeat(200), '{"remark":"runtime error: timed out","elements":[]}', 'rate_limited', '{}', 'not json'].entries()) {
|
||||
const endpoints = [`https://bad-${i}.example/`, `https://good-${i}.example/`];
|
||||
const seen = [];
|
||||
const payload = await fetchOverpassPayload('data=x', 100, { endpoints, fetchImpl: async (url) => {
|
||||
seen.push(url); return new Response(seen.length === 1 ? body : DATA.body);
|
||||
} });
|
||||
assert.equal(payload.body, DATA.body); assert.deepEqual(seen, endpoints);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -267,7 +146,10 @@ function invoke(handler, body) {
|
||||
|
||||
test('coalesced outage callers both receive last-good data, never a cached refusal', async (t) => {
|
||||
const handler = proxyHandler();
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
t.after(() => { if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS; else process.env.OVERPASS_UPSTREAMS = prior; });
|
||||
for (const status of [406, 503, 429]) {
|
||||
process.env.OVERPASS_UPSTREAMS = `https://outage-${status}.example/api`;
|
||||
const query = `[out:json][timeout:12];node(around:10,30.27,-97.74)["name"="${randomUUID()}"];out;`;
|
||||
const body = `data=${encodeURIComponent(query)}`;
|
||||
const directory = path.join(process.cwd(), '.gev-cache', 'overpass');
|
||||
@@ -297,7 +179,7 @@ test('coalesced outage callers both receive last-good data, never a cached refus
|
||||
assert.equal(response.body, DATA.body);
|
||||
assert.equal(response.headers['X-Overpass-Cache'], 'STALE');
|
||||
}
|
||||
assert.equal(fetches, 4, 'one shared, bounded mirror sequence');
|
||||
assert.equal(fetches, 1, 'one shared configured request');
|
||||
assert.deepEqual(JSON.parse(await readFile(file, 'utf8')), stale);
|
||||
} finally {
|
||||
release.resolve();
|
||||
|
||||
Reference in new issue
Block a user