mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
Merge pull request #173: Overpass mirror and cache recovery
Recover across existing Overpass mirrors, reject cached refusals, and serve consistent last-good data to coalesced callers. Preserve the original contribution from PR #165.
This commit is contained in:
5 files changed
+335
-23
No files matched your search
@@ -5,6 +5,19 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- The Overpass proxy now rotates to the next mirror on any non-2xx upstream
|
||||
response, not only on 5xx. `overpass-api.de` and its `lz4` alias answer 406 to
|
||||
the proxy's User-Agent while two of the configured mirrors answer 200 to the
|
||||
identical request, so the fan-out stopped at the first refusal with healthy
|
||||
mirrors untried. The refusal was also cached to memory and disk and served as
|
||||
data — boundary-class queries hold a month-long TTL — which affected every
|
||||
Overpass-backed feature: road geometry, annotation outlines and place lookup.
|
||||
- Existing cached refusals are now ignored immediately, including during
|
||||
stale-data fallback. Concurrent identical requests share the same last-good
|
||||
fallback when all mirrors refuse, without duplicating upstream requests.
|
||||
|
||||
## [0.1.1] — 2026-09-01 — Installation and live-data fixes
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -2006,6 +2006,12 @@ silently demoting every later lookup for the session.
|
||||
- **Track trails**: server accumulates per-MMSI ring buffers (`/api/ais-live/track?mmsi=`, Float32+Uint32, 64 samples, 30s/25m thinning); aircraft backfill proxies `/api/opensky-track` (OAuth, own credit bucket) and `/api/adsblol/trace` (tar1090 readsb, ~24h history, ODbL — credit adsb.lol).
|
||||
- Shared `src/data/pickRegistry.js` stops the two flight layers' click handlers from fighting over the camera.
|
||||
|
||||
### Overpass proxy mirror rotation (September 2026)
|
||||
|
||||
- `/api/overpass` fans out across four public mirrors. `overpassPayloadIsData()` governs cache reads, writes, and stale fallback: only a 2xx that is neither rate-limited nor a body-level runtime error qualifies. Previously stored refusals are ignored on both fresh and stale reads, so upgrading does not require manually clearing the disk cache.
|
||||
- HTTP refusals such as 406 now rotate alongside the existing network, rate-limit, and runtime-error cases. A refusal from one mirror no longer prevents reaching healthy alternatives or persists under the seven-day road/month-long boundary cache TTLs. Concurrent identical queries share one mirror sequence; if it fails, both the initiating and joined callers can use the same last-good data.
|
||||
- A refusal every mirror agrees on is still reported with the first mirror's status and body, so a genuinely malformed query says what upstream said — but only after every mirror has had the chance to answer it. `fetchOverpassPayload` takes injectable endpoints and fetch so the rotation is tested without a live mirror (`src/overpassProxy.test.mjs`).
|
||||
|
||||
### Share-link v2 layer state (August 2026)
|
||||
|
||||
- Generated share links use a deterministic v2 hash. Existing camera, visual,
|
||||
|
||||
@@ -16,9 +16,9 @@ import {
|
||||
|
||||
test('preflight checks memory, in-flight, then disk before consuming limiter quota', async () => {
|
||||
const key = 'normalized query';
|
||||
const fresh = { id: 'memory', cachedAt: 900 };
|
||||
const joined = { id: 'inflight', cachedAt: 950 };
|
||||
const disk = { id: 'disk', cachedAt: 975 };
|
||||
const fresh = { id: 'memory', status: 200, cachedAt: 900 };
|
||||
const joined = { id: 'inflight', status: 200, cachedAt: 950 };
|
||||
const disk = { id: 'disk', status: 200, cachedAt: 975 };
|
||||
let diskReads = 0;
|
||||
let limiterCalls = 0;
|
||||
const allowUpstream = () => { limiterCalls += 1; return true; };
|
||||
@@ -39,7 +39,7 @@ test('preflight checks memory, in-flight, then disk before consuming limiter quo
|
||||
|
||||
const inFlightHit = await resolveOverpassPreflight({
|
||||
cacheKey: key,
|
||||
memoryCache: new Map([[key, { id: 'stale', cachedAt: 0 }]]),
|
||||
memoryCache: new Map([[key, { id: 'stale', status: 200, cachedAt: 0 }]]),
|
||||
inFlight: new Map([[key, Promise.resolve(joined)]]),
|
||||
readDisk: async () => { diskReads += 1; return disk; },
|
||||
allowUpstream,
|
||||
@@ -84,6 +84,21 @@ test('preflight checks memory, in-flight, then disk before consuming limiter quo
|
||||
assert.equal(denied.source, 'RATE_LIMITED');
|
||||
});
|
||||
|
||||
test('preflight treats cached refusals as misses without spending extra quota', async () => {
|
||||
for (const invalid of [{ status: 406 }, { status: 200, runtimeError: true }]) {
|
||||
let admissions = 0;
|
||||
const result = await resolveOverpassPreflight({
|
||||
cacheKey: 'refused',
|
||||
memoryCache: new Map([['refused', { ...invalid, cachedAt: Date.now() }]]),
|
||||
inFlight: new Map(),
|
||||
readDisk: async () => ({ ...invalid, cachedAt: Date.now() }),
|
||||
allowUpstream: () => { admissions++; return true; },
|
||||
});
|
||||
assert.equal(result.source, 'UPSTREAM');
|
||||
assert.equal(admissions, 1);
|
||||
}
|
||||
});
|
||||
|
||||
/** Synthetic dense ring: N points on a circle with sub-tolerance jitter. */
|
||||
function denseRing(n, { latC = 37.5, lonC = 14.2, radiusDeg = 0.5 } = {}) {
|
||||
const pts = [];
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
// OVERPASS PROXY — which upstream answers count as an answer.
|
||||
//
|
||||
// One predicate governs cache reads, writes, and stale fallback. A mirror's
|
||||
// refusal must neither end the search for healthy alternatives nor persist as
|
||||
// data under the week/month-long cache TTLs. These cases use no live providers.
|
||||
//
|
||||
// Run with: npm test
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdir, readFile, writeFile, unlink } from 'node:fs/promises';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import createViteConfig, { fetchOverpassPayload, overpassPayloadIsData, readOverpassDisk } from '../vite.config.js';
|
||||
|
||||
const ENDPOINTS = ['https://a.example/api', 'https://b.example/api', 'https://c.example/api'];
|
||||
|
||||
/** Answer each endpoint from a map of url → {status, body}; record the order. */
|
||||
function mirrors(byUrl) {
|
||||
const tried = [];
|
||||
const fetchImpl = async (url) => {
|
||||
tried.push(url);
|
||||
const answer = byUrl[url];
|
||||
if (answer instanceof Error) throw answer;
|
||||
return { status: answer.status, headers: { get: () => answer.contentType || 'application/json' } };
|
||||
};
|
||||
return { fetchImpl, tried, readBody: async (_, __) => byUrl[tried[tried.length - 1]]?.body ?? '' };
|
||||
}
|
||||
|
||||
const run = (byUrl) => {
|
||||
const m = mirrors(byUrl);
|
||||
return fetchOverpassPayload('data=x', 1e6, {
|
||||
endpoints: ENDPOINTS,
|
||||
fetchImpl: m.fetchImpl,
|
||||
readBody: m.readBody,
|
||||
simplify: (body) => body,
|
||||
}).then((payload) => ({ payload, tried: m.tried }), (error) => ({ error, tried: m.tried }));
|
||||
};
|
||||
|
||||
const DATA = { status: 200, body: '{"elements":[]}' };
|
||||
|
||||
test('disk cache rejects old refusals for fresh and stale reads but preserves last-good data', async () => {
|
||||
const key = `overpass-cache-regression-${randomUUID()}`;
|
||||
const directory = path.join(process.cwd(), '.gev-cache', 'overpass');
|
||||
const file = path.join(directory, `${createHash('sha1').update(key).digest('hex')}.json`);
|
||||
await mkdir(directory, { recursive: true });
|
||||
try {
|
||||
for (const refusal of [
|
||||
{ status: 406 }, { status: 429 }, { status: 503 },
|
||||
{ status: 200, rateLimited: true }, { status: 200, runtimeError: true },
|
||||
]) {
|
||||
await writeFile(file, JSON.stringify({ ...DATA, cachedAt: Date.now(), ...refusal }));
|
||||
assert.equal(await readOverpassDisk(key, 60000), null, `fresh ${JSON.stringify(refusal)}`);
|
||||
assert.equal(await readOverpassDisk(key, Infinity), null, `stale ${JSON.stringify(refusal)}`);
|
||||
}
|
||||
const good = { ...DATA, cachedAt: Date.now() - 120000 };
|
||||
await writeFile(file, JSON.stringify(good));
|
||||
assert.equal(await readOverpassDisk(key, 60000), null, 'expired good data misses normal TTL');
|
||||
assert.deepEqual(await readOverpassDisk(key, Infinity), good, 'last-good data survives an outage');
|
||||
await writeFile(file, '{invalid');
|
||||
assert.equal(await readOverpassDisk(key, Infinity), null, 'corrupt cache is ignored');
|
||||
} finally {
|
||||
await unlink(file);
|
||||
}
|
||||
});
|
||||
|
||||
// ── The predicate ────────────────────────────────────────────────────────────
|
||||
|
||||
test('only a 2xx that is neither rate-limited nor a runtime error is data', () => {
|
||||
assert.equal(overpassPayloadIsData({ status: 200 }), true);
|
||||
assert.equal(overpassPayloadIsData({ status: 204 }), true);
|
||||
|
||||
// The measured refusal, and its neighbours. `< 500` admitted every one.
|
||||
for (const status of [400, 403, 406, 410, 429]) {
|
||||
assert.equal(overpassPayloadIsData({ status }), false, `${status} is not data`);
|
||||
}
|
||||
assert.equal(overpassPayloadIsData({ status: 502 }), false);
|
||||
// A 200 can still not be data: Overpass reports runtime failures in the body.
|
||||
assert.equal(overpassPayloadIsData({ status: 200, runtimeError: true }), false);
|
||||
assert.equal(overpassPayloadIsData({ status: 200, rateLimited: true }), false);
|
||||
assert.equal(overpassPayloadIsData({}), false);
|
||||
assert.equal(overpassPayloadIsData(null), false);
|
||||
});
|
||||
|
||||
// ── The fan-out ──────────────────────────────────────────────────────────────
|
||||
|
||||
test('a refusal moves to the next mirror instead of ending the fan-out', async () => {
|
||||
// The exact shape measured against the live mirrors.
|
||||
const { payload, tried } = await run({
|
||||
[ENDPOINTS[0]]: { status: 406, contentType: 'text/html', body: '<!DOCTYPE HTML><title>406</title>' },
|
||||
[ENDPOINTS[1]]: DATA,
|
||||
[ENDPOINTS[2]]: DATA,
|
||||
});
|
||||
|
||||
assert.equal(payload.status, 200);
|
||||
assert.equal(payload.endpoint, ENDPOINTS[1]);
|
||||
assert.deepEqual(tried, ENDPOINTS.slice(0, 2), 'the healthy mirror must be reached, and no further');
|
||||
});
|
||||
|
||||
test('the first mirror to answer wins, and the rest are left alone', async () => {
|
||||
const { payload, tried } = await run({
|
||||
[ENDPOINTS[0]]: DATA, [ENDPOINTS[1]]: DATA, [ENDPOINTS[2]]: DATA,
|
||||
});
|
||||
|
||||
assert.equal(payload.endpoint, ENDPOINTS[0]);
|
||||
assert.deepEqual(tried, [ENDPOINTS[0]]);
|
||||
});
|
||||
|
||||
test('a refusal every mirror agrees on is reported, not swallowed', async () => {
|
||||
// A genuinely bad query must still say what upstream said — but only after
|
||||
// every mirror has had its chance to answer it.
|
||||
const refusal = { status: 400, body: 'line 1: parse error' };
|
||||
const { payload, tried } = await run({
|
||||
[ENDPOINTS[0]]: refusal, [ENDPOINTS[1]]: refusal, [ENDPOINTS[2]]: refusal,
|
||||
});
|
||||
|
||||
assert.equal(payload.status, 400);
|
||||
assert.equal(payload.endpoint, ENDPOINTS[0], 'the FIRST refusal is the one reported');
|
||||
assert.deepEqual(tried, ENDPOINTS);
|
||||
assert.equal(overpassPayloadIsData(payload), false, 'so it is neither cached nor served as data');
|
||||
});
|
||||
|
||||
test('a mirror that throws is no different from one that refuses', async () => {
|
||||
const { payload, tried } = await run({
|
||||
[ENDPOINTS[0]]: new Error('ECONNRESET'),
|
||||
[ENDPOINTS[1]]: { status: 503, body: 'busy' },
|
||||
[ENDPOINTS[2]]: DATA,
|
||||
});
|
||||
|
||||
assert.equal(payload.endpoint, ENDPOINTS[2]);
|
||||
assert.deepEqual(tried, ENDPOINTS);
|
||||
});
|
||||
|
||||
test('when every mirror is unreachable the caller gets a throw, not a payload', async () => {
|
||||
const { error, payload } = await run({
|
||||
[ENDPOINTS[0]]: new Error('ECONNRESET'),
|
||||
[ENDPOINTS[1]]: new Error('ETIMEDOUT'),
|
||||
[ENDPOINTS[2]]: new Error('ENOTFOUND'),
|
||||
});
|
||||
|
||||
assert.equal(payload, undefined);
|
||||
assert.match(error.message, /ENOTFOUND/);
|
||||
});
|
||||
|
||||
test('production reader rotates past oversized, runtime-error and rate-limited bodies', async () => {
|
||||
for (const [status, body] of [
|
||||
[200, 'x'.repeat(200)], [200, '{"remark":"runtime error: timed out","elements":[]}'],
|
||||
[200, 'rate_limited'], [429, 'busy'], [403, 'forbidden'],
|
||||
]) {
|
||||
const tried = [];
|
||||
const payload = await fetchOverpassPayload('data=x', 100, {
|
||||
endpoints: ENDPOINTS,
|
||||
fetchImpl: async (url) => {
|
||||
tried.push(url);
|
||||
return new Response(tried.length === 1 ? body : DATA.body, {
|
||||
status: tried.length === 1 ? status : 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
},
|
||||
});
|
||||
assert.equal(payload.body, DATA.body);
|
||||
assert.deepEqual(tried, ENDPOINTS.slice(0, 2));
|
||||
}
|
||||
});
|
||||
|
||||
function proxyHandler() {
|
||||
const plugin = createViteConfig({ mode: 'test' }).plugins.find(p => p.name === 'overpass-proxy');
|
||||
const routes = new Map();
|
||||
plugin.configureServer({ middlewares: { use: (route, handler) => routes.set(route, handler) } });
|
||||
return routes.get('/api/overpass');
|
||||
}
|
||||
|
||||
function invoke(handler, body) {
|
||||
const req = Readable.from([Buffer.from(body)]);
|
||||
Object.assign(req, { method: 'POST', headers: {}, socket: { remoteAddress: '127.0.0.1' } });
|
||||
return new Promise((resolve, reject) => {
|
||||
const res = {
|
||||
writeHead(status, headers) { this.status = status; this.headers = headers; },
|
||||
end(body) { resolve({ status: this.status, headers: this.headers, body }); },
|
||||
};
|
||||
Promise.resolve(handler(req, res)).catch(reject);
|
||||
});
|
||||
}
|
||||
|
||||
test('coalesced outage callers both receive last-good data, never a cached refusal', async (t) => {
|
||||
const handler = proxyHandler();
|
||||
for (const status of [406, 503, 429]) {
|
||||
const query = `[out:json][timeout:12];node(around:10,30.27,-97.74)["name"="${randomUUID()}"];out;`;
|
||||
const body = `data=${encodeURIComponent(query)}`;
|
||||
const directory = path.join(process.cwd(), '.gev-cache', 'overpass');
|
||||
const file = path.join(directory, `${createHash('sha1').update(body).digest('hex')}.json`);
|
||||
await mkdir(directory, { recursive: true });
|
||||
const stale = { ...DATA, cachedAt: Date.now() - 40 * 86400000 };
|
||||
await writeFile(file, JSON.stringify(stale));
|
||||
const entered = Promise.withResolvers();
|
||||
const release = Promise.withResolvers();
|
||||
let fetches = 0;
|
||||
const mock = t.mock.method(globalThis, 'fetch', async () => {
|
||||
fetches++;
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
return new Response('upstream unavailable', { status });
|
||||
});
|
||||
try {
|
||||
const first = invoke(handler, body);
|
||||
await entered.promise;
|
||||
const second = invoke(handler, body);
|
||||
// The second request consumes its in-memory stream and joins the pending
|
||||
// promise before releasing upstream. No network or elapsed-time sleep.
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
release.resolve();
|
||||
for (const response of await Promise.all([first, second])) {
|
||||
assert.equal(response.status, 200, `${status}: both callers use last-good data`);
|
||||
assert.equal(response.body, DATA.body);
|
||||
assert.equal(response.headers['X-Overpass-Cache'], 'STALE');
|
||||
}
|
||||
assert.equal(fetches, 4, 'one shared, bounded mirror sequence');
|
||||
assert.deepEqual(JSON.parse(await readFile(file, 'utf8')), stale);
|
||||
} finally {
|
||||
release.resolve();
|
||||
mock.mock.restore();
|
||||
await unlink(file);
|
||||
}
|
||||
}
|
||||
});
|
||||
+72
-19
@@ -342,11 +342,14 @@ function overpassDiskPath(cacheKey) {
|
||||
* serve-stale path when every mirror is down).
|
||||
* @returns {Promise<?Object>} Payload with cachedAt, or null.
|
||||
*/
|
||||
async function readOverpassDisk(cacheKey, maxAgeMs) {
|
||||
export async function readOverpassDisk(cacheKey, maxAgeMs) {
|
||||
try {
|
||||
const raw = await fsp.readFile(overpassDiskPath(cacheKey), 'utf8');
|
||||
const payload = JSON.parse(raw);
|
||||
if (!payload || typeof payload.body !== 'string' || !Number.isFinite(payload.cachedAt)) return null;
|
||||
// Older versions persisted 4xx refusals with normal data TTLs. Ignore
|
||||
// them on both fresh and stale reads so an upgrade can recover immediately.
|
||||
if (!overpassPayloadIsData(payload)) return null;
|
||||
if (Date.now() - payload.cachedAt > maxAgeMs) return null;
|
||||
return payload;
|
||||
} catch {
|
||||
@@ -387,18 +390,24 @@ export async function resolveOverpassPreflight({
|
||||
cacheMs = OVERPASS_CACHE_MS,
|
||||
}) {
|
||||
const cached = memoryCache.get(cacheKey);
|
||||
if (cached && now - cached.cachedAt <= cacheMs) return { source: 'HIT', payload: cached };
|
||||
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs) return { source: 'HIT', payload: cached };
|
||||
|
||||
const pending = inFlight.get(cacheKey);
|
||||
if (pending) return { source: 'INFLIGHT', payload: await pending };
|
||||
|
||||
const disk = await readDisk();
|
||||
if (disk) return { source: 'DISK', payload: disk };
|
||||
if (overpassPayloadIsData(disk)) return { source: 'DISK', payload: disk };
|
||||
|
||||
return allowUpstream()
|
||||
? { source: 'UPSTREAM', payload: null }
|
||||
: { source: 'RATE_LIMITED', payload: null };
|
||||
}
|
||||
|
||||
/** Return only last-good Overpass data, regardless of its age. */
|
||||
async function readStaleOverpass(cacheKey) {
|
||||
const cached = _overpassCache.get(cacheKey);
|
||||
return overpassPayloadIsData(cached) ? cached : readOverpassDisk(cacheKey, Infinity);
|
||||
}
|
||||
/** OSM routing (FOSSGIS OSRM) cache: profile|coords -> { payload, cachedAt }. */
|
||||
const ROUTE_CACHE_MS = 600000;
|
||||
const _routeCache = new Map();
|
||||
@@ -2548,26 +2557,47 @@ function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
|
||||
}
|
||||
|
||||
/**
|
||||
* Try each Overpass upstream in order until one succeeds.
|
||||
* True only for an upstream response that is actually Overpass data.
|
||||
*
|
||||
* Skips rate-limited or 5xx responses and falls through to the next
|
||||
* mirror. If all mirrors fail, returns the last rate-limited payload
|
||||
* (if any) or throws the last error.
|
||||
*
|
||||
* @param {string} body - URL-encoded Overpass QL query body.
|
||||
* The proxy caches on this and serves stale on its negation, so the two
|
||||
* decisions cannot drift apart: a payload that is not data must never be
|
||||
* written to the cache and must always be eligible for a stale replacement.
|
||||
* @param {{status: number, rateLimited?: boolean, runtimeError?: boolean}} payload
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function overpassPayloadIsData(payload) {
|
||||
const status = Number(payload?.status);
|
||||
return Number.isFinite(status)
|
||||
&& status >= 200 && status < 300
|
||||
&& !payload.rateLimited
|
||||
&& !payload.runtimeError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try each mirror once, retaining response-size and per-mirror timeout caps.
|
||||
* Refusals and body-level failures rotate; total failure returns the last
|
||||
* rate-limit payload, otherwise the first refusal, or throws a network error.
|
||||
* @param {string} body URL-encoded Overpass QL query body.
|
||||
* @param {number} [maxResponseBytes] Endpoint-specific response cap.
|
||||
* @param {object} [options] Server-only endpoint and I/O overrides for tests.
|
||||
* @returns {Promise<{status:number,body:string,contentType:string,endpoint:string,rateLimited:boolean}>}
|
||||
*/
|
||||
async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPONSE_BYTES) {
|
||||
export async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPONSE_BYTES, {
|
||||
endpoints = OVERPASS_UPSTREAMS,
|
||||
fetchImpl = fetch,
|
||||
readBody = readResponseTextCapped,
|
||||
simplify = simplifyOverpassPayloadBody,
|
||||
} = {}) {
|
||||
let lastError = null;
|
||||
let lastRateLimitPayload = null;
|
||||
let lastRefusalPayload = null;
|
||||
|
||||
for (const endpoint of OVERPASS_UPSTREAMS) {
|
||||
for (const endpoint of endpoints) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const upstream = await fetch(endpoint, {
|
||||
const upstream = await fetchImpl(endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
@@ -2577,7 +2607,7 @@ async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPON
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const responseBody = await readResponseTextCapped(upstream, maxResponseBytes);
|
||||
const responseBody = await readBody(upstream, maxResponseBytes);
|
||||
const contentType = upstream.headers.get('content-type') || 'application/json';
|
||||
const status = upstream.status;
|
||||
const rateLimited = status === 429 || overpassLooksRateLimited(responseBody);
|
||||
@@ -2601,14 +2631,23 @@ async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPON
|
||||
lastError = new Error(`Overpass runtime error (${endpoint})`);
|
||||
continue;
|
||||
}
|
||||
if (status >= 500) {
|
||||
// Anything but 2xx is this mirror declining, not an answer. Only 5xx used
|
||||
// to rotate, so a 4xx ended the fan-out and was returned — and cached —
|
||||
// as data: overpass-api.de and its lz4 alias answer 406 to this proxy's
|
||||
// User-Agent while kumi.systems and private.coffee answer 200 to the very
|
||||
// same request, so every Overpass-backed layer failed on an Apache error
|
||||
// page with two healthy mirrors untried. The first refusal is kept so a
|
||||
// genuinely bad query still reports what upstream said, but only after
|
||||
// every mirror has had the chance to answer it.
|
||||
if (status < 200 || status >= 300) {
|
||||
if (!lastRefusalPayload) lastRefusalPayload = payload;
|
||||
lastError = new Error(`Overpass upstream returned ${status} (${endpoint})`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Success: decimate giant boundary geometry before it reaches the cache,
|
||||
// the disk, or the client (what makes the 32 MB read cap safe to hold).
|
||||
payload.body = simplifyOverpassPayloadBody(payload.body);
|
||||
payload.body = simplify(payload.body);
|
||||
return payload;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
@@ -2618,6 +2657,7 @@ async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPON
|
||||
}
|
||||
|
||||
if (lastRateLimitPayload) return lastRateLimitPayload;
|
||||
if (lastRefusalPayload) return lastRefusalPayload;
|
||||
throw lastError || new Error('All Overpass upstreams failed');
|
||||
}
|
||||
|
||||
@@ -2692,6 +2732,15 @@ function overpassProxy() {
|
||||
return;
|
||||
}
|
||||
if (preflight.source !== 'UPSTREAM') {
|
||||
// A coalesced caller sees the same failure as the original request
|
||||
// and must get the same last-good fallback, not the raw refusal.
|
||||
if (!overpassPayloadIsData(preflight.payload)) {
|
||||
const stale = await readStaleOverpass(cacheKey);
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (preflight.source === 'DISK') {
|
||||
_overpassCache.set(cacheKey, preflight.payload);
|
||||
trimOverpassCache();
|
||||
@@ -2710,7 +2759,11 @@ function overpassProxy() {
|
||||
_overpassConcurrent += 1;
|
||||
const requestPromise = fetchOverpassPayload(safeBody)
|
||||
.then((payload) => {
|
||||
if (payload.status < 500 && !payload.rateLimited && !payload.runtimeError) {
|
||||
// Only a 2xx is data. `< 500` cached every 4xx, so one mirror's
|
||||
// refusal was written to memory AND disk — and boundary-class
|
||||
// queries hold a month-long TTL, so a single 406 outlived the
|
||||
// outage that caused it.
|
||||
if (overpassPayloadIsData(payload)) {
|
||||
const entry = { ...payload, cachedAt: Date.now() };
|
||||
_overpassCache.set(cacheKey, entry);
|
||||
trimOverpassCache();
|
||||
@@ -2728,8 +2781,8 @@ function overpassProxy() {
|
||||
// Degraded upstream (rate-limited on every mirror / 5xx / runtime
|
||||
// error): last-good roads beat an empty layer — serve stale from
|
||||
// memory or disk at ANY age before surfacing the failure.
|
||||
if (payload.rateLimited || payload.runtimeError || payload.status >= 500) {
|
||||
const stale = _overpassCache.get(cacheKey) || await readOverpassDisk(cacheKey, Infinity);
|
||||
if (!overpassPayloadIsData(payload)) {
|
||||
const stale = await readStaleOverpass(cacheKey);
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
@@ -2739,7 +2792,7 @@ function overpassProxy() {
|
||||
} catch (e) {
|
||||
// Every mirror threw (network-level). Same serve-stale rule.
|
||||
const stale = cacheKey
|
||||
? (_overpassCache.get(cacheKey) || await readOverpassDisk(cacheKey, Infinity).catch(() => null))
|
||||
? await readStaleOverpass(cacheKey)
|
||||
: null;
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
|
||||
Reference in new issue
Block a user