From fb5ee5aa36272f6463385f9ebe1e5085bd0164e6 Mon Sep 17 00:00:00 2001 From: Bilawal Sidhu <106619546+bilawalsidhu@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:11:32 -0500 Subject: [PATCH] fix(installations): send an identifying User-Agent to Overpass (#570) Supersedes #420 by @GladiatorrX9, ported onto the current component layout with fixes from review. Verified before merge: format, package boundaries, unit suite, build, and the browser tracking suite; a browser harness for the change where one applies; adversarial review applied. Co-authored-by: Cyber Knight --- CHANGELOG.md | 7 +++ docs/CURRENT-STATE.md | 1 + server/providers/overpass/constants.js | 14 +++++ server/providers/overpass/transport.js | 14 ++--- src/overpassProxy.test.mjs | 83 ++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c8b828b2..168d709cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,13 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md proxy as `/api/gbfs?url=`, but the proxy reads its upstream target from the path, so every request answered 400 and the layer reported a fetch error for every city (#441 — thanks @MiguelGFerreira). +- Overpass requests now carry a User-Agent that names the application, its + version and the project address, which is what the OpenStreetMap API usage + policy asks for; the previous string identified neither. A mirror may refuse + a client it cannot identify, and a refused mirror is one the fan-out has to + skip, so this affects every Overpass-backed layer: Mapped Installations, + traffic roads and annotation geometry. Mirror rotation, cooldown and cache + admission are unchanged (#420 — thanks @GladiatorrX9). - Draped annotation geometry — area fills and outlines, routes and arrows — classifies onto terrain as well as 3D tiles. On a keyless boot, where Cesium's diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fde4d4c59..4305e0356 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2884,6 +2884,7 @@ silently demoting every later lookup for the session. - `/api/overpass` fans out across four public mirrors. `overpassPayloadIsData()` governs cache reads, writes, and stale fallback: only a 2xx that is neither rate-limited nor a body-level runtime error qualifies. Previously stored refusals are ignored on both fresh and stale reads, so upgrading does not require manually clearing the disk cache. - HTTP refusals such as 406 now rotate alongside the existing network, rate-limit, and runtime-error cases. A refusal from one mirror no longer prevents reaching healthy alternatives or persists under the seven-day road/month-long boundary cache TTLs. Concurrent identical queries share one mirror sequence; if it fails, both the initiating and joined callers can use the same last-good data. - A refusal every mirror agrees on is still reported with the first mirror's status and body, so a genuinely malformed query says what upstream said — but only after every mirror has had the chance to answer it. `fetchOverpassPayload` takes injectable endpoints and fetch so the rotation is tested without a live mirror (`src/overpassProxy.test.mjs`). +- Every mirror is asked with a User-Agent that names the application, its version and the project address (`OVERPASS_USER_AGENT` in `server/providers/overpass/constants.js`), which is what the OSM API usage policy asks for. Mirrors may refuse a client they cannot identify; such a refusal is never treated as data and costs the fan-out that mirror, so the header is what keeps the list at full strength. ### Share-link v2 layer state (August 2026) diff --git a/server/providers/overpass/constants.js b/server/providers/overpass/constants.js index aef0267eb..ddb53c01f 100644 --- a/server/providers/overpass/constants.js +++ b/server/providers/overpass/constants.js @@ -3,6 +3,19 @@ import path from 'node:path'; // --------------------------------------------------------------------------- // Overpass API proxy constants and cache state // --------------------------------------------------------------------------- +/** + * User-Agent sent to every Overpass mirror. + * + * The OSM API usage policy asks for a "Valid User-Agent identifying application + * and version"; a generic proxy label is not one. A mirror is free to refuse a + * client it cannot identify, and `src/overpassProxy.test.mjs` pins what that + * costs: a refusal is never data, so the query falls through to whatever + * mirrors are left. Keep this honest and stable — if it is ever refused, the + * answer is less query volume, not a new name. + */ +const OVERPASS_USER_AGENT = + 'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)'; + /** Ordered list of Overpass API mirrors; tried sequentially on failure/rate-limit. */ const OVERPASS_UPSTREAMS = [ 'https://overpass-api.de/api/interpreter', @@ -130,5 +143,6 @@ export { OVERPASS_SIMPLIFY_TOLERANCE_DEG, OVERPASS_MAX_RESPONSE_BYTES, OVERPASS_UPSTREAMS, + OVERPASS_USER_AGENT, OVERPASS_TIMEOUT_MS, }; diff --git a/server/providers/overpass/transport.js b/server/providers/overpass/transport.js index ff15ae0ff..28a608956 100644 --- a/server/providers/overpass/transport.js +++ b/server/providers/overpass/transport.js @@ -1,6 +1,7 @@ import { OVERPASS_MAX_RESPONSE_BYTES, OVERPASS_UPSTREAMS, + OVERPASS_USER_AGENT, OVERPASS_TIMEOUT_MS, } from './constants.js'; import { readResponseTextCapped } from '../common/http.js'; @@ -91,7 +92,7 @@ async function fetchOverpassPayload( method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', - 'User-Agent': 'gods-eye-view-overpass-proxy/1.0', + 'User-Agent': OVERPASS_USER_AGENT, }, body, signal: controller.signal, @@ -125,12 +126,11 @@ async function fetchOverpassPayload( } // Anything but 2xx is this mirror declining, not an answer. Only 5xx used // to rotate, so a 4xx ended the fan-out and was returned — and cached — - // as data: overpass-api.de and its lz4 alias answer 406 to this proxy's - // User-Agent while kumi.systems and private.coffee answer 200 to the very - // same request, so every Overpass-backed layer failed on an Apache error - // page with two healthy mirrors untried. The first refusal is kept so a - // genuinely bad query still reports what upstream said, but only after - // every mirror has had the chance to answer it. + // as data: a mirror refusing this client answers 406 while the others + // answer 200 to the very same request, so every Overpass-backed layer + // failed on an error page with healthy mirrors untried. The first + // refusal is kept so a genuinely bad query still reports what upstream + // said, but only after every mirror has had the chance to answer it. if (status < 200 || status >= 300) { if (!lastRefusalPayload) lastRefusalPayload = payload; lastError = new Error( diff --git a/src/overpassProxy.test.mjs b/src/overpassProxy.test.mjs index 106946129..f25f20b16 100644 --- a/src/overpassProxy.test.mjs +++ b/src/overpassProxy.test.mjs @@ -97,6 +97,89 @@ test('a refusal moves to the next mirror instead of ending the fan-out', async ( assert.deepEqual(tried, ENDPOINTS.slice(0, 2), 'the healthy mirror must be reached, and no further'); }); +test('every mirror is asked with a User-Agent that identifies the application', async () => { + // The OSM API usage policy asks for a "Valid User-Agent identifying + // application and version". Every outbound request must carry it, not just + // the first: a mirror further down the list refusing an unidentified client + // is exactly the case the fan-out exists to survive. + const seen = []; + const fetchImpl = async (url, options) => { + seen.push({ url, agent: options?.headers?.['User-Agent'] }); + // Refuse everywhere, so the loop is forced through the whole list. + return { status: 503, headers: { get: () => 'text/html' } }; + }; + await fetchOverpassPayload('data=x', 1e6, { + endpoints: ENDPOINTS, + fetchImpl, + readBody: async () => 'upstream down', + simplify: (body) => body, + }); + + assert.deepEqual( + seen.map((request) => request.url), + ENDPOINTS, + 'the fan-out must reach every mirror', + ); + for (const request of seen) { + const agent = String(request.agent || ''); + assert.match( + agent, + /^gods-eye-view\/\d/, + `${request.url} must name the application and its version`, + ); + assert.ok( + !/proxy\/1\.0$/.test(agent), + `${request.url} must not fall back to the unidentified label`, + ); + assert.match( + agent, + /github\.com\/bilawalsidhu\/gods-eye-view/, + `${request.url} must carry a route back to the project`, + ); + } +}); + +test('a mirror that refuses the old label serves the same query under the identifying one', async () => { + // Recorded from the canonical instance on 2026-09-14: the unidentified label + // is answered with a 406 Not Acceptable HTML page before the query is read, + // and the identifying one is served. This stub replays that shape so the + // behaviour the header change buys is pinned without a live mirror. + const REFUSED = 'gods-eye-view-overpass-proxy/1.0'; + const answer = (agent) => + String(agent || '').startsWith(REFUSED) + ? { + status: 406, + body: '406 Not Acceptable', + contentType: 'text/html', + } + : { status: 200, body: DATA.body, contentType: 'application/json' }; + + let last = null; + const run = (agentOverride) => + fetchOverpassPayload('data=x', 1e6, { + endpoints: [ENDPOINTS[0]], + fetchImpl: async (url, options) => { + last = answer(agentOverride ?? options?.headers?.['User-Agent']); + return { status: last.status, headers: { get: () => last.contentType } }; + }, + readBody: async () => last.body, + simplify: (body) => body, + }); + + const refused = await run(REFUSED); + assert.equal(refused.status, 406, 'the old label is refused by the mirror'); + assert.equal( + overpassPayloadIsData(refused), + false, + 'a refusal is never treated as data', + ); + + const served = await run(undefined); + assert.equal(served.status, 200, 'the header the proxy now sends is served'); + assert.equal(overpassPayloadIsData(served), true); + assert.equal(served.body, DATA.body); +}); + test('the first mirror to answer wins, and the rest are left alone', async () => { const { payload, tried } = await run({ [ENDPOINTS[0]]: DATA, [ENDPOINTS[1]]: DATA, [ENDPOINTS[2]]: DATA,