diff --git a/CHANGELOG.md b/CHANGELOG.md index dd76d7a25..c2b02f0b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +- Overpass data whose element tags contain a failure phrase ("timed out", + "runtime error", "too many requests" and similar, e.g. a place named "Timed + Out Escape Rooms") is returned as data instead of being refused and putting + the endpoint on cooldown; upstream failures are read from the remark. + - Add Barcelona's bike share to the GBFS registry: Bicing (543 stations, the city's PBSC system) and AMBici (237 e-bike stations across the metropolitan area, run by nextbike). Both are keyless GBFS 2.3 feeds listed in diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 4c06dd3cb..3424560eb 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -525,7 +525,7 @@ are protected from fallback writes. Version 6 is the export format, with zero pitch, low camera heights, scope and detection edits preserved. See [the document contract](SCENE-DOCUMENT.md); see [Director](DIRECTOR.md) for camera, pack, action and sharing support. -Search framing and annotations request semantic map features from an explicit source. The Overpass adapter owns bounded queries, member/tag decoding and request deadlines; callers retain candidate ranking, outline caching, deferred retries and scene placement. Empty, unavailable, transient and throttled outcomes remain distinct. Traffic sources return road records and installation sources return mapped records with freshness/saturation metadata, so their layers no longer decode upstream elements. ALPR already normalizes its records in the source. Traffic uses OpenFreeMap roads with optional matched TomTom congestion; ALPR uses an hourly OpenStreetMap extract. Keyless installations use military-area tiles. Detailed feature queries require operator-configured Overpass. +Search framing and annotations request semantic map features from an explicit source. The Overpass adapter owns bounded queries, member/tag decoding and request deadlines; callers retain candidate ranking, outline caching, deferred retries and scene placement. Empty, unavailable, transient and throttled outcomes remain distinct; a data reply is refused only for its `remark`, never for text in element tags. Traffic sources return road records and installation sources return mapped records with freshness/saturation metadata, so their layers no longer decode upstream elements. ALPR already normalizes its records in the source. Traffic uses OpenFreeMap roads with optional matched TomTom congestion; ALPR uses an hourly OpenStreetMap extract. Keyless installations use military-area tiles. Detailed feature queries require operator-configured Overpass. Nepal media preloads survive repeated camera-flight updates, but Stop, event disable and replacement remove abandoned frames and revoke pending Facebook sessions. Fallback evidence-card clicks respect the shared drawing-tool pointer lease. diff --git a/server/providers/overpass/transport.js b/server/providers/overpass/transport.js index 38c31bb08..e3776e291 100644 --- a/server/providers/overpass/transport.js +++ b/server/providers/overpass/transport.js @@ -14,7 +14,7 @@ import { simplifyOverpassPayloadBody } from './geometry.js'; * Checks for known rate-limit phrases in the body text regardless of * HTTP status code, since some mirrors return 200 with an error payload. * - * @param {string} bodyText - Upstream response body. + * @param {string} bodyText - The remark of a data reply, or a non-data body. * @returns {boolean} True if the body looks rate-limited. */ function overpassLooksRateLimited(bodyText) { @@ -41,6 +41,14 @@ function overpassLooksRuntimeError(bodyText) { ); } +function parseJson(text) { + try { + return JSON.parse(text); + } catch { + return undefined; + } +} + /** * True only for an upstream response that is actually Overpass data. * @@ -151,15 +159,22 @@ async function fetchOverpassPayload( signal: controller.signal, }); const responseBody = await readBody(upstream, maxResponseBytes); + const parsed = parseJson(responseBody); + // An Overpass data reply reports failure in its remark; its element + // tags are data and can contain any phrase. Any other body (an error + // page, or JSON without elements) is matched whole. + const failureText = Array.isArray(parsed?.elements) + ? String(parsed.remark || '') + : responseBody; const rateLimited = upstream.status === 429 || upstream.status === 406 || - overpassLooksRateLimited(responseBody); + overpassLooksRateLimited(failureText); if ( rateLimited || upstream.status < 200 || upstream.status >= 300 || - overpassLooksRuntimeError(responseBody) + overpassLooksRuntimeError(failureText) ) { const failures = (previous?.failures || 0) + 1; const delay = retryDelay( @@ -178,7 +193,6 @@ async function fetchOverpassPayload( failure = refusal(status, delay); continue; } - const parsed = JSON.parse(responseBody); if (!Array.isArray(parsed?.elements) || parsed.remark) throw new Error('Malformed Overpass response'); cooldowns.delete(endpoint); diff --git a/src/overpassProxy.test.mjs b/src/overpassProxy.test.mjs index ed4539fda..788846db7 100644 --- a/src/overpassProxy.test.mjs +++ b/src/overpassProxy.test.mjs @@ -126,6 +126,32 @@ test('production reader rejects oversized, malformed and runtime-failure bodies' } }); +test('element data that mentions a failure phrase is data, and sets no cooldown', async () => { + const names = ['Timed Out Escape Rooms', 'Runtime Error Bar', 'Out of Memory Lane', 'Too Many Requests Cafe', 'rate_limited']; + for (const [i, name] of names.entries()) { + const body = JSON.stringify({ elements: [{ type: 'node', id: 1, lat: 1, lon: 2, tags: { name } }] }); + let count = 0; + const options = { endpoints: [`https://phrase-${i}.example/`], fetchImpl: async () => { + count++; return new Response(body); + } }; + const payload = await fetchOverpassPayload('first', 1024, options); + assert.equal(payload.status, 200, name); + assert.ok(overpassPayloadIsData(payload), name); + assert.ok(payload.body.includes(name), name); + await fetchOverpassPayload('different', 1024, options); + assert.equal(count, 2, name); + } + const remark = JSON.stringify({ elements: [], remark: 'runtime error: rate_limited' }); + const limited = await fetchOverpassPayload('x', 1024, { endpoints: ['https://remark.example/'], + fetchImpl: async () => new Response(remark, { headers: { 'Retry-After': '90' } }) }); + assert.equal(limited.status, 429); + assert.equal(limited.retryAfterMs, 90_000); + const error = await fetchOverpassPayload('x', 1024, { endpoints: ['https://error.example/'], + fetchImpl: async () => new Response('{"error":"rate_limited"}', { headers: { 'Retry-After': '120' } }) }); + assert.equal(error.status, 429); + assert.equal(error.retryAfterMs, 120_000); +}); + function proxyHandler() { const plugin = createViteConfig({ mode: 'test' }).plugins.find(p => p.name === 'overpass-proxy'); const routes = new Map();