diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 7674c2841..9dfad2848 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3100,7 +3100,11 @@ The server proxy (`server/providers/mapillary.js`) adds the token to coverage-tile requests (the browser holds it too, for MapillaryJS and the Graph API), serves only z11–14 coverage, refuses upstream redirects, strips the unused `image` layer, shares one fetch per tile, runs at most six at once -(cancelling those every client has left) and keeps tiles in memory for 24 h. +(cancelling those every client has left) and keeps tiles in memory for 24 h +(a body too big to be a trimmed tile is served but not kept). While Mapillary +rate-limits, the proxy answers 429 itself until the Retry-After (seconds or an +HTTP date, at most 10 minutes) is over, so the rest of a view does not go on +asking. MapillaryJS loads on the first photo. Opening one frames it with a single camera flight through the application's deferred navigation (`begin` / diff --git a/server/providers/mapillary.js b/server/providers/mapillary.js index c27241900..9ee5bdb6c 100644 --- a/server/providers/mapillary.js +++ b/server/providers/mapillary.js @@ -4,6 +4,7 @@ import { fetchTile, mapillaryToken, parseTilePath, + retryAfterWait, TileUpstreamError, } from './mapillary/tiles.js'; @@ -12,8 +13,6 @@ import { * 9-tile views a minute, more than a person flying the camera reaches. */ export const TILE_ROUTE_MAX_PER_MIN = 600; -/** The longest upstream Retry-After passed on, in seconds. */ -const MAX_RETRY_AFTER_SEC = 600; function sendJson(res, status, payload) { res.statusCode = status; @@ -69,11 +68,7 @@ async function handleTile(req, res, allow) { keyRejected: true, }); if (status === 429) { - const sent = Number(error.retryAfter); - const retryAfter = - Number.isInteger(sent) && sent > 0 - ? Math.min(sent, MAX_RETRY_AFTER_SEC) - : 60; + const retryAfter = retryAfterWait(error.retryAfter); res.setHeader('Retry-After', String(retryAfter)); return sendJson(res, 429, { error: 'Mapillary is rate-limiting tile requests', diff --git a/server/providers/mapillary/tiles.js b/server/providers/mapillary/tiles.js index 4ef77090d..b7bfcf9fd 100644 --- a/server/providers/mapillary/tiles.js +++ b/server/providers/mapillary/tiles.js @@ -20,6 +20,10 @@ export const TILE_UPSTREAM_CONCURRENCY = 6; export const TILE_TTL_MS = 24 * 60 * 60 * 1000; /** Trimmed tiles are ~80 KB: a dozen 9-tile views, well under 100 MB. */ export const TILE_MEMORY_MAX_TILES = 120; +/** A trimmed tile is ~80 KB; a body this big failed to trim and is not kept. */ +export const TILE_MEMORY_MAX_TILE_BYTES = 4 * 1024 * 1024; +/** The longest upstream Retry-After honoured, in seconds. */ +export const MAX_RETRY_AFTER_SEC = 600; /** The client token lives in the browser by design; the server adds it to tile URLs too. */ export function mapillaryToken() { @@ -37,6 +41,40 @@ export class TileUpstreamError extends Error { } } +/** Seconds a Retry-After header asks for (an integer or an HTTP date), else null. */ +export function retryAfterSeconds(header, now = Date.now()) { + const text = String(header ?? '').trim(); + if (!text) return null; + const seconds = /^\d+$/.test(text) + ? Number(text) + : (Date.parse(text) - now) / 1000; + return Number.isFinite(seconds) && seconds > 0 ? Math.ceil(seconds) : null; +} + +/** The wait a 429 asks for, bounded; a missing or malformed header means a minute. */ +export function retryAfterWait(header, now = Date.now()) { + return Math.min(retryAfterSeconds(header, now) ?? 60, MAX_RETRY_AFTER_SEC); +} + +/** + * While Mapillary rate-limits, tile misses are refused here until the wait is + * over, so the rest of a view (and the fetches queued behind the first 429) + * do not go on asking. Scoped to the token that was refused. + * @type {{until: number, token: string}|null} + */ +let refusal = null; + +/** The refusal still in force for `token`, as the error to answer with. */ +function refusedNow(token) { + if (!refusal) return null; + const left = refusal.until - Date.now(); + if (left <= 0 || refusal.token !== token) { + refusal = null; + return null; + } + return new TileUpstreamError(429, undefined, String(Math.ceil(left / 1000))); +} + /** * Parse `/coverage/{z}/{x}/{y}`; null unless z is 11–14 and x/y are on the grid. * @returns {{z: number, x: number, y: number, key: string}|null} @@ -60,9 +98,11 @@ function trim(bytes) { /** One tile from Mapillary, trimmed; empty bytes mean no coverage. */ async function fetchUpstream({ z, x, y }, signal) { - const token = encodeURIComponent(mapillaryToken()); + const token = mapillaryToken(); + const refused = refusedNow(token); + if (refused) throw refused; const response = await fetch( - `${TILE_HOST}/mly1_public/2/${z}/${x}/${y}?access_token=${token}`, + `${TILE_HOST}/mly1_public/2/${z}/${x}/${y}?access_token=${encodeURIComponent(token)}`, { signal: AbortSignal.any([signal, AbortSignal.timeout(FETCH_TIMEOUT_MS)]), // The token is in the query: a redirect is never followed (a 3xx is a 502). @@ -74,11 +114,13 @@ async function fetchUpstream({ z, x, y }, signal) { await response.body?.cancel().catch(() => {}); if (response.status === 204 || response.status === 404) return Buffer.alloc(0); - throw new TileUpstreamError( - response.status, - undefined, - response.headers.get('retry-after'), - ); + const retryAfter = response.headers.get('retry-after'); + if (response.status === 429) + refusal = { + token, + until: Date.now() + retryAfterWait(retryAfter) * 1000, + }; + throw new TileUpstreamError(response.status, undefined, retryAfter); } try { const bytes = await readResponseBytesCapped(response, TILE_MAX_BYTES); @@ -147,6 +189,7 @@ const memory = new Map(); const inFlight = new Map(); function remember(key, bytes, at = Date.now()) { + if (bytes.length > TILE_MEMORY_MAX_TILE_BYTES) return; memory.delete(key); memory.set(key, { bytes, at }); for (const oldest of memory.keys()) { @@ -216,4 +259,5 @@ export async function fetchTile(address, { signal } = {}) { export function _resetTileCacheForTest() { memory.clear(); inFlight.clear(); + refusal = null; } diff --git a/src/tooling/mapillaryProvider.test.mjs b/src/tooling/mapillaryProvider.test.mjs index b560eb6fe..f55d73397 100644 --- a/src/tooling/mapillaryProvider.test.mjs +++ b/src/tooling/mapillaryProvider.test.mjs @@ -7,7 +7,9 @@ import { } from 'gods-eye-view/server/providers/mapillary'; import { stripTileLayers } from '../../server/providers/mapillary/trim.js'; import { + MAX_RETRY_AFTER_SEC, TILE_MAX_BYTES, + TILE_MEMORY_MAX_TILE_BYTES, TILE_MEMORY_MAX_TILES, TILE_UPSTREAM_CONCURRENCY, TILE_TTL_MS, @@ -244,7 +246,6 @@ for (const status of [401, 403]) for (const [sent, expected] of [ ['30', 30], [null, 60], - ['Wed, 21 Oct 2026 07:28:00 GMT', 60], ['999999', 600], ]) test(`an upstream 429 (Retry-After ${sent}) is passed on as ${expected} s`, async () => { @@ -262,9 +263,10 @@ for (const [sent, expected] of [ error: 'Mapillary is rate-limiting tile requests', retryAfter: expected, }); - // Errors are not cached: the next request asks again. - await call('/api/mapillary/tiles', '/coverage/14/5/5'); - assert.equal(calls.length, 2); + // Not cached as a tile, but refused here until the wait is over. + const again = await call('/api/mapillary/tiles', '/coverage/14/5/5'); + assert.equal(again.statusCode, 429); + assert.equal(calls.length, 1); }, ); }); @@ -557,6 +559,126 @@ test('the memory cache evicts the least recently used tile past its size', async ); }); +// ── Upstream refusals ── + +const rateLimited = (retryAfter) => + new Response('', { status: 429, headers: { 'retry-after': retryAfter } }); + +test('after a 429 the rest of the view waits out Retry-After here instead of asking Mapillary', async (t) => { + const now = Date.now(); + t.mock.method(Date, 'now', () => now); + await withUpstream( + (n) => (n === 1 ? rateLimited('120') : ok()), + async ({ calls, call }) => { + const first = await call('/api/mapillary/tiles', '/coverage/14/1/1'); + assert.equal(first.statusCode, 429); + assert.equal(json(first).retryAfter, 120); + const rest = await Promise.all( + Array.from({ length: 8 }, (_, i) => + call('/api/mapillary/tiles', `/coverage/14/2/${i}`), + ), + ); + assert.equal(calls.length, 1, 'nothing else reached Mapillary'); + for (const res of rest) { + assert.equal(res.statusCode, 429); + assert.equal(json(res).retryAfter, 120); + } + // Half way through the wait, the answer says what is left. + Date.now.mock.mockImplementation(() => now + 50_000); + const midway = await call('/api/mapillary/tiles', '/coverage/14/3/3'); + assert.equal(json(midway).retryAfter, 70); + assert.equal(calls.length, 1); + // Once it is over, requests go upstream again. + Date.now.mock.mockImplementation(() => now + 121_000); + const later = await call('/api/mapillary/tiles', '/coverage/14/3/3'); + assert.equal(later.statusCode, 200); + assert.equal(calls.length, 2); + }, + ); +}); + +test('a refusal is scoped to the token that was refused', async () => { + await withUpstream( + (n) => (n === 1 ? rateLimited('120') : ok()), + async ({ calls, call }) => { + await call('/api/mapillary/tiles', '/coverage/14/1/1'); + process.env.MAPILLARY_CLIENT_TOKEN = 'MLY|replaced|token'; + const res = await call('/api/mapillary/tiles', '/coverage/14/1/2'); + assert.equal(res.statusCode, 200, 'a new token is tried at once'); + assert.equal(calls.length, 2); + }, + ); +}); + +test( + 'fetches queued behind a 429 never reach Mapillary', + { timeout: 5000 }, + async () => { + let release; + const gate = new Promise((resolve) => (release = resolve)); + const answer = (n) => gate.then(() => (n === 1 ? rateLimited('60') : ok())); + await withUpstream(answer, async ({ calls }) => { + const busy = Array.from({ length: TILE_UPSTREAM_CONCURRENCY }, (_, i) => + fetchTile(parseTilePath(`/coverage/14/${i}/6`)).catch((e) => e), + ); + const queued = Array.from({ length: 3 }, (_, i) => + fetchTile(parseTilePath(`/coverage/14/${i}/7`)).catch((e) => e), + ); + await settleSoon(); + assert.equal(calls.length, TILE_UPSTREAM_CONCURRENCY, 'three wait'); + release(); + const [first] = await Promise.all(busy); + assert.equal(first.status, 429); + for (const outcome of await Promise.all(queued)) { + assert.equal(outcome.status, 429, 'refused here'); + assert.equal(outcome.retryAfter, '60'); + } + assert.equal(calls.length, TILE_UPSTREAM_CONCURRENCY, 'and never ran'); + }); + }, +); + +test('an HTTP-date Retry-After is passed on as the seconds left; garbage means a minute', async (t) => { + const now = Date.UTC(2026, 9, 13, 12, 0, 0); + t.mock.method(Date, 'now', () => now); + const inFiveMinutes = new Date(now + 5 * 60_000).toUTCString(); + const inAnHour = new Date(now + 60 * 60_000).toUTCString(); + const anHourAgo = new Date(now - 60 * 60_000).toUTCString(); + const sent = [inFiveMinutes, inAnHour, anHourAgo, 'soon', '']; + await withUpstream( + (n) => rateLimited(sent[n - 1]), + async ({ call }) => { + const retryAfter = async (x) => { + _resetTileCacheForTest(); + const res = await call('/api/mapillary/tiles', `/coverage/14/${x}/9`); + assert.equal(res.statusCode, 429); + assert.equal(res.headers['retry-after'], String(json(res).retryAfter)); + return json(res).retryAfter; + }; + assert.equal(await retryAfter(1), 300); + assert.equal(await retryAfter(2), MAX_RETRY_AFTER_SEC, 'capped'); + assert.equal(await retryAfter(3), 60, 'a date in the past'); + assert.equal(await retryAfter(4), 60, 'not a date'); + assert.equal(await retryAfter(5), 60, 'no header'); + }, + ); +}); + +test('a body too big to be a trimmed tile is served but not kept in memory', async () => { + const huge = Buffer.alloc(TILE_MEMORY_MAX_TILE_BYTES + 1, 0xff); + await withUpstream( + () => ok(huge), + async ({ calls, call }) => { + const first = await call('/api/mapillary/tiles', '/coverage/14/5/5'); + assert.equal(first.statusCode, 200); + assert.equal(first.body.length, huge.length, 'served as it came'); + const second = await call('/api/mapillary/tiles', '/coverage/14/5/5'); + assert.equal(second.headers['x-gev-cache'], 'upstream', 'not cached'); + assert.equal(calls.length, 2); + }, + ); +}); + // ── Tile trimming ── test('dropping a layer keeps the others byte-for-byte', () => {