diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 3e648a62e..4091b56ad 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2956,7 +2956,8 @@ key reads KEY REJECTED and stops tile requests until the layer goes off; a 429 pauses them for its Retry-After. The server proxy (`server/providers/mapillary.js`) keeps the token server-side, -serves only z11–14 coverage, strips the unused `image` layer, coalesces +serves only z11–14 coverage, refuses upstream redirects, strips the unused +`image` layer, coalesces concurrent requests and keeps tiles in a 24 h memory cache. MapillaryJS loads on the first photo. Opening one frames it with a single diff --git a/server/providers/mapillary/tiles.js b/server/providers/mapillary/tiles.js index 76e4c90c5..89d434875 100644 --- a/server/providers/mapillary/tiles.js +++ b/server/providers/mapillary/tiles.js @@ -6,7 +6,6 @@ import { stripTileLayers } from './trim.js'; /** Mapillary vector tiles, /{layer}/2/{z}/{x}/{y}: the only host the token goes to. */ const TILE_HOST = 'https://tiles.mapillary.com/maps/vtp'; -const TILE_ORIGIN = new URL(TILE_HOST).origin; /** Only the street zooms the app draws are proxied. */ const MIN_ZOOM = 11; const MAX_ZOOM = 14; @@ -17,21 +16,19 @@ const MAX_ZOOM = 14; const DROP_LAYERS = ['image']; /** A z14 tile with its image layer is ~11 MB; anything past this is wrong. */ export const TILE_MAX_BYTES = 48 * 1024 * 1024; -export const TILE_MAX_REDIRECTS = 3; const FETCH_TIMEOUT_MS = 60_000; /** Tiles change only when new imagery is processed. */ export const TILE_TTL_MS = 24 * 60 * 60 * 1000; export const TILE_MEMORY_BUDGET_BYTES = 96 * 1024 * 1024; /** Per-entry cost on top of its bytes, so empty tiles still count. */ const ENTRY_OVERHEAD_BYTES = 1024; -const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); /** The client token lives in the browser by design; the server adds it to tile URLs too. */ export function mapillaryToken() { return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim(); } -/** Mapillary answered with an error status (or a redirect/size this proxy refuses). */ +/** Mapillary answered with an error status (or a size this proxy refuses). */ export class TileUpstreamError extends Error { constructor(status, message, retryAfter = null) { super(message || `Mapillary tiles HTTP ${status}`); @@ -54,27 +51,6 @@ export function parseTilePath(pathname) { return { z, x, y, key: `${z}/${x}/${y}` }; } -/** The next hop, if it stays HTTPS on the tile origin (the token is in the query). */ -function pinnedRedirect(location, base) { - let target = null; - try { - target = location ? new URL(location, base) : null; - } catch { - /* unparseable Location */ - } - if ( - target?.protocol !== 'https:' || - target.origin !== TILE_ORIGIN || - target.username || - target.password - ) - throw new TileUpstreamError( - 502, - 'Mapillary tile redirect left the tile origin', - ); - return target.href; -} - function trim(bytes) { try { return stripTileLayers(bytes, DROP_LAYERS); @@ -87,36 +63,31 @@ function trim(bytes) { /** One tile from Mapillary, trimmed; empty bytes mean no coverage. */ async function fetchUpstream({ z, x, y }) { const token = encodeURIComponent(mapillaryToken()); - let url = `${TILE_HOST}/mly1_public/2/${z}/${x}/${y}?access_token=${token}`; - const signal = AbortSignal.timeout(FETCH_TIMEOUT_MS); - for (let hop = 0; ; hop++) { - // Redirects are followed by hand so every hop is checked. - const response = await fetch(url, { - signal, + const response = await fetch( + `${TILE_HOST}/mly1_public/2/${z}/${x}/${y}?access_token=${token}`, + { + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + // The token is in the query: a redirect is never followed (a 3xx is a 502). redirect: 'manual', headers: { Accept: 'application/x-protobuf' }, - }); - if (response.ok && response.status !== 204) { - try { - const bytes = await readResponseBytesCapped(response, TILE_MAX_BYTES); - return trim(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length)); - } catch (error) { - if (error?.code !== 'RESPONSE_TOO_LARGE') throw error; - throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap'); - } - } + }, + ); + if (!response.ok || response.status === 204) { await response.body?.cancel().catch(() => {}); if (response.status === 204 || response.status === 404) return Buffer.alloc(0); - if (!REDIRECT_STATUSES.has(response.status)) - throw new TileUpstreamError( - response.status, - undefined, - response.headers.get('retry-after'), - ); - if (hop >= TILE_MAX_REDIRECTS) - throw new TileUpstreamError(502, 'Mapillary tile redirected too often'); - url = pinnedRedirect(response.headers.get('location'), url); + throw new TileUpstreamError( + response.status, + undefined, + response.headers.get('retry-after'), + ); + } + try { + const bytes = await readResponseBytesCapped(response, TILE_MAX_BYTES); + return trim(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length)); + } catch (error) { + if (error?.code !== 'RESPONSE_TOO_LARGE') throw error; + throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap'); } } diff --git a/src/tooling/mapillaryProvider.test.mjs b/src/tooling/mapillaryProvider.test.mjs index 8d7363ff3..949950f1f 100644 --- a/src/tooling/mapillaryProvider.test.mjs +++ b/src/tooling/mapillaryProvider.test.mjs @@ -11,7 +11,6 @@ import { } from '../../server/providers/mapillary/trim.js'; import { TILE_MAX_BYTES, - TILE_MAX_REDIRECTS, TILE_MEMORY_BUDGET_BYTES, TILE_TTL_MS, _resetTileCacheForTest, @@ -260,7 +259,7 @@ for (const [sent, expected] of [ ); }); -for (const status of [400, 410, 500]) +for (const status of [302, 400, 410, 500]) test(`an upstream ${status} is a 502, not the client's fault`, async () => { await withUpstream( () => new Response('{}', { status }), @@ -274,62 +273,6 @@ for (const status of [400, 410, 500]) ); }); -const redirect = (location) => - new Response(null, { - status: 302, - headers: location === null ? {} : { location }, - }); - -test('a redirect within the tile origin is followed by hand; endless ones stop', async () => { - await withUpstream( - (n) => - n === 1 ? redirect('/maps/vtp/mly1_public/2/14/13/13?moved=1') : ok(), - async ({ calls, inits, call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/14/13/13'); - assert.equal(res.statusCode, 200); - assert.equal( - calls[1], - 'https://tiles.mapillary.com/maps/vtp/mly1_public/2/14/13/13?moved=1', - ); - for (const init of inits) assert.equal(init.redirect, 'manual'); - }, - ); - await withUpstream( - (n) => redirect(`/maps/vtp/mly1_public/2/14/15/15?hop=${n}`), - async ({ calls, call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/14/15/15'); - assert.equal(res.statusCode, 502); - assert.deepEqual(json(res), { - error: 'Mapillary tile redirected too often', - }); - assert.equal(calls.length, TILE_MAX_REDIRECTS + 1); - }, - ); -}); - -for (const [why, location] of [ - ['another host', 'https://evil.example/collect'], - ['a look-alike host', 'https://tiles.mapillary.com.evil.example/t'], - ['another port', 'https://tiles.mapillary.com:8443/t'], - ['plain HTTP', 'http://tiles.mapillary.com/maps/vtp/t'], - ['credentials in the URL', 'https://user:pw@tiles.mapillary.com/t'], - ['a scheme-relative host', '//evil.example/t'], - ['no Location', null], -]) - test(`a redirect off the tile origin (${why}) is a 502, and the token stays home`, async () => { - await withUpstream( - (n) => (n === 1 ? redirect(location) : ok()), - async ({ calls, call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/14/14/14'); - assert.equal(res.statusCode, 502); - assert.deepEqual(json(res), { - error: 'Mapillary tile redirect left the tile origin', - }); - assert.equal(calls.length, 1, 'the redirect was not followed'); - }, - ); - }); - test('a streamed body past the size cap is cancelled and answered 502', async () => { const MB = 1024 * 1024; let pulled = 0;