import assert from 'node:assert/strict'; import test from 'node:test'; import { PbfWriter } from 'pbf'; import { mapillaryProxy, TILE_ROUTE_MAX_PER_MIN, } from 'gods-eye-view/server/providers/mapillary'; import { stripTileLayers } from '../../server/providers/mapillary/trim.js'; import { MAX_RETRY_AFTER_SEC, TILE_MAX_BYTES, TILE_MEMORY_MAX_TILE_BYTES, TILE_MEMORY_MAX_TILES, TILE_UPSTREAM_CONCURRENCY, TILE_TTL_MS, fetchTile, parseTilePath, _resetTileCacheForTest, } from '../../server/providers/mapillary/tiles.js'; /** Build a minimal MVT: layers with a name, a version and one opaque feature. */ function tile(layers) { const writer = new PbfWriter(); for (const { name, payload } of layers) { writer.writeMessage( 3, (layer, pbf) => { pbf.writeVarintField(15, 2); // version pbf.writeStringField(1, layer.name); pbf.writeMessage(2, (_f, p) => p.writeVarintField(1, 7), null); // feature pbf.writeVarintField(5, 4096); // extent if (layer.payload) pbf.writeBytesField(4, layer.payload); // a big key }, { name, payload }, ); } return Buffer.from(writer.finish()); } const SEQUENCE = tile([{ name: 'sequence' }]); const UNTRIMMED = tile([ { name: 'sequence' }, { name: 'image', payload: Buffer.alloc(4000, 1) }, ]); /** Mount the plugin and return `call(route, url, method, headers, onResponse)`. */ function install(mode = 'configureServer') { const routes = new Map(); mapillaryProxy()[mode]({ middlewares: { use: (route, handler) => routes.set(route, handler) }, }); const call = async ( route, url = '/', method = 'GET', reqHeaders = {}, onResponse = null, ) => { const handler = routes.get(route); assert.ok(handler, `route ${route} is mounted`); const headers = {}; const listeners = new Map(); const res = { statusCode: 200, headers, on: (type, listener) => listeners.set(type, listener), /** The client goes away, as Node reports it. */ close: () => listeners.get('close')?.(), setHeader: (name, value) => (headers[name.toLowerCase()] = value), getHeader: (name) => headers[name.toLowerCase()], writeHead(status, extra) { this.statusCode = status; Object.assign(headers, extra || {}); }, end(payload) { this.body = payload; this.writableEnded = true; }, }; onResponse?.(res); await handler({ url, method, headers: reqHeaders }, res); return res; }; return { routes, call }; } const json = (res) => JSON.parse(String(res.body)); /** Every header value and the body, as text, for a token-leak check. */ const leakText = (res) => [ ...Object.values(res.headers).map(String), Buffer.isBuffer(res.body) ? res.body.toString('latin1') : String(res.body), ].join('\n'); function withToken(token, run) { const saved = process.env.MAPILLARY_CLIENT_TOKEN; if (token === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; else process.env.MAPILLARY_CLIENT_TOKEN = token; return Promise.resolve() .then(run) .finally(() => { if (saved === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; else process.env.MAPILLARY_CLIENT_TOKEN = saved; }); } /** Drive the tile route against a scripted upstream `answer(n, url)`. */ async function withUpstream(answer, run) { const savedFetch = globalThis.fetch; _resetTileCacheForTest(); const calls = []; const inits = []; globalThis.fetch = async (url, init) => { calls.push(String(url)); inits.push(init); return answer(calls.length, String(url), init); }; try { await withToken('MLY|test|token', () => run({ calls, inits, call: install().call }), ); } finally { globalThis.fetch = savedFetch; _resetTileCacheForTest(); } } const ok = (body = SEQUENCE) => new Response(body, { status: 200 }); test('the plugin mounts the status and tile routes for dev and preview servers', () => { for (const mode of ['configureServer', 'configurePreviewServer']) assert.deepEqual([...install(mode).routes.keys()].sort(), [ '/api/mapillary/status', '/api/mapillary/tiles', ]); }); test('status reports whether a token exists, never its value', async () => { const { call } = install(); await withToken(undefined, async () => { const res = await call('/api/mapillary/status'); assert.equal(res.statusCode, 200); assert.deepEqual(json(res), { configured: false }); assert.equal( (await call('/api/mapillary/status', '/', 'POST')).statusCode, 405, ); }); await withToken('MLY|secret', async () => { const res = await call('/api/mapillary/status'); assert.deepEqual(json(res), { configured: true }); assert.doesNotMatch(leakText(res), /MLY\|/); }); }); test('the tile route validates the path, then needs a token', async () => { await withUpstream( () => assert.fail('Mapillary is not asked'), async ({ call }) => { for (const bad of [ '/coverage/14/1/x', '/signs/14/1/2', '/coverage/10/0/0', '/coverage/15/0/0', '/coverage/11/2048/0', '/coverage/11/0/2048', '/coverage/14/1/2/3', ]) { const res = await call('/api/mapillary/tiles', bad); assert.equal(res.statusCode, 400, bad); } const post = await call( '/api/mapillary/tiles', '/coverage/14/1/2', 'POST', ); assert.equal(post.statusCode, 405); }, ); await withToken(undefined, async () => { const res = await install().call( '/api/mapillary/tiles', '/coverage/14/1/2', ); assert.equal(res.statusCode, 503); assert.deepEqual(json(res), { error: 'no_key', keyRequired: true }); }); }); test('a tile is trimmed, served with browser caching, then from memory', async () => { await withUpstream( () => ok(UNTRIMMED), async ({ calls, inits, call }) => { const res = await call('/api/mapillary/tiles', '/coverage/14/9/9'); assert.equal(res.statusCode, 200); assert.equal(res.headers['content-type'], 'application/x-protobuf'); assert.equal(res.headers['cache-control'], 'public, max-age=3600'); assert.equal(res.headers['x-gev-cache'], 'upstream'); assert.deepEqual(res.body, SEQUENCE, 'the image layer is dropped'); assert.doesNotMatch(leakText(res), /MLY\|/); assert.equal( calls[0], 'https://tiles.mapillary.com/maps/vtp/mly1_public/2/14/9/9?access_token=MLY%7Ctest%7Ctoken', ); assert.equal(inits[0].redirect, 'manual'); const again = await call('/api/mapillary/tiles', '/coverage/14/9/9?v=2'); assert.equal(again.headers['x-gev-cache'], 'memory'); assert.deepEqual(again.body, SEQUENCE); assert.equal(calls.length, 1); }, ); }); for (const status of [204, 404]) test(`an empty upstream tile (${status}) is a 204 with no body, and is cached`, async () => { await withUpstream( () => new Response(null, { status }), async ({ calls, call }) => { for (const source of ['upstream', 'memory']) { const res = await call('/api/mapillary/tiles', '/coverage/14/10/10'); assert.equal(res.statusCode, 204); assert.equal(res.body, undefined); assert.equal(res.headers['x-gev-cache'], source); } assert.equal(calls.length, 1); }, ); }); for (const status of [401, 403]) test(`an upstream ${status} is a rejected key`, async () => { await withUpstream( () => new Response('{}', { status }), async ({ call }) => { const res = await call('/api/mapillary/tiles', '/coverage/14/5/5'); assert.equal(res.statusCode, 403); assert.deepEqual(json(res), { error: 'Mapillary rejected the access token', keyRejected: true, }); assert.doesNotMatch(leakText(res), /MLY\|/); }, ); }); for (const [sent, expected] of [ ['30', 30], [null, 60], ['999999', 600], ]) test(`an upstream 429 (Retry-After ${sent}) is passed on as ${expected} s`, async () => { await withUpstream( () => new Response('{}', { status: 429, headers: sent === null ? {} : { 'Retry-After': sent }, }), async ({ calls, call }) => { const res = await call('/api/mapillary/tiles', '/coverage/14/5/5'); assert.equal(res.statusCode, 429); assert.equal(res.headers['retry-after'], String(expected)); assert.deepEqual(json(res), { error: 'Mapillary is rate-limiting tile requests', retryAfter: expected, }); // Not cached as a tile, but refused here until the wait is over. const again = await call('/api/mapillary/tiles', '/coverage/14/5/5'); assert.equal(again.statusCode, 429); assert.equal(calls.length, 1); }, ); }); for (const status of [302, 400, 410, 500]) test(`an upstream ${status} is a 502, not the client's fault`, async () => { await withUpstream( () => new Response('{}', { status }), async ({ call }) => { const res = await call('/api/mapillary/tiles', '/coverage/14/12/12'); assert.equal(res.statusCode, 502); assert.deepEqual(json(res), { error: `Mapillary tiles HTTP ${status}`, }); }, ); }); test('a streamed body past the size cap is cancelled and answered 502', async () => { const MB = 1024 * 1024; let pulled = 0; let cancelled = false; // Chunked, no Content-Length: only a running count can see it is too big. const body = () => new ReadableStream({ pull(controller) { controller.enqueue(new Uint8Array(MB)); pulled += MB; if (pulled >= TILE_MAX_BYTES + 8 * MB) controller.close(); }, cancel() { cancelled = true; }, }); await withUpstream( () => new Response(body(), { status: 200 }), async ({ call }) => { const res = await call('/api/mapillary/tiles', '/coverage/14/19/19'); assert.equal(res.statusCode, 502); assert.deepEqual(json(res), { error: 'Mapillary tile exceeds size cap' }); assert.equal(cancelled, true); assert.ok(pulled <= TILE_MAX_BYTES + 2 * MB, `stopped after ${pulled}`); }, ); }); test('cross-site requests are refused on both routes; the app itself passes', async () => { await withUpstream( () => ok(), async ({ calls, call }) => { const host = 'localhost:5173'; for (const headers of [ { host, 'sec-fetch-site': 'cross-site' }, // , navigation { host, 'sec-fetch-site': 'same-site' }, { host, origin: 'https://evil.example' }, // fetch from another page { host, origin: 'null' }, // sandboxed frame { host, 'x-forwarded-for': '203.0.113.9' }, // through a proxy ]) for (const [route, url] of [ ['/api/mapillary/status', '/'], ['/api/mapillary/tiles', '/coverage/14/13/13'], ]) { const res = await call(route, url, 'GET', headers); assert.equal( res.statusCode, 403, `${route} ${JSON.stringify(headers)}`, ); } assert.equal(calls.length, 0, 'Mapillary is never asked'); const app = { host, 'sec-fetch-site': 'same-origin' }; const status = await call('/api/mapillary/status', '/', 'GET', { ...app, origin: `http://${host}`, }); assert.deepEqual(json(status), { configured: true }); const res = await call( '/api/mapillary/tiles', '/coverage/14/13/14', 'GET', app, ); assert.equal(res.statusCode, 200); }, ); }); test('the tile route answers 429 past its per-IP budget', async () => { await withToken(undefined, async () => { const { call } = install(); for (let i = 0; i < TILE_ROUTE_MAX_PER_MIN; i++) assert.equal( (await call('/api/mapillary/tiles', '/coverage/14/1/2')).statusCode, 503, ); const over = await call('/api/mapillary/tiles', '/coverage/14/1/2'); assert.equal(over.statusCode, 429); assert.equal(over.headers['retry-after'], '5'); assert.deepEqual(json(over), { error: 'Too many tile requests', retryAfter: 5, }); const status = await call('/api/mapillary/status'); assert.equal(status.statusCode, 200, 'the status route is not limited'); }); }); test('identical concurrent tile requests share one upstream fetch', async () => { let release; const gate = new Promise((resolve) => (release = resolve)); await withUpstream( async (n) => n === 1 ? (await gate, ok()) : new Response('{}', { status: 500 }), async ({ calls, call }) => { const pending = [1, 2, 3].map(() => call('/api/mapillary/tiles', '/coverage/14/1/1'), ); await new Promise((resolve) => setTimeout(resolve, 10)); assert.equal(calls.length, 1); release(); const sources = (await Promise.all(pending)).map((res) => { assert.deepEqual(res.body, SEQUENCE); return res.headers['x-gev-cache']; }); assert.deepEqual(sources, ['upstream', 'inflight', 'inflight']); // A failed flight is not reused either. const failed = await call('/api/mapillary/tiles', '/coverage/14/2/2'); assert.equal(failed.statusCode, 502); await call('/api/mapillary/tiles', '/coverage/14/2/2'); assert.equal(calls.length, 3); }, ); }); test('at most a few upstream fetches run at once; the rest wait their turn', async () => { let release; let gate = new Promise((resolve) => (release = resolve)); await withUpstream( async () => (await gate, ok()), async ({ calls, call }) => { const tiles = TILE_UPSTREAM_CONCURRENCY + 4; const pending = Array.from({ length: tiles }, (_, i) => call('/api/mapillary/tiles', `/coverage/14/${i}/3`), ); await new Promise((resolve) => setTimeout(resolve, 10)); assert.equal(calls.length, TILE_UPSTREAM_CONCURRENCY); release(); const done = await Promise.all(pending); assert.ok(done.every((res) => res.statusCode === 200)); assert.equal(calls.length, tiles, 'the waiting ones ran too'); // Every slot came back: a second burst runs as many at once again. let reopen; gate = new Promise((resolve) => (reopen = resolve)); const again = Array.from({ length: tiles }, (_, i) => call('/api/mapillary/tiles', `/coverage/14/${i}/4`), ); await new Promise((resolve) => setTimeout(resolve, 10)); assert.equal(calls.length, tiles + TILE_UPSTREAM_CONCURRENCY); reopen(); await Promise.all(again); assert.equal(calls.length, 2 * tiles); }, ); }); /** An upstream that answers when released, and fails as a fetch does when aborted. */ function heldUpstream() { let release; const gate = new Promise((resolve) => (release = resolve)); const answer = (n, url, init) => new Promise((resolve, reject) => { init?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')), ); gate.then(() => resolve(ok())); }); return { answer, release }; } const settleSoon = () => new Promise((resolve) => setTimeout(resolve, 10)); test( 'a queued fetch whose client left never reaches Mapillary', { timeout: 5000 }, async () => { const upstream = heldUpstream(); await withUpstream(upstream.answer, async ({ calls }) => { const busy = Array.from({ length: TILE_UPSTREAM_CONCURRENCY }, (_, i) => fetchTile(parseTilePath(`/coverage/14/${i}/5`)), ); const client = new AbortController(); const queued = fetchTile(parseTilePath('/coverage/14/99/5'), { signal: client.signal, }); await settleSoon(); assert.equal(calls.length, TILE_UPSTREAM_CONCURRENCY, 'the 7th waits'); client.abort(); await assert.rejects(queued, { name: 'AbortError' }); upstream.release(); await Promise.all(busy); await settleSoon(); assert.equal(calls.length, TILE_UPSTREAM_CONCURRENCY, 'and never ran'); // The freed slots still serve a live request. await fetchTile(parseTilePath('/coverage/14/98/5')); assert.equal(calls.length, TILE_UPSTREAM_CONCURRENCY + 1); }); }, ); test( 'a client that disconnects cancels its tile fetch and gets no answer', { timeout: 5000 }, async () => { const upstream = heldUpstream(); await withUpstream(upstream.answer, async ({ inits, call }) => { let res; const done = call( '/api/mapillary/tiles', '/coverage/14/8/8', 'GET', {}, (r) => (res = r), ); await settleSoon(); res.close(); await done; assert.equal(inits[0].signal.aborted, true, 'the upstream fetch stopped'); assert.equal(res.body, undefined, 'nothing written to a closed socket'); }); }, ); test( 'a shared fetch is cancelled only when its last client leaves', { timeout: 5000 }, async () => { const upstream = heldUpstream(); await withUpstream(upstream.answer, async ({ calls, inits }) => { const address = parseTilePath('/coverage/14/7/7'); const first = new AbortController(); const second = new AbortController(); const a = fetchTile(address, { signal: first.signal }); const b = fetchTile(address, { signal: second.signal }); await settleSoon(); assert.equal(calls.length, 1, 'one fetch for both'); first.abort(); await assert.rejects(a, { name: 'AbortError' }); assert.equal(inits[0].signal.aborted, false, 'the other still waits'); second.abort(); await assert.rejects(b, { name: 'AbortError' }); assert.equal(inits[0].signal.aborted, true, 'the last one left'); // A new request starts afresh rather than joining the cancelled fetch. upstream.release(); const again = await fetchTile(address); assert.equal(again.source, 'upstream'); assert.equal(calls.length, 2); }); }, ); test('a tile held in memory past its TTL is fetched again', async (t) => { await withUpstream( () => ok(), async ({ calls, call }) => { await call('/api/mapillary/tiles', '/coverage/14/3/3'); await call('/api/mapillary/tiles', '/coverage/14/3/3'); assert.equal(calls.length, 1); const now = Date.now(); t.mock.method(Date, 'now', () => now + TILE_TTL_MS + 1000); const res = await call('/api/mapillary/tiles', '/coverage/14/3/3'); assert.equal(res.headers['x-gev-cache'], 'upstream'); assert.equal(calls.length, 2); }, ); }); test('the memory cache evicts the least recently used tile past its size', async () => { await withUpstream( () => ok(), async ({ calls }) => { const get = async (x) => (await fetchTile(parseTilePath(`/coverage/14/${x}/20`))).source; for (let x = 0; x < TILE_MEMORY_MAX_TILES; x++) await get(x); assert.equal(await get(0), 'memory', 'the first is used again'); await get(TILE_MEMORY_MAX_TILES); assert.equal(await get(0), 'memory', 'so it stays'); assert.equal(await get(1), 'upstream', 'the least recently used went'); assert.equal(calls.length, TILE_MEMORY_MAX_TILES + 2); }, ); }); // ── Upstream refusals ── const rateLimited = (retryAfter) => new Response('', { status: 429, headers: { 'retry-after': retryAfter } }); test('after a 429 the rest of the view waits out Retry-After here instead of asking Mapillary', async (t) => { const now = Date.now(); t.mock.method(Date, 'now', () => now); await withUpstream( (n) => (n === 1 ? rateLimited('120') : ok()), async ({ calls, call }) => { const first = await call('/api/mapillary/tiles', '/coverage/14/1/1'); assert.equal(first.statusCode, 429); assert.equal(json(first).retryAfter, 120); const rest = await Promise.all( Array.from({ length: 8 }, (_, i) => call('/api/mapillary/tiles', `/coverage/14/2/${i}`), ), ); assert.equal(calls.length, 1, 'nothing else reached Mapillary'); for (const res of rest) { assert.equal(res.statusCode, 429); assert.equal(json(res).retryAfter, 120); } // Half way through the wait, the answer says what is left. Date.now.mock.mockImplementation(() => now + 50_000); const midway = await call('/api/mapillary/tiles', '/coverage/14/3/3'); assert.equal(json(midway).retryAfter, 70); assert.equal(calls.length, 1); // Once it is over, requests go upstream again. Date.now.mock.mockImplementation(() => now + 121_000); const later = await call('/api/mapillary/tiles', '/coverage/14/3/3'); assert.equal(later.statusCode, 200); assert.equal(calls.length, 2); }, ); }); test('a refusal is scoped to the token that was refused', async () => { await withUpstream( (n) => (n === 1 ? rateLimited('120') : ok()), async ({ calls, call }) => { await call('/api/mapillary/tiles', '/coverage/14/1/1'); process.env.MAPILLARY_CLIENT_TOKEN = 'MLY|replaced|token'; const res = await call('/api/mapillary/tiles', '/coverage/14/1/2'); assert.equal(res.statusCode, 200, 'a new token is tried at once'); assert.equal(calls.length, 2); }, ); }); test( 'fetches queued behind a 429 never reach Mapillary', { timeout: 5000 }, async () => { let release; const gate = new Promise((resolve) => (release = resolve)); const answer = (n) => gate.then(() => (n === 1 ? rateLimited('60') : ok())); await withUpstream(answer, async ({ calls }) => { const busy = Array.from({ length: TILE_UPSTREAM_CONCURRENCY }, (_, i) => fetchTile(parseTilePath(`/coverage/14/${i}/6`)).catch((e) => e), ); const queued = Array.from({ length: 3 }, (_, i) => fetchTile(parseTilePath(`/coverage/14/${i}/7`)).catch((e) => e), ); await settleSoon(); assert.equal(calls.length, TILE_UPSTREAM_CONCURRENCY, 'three wait'); release(); const [first] = await Promise.all(busy); assert.equal(first.status, 429); for (const outcome of await Promise.all(queued)) { assert.equal(outcome.status, 429, 'refused here'); assert.equal(outcome.retryAfter, '60'); } assert.equal(calls.length, TILE_UPSTREAM_CONCURRENCY, 'and never ran'); }); }, ); test('an HTTP-date Retry-After is passed on as the seconds left; garbage means a minute', async (t) => { const now = Date.UTC(2026, 9, 13, 12, 0, 0); t.mock.method(Date, 'now', () => now); const inFiveMinutes = new Date(now + 5 * 60_000).toUTCString(); const inAnHour = new Date(now + 60 * 60_000).toUTCString(); const anHourAgo = new Date(now - 60 * 60_000).toUTCString(); const sent = [inFiveMinutes, inAnHour, anHourAgo, 'soon', '']; await withUpstream( (n) => rateLimited(sent[n - 1]), async ({ call }) => { const retryAfter = async (x) => { _resetTileCacheForTest(); const res = await call('/api/mapillary/tiles', `/coverage/14/${x}/9`); assert.equal(res.statusCode, 429); assert.equal(res.headers['retry-after'], String(json(res).retryAfter)); return json(res).retryAfter; }; assert.equal(await retryAfter(1), 300); assert.equal(await retryAfter(2), MAX_RETRY_AFTER_SEC, 'capped'); assert.equal(await retryAfter(3), 60, 'a date in the past'); assert.equal(await retryAfter(4), 60, 'not a date'); assert.equal(await retryAfter(5), 60, 'no header'); }, ); }); test('a body too big to be a trimmed tile is served but not kept in memory', async () => { const huge = Buffer.alloc(TILE_MEMORY_MAX_TILE_BYTES + 1, 0xff); await withUpstream( () => ok(huge), async ({ calls, call }) => { const first = await call('/api/mapillary/tiles', '/coverage/14/5/5'); assert.equal(first.statusCode, 200); assert.equal(first.body.length, huge.length, 'served as it came'); const second = await call('/api/mapillary/tiles', '/coverage/14/5/5'); assert.equal(second.headers['x-gev-cache'], 'upstream', 'not cached'); assert.equal(calls.length, 2); }, ); }); // ── Tile trimming ── test('dropping a layer keeps the others byte-for-byte', () => { const bytes = tile([ { name: 'sequence' }, { name: 'image', payload: Buffer.alloc(50_000, 7) }, { name: 'overview' }, ]); const trimmed = stripTileLayers(bytes, ['image']); assert.deepEqual(trimmed, tile([{ name: 'sequence' }, { name: 'overview' }])); }); test('a tile without the layer is returned untouched', () => { assert.equal(stripTileLayers(SEQUENCE, ['image']), SEQUENCE); assert.equal(stripTileLayers(SEQUENCE, []), SEQUENCE); assert.equal(stripTileLayers(Buffer.alloc(0), ['image']).length, 0); });