From b5a78826f8c99ae7d0bc818b45e2abd8f02206e2 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Wed, 7 Oct 2026 22:27:34 +0400 Subject: [PATCH] =?UTF-8?q?refactor(street-level):=20slim=20Mapillary=20ti?= =?UTF-8?q?le=20proxy=20=E2=80=94=20memory=20cache=20only,=20no=20server?= =?UTF-8?q?=20holds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the disk tile cache (.gev-cache/mapillary/tiles, background sweep, size cap, legacy retrim), the server-side 401/403/429 holds, the upstream concurrency semaphore and per-waiter abort bookkeeping. The client already pauses on 429 and stops on a rejected key. Keep: server-side token, {configured}-only status, strict coverage path validation (z11-14), image-layer trimming, same-site gate and per-IP limit, HTTPS redirect pinning to the tile origin, the streamed size cap, the error mapping the client reads, a 24 h byte-budget LRU memory cache, in-flight coalescing via the shared coalesceProxyRequest, and browser Cache-Control. Routes and constants fold into mapillary.js and mapillary/tiles.js. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +- SECURITY.md | 2 +- docs/CURRENT-STATE.md | 2 +- scripts/format-scope.json | 2 - scripts/package-boundaries.json | 2 - server/providers/mapillary.js | 114 +- server/providers/mapillary/constants.js | 72 - server/providers/mapillary/routes.js | 95 -- server/providers/mapillary/tiles.js | 627 ++------ src/tooling/mapillaryProvider.test.mjs | 1732 ++++++----------------- 10 files changed, 717 insertions(+), 1938 deletions(-) delete mode 100644 server/providers/mapillary/constants.js delete mode 100644 server/providers/mapillary/routes.js diff --git a/CHANGELOG.md b/CHANGELOG.md index b312678ca..6a97c3368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,10 @@ the terrain and on Google 3D tiles; MapillaryJS loads on the first photo, and opening one frames it with one camera flight. Share links carry the Mapillary switch, the filters and the panel's collapsed - state. A server proxy caches Mapillary coverage tiles (24 h, 1 GiB on disk), - refuses cross-site requests, rate-limits per IP and backs off when Mapillary - rejects the token or rate-limits. + state. A server proxy adds the token to Mapillary coverage tiles, drops + their unused image layer and keeps them in memory (24 h); it refuses + cross-site requests and rate-limits per IP, and the panel stops asking when + Mapillary rejects the token or rate-limits. - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its diff --git a/SECURITY.md b/SECURITY.md index eb5fcd6e0..2b449e115 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,7 +28,7 @@ These are designed to be used directly in the browser (like a Mapbox public toke 1. **Google Maps API key** — loads Photorealistic 3D Tiles directly and powers GEV place search. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. 2. **Cesium ion token** (`CESIUM_ION_TOKEN`, optional — for ion-hosted Google Photorealistic 3D Tiles, Bing world imagery, and world terrain) — used as `Cesium.Ion.defaultAccessToken` client-side. Use a public **`assets:read`** token with **URL restrictions** for any hosted deployment. The Community plan has eligibility and usage limits; a public token is not a secret, but it can still consume the account's quota. -3. **Mapillary client token** (`MAPILLARY_CLIENT_TOKEN`, optional, Street Level) — used by the MapillaryJS viewer and Graph API lookups in the browser, and by the server's coverage-tile proxy. Mapillary client tokens are meant for browser use. +3. **Mapillary client token** (`MAPILLARY_CLIENT_TOKEN`, optional, Street Level) — used by the MapillaryJS viewer and Graph API lookups in the browser, and by the server's coverage-tile proxy, which sends it only to `tiles.mapillary.com` over HTTPS (redirects are followed by hand and pinned to that origin) and caches tiles in memory only. Mapillary client tokens are meant for browser use. > The explicit browser `define` block in `build/vite.js` controls exactly what reaches the client: only these three keys. Everything else stays server-side. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 27e6b9993..57c163299 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2928,7 +2928,7 @@ its criteria cannot be silently ignored. | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | | FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | -| Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, cached 24 h) | +| Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, server memory cache 24 h) | | Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | | Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 6f4d7cac3..d8bb556e0 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -884,8 +884,6 @@ "src/layers/streetLevel/providers/mapillary/source.js", "src/layers/streetLevel/providers/mapillary/nearest.js", "server/providers/mapillary.js", - "server/providers/mapillary/routes.js", - "server/providers/mapillary/constants.js", "server/providers/mapillary/tiles.js", "server/providers/mapillary/trim.js", "src/layers/streetLevel/tileMath.test.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index fc907e21c..f1285c4eb 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -188,8 +188,6 @@ "exports": ["./server/providers/mapillary"], "modules": [ "server/providers/mapillary.js", - "server/providers/mapillary/routes.js", - "server/providers/mapillary/constants.js", "server/providers/mapillary/tiles.js", "server/providers/mapillary/trim.js", "server/providers/common/same-site.js", diff --git a/server/providers/mapillary.js b/server/providers/mapillary.js index 401be28ae..dc78f3611 100644 --- a/server/providers/mapillary.js +++ b/server/providers/mapillary.js @@ -1,19 +1,105 @@ -import { installMapillaryRoutes } from './mapillary/routes.js'; +import { sameSiteGated } from './common/same-site.js'; +import { makeRateLimiter, clientKey } from './common/rate-limit.js'; +import { + fetchTile, + mapillaryToken, + parseTilePath, + TileUpstreamError, +} from './mapillary/tiles.js'; /** - * Vite plugin: cached Mapillary coverage tiles (token added server-side) and a - * status route that says whether a token is configured. + * Tile requests per client IP per minute, cache hits included: 24 new 25-tile + * views a minute, more than a person flying the camera reaches. */ -function mapillaryProxy() { - return { - name: 'mapillary-proxy', - configureServer(server) { - installMapillaryRoutes(server.middlewares); - }, - configurePreviewServer(server) { - installMapillaryRoutes(server.middlewares); - }, - }; +export const TILE_ROUTE_MAX_PER_MIN = 600; +/** The longest upstream Retry-After passed on, in seconds. */ +const MAX_RETRY_AFTER_SEC = 600; + +function sendJson(res, status, payload) { + res.statusCode = status; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('Cache-Control', 'no-store'); + res.end(JSON.stringify(payload)); } -export { mapillaryProxy }; +/** GET /api/mapillary/status — whether a token is configured, never its value. */ +function handleStatus(req, res) { + if (req.method !== 'GET') + return sendJson(res, 405, { error: 'Method not allowed' }); + sendJson(res, 200, { configured: Boolean(mapillaryToken()) }); +} + +/** GET /api/mapillary/tiles/coverage/{z}/{x}/{y} — trimmed protobuf tile. */ +async function handleTile(req, res, allow) { + if (req.method !== 'GET') + return sendJson(res, 405, { error: 'Method not allowed' }); + if (!allow(clientKey(req))) { + res.setHeader('Retry-After', '5'); + return sendJson(res, 429, { + error: 'Too many tile requests', + retryAfter: 5, + }); + } + const address = parseTilePath((req.url || '').split('?')[0]); + if (!address) + return sendJson(res, 400, { + error: 'Tile path must be /coverage/{z}/{x}/{y} with z 11–14', + }); + if (!mapillaryToken()) + return sendJson(res, 503, { error: 'no_key', keyRequired: true }); + try { + const { bytes, source } = await fetchTile(address); + res.statusCode = bytes.length ? 200 : 204; + res.setHeader('Content-Type', 'application/x-protobuf'); + res.setHeader('Cache-Control', 'public, max-age=3600'); + res.setHeader('X-Gev-Cache', source); + res.end(bytes.length ? bytes : undefined); + } catch (error) { + const status = error instanceof TileUpstreamError ? error.status : 0; + // A rejected token is a key problem the panel can name, not a fault. + if (status === 401 || status === 403) + return sendJson(res, 403, { + error: 'Mapillary rejected the access token', + keyRejected: true, + }); + if (status === 429) { + const sent = Number(error.retryAfter); + const retryAfter = + Number.isInteger(sent) && sent > 0 + ? Math.min(sent, MAX_RETRY_AFTER_SEC) + : 60; + res.setHeader('Retry-After', String(retryAfter)); + return sendJson(res, 429, { + error: 'Mapillary is rate-limiting tile requests', + retryAfter, + }); + } + // An upstream 400 must not read as a malformed request to this proxy. + sendJson(res, 502, { error: error?.message || 'Tile fetch failed' }); + } +} + +/** + * Vite plugin: Mapillary coverage tiles with the token added server-side, and + * a status route. Both refuse cross-site requests so another page cannot + * spend the token. + */ +export function mapillaryProxy() { + const install = (middlewares) => { + const allow = makeRateLimiter({ + windowMs: 60_000, + max: TILE_ROUTE_MAX_PER_MIN, + globalMax: TILE_ROUTE_MAX_PER_MIN * 4, + }); + middlewares.use('/api/mapillary/status', sameSiteGated(handleStatus)); + middlewares.use( + '/api/mapillary/tiles', + sameSiteGated((req, res) => handleTile(req, res, allow)), + ); + }; + return { + name: 'mapillary-proxy', + configureServer: (server) => install(server.middlewares), + configurePreviewServer: (server) => install(server.middlewares), + }; +} diff --git a/server/providers/mapillary/constants.js b/server/providers/mapillary/constants.js deleted file mode 100644 index 202d2846a..000000000 --- a/server/providers/mapillary/constants.js +++ /dev/null @@ -1,72 +0,0 @@ -import path from 'node:path'; - -/** Mapillary vector tile root; the path template is /{layer}/2/{z}/{x}/{y}. */ -export const MAPILLARY_TILE_HOST = 'https://tiles.mapillary.com/maps/vtp'; - -/** Public tile layer names this proxy exposes, mapped to Mapillary's ids. */ -export const TILE_LAYERS = Object.freeze({ - // Only the street zooms the app draws (z11–14); the rest are refused. The - // z14 `image` layer is ~98% of a tile and unused (positions come from the - // graph API), so it is dropped in transit. - coverage: Object.freeze({ - upstream: 'mly1_public', - zoomRanges: Object.freeze([Object.freeze([11, 14])]), - dropLayers: Object.freeze(['image']), - }), -}); - -/** Disk cache root, alongside the other providers' caches. */ -export const MAPILLARY_CACHE_DIR = path.join( - process.cwd(), - '.gev-cache', - 'mapillary', -); -export const TILE_DISK_DIR = path.join(MAPILLARY_CACHE_DIR, 'tiles'); - -/** - * Longer than the 15-minute upstream Cache-Control: tiles change only when new - * imagery is processed, and a day avoids re-downloading a city per camera move. - */ -export const TILE_DISK_TTL_MS = 24 * 60 * 60 * 1000; - -/** - * A tile write starts a background sweep at most this often: expired files go - * first, then the oldest until the cache fits in TILE_DISK_MAX_BYTES. - */ -export const TILE_DISK_SWEEP_INTERVAL_MS = 15 * 60 * 1000; -export const TILE_DISK_MAX_BYTES = 1024 * 1024 * 1024; -/** Per-file cost on top of its size, so empty tiles still count. */ -export const TILE_DISK_FILE_OVERHEAD_BYTES = 4096; - -/** A z14 image tile over a dense city is ~11 MB; anything past this is wrong. */ -export const TILE_MAX_BYTES = 48 * 1024 * 1024; - -export const TILE_MEMORY_BUDGET_BYTES = 96 * 1024 * 1024; -/** Per-entry cost on top of its bytes, so empty tiles still count. */ -export const TILE_MEMORY_ENTRY_OVERHEAD_BYTES = 1024; -export const TILE_FETCH_TIMEOUT_MS = 60_000; -/** - * Upstream fetches at once; further misses queue. A view needs at most 25 - * tiles, and the cap stops a burst of misses opening thousands of requests. - */ -export const TILE_UPSTREAM_CONCURRENCY = 6; -/** - * Tile requests per client IP per minute, cache hits included. 600 is 24 new - * 25-tile views a minute, more than a person flying the camera reaches, but it - * stops one page draining the shared token into a Mapillary rate limit. - */ -export const TILE_ROUTE_MAX_PER_MIN = 600; -/** - * How long a 401/403 answers tile misses before asking Mapillary again. A - * changed token is tried at once. - */ -export const TILE_KEY_REJECTED_HOLD_MS = 5 * 60 * 1000; -/** After a 429 without a usable Retry-After, hold tile misses this long. */ -export const TILE_RATE_LIMIT_HOLD_MS = 60_000; -/** The longest Retry-After honoured, so a bad header cannot stall coverage. */ -export const TILE_RATE_LIMIT_MAX_HOLD_MS = 10 * 60 * 1000; - -/** The client token lives in the browser by design; the server adds it to tile URLs too. */ -export function mapillaryToken() { - return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim(); -} diff --git a/server/providers/mapillary/routes.js b/server/providers/mapillary/routes.js deleted file mode 100644 index 1721dd40a..000000000 --- a/server/providers/mapillary/routes.js +++ /dev/null @@ -1,95 +0,0 @@ -import { sameSiteGated } from '../common/same-site.js'; -import { makeRateLimiter, clientKey } from '../common/rate-limit.js'; -import { fetchTile, TileRequestError, TileUpstreamError } from './tiles.js'; -import { mapillaryToken, TILE_ROUTE_MAX_PER_MIN } from './constants.js'; - -function sendJson(res, status, payload) { - res.statusCode = status; - res.setHeader('Content-Type', 'application/json; charset=utf-8'); - res.setHeader('Cache-Control', 'no-store'); - res.end(JSON.stringify(payload)); -} - -/** GET /api/mapillary/status — whether a token is configured, never its value. */ -function handleStatus(req, res) { - if (req.method !== 'GET') - return sendJson(res, 405, { error: 'Method not allowed' }); - sendJson(res, 200, { configured: Boolean(mapillaryToken()) }); -} - -/** GET /api/mapillary/tiles/coverage/{z}/{x}/{y} — cached protobuf tile. */ -async function handleTile(req, res, allow) { - if (req.method !== 'GET') - return sendJson(res, 405, { error: 'Method not allowed' }); - if (!allow(clientKey(req))) { - res.setHeader('Retry-After', '5'); - return sendJson(res, 429, { - error: 'Too many tile requests', - retryAfter: 5, - }); - } - const match = /^\/(coverage)\/(\d{1,2})\/(\d{1,6})\/(\d{1,6})$/.exec( - (req.url || '').split('?')[0], - ); - if (!match) - return sendJson(res, 400, { - error: 'Tile path must be /coverage/{z}/{x}/{y}', - }); - if (!mapillaryToken()) - return sendJson(res, 503, { error: 'no_key', keyRequired: true }); - const abandoned = new AbortController(); - req.on?.('aborted', () => abandoned.abort()); - res.on?.('close', () => abandoned.abort()); - try { - const { bytes, source } = await fetchTile( - { layer: match[1], z: match[2], x: match[3], y: match[4] }, - { signal: abandoned.signal }, - ); - if (res.writableEnded) return; - res.statusCode = bytes.length ? 200 : 204; - res.setHeader('Content-Type', 'application/x-protobuf'); - res.setHeader('Cache-Control', 'public, max-age=3600'); - res.setHeader('X-Gev-Cache', source); - res.end(bytes.length ? bytes : undefined); - } catch (error) { - if (res.writableEnded || abandoned.signal.aborted) return; - if (error instanceof TileRequestError) - return sendJson(res, error.status, { error: error.message }); - if (error instanceof TileUpstreamError) { - // A rejected token is a key problem the panel can name, not a fault. - if (error.keyRejected) - return sendJson(res, 403, { - error: 'Mapillary rejected the access token', - keyRejected: true, - }); - if (error.status === 429) { - const retryAfter = error.retryAfterSec || 60; - res.setHeader('Retry-After', String(retryAfter)); - return sendJson(res, 429, { - error: 'Mapillary is rate-limiting tile requests', - retryAfter, - }); - } - // An upstream 400 must not read as a malformed request to this proxy. - return sendJson(res, 502, { error: error.message }); - } - sendJson(res, 502, { error: error?.message || 'Tile fetch failed' }); - } -} - -/** - * Attach the Mapillary routes. Both refuse cross-site requests so another page - * cannot spend the token or provoke a 429 that holds every tile miss. - */ -export function installMapillaryRoutes(middlewares) { - const allow = makeRateLimiter({ - windowMs: 60_000, - max: TILE_ROUTE_MAX_PER_MIN, - globalMax: TILE_ROUTE_MAX_PER_MIN * 4, - }); - middlewares.use('/api/mapillary/status', sameSiteGated(handleStatus)); - middlewares.use( - '/api/mapillary/tiles', - sameSiteGated((req, res) => handleTile(req, res, allow)), - ); -} diff --git a/server/providers/mapillary/tiles.js b/server/providers/mapillary/tiles.js index 832e8e483..76e4c90c5 100644 --- a/server/providers/mapillary/tiles.js +++ b/server/providers/mapillary/tiles.js @@ -1,518 +1,187 @@ -import path from 'node:path'; -import { promises as fsp } from 'node:fs'; -import { readResponseBytesCapped } from '../common/http.js'; -import { stripTileLayers } from './trim.js'; import { - MAPILLARY_TILE_HOST, - TILE_LAYERS, - TILE_DISK_DIR, - TILE_DISK_TTL_MS, - TILE_DISK_SWEEP_INTERVAL_MS, - TILE_DISK_MAX_BYTES, - TILE_DISK_FILE_OVERHEAD_BYTES, - TILE_MAX_BYTES, - TILE_MEMORY_BUDGET_BYTES, - TILE_MEMORY_ENTRY_OVERHEAD_BYTES, - TILE_FETCH_TIMEOUT_MS, - TILE_UPSTREAM_CONCURRENCY, - TILE_KEY_REJECTED_HOLD_MS, - TILE_RATE_LIMIT_HOLD_MS, - TILE_RATE_LIMIT_MAX_HOLD_MS, - mapillaryToken, -} from './constants.js'; + coalesceProxyRequest, + readResponseBytesCapped, +} from '../common/http.js'; +import { stripTileLayers } from './trim.js'; -/** Log without the stack; the token is never in scope here. */ -function warn(what, error) { - console.warn(`[Mapillary Proxy] ${what}:`, error?.message || error); +/** Mapillary vector tiles, /{layer}/2/{z}/{x}/{y}: the only host the token goes to. */ +const TILE_HOST = 'https://tiles.mapillary.com/maps/vtp'; +const TILE_ORIGIN = new URL(TILE_HOST).origin; +/** Only the street zooms the app draws are proxied. */ +const MIN_ZOOM = 11; +const MAX_ZOOM = 14; +/** + * The z14 `image` layer is ~98% of a tile (12 MB over a dense city) and unused + * (positions come from the Graph API), so it is dropped in transit. + */ +const DROP_LAYERS = ['image']; +/** A z14 tile with its image layer is ~11 MB; anything past this is wrong. */ +export const TILE_MAX_BYTES = 48 * 1024 * 1024; +export const TILE_MAX_REDIRECTS = 3; +const FETCH_TIMEOUT_MS = 60_000; +/** Tiles change only when new imagery is processed. */ +export const TILE_TTL_MS = 24 * 60 * 60 * 1000; +export const TILE_MEMORY_BUDGET_BYTES = 96 * 1024 * 1024; +/** Per-entry cost on top of its bytes, so empty tiles still count. */ +const ENTRY_OVERHEAD_BYTES = 1024; +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); + +/** The client token lives in the browser by design; the server adds it to tile URLs too. */ +export function mapillaryToken() { + return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim(); } -/** Thrown for a request this proxy refuses before contacting Mapillary. */ -export class TileRequestError extends Error { - constructor(message, status = 400) { - super(message); - this.name = 'TileRequestError'; - this.status = status; - } -} - -/** Mapillary answered with an error status; 401/403 set `keyRejected`. */ +/** Mapillary answered with an error status (or a redirect/size this proxy refuses). */ export class TileUpstreamError extends Error { - constructor(status, message, { retryAfterSec = null } = {}) { + constructor(status, message, retryAfter = null) { super(message || `Mapillary tiles HTTP ${status}`); this.name = 'TileUpstreamError'; this.status = status; - this.keyRejected = status === 401 || status === 403; - this.retryAfterSec = retryAfterSec; + /** The upstream Retry-After header, as sent. */ + this.retryAfter = retryAfter; } } /** - * A rejected token or rate limit that answers every tile miss until it expires - * (or the token changes). Cached tiles are still served meanwhile. - * @type {{status: number, until: number, token: string}|null} + * Parse `/coverage/{z}/{x}/{y}`; null unless z is 11–14 and x/y are on the grid. + * @returns {{z: number, x: number, y: number, key: string}|null} */ -let _upstreamHold = null; - -/** Milliseconds to wait from a Retry-After header (seconds or an HTTP date). */ -function retryAfterMs(header, now = Date.now()) { - const text = String(header ?? '').trim(); - let ms = NaN; - if (/^\d+$/.test(text)) ms = Number(text) * 1000; - else if (text) ms = Date.parse(text) - now; - if (!Number.isFinite(ms) || ms <= 0) return TILE_RATE_LIMIT_HOLD_MS; - return Math.min(ms, TILE_RATE_LIMIT_MAX_HOLD_MS); +export function parseTilePath(pathname) { + const match = /^\/coverage\/(\d{1,2})\/(\d{1,6})\/(\d{1,6})$/.exec(pathname); + if (!match) return null; + const [z, x, y] = match.slice(1).map(Number); + if (z < MIN_ZOOM || z > MAX_ZOOM || x >= 2 ** z || y >= 2 ** z) return null; + return { z, x, y, key: `${z}/${x}/${y}` }; } -function heldRefusal(token) { - const hold = _upstreamHold; - if (!hold) return null; - const left = hold.until - Date.now(); - if (left <= 0 || hold.token !== token) { - _upstreamHold = null; - return null; - } - return new TileUpstreamError(hold.status, undefined, { - retryAfterSec: Math.ceil(left / 1000), - }); -} - -/** - * Validate a tile address against the layer's zoom ranges. - * @returns {{layer:string, upstream:string, z:number, x:number, y:number, key:string}} - */ -export function normalizeTileAddress({ layer, z, x, y }) { - const spec = Object.hasOwn(TILE_LAYERS, layer) ? TILE_LAYERS[layer] : null; - if (!spec) throw new TileRequestError(`Unknown tile layer: ${layer}`); - const zi = Number(z); - const xi = Number(x); - const yi = Number(y); - if ( - ![zi, xi, yi].every((v) => Number.isInteger(v) && v >= 0) || - !spec.zoomRanges.some(([min, max]) => zi >= min && zi <= max) - ) - throw new TileRequestError( - `Tile zoom for ${layer} must be ${spec.zoomRanges - .map(([min, max]) => `${min}–${max}`) - .join(' or ')}`, - ); - const n = 2 ** zi; - if (xi >= n || yi >= n) - throw new TileRequestError('Tile address out of range'); - return { - layer, - upstream: spec.upstream, - dropLayers: spec.dropLayers || [], - z: zi, - x: xi, - y: yi, - key: `${layer}/${zi}/${xi}/${yi}`, - }; -} - -/** @type {Map} insertion-ordered LRU */ -const _memory = new Map(); -let _memoryBytes = 0; -const memoryCost = (bytes) => bytes.length + TILE_MEMORY_ENTRY_OVERHEAD_BYTES; -/** - * Upstream fetches shared by every request for the same tile; the last waiter - * to leave cancels it (see `joinFlight`). - * @type {Map, waiters: number}>} - */ -const _inFlight = new Map(); - -function memoryGet(key) { - const hit = _memory.get(key); - if (!hit) return null; - // Same life as the disk copy, so a long-running server never serves stale - // coverage. - if (Date.now() - hit.at > TILE_DISK_TTL_MS) { - _memory.delete(key); - _memoryBytes -= memoryCost(hit.bytes); - return null; - } - _memory.delete(key); - _memory.set(key, hit); - return hit.bytes; -} - -/** `at` is when Mapillary served the bytes (a disk tile's mtime). */ -function memoryPut(key, bytes, at = Date.now()) { - if (memoryCost(bytes) > TILE_MEMORY_BUDGET_BYTES / 2) return; - const existing = _memory.get(key); - if (existing) _memoryBytes -= memoryCost(existing.bytes); - _memory.set(key, { bytes, at }); - _memoryBytes += memoryCost(bytes); - while (_memoryBytes > TILE_MEMORY_BUDGET_BYTES && _memory.size) { - const [oldest, entry] = _memory.entries().next().value; - _memory.delete(oldest); - _memoryBytes -= memoryCost(entry.bytes); - } -} - -/** Disk cache root; tests point it at a temporary directory. */ -let _diskDir = TILE_DISK_DIR; - -function diskPath({ layer, z, x, y }, root = _diskDir) { - return path.join(root, layer, String(z), `${x}-${y}.pbf`); -} - -async function readDisk(address) { - const file = diskPath(address); +/** The next hop, if it stays HTTPS on the tile origin (the token is in the query). */ +function pinnedRedirect(location, base) { + let target = null; try { - const stat = await fsp.stat(file); - if (Date.now() - stat.mtimeMs > TILE_DISK_TTL_MS) return null; - return { bytes: await fsp.readFile(file), at: stat.mtimeMs }; + target = location ? new URL(location, base) : null; } catch { - return null; + /* unparseable Location */ } -} - -/** Unsettled background writes and sweeps; requests never wait on them. */ -const _background = new Set(); - -function track(task) { - const settled = task - .catch(() => {}) - .finally(() => _background.delete(settled)); - _background.add(settled); - return settled; -} - -/** @type {Map} cache file to its newest write's sequence */ -const _diskWrites = new Map(); -let _diskWriteSeq = 0; - -function diskWritePending(address) { - return _diskWrites.has(diskPath(address)); -} - -/** - * Write a tile in the background. `at` keeps a rewrite's original fetch time - * so it does not extend the tile's life. An overtaken write steps aside. - */ -function writeDisk(address, bytes, at = null) { - // Sweep this write's root even if the cache dir has moved since. - const root = _diskDir; - const file = diskPath(address, root); - const seq = ++_diskWriteSeq; - const tmp = `${file}.${process.pid}-${seq}.tmp`; - const newest = () => _diskWrites.get(file) === seq; - _diskWrites.set(file, seq); - return track( - fsp - .mkdir(path.dirname(file), { recursive: true }) - .then(() => fsp.writeFile(tmp, bytes)) - .then(() => at != null && fsp.utimes(tmp, new Date(), new Date(at))) - .then(() => { - if (!newest()) return fsp.rm(tmp, { force: true }); - return fsp.rename(tmp, file).then(() => scheduleSweep(root)); - }) - .catch(async (error) => { - await fsp.rm(tmp, { force: true }).catch(() => {}); - warn('tile cache write failed', error); - }) - .finally(() => { - if (newest()) _diskWrites.delete(file); - }), - ); -} - -let _lastSweepAt = 0; -let _sweeping = null; -let _sweepWarned = false; - -function scheduleSweep(root) { - if (_sweeping || Date.now() - _lastSweepAt < TILE_DISK_SWEEP_INTERVAL_MS) - return; - _lastSweepAt = Date.now(); - const sweep = sweepTileDisk({ root }) - .catch((error) => { - if (_sweepWarned) return; - _sweepWarned = true; - warn('tile cache sweep failed', error); - }) - .finally(() => { - if (_sweeping === tracked) _sweeping = null; - }); - const tracked = track(sweep); - _sweeping = tracked; -} - -/** Every file under `dir` with size and mtime; a missing dir is empty. */ -async function listCacheFiles(dir) { - let entries; - try { - entries = await fsp.readdir(dir, { withFileTypes: true }); - } catch (error) { - if (error?.code === 'ENOENT') return []; - throw error; - } - const files = []; - for (const entry of entries) { - const file = path.join(dir, entry.name); - if (entry.isDirectory()) files.push(...(await listCacheFiles(file))); - else if (entry.isFile()) { - const stat = await fsp.stat(file).catch(() => null); - if (stat) files.push({ file, size: stat.size, at: stat.mtimeMs }); - } - } - return files; -} - -/** - * Delete expired tiles, then the oldest until the rest fit in `maxBytes`. - * @returns {Promise<{removed: number, bytes: number}>} files deleted, charged bytes kept - */ -export async function sweepTileDisk({ - root = _diskDir, - maxBytes = TILE_DISK_MAX_BYTES, -} = {}) { - const now = Date.now(); - const files = (await listCacheFiles(root)).sort((a, b) => a.at - b.at); - const cost = (size) => size + TILE_DISK_FILE_OVERHEAD_BYTES; - let bytes = files.reduce((sum, { size }) => sum + cost(size), 0); - let removed = 0; - for (const { file, size, at } of files) { - if (now - at <= TILE_DISK_TTL_MS && bytes <= maxBytes) break; - await fsp.rm(file, { force: true }); - bytes -= cost(size); - removed++; - } - return { removed, bytes }; -} - -/** The one origin the tile token may be sent to, at every redirect hop. */ -const TILE_ORIGIN = new URL(MAPILLARY_TILE_HOST).origin; -export const TILE_MAX_REDIRECTS = 3; -const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); - -/** - * Follow redirects by hand so every hop stays HTTPS on the tile origin and the - * token in the query never reaches another host. A bad redirect is a 502. - */ -async function fetchPinned(url, init) { - let next = url; - for (let hop = 0; ; hop++) { - const response = await fetch(next, { ...init, redirect: 'manual' }); - if (!REDIRECT_STATUSES.has(response.status)) return response; - await response.body?.cancel().catch(() => {}); - if (hop >= TILE_MAX_REDIRECTS) - throw new TileUpstreamError(502, 'Mapillary tile redirected too often'); - let target = null; - try { - target = new URL(response.headers.get('location') ?? '', next); - } catch { - /* unparseable Location */ - } - if ( - !response.headers.get('location') || - target?.protocol !== 'https:' || - target.origin !== TILE_ORIGIN || - target.username || - target.password - ) - throw new TileUpstreamError( - 502, - 'Mapillary tile redirect left the tile origin', - ); - next = target.href; - } -} - -async function fetchUpstream(address, signal) { - const token = mapillaryToken(); - if (!token) throw new TileRequestError('Mapillary token not configured', 503); - const held = heldRefusal(token); - if (held) throw held; - const url = `${MAPILLARY_TILE_HOST}/${address.upstream}/2/${address.z}/${address.x}/${address.y}?access_token=${encodeURIComponent(token)}`; - const timeout = AbortSignal.timeout(TILE_FETCH_TIMEOUT_MS); - const oversize = new AbortController(); - const response = await fetchPinned(url, { - signal: AbortSignal.any([signal, timeout, oversize.signal].filter(Boolean)), - headers: { Accept: 'application/x-protobuf' }, - }); - if (response.status === 404 || response.status === 204) { - await response.body?.cancel().catch(() => {}); - return Buffer.alloc(0); - } - if (!response.ok) { - await response.body?.cancel().catch(() => {}); - const error = new TileUpstreamError(response.status); - if (error.keyRejected) - _upstreamHold = { - status: response.status, - until: Date.now() + TILE_KEY_REJECTED_HOLD_MS, - token, - }; - else if (response.status === 429) { - const wait = retryAfterMs(response.headers.get('retry-after')); - _upstreamHold = { status: 429, until: Date.now() + wait, token }; - error.retryAfterSec = Math.ceil(wait / 1000); - } - throw error; - } - // Chunked or compressed bodies have no Content-Length: the cap applies as - // the body streams, and an oversized one also aborts the request. - try { - const bytes = await readResponseBytesCapped( - response, - TILE_MAX_BYTES, - signal, + if ( + target?.protocol !== 'https:' || + target.origin !== TILE_ORIGIN || + target.username || + target.password + ) + throw new TileUpstreamError( + 502, + 'Mapillary tile redirect left the tile origin', ); - return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); - } catch (error) { - if (error?.code !== 'RESPONSE_TOO_LARGE') throw error; - oversize.abort(); - throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap'); - } + return target.href; } -/** @type {{active: number, queue: Array<() => void>}} upstream slots */ -let _slots = { active: 0, queue: [] }; - -/** - * Resolve with a release function once a slot is free. An abort while - * queued leaves the queue. - * @returns {Promise<() => void>} - */ -function acquireUpstreamSlot(signal) { - const slots = _slots; - return new Promise((resolve, reject) => { - if (signal.aborted) return reject(signal.reason); - const onAbort = () => { - const index = slots.queue.indexOf(grant); - if (index !== -1) slots.queue.splice(index, 1); - reject(signal.reason); - }; - function grant() { - signal.removeEventListener('abort', onAbort); - slots.active++; - let released = false; - resolve(() => { - if (released) return; - released = true; - slots.active--; - slots.queue.shift()?.(); - }); - } - if (slots.active < TILE_UPSTREAM_CONCURRENCY) return grant(); - signal.addEventListener('abort', onAbort, { once: true }); - slots.queue.push(grant); - }); -} - -function trim(address, bytes) { - if (!address.dropLayers.length || !bytes.length) return bytes; +function trim(bytes) { try { - return stripTileLayers(bytes, address.dropLayers); + return stripTileLayers(bytes, DROP_LAYERS); } catch (error) { - warn('tile trim failed, serving raw', error); + console.warn('[Mapillary Proxy] tile trim failed:', error?.message); return bytes; } } -/** - * Fetch one tile from memory, disk, a shared flight, then Mapillary. Empty - * bytes mean no coverage. - * @param {{layer:string,z:number|string,x:number|string,y:number|string}} request - * @param {{signal?: AbortSignal}} [options] - * @returns {Promise<{bytes: Buffer, source: 'memory'|'disk'|'inflight'|'upstream', address: object}>} - */ -export async function fetchTile(request, { signal } = {}) { - const address = normalizeTileAddress(request); - const memory = memoryGet(address.key); - if (memory) return { bytes: memory, source: 'memory', address }; - const disk = await readDisk(address); - if (disk) { - // Older cache files may be untrimmed: rewrite once, keeping the fetch time. - const bytes = trim(address, disk.bytes); - if (bytes !== disk.bytes && !diskWritePending(address)) - writeDisk(address, bytes, disk.at); - memoryPut(address.key, bytes, disk.at); - return { bytes, source: 'disk', address }; - } - // A caller gone while the disk was read must not start a fetch nobody joins. - signal?.throwIfAborted(); - let flight = _inFlight.get(address.key); - const joined = Boolean(flight) && !flight.controller.signal.aborted; - if (!joined) flight = startFlight(address); - const bytes = await joinFlight(flight, signal); - return { bytes, source: joined ? 'inflight' : 'upstream', address }; -} - -/** Start the shared upstream fetch; it holds a slot until the body is read. */ -function startFlight(address) { - const controller = new AbortController(); - const flight = { controller, waiters: 0, promise: null }; - flight.promise = acquireUpstreamSlot(controller.signal) - .then((release) => - fetchUpstream(address, controller.signal).finally(release), - ) - .then((raw) => trim(address, raw)) - .then((bytes) => { - memoryPut(address.key, bytes); - writeDisk(address, bytes); - return bytes; - }) - .finally(() => { - if (_inFlight.get(address.key) === flight) _inFlight.delete(address.key); +/** One tile from Mapillary, trimmed; empty bytes mean no coverage. */ +async function fetchUpstream({ z, x, y }) { + const token = encodeURIComponent(mapillaryToken()); + let url = `${TILE_HOST}/mly1_public/2/${z}/${x}/${y}?access_token=${token}`; + const signal = AbortSignal.timeout(FETCH_TIMEOUT_MS); + for (let hop = 0; ; hop++) { + // Redirects are followed by hand so every hop is checked. + const response = await fetch(url, { + signal, + redirect: 'manual', + headers: { Accept: 'application/x-protobuf' }, }); - // Every waiter may have left before it settles; nobody awaits it then. - flight.promise.catch(() => {}); - _inFlight.set(address.key, flight); - return flight; + if (response.ok && response.status !== 204) { + try { + const bytes = await readResponseBytesCapped(response, TILE_MAX_BYTES); + return trim(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length)); + } catch (error) { + if (error?.code !== 'RESPONSE_TOO_LARGE') throw error; + throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap'); + } + } + await response.body?.cancel().catch(() => {}); + if (response.status === 204 || response.status === 404) + return Buffer.alloc(0); + if (!REDIRECT_STATUSES.has(response.status)) + throw new TileUpstreamError( + response.status, + undefined, + response.headers.get('retry-after'), + ); + if (hop >= TILE_MAX_REDIRECTS) + throw new TileUpstreamError(502, 'Mapillary tile redirected too often'); + url = pinnedRedirect(response.headers.get('location'), url); + } +} + +/** @type {Map} insertion-ordered LRU */ +const memory = new Map(); +let memoryBytes = 0; +/** @type {Map>} upstream fetches shared per tile */ +const inFlight = new Map(); +const cost = (bytes) => bytes.length + ENTRY_OVERHEAD_BYTES; + +function forget(key) { + const hit = memory.get(key); + if (!hit) return null; + memory.delete(key); + memoryBytes -= cost(hit.bytes); + return hit; +} + +function remember(key, bytes, at = Date.now()) { + // One huge (untrimmed) tile must not flush everything else. + if (cost(bytes) > TILE_MEMORY_BUDGET_BYTES / 2) return; + forget(key); + memory.set(key, { bytes, at }); + memoryBytes += cost(bytes); + for (const [oldest] of memory) { + if (memoryBytes <= TILE_MEMORY_BUDGET_BYTES) break; + forget(oldest); + } +} + +function recall(key) { + const hit = forget(key); + if (!hit || Date.now() - hit.at > TILE_TTL_MS) return null; + remember(key, hit.bytes, hit.at); // most recently used again + return hit.bytes; } /** - * Wait on a shared flight. A caller's abort rejects only that caller; the - * fetch is cancelled when the last waiter leaves. + * One tile from memory, a shared in-flight fetch, or Mapillary. + * @param {{z: number, x: number, y: number, key: string}} address from parseTilePath + * @returns {Promise<{bytes: Buffer, source: 'memory'|'inflight'|'upstream'}>} */ -function joinFlight(flight, signal) { - signal?.throwIfAborted(); - flight.waiters++; - return new Promise((resolve, reject) => { - let done = false; - const leave = () => { - if (done) return false; - done = true; - signal?.removeEventListener('abort', onAbort); - flight.waiters--; - return true; - }; - const onAbort = () => { - if (!leave()) return; - if (flight.waiters === 0) flight.controller.abort(); - reject(signal.reason); - }; - signal?.addEventListener('abort', onAbort, { once: true }); - flight.promise.then( - (bytes) => leave() && resolve(bytes), - (error) => leave() && reject(error), - ); - }); +export async function fetchTile(address) { + const cached = recall(address.key); + if (cached) return { bytes: cached, source: 'memory' }; + const { promise, shared } = coalesceProxyRequest( + inFlight, + address.key, + async () => { + const bytes = await fetchUpstream(address); + remember(address.key, bytes); + return bytes; + }, + ); + return { bytes: await promise, source: shared ? 'inflight' : 'upstream' }; } -/** Test seam: forget every cached tile held in memory. */ -export function _resetTileMemoryForTest() { - _memory.clear(); - _memoryBytes = 0; - _inFlight.clear(); - _upstreamHold = null; - _slots = { active: 0, queue: [] }; +/** Test seam: forget every cached and in-flight tile. */ +export function _resetTileCacheForTest() { + memory.clear(); + memoryBytes = 0; + inFlight.clear(); } /** Test seam: how many tiles memory holds and the bytes charged for them. */ export function _tileMemoryForTest() { - return { entries: _memory.size, bytes: _memoryBytes }; -} - -/** Test seam: wait for every background write and the sweeps they start. */ -export async function _settleTileWritesForTest() { - while (_background.size) await Promise.allSettled([..._background]); -} - -/** Test seam: keep the disk cache in `dir` (null restores the default). */ -export function _setTileCacheDirForTest(dir) { - _diskDir = dir || TILE_DISK_DIR; - _lastSweepAt = 0; + return { entries: memory.size, bytes: memoryBytes }; } diff --git a/src/tooling/mapillaryProvider.test.mjs b/src/tooling/mapillaryProvider.test.mjs index d6e5a0e9e..8d7363ff3 100644 --- a/src/tooling/mapillaryProvider.test.mjs +++ b/src/tooling/mapillaryProvider.test.mjs @@ -1,1270 +1,23 @@ import assert from 'node:assert/strict'; -import { promises as fsp } from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import test, { after } from 'node:test'; +import test from 'node:test'; import { PbfWriter } from 'pbf'; -import { mapillaryProxy } from 'gods-eye-view/server/providers/mapillary'; +import { + mapillaryProxy, + TILE_ROUTE_MAX_PER_MIN, +} from 'gods-eye-view/server/providers/mapillary'; import { listTileLayers, stripTileLayers, } from '../../server/providers/mapillary/trim.js'; -import * as tiles from '../../server/providers/mapillary/tiles.js'; import { - TILE_DISK_FILE_OVERHEAD_BYTES, TILE_MAX_BYTES, + TILE_MAX_REDIRECTS, TILE_MEMORY_BUDGET_BYTES, - TILE_MEMORY_ENTRY_OVERHEAD_BYTES, - TILE_RATE_LIMIT_MAX_HOLD_MS, - TILE_ROUTE_MAX_PER_MIN, - TILE_UPSTREAM_CONCURRENCY, -} from '../../server/providers/mapillary/constants.js'; -import { - fetchTile, - normalizeTileAddress, - TileRequestError, - _resetTileMemoryForTest, - _setTileCacheDirForTest, - _settleTileWritesForTest, + TILE_TTL_MS, + _resetTileCacheForTest, + _tileMemoryForTest, } from '../../server/providers/mapillary/tiles.js'; -// Keep test tiles out of the developer's .gev-cache. -const cacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'gev-mly-tiles-')); -_setTileCacheDirForTest(cacheDir); -// Not switched back, so a late background write can never sweep the real -// cache; writes settle before the directory is removed. -after(async () => { - await tiles._settleTileWritesForTest(); - await fsp.rm(cacheDir, { recursive: true, force: true }); -}); -const tileFile = ({ z, x, y }, root = cacheDir) => - path.join(root, 'coverage', String(z), `${x}-${y}.pbf`); -const HOUR = 60 * 60 * 1000; - -async function writeAged(file, bytes, ageMs) { - await fsp.mkdir(path.dirname(file), { recursive: true }); - await fsp.writeFile(file, bytes); - const at = new Date(Date.now() - ageMs); - await fsp.utimes(file, at, at); -} - -/** Poll until `check()` holds (background cache writes are not awaited). */ -async function waitFor(check, what) { - for (let i = 0; i < 200; i++) { - if (await check()) return; - await new Promise((resolve) => setTimeout(resolve, 10)); - } - assert.fail(`timed out waiting for ${what}`); -} - -const exists = (file) => - fsp.stat(file).then( - () => true, - () => false, - ); - -/** - * Mount the plugin. `call` waits for the handler; `start` returns at once with - * the response, its `close` trigger and the handler's `done` promise. - */ -function install(plugin, mode = 'configureServer') { - const routes = new Map(); - plugin[mode]({ - middlewares: { - use(route, handler) { - routes.set(route, handler); - }, - }, - }); - const start = ( - route, - url = '/', - method = 'GET', - body, - requestHeaders = {}, - ) => { - const handler = routes.get(route); - assert.ok(handler, `route ${route} is mounted`); - const headers = {}; - const resListeners = {}; - const res = { - statusCode: 200, - headersSent: false, - writableEnded: false, - setHeader(name, value) { - headers[name.toLowerCase()] = value; - }, - getHeader(name) { - return headers[name.toLowerCase()]; - }, - writeHead(status, extra) { - this.statusCode = status; - Object.assign(headers, extra || {}); - }, - end(payload) { - this.body = payload; - this.writableEnded = true; - }, - on(event, fn) { - resListeners[event] = fn; - }, - }; - const listeners = {}; - const req = { - url, - method, - headers: requestHeaders, - on(event, fn) { - listeners[event] = fn; - }, - [Symbol.asyncIterator]: async function* () { - if (body) yield Buffer.from(body); - }, - }; - const done = Promise.resolve(handler(req, res)); - return { res, headers, done, close: () => resListeners.close?.() }; - }; - const call = async (...args) => { - const { res, headers, done } = start(...args); - await done; - return { ...res, headers }; - }; - return { routes, call, start }; -} - -const json = (res) => JSON.parse(String(res.body)); - -test('the plugin mounts the status and tile routes for dev and preview servers', () => { - for (const mode of ['configureServer', 'configurePreviewServer']) { - const { routes } = install(mapillaryProxy(), mode); - assert.deepEqual([...routes.keys()].sort(), [ - '/api/mapillary/status', - '/api/mapillary/tiles', - ]); - } -}); - -test('status reports whether a token exists, never its value, and rejects non-GET', async () => { - const saved = { MAPILLARY_CLIENT_TOKEN: process.env.MAPILLARY_CLIENT_TOKEN }; - delete process.env.MAPILLARY_CLIENT_TOKEN; - try { - const { call } = install(mapillaryProxy()); - const res = await call('/api/mapillary/status'); - assert.equal(res.statusCode, 200); - assert.deepEqual(json(res), { configured: false }); - assert.doesNotMatch(String(res.body), /MLY\||planner/); - const post = await call('/api/mapillary/status', '/', 'POST'); - assert.equal(post.statusCode, 405); - } finally { - for (const [key, value] of Object.entries(saved)) - if (value === undefined) delete process.env[key]; - else process.env[key] = value; - } -}); - -test('tile route validates the path and refuses to proxy without a token', async () => { - const saved = process.env.MAPILLARY_CLIENT_TOKEN; - delete process.env.MAPILLARY_CLIENT_TOKEN; - try { - const { call } = install(mapillaryProxy()); - const bad = await call('/api/mapillary/tiles', '/coverage/14/1/x'); - assert.equal(bad.statusCode, 400); - const signs = await call('/api/mapillary/tiles', '/signs/14/1/2'); - assert.equal(signs.statusCode, 400, 'only coverage tiles are proxied'); - const noKey = await call('/api/mapillary/tiles', '/coverage/14/1/2'); - assert.equal(noKey.statusCode, 503); - assert.deepEqual(json(noKey), { error: 'no_key', keyRequired: true }); - const post = await call('/api/mapillary/tiles', '/coverage/14/1/2', 'POST'); - assert.equal(post.statusCode, 405); - } finally { - if (saved === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; - else process.env.MAPILLARY_CLIENT_TOKEN = saved; - } -}); - -/** Drive the tile route against a scripted upstream `answer(call, url)`. */ -async function withUpstream(answer, run) { - const savedFetch = globalThis.fetch; - const savedToken = process.env.MAPILLARY_CLIENT_TOKEN; - process.env.MAPILLARY_CLIENT_TOKEN = 'MLY|test|token'; - _resetTileMemoryForTest(); - const calls = []; - const inits = []; - globalThis.fetch = async (url, init) => { - calls.push(String(url)); - inits.push(init); - return answer(calls.length, String(url)); - }; - try { - const { call, start } = install(mapillaryProxy()); - await run({ calls, inits, call, start }); - } finally { - // No background write or sweep (dated by a mocked clock) outlives the test. - await tiles._settleTileWritesForTest(); - globalThis.fetch = savedFetch; - if (savedToken === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; - else process.env.MAPILLARY_CLIENT_TOKEN = savedToken; - _resetTileMemoryForTest(); - } -} - -for (const status of [401, 403]) - test(`an upstream ${status} is a rejected key, and misses stop asking Mapillary`, async () => { - await withUpstream( - () => new Response('{}', { status }), - async ({ calls, call }) => { - for (const tile of ['/coverage/14/5/5', '/coverage/14/6/6']) { - const res = await call('/api/mapillary/tiles', tile); - assert.equal(res.statusCode, 403); - assert.deepEqual(json(res), { - error: 'Mapillary rejected the access token', - keyRejected: true, - }); - assert.doesNotMatch(String(res.body), /MLY\|/); - } - assert.equal(calls.length, 1, 'the second miss is answered locally'); - // A new token is asked at once. - process.env.MAPILLARY_CLIENT_TOKEN = 'MLY|new|token'; - await call('/api/mapillary/tiles', '/coverage/14/7/7'); - assert.equal(calls.length, 2); - }, - ); - }); - -test('a 429 passes its Retry-After on and holds misses until it is over', async (t) => { - await withUpstream( - () => new Response('{}', { status: 429, headers: { 'Retry-After': '30' } }), - async ({ calls, call }) => { - const first = await call('/api/mapillary/tiles', '/coverage/14/5/5'); - assert.equal(first.statusCode, 429); - assert.equal(first.headers['retry-after'], '30'); - assert.deepEqual(json(first), { - error: 'Mapillary is rate-limiting tile requests', - retryAfter: 30, - }); - const held = await call('/api/mapillary/tiles', '/coverage/14/6/6'); - assert.equal(held.statusCode, 429); - assert.equal(calls.length, 1, 'held: Mapillary is not asked again'); - const now = Date.now(); - t.mock.method(Date, 'now', () => now + 31_000); - await call('/api/mapillary/tiles', '/coverage/14/6/6'); - assert.equal(calls.length, 2, 'asked again once the wait is over'); - }, - ); -}); - -for (const [why, retryAfter] of [ - ['seconds', () => '999999'], - ['an HTTP date', () => new Date(Date.now() + 365 * 24 * HOUR).toUTCString()], -]) - test(`a far-off Retry-After (${why}) holds misses for at most TILE_RATE_LIMIT_MAX_HOLD_MS`, async (t) => { - const maxSec = TILE_RATE_LIMIT_MAX_HOLD_MS / 1000; - await withUpstream( - () => - new Response('{}', { - status: 429, - headers: { 'Retry-After': retryAfter() }, - }), - async ({ calls, call }) => { - const first = await call('/api/mapillary/tiles', '/coverage/14/5/5'); - assert.equal(first.statusCode, 429); - assert.equal(first.headers['retry-after'], String(maxSec)); - assert.equal(json(first).retryAfter, maxSec); - const now = Date.now(); - t.mock.method( - Date, - 'now', - () => now + TILE_RATE_LIMIT_MAX_HOLD_MS - 5000, - ); - const held = await call('/api/mapillary/tiles', '/coverage/14/6/6'); - assert.equal(held.statusCode, 429); - assert.ok( - json(held).retryAfter <= 5, - 'the hold counts down from the cap', - ); - assert.equal(calls.length, 1, 'held: Mapillary is not asked again'); - Date.now.mock.mockImplementation( - () => now + TILE_RATE_LIMIT_MAX_HOLD_MS + 1000, - ); - await call('/api/mapillary/tiles', '/coverage/14/6/6'); - assert.equal(calls.length, 2, 'asked again once the capped hold ends'); - }, - ); - }); - -test('normalizeTileAddress enforces layer names, zoom ranges and tile bounds', () => { - assert.equal( - normalizeTileAddress({ layer: 'coverage', z: 11, x: 1, y: 2 }).key, - 'coverage/11/1/2', - ); - assert.deepEqual( - normalizeTileAddress({ layer: 'coverage', z: '14', x: '5', y: '6' }) - .dropLayers, - ['image'], - ); - for (const bad of [ - { layer: 'image', z: 14, x: 1, y: 1 }, - { layer: 'points', z: 14, x: 1, y: 1 }, - { layer: 'signs', z: 14, x: 1, y: 1 }, - { layer: 'coverage', z: 15, x: 1, y: 1 }, - { layer: 'coverage', z: 11, x: 2048, y: 0 }, - { layer: 'coverage', z: 11, x: 1.5, y: 0 }, - { layer: 'coverage', z: -1, x: 0, y: 0 }, - ]) - assert.throws( - () => normalizeTileAddress(bad), - TileRequestError, - JSON.stringify(bad), - ); -}); - -test('normalizeTileAddress refuses prototype keys as layer names', () => { - for (const layer of ['constructor', '__proto__', 'toString']) - assert.throws( - () => normalizeTileAddress({ layer, z: 1, x: 0, y: 0 }), - TileRequestError, - layer, - ); -}); - -test('coverage zooms the app never requests (below z11) are refused with a 400', async () => { - for (const z of [0, 3, 5, 6, 7, 8, 9, 10]) - assert.throws( - () => normalizeTileAddress({ layer: 'coverage', z, x: 0, y: 0 }), - TileRequestError, - `z${z}`, - ); - for (const z of [11, 12, 13, 14]) - assert.equal( - normalizeTileAddress({ layer: 'coverage', z, x: 0, y: 0 }).z, - z, - ); - await withUpstream( - () => new Response(null, { status: 204 }), - async ({ calls, call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/9/5/5'); - assert.equal(res.statusCode, 400); - assert.deepEqual(json(res), { - error: 'Tile zoom for coverage must be 11–14', - }); - assert.equal(calls.length, 0, 'Mapillary is not asked'); - }, - ); -}); - -test('fetchTile serves from memory after one upstream fetch and strips the image layer', async () => { - const savedFetch = globalThis.fetch; - const savedToken = process.env.MAPILLARY_CLIENT_TOKEN; - process.env.MAPILLARY_CLIENT_TOKEN = 'MLY|test|token'; - _resetTileMemoryForTest(); - const tile = (() => { - const writer = new PbfWriter(); - for (const name of ['sequence', 'image']) { - writer.writeMessage( - 3, - (layer, pbf) => { - pbf.writeVarintField(15, 2); - pbf.writeStringField(1, layer.name); - if (layer.name === 'image') - pbf.writeBytesField(4, Buffer.alloc(4000, 1)); - }, - { name }, - ); - } - return Buffer.from(writer.finish()); - })(); - let upstreamCalls = 0; - globalThis.fetch = async (url) => { - upstreamCalls++; - assert.match( - String(url), - /tiles\.mapillary\.com\/maps\/vtp\/mly1_public\/2\/14\/0\/0\?access_token=/, - ); - return new Response(tile, { - status: 200, - headers: { 'content-type': 'application/x-protobuf' }, - }); - }; - try { - // The file is removed first so an earlier test cannot leave a disk hit. - await fsp.rm(tileFile({ z: 14, x: 0, y: 0 }), { force: true }); - const first = await fetchTile({ layer: 'coverage', z: 14, x: 0, y: 0 }); - assert.equal(first.source, 'upstream'); - assert.deepEqual(listTileLayers(first.bytes), ['sequence']); - assert.ok(first.bytes.length < 100, 'image layer stripped in transit'); - const second = await fetchTile({ layer: 'coverage', z: 14, x: 0, y: 0 }); - assert.equal(second.source, 'memory'); - assert.equal(upstreamCalls, 1); - } finally { - globalThis.fetch = savedFetch; - if (savedToken === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; - else process.env.MAPILLARY_CLIENT_TOKEN = savedToken; - _resetTileMemoryForTest(); - } -}); - -test('a tile held in memory past the 24 h TTL is fetched again', async (t) => { - const tile = { layer: 'coverage', z: 14, x: 3, y: 3 }; - await withDeferredUpstream([tile], async ({ upstream, settle }) => { - const first = fetchTile(tile); - await waitFor(() => upstream.length === 1, 'the upstream fetch'); - upstream[0].release(); - assert.equal((await first).source, 'upstream'); - assert.equal((await fetchTile(tile)).source, 'memory'); - const now = Date.now(); - t.mock.method(Date, 'now', () => now + 25 * 60 * 60 * 1000); - const again = fetchTile(tile); - // Neither cache answers a stale tile: Mapillary is asked again. - await waitFor(() => upstream.length === 2, 'a second upstream fetch'); - upstream[1].release(); - assert.equal((await again).source, 'upstream'); - }); -}); - -test('a disk tile keeps its age in memory and through a retrim rewrite', async (t) => { - const address = { layer: 'coverage', z: 14, x: 8, y: 8 }; - const file = tileFile(address); - const untrimmed = tile([ - { name: 'sequence' }, - { name: 'image', payload: Buffer.alloc(4000, 1) }, - ]); - await writeAged(file, untrimmed, 23 * HOUR); - const writtenAt = (await fsp.stat(file)).mtimeMs; - await withUpstream( - () => new Response(tile([{ name: 'sequence' }]), { status: 200 }), - async ({ calls }) => { - const first = await fetchTile(address); - assert.equal(first.source, 'disk'); - assert.deepEqual(listTileLayers(first.bytes), ['sequence']); - // The untrimmed file is replaced by the trimmed tile, still dated when - // Mapillary served it. - await tiles._settleTileWritesForTest(); - assert.equal((await fsp.stat(file)).size, first.bytes.length); - assert.ok( - Math.abs((await fsp.stat(file)).mtimeMs - writtenAt) < 1000, - 'the rewrite keeps the original mtime', - ); - const now = Date.now(); - t.mock.method(Date, 'now', () => now + 2 * HOUR); - const later = await fetchTile(address); - assert.equal(later.source, 'upstream', '25 h old: no cache serves it'); - assert.equal(calls.length, 1); - await tiles._settleTileWritesForTest(); - assert.ok((await fsp.stat(file)).mtimeMs > writtenAt + HOUR); - }, - ); -}); - -/** Point the disk cache at a fresh directory for one test. */ -async function withCacheDir(run) { - const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'gev-mly-sweep-')); - tiles._setTileCacheDirForTest(dir); - try { - await run(dir); - } finally { - await tiles._settleTileWritesForTest(); - tiles._setTileCacheDirForTest(cacheDir); - await fsp.rm(dir, { recursive: true, force: true }); - } -} - -test('the disk sweep removes expired tiles and keeps fresh ones', async () => { - await withCacheDir(async (dir) => { - const expired = tileFile({ z: 14, x: 1, y: 1 }, dir); - const fresh = tileFile({ z: 14, x: 2, y: 2 }, dir); - await writeAged(expired, Buffer.alloc(100, 1), 25 * HOUR); - await writeAged(fresh, Buffer.alloc(100, 2), 23 * HOUR); - const result = await tiles.sweepTileDisk(); - assert.equal(result.removed, 1); - assert.equal(await exists(expired), false); - assert.equal(await exists(fresh), true); - }); -}); - -test('the disk sweep evicts the oldest tiles past the size cap', async () => { - await withCacheDir(async (dir) => { - const files = [3, 2, 1].map((age) => { - const file = tileFile({ z: 13, x: age, y: age }, dir); - return { file, age }; - }); - for (const { file, age } of files) - await writeAged(file, Buffer.alloc(100, age), age * HOUR); - const charged = 100 + TILE_DISK_FILE_OVERHEAD_BYTES; - const result = await tiles.sweepTileDisk({ maxBytes: 2.5 * charged }); - assert.deepEqual(result, { removed: 1, bytes: 2 * charged }); - assert.equal(await exists(files[0].file), false, 'the oldest goes first'); - assert.equal(await exists(files[1].file), true); - assert.equal(await exists(files[2].file), true); - }); -}); - -test('caching a tile sweeps expired files off the disk in the background', async () => { - await withCacheDir(async (dir) => { - const expired = tileFile({ z: 12, x: 1, y: 1 }, dir); - const fresh = tileFile({ z: 12, x: 2, y: 2 }, dir); - await writeAged(expired, Buffer.alloc(100, 1), 30 * HOUR); - await writeAged(fresh, Buffer.alloc(100, 2), HOUR); - await withUpstream( - () => new Response(tile([{ name: 'sequence' }]), { status: 200 }), - async () => { - const address = { layer: 'coverage', z: 14, x: 4, y: 4 }; - assert.equal((await fetchTile(address)).source, 'upstream'); - await waitFor(() => exists(tileFile(address, dir)), 'the tile write'); - await waitFor(async () => !(await exists(expired)), 'the sweep'); - assert.equal(await exists(fresh), true); - }, - ); - }); -}); - -/** An upstream whose fetches wait until released and record their abort. */ -async function withDeferredUpstream(tiles, run) { - const savedFetch = globalThis.fetch; - const savedToken = process.env.MAPILLARY_CLIENT_TOKEN; - process.env.MAPILLARY_CLIENT_TOKEN = 'MLY|test|token'; - _resetTileMemoryForTest(); - for (const tile of tiles) await fsp.rm(tileFile(tile), { force: true }); - const body = (() => { - const writer = new PbfWriter(); - writer.writeMessage( - 3, - (layer, pbf) => { - pbf.writeVarintField(15, 2); - pbf.writeStringField(1, layer.name); - }, - { name: 'sequence' }, - ); - return Buffer.from(writer.finish()); - })(); - const upstream = []; - globalThis.fetch = (url, { signal } = {}) => - new Promise((resolve, reject) => { - const call = { url: String(url), signal, aborted: false }; - call.release = () => - resolve( - new Response(body, { - status: 200, - headers: { 'content-type': 'application/x-protobuf' }, - }), - ); - signal?.addEventListener( - 'abort', - () => { - call.aborted = true; - reject(signal.reason); - }, - { once: true }, - ); - upstream.push(call); - }); - const settle = () => new Promise((resolve) => setTimeout(resolve, 30)); - try { - await run({ upstream, settle, body }); - } finally { - await _settleTileWritesForTest(); - globalThis.fetch = savedFetch; - if (savedToken === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; - else process.env.MAPILLARY_CLIENT_TOKEN = savedToken; - _resetTileMemoryForTest(); - } -} - -test('a joined tile request survives the first caller abandoning it', async () => { - const tile = { layer: 'coverage', z: 14, x: 1, y: 1 }; - await withDeferredUpstream([tile], async ({ upstream, settle }) => { - const first = new AbortController(); - const second = new AbortController(); - const a = fetchTile(tile, { signal: first.signal }); - await waitFor(() => upstream.length === 1, 'the upstream fetch'); - const b = fetchTile(tile, { signal: second.signal }); - await settle(); - assert.equal(upstream.length, 1, 'one upstream request for both'); - first.abort(); - await assert.rejects(a, { name: 'AbortError' }); - assert.equal(upstream[0].aborted, false, 'the shared fetch keeps going'); - upstream[0].release(); - const joined = await b; - assert.equal(joined.source, 'inflight'); - assert.deepEqual(listTileLayers(joined.bytes), ['sequence']); - assert.equal(upstream.length, 1); - }); -}); - -test('a joined caller that aborts leaves at once and the first still gets the tile', async () => { - const tile = { layer: 'coverage', z: 14, x: 2, y: 2 }; - await withDeferredUpstream([tile], async ({ upstream, settle }) => { - const first = new AbortController(); - const second = new AbortController(); - const a = fetchTile(tile, { signal: first.signal }); - await waitFor(() => upstream.length === 1, 'the upstream fetch'); - const b = fetchTile(tile, { signal: second.signal }); - await settle(); - second.abort(); - await assert.rejects(b, { name: 'AbortError' }); - assert.equal(upstream[0].aborted, false); - upstream[0].release(); - assert.equal((await a).source, 'upstream'); - }); -}); - -test('the upstream tile fetch is cancelled once every waiter has left', async () => { - const tile = { layer: 'coverage', z: 14, x: 3, y: 3 }; - await withDeferredUpstream([tile], async ({ upstream, settle }) => { - const first = new AbortController(); - const second = new AbortController(); - const a = fetchTile(tile, { signal: first.signal }); - await waitFor(() => upstream.length === 1, 'the upstream fetch'); - const b = fetchTile(tile, { signal: second.signal }); - await settle(); - first.abort(); - await assert.rejects(a, { name: 'AbortError' }); - assert.equal(upstream[0].aborted, false, 'one waiter is still there'); - second.abort(); - await assert.rejects(b, { name: 'AbortError' }); - assert.equal(upstream[0].aborted, true, 'the last waiter cancels it'); - // A later request for the same tile starts a fresh fetch. - const again = fetchTile(tile); - await waitFor(() => upstream.length === 2, 'a fresh fetch'); - upstream[1].release(); - assert.equal((await again).source, 'upstream'); - }); -}); - -test('a request arriving before a cancelled fetch has wound down starts a fresh one', async () => { - const tile = { layer: 'coverage', z: 13, x: 130, y: 100 }; - await withDeferredUpstream([tile], async ({ body, settle }) => { - // A real fetch rejects some time after its abort; until then the - // cancelled flight is still registered for its tile. - const upstream = []; - globalThis.fetch = (url, { signal } = {}) => - new Promise((resolve, reject) => { - const call = { aborted: false }; - call.release = () => resolve(new Response(body, { status: 200 })); - call.windDown = () => reject(signal.reason); - signal?.addEventListener('abort', () => (call.aborted = true), { - once: true, - }); - upstream.push(call); - }); - const first = new AbortController(); - const a = fetchTile(tile, { signal: first.signal }); - await waitFor(() => upstream.length === 1, 'the upstream fetch'); - first.abort(); - await assert.rejects(a, { name: 'AbortError' }); - assert.equal(upstream[0].aborted, true, 'the last waiter cancelled it'); - // Joining the cancelled flight would end in its AbortError (a 502). - const b = fetchTile(tile); - await waitFor(() => upstream.length === 2, 'a fresh upstream fetch'); - // The cancelled flight winding down must not unregister the fresh one. - upstream[0].windDown(); - await settle(); - const c = fetchTile(tile); - await settle(); - assert.equal(upstream.length, 2, 'a third request joins the fresh fetch'); - upstream[1].release(); - const fresh = await b; - assert.equal(fresh.source, 'upstream'); - assert.deepEqual(listTileLayers(fresh.bytes), ['sequence']); - assert.equal((await c).source, 'inflight'); - }); -}); - -// ── Route: success path, refusals, gate and limiter ── - -/** Every header value and the body, as text, for a token-leak check. */ -const leakText = (res) => - [ - ...Object.values(res.headers).map(String), - Buffer.isBuffer(res.body) ? res.body.toString('latin1') : String(res.body), - ].join('\n'); - -test('the tile route serves a trimmed protobuf tile, then from memory', async () => { - const address = { layer: 'coverage', z: 14, x: 9, y: 9 }; - await fsp.rm(tileFile(address), { force: true }); - const upstream = tile([ - { name: 'sequence' }, - { name: 'image', payload: Buffer.alloc(4000, 1) }, - ]); - await withUpstream( - () => - new Response(upstream, { - status: 200, - headers: { 'content-type': 'application/x-protobuf' }, - }), - async ({ calls, call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/14/9/9'); - assert.equal(res.statusCode, 200); - assert.equal(res.headers['content-type'], 'application/x-protobuf'); - assert.equal(res.headers['cache-control'], 'public, max-age=3600'); - assert.equal(res.headers['x-gev-cache'], 'upstream'); - assert.ok(Buffer.isBuffer(res.body)); - assert.deepEqual(listTileLayers(res.body), ['sequence']); - assert.deepEqual(res.body, tile([{ name: 'sequence' }])); - assert.doesNotMatch(leakText(res), /MLY\|/); - const again = await call('/api/mapillary/tiles', '/coverage/14/9/9'); - assert.equal(again.statusCode, 200); - assert.equal(again.headers['x-gev-cache'], 'memory'); - assert.deepEqual(again.body, res.body); - assert.equal(calls.length, 1); - }, - ); -}); - -for (const status of [204, 404]) - test(`an empty upstream tile (${status}) is a 204 with no body`, async () => { - const address = { layer: 'coverage', z: 14, x: 10, y: status }; - await fsp.rm(tileFile(address), { force: true }); - await withUpstream( - () => new Response(null, { status }), - async ({ call }) => { - const res = await call( - '/api/mapillary/tiles', - `/coverage/14/10/${status}`, - ); - assert.equal(res.statusCode, 204); - assert.equal(res.body, undefined); - assert.equal(res.headers['content-type'], 'application/x-protobuf'); - assert.equal(res.headers['x-gev-cache'], 'upstream'); - assert.doesNotMatch(leakText(res), /MLY\|/); - }, - ); - }); - -test('closing the response mid-flight aborts the upstream fetch', async () => { - const address = { layer: 'coverage', z: 14, x: 11, y: 11 }; - await withDeferredUpstream([address], async ({ upstream }) => { - const { start } = install(mapillaryProxy()); - const pending = start('/api/mapillary/tiles', '/coverage/14/11/11'); - await waitFor(() => upstream.length === 1, 'the upstream fetch'); - assert.equal(upstream[0].aborted, false); - pending.close(); - await pending.done; - assert.equal(upstream[0].aborted, true, 'the last waiter left'); - assert.equal(pending.res.writableEnded, false, 'nothing is written'); - }); -}); - -for (const status of [400, 410, 422]) - test(`an upstream ${status} is a 502, not the client's fault`, async () => { - await withUpstream( - () => new Response('{}', { status }), - async ({ call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/14/12/12'); - assert.equal(res.statusCode, 502); - assert.deepEqual(json(res), { - error: `Mapillary tiles HTTP ${status}`, - }); - }, - ); - }); - -const redirect = (location) => - new Response(null, { - status: 302, - headers: location === null ? {} : { location }, - }); - -test('a redirect within the Mapillary tile origin is followed by hand', async () => { - await fsp.rm(tileFile({ z: 14, x: 13, y: 13 }), { force: true }); - await withUpstream( - (n) => - n === 1 - ? redirect('/maps/vtp/mly1_public/2/14/13/13?moved=1') - : new Response(tile([{ name: 'sequence' }]), { status: 200 }), - async ({ calls, inits, call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/14/13/13'); - assert.equal(res.statusCode, 200); - assert.deepEqual(listTileLayers(res.body), ['sequence']); - assert.equal(calls.length, 2); - assert.equal( - calls[1], - 'https://tiles.mapillary.com/maps/vtp/mly1_public/2/14/13/13?moved=1', - ); - for (const init of inits) - assert.equal(init.redirect, 'manual', 'fetch never follows itself'); - assert.doesNotMatch(leakText(res), /MLY\|/); - }, - ); -}); - -for (const [why, location] of [ - ['another host', 'https://evil.example/collect'], - ['a look-alike host', 'https://tiles.mapillary.com.evil.example/t'], - ['another port', 'https://tiles.mapillary.com:8443/t'], - ['plain HTTP', 'http://tiles.mapillary.com/maps/vtp/t'], - ['credentials in the URL', 'https://user:pw@tiles.mapillary.com/t'], - ['a scheme-relative host', '//evil.example/t'], - ['no Location', null], -]) - test(`a redirect off the tile origin (${why}) is a 502, and the token stays home`, async () => { - await withUpstream( - (n) => - n === 1 - ? redirect(location) - : new Response(tile([{ name: 'sequence' }]), { status: 200 }), - async ({ calls, inits, call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/14/14/14'); - assert.equal(res.statusCode, 502); - assert.deepEqual(json(res), { - error: 'Mapillary tile redirect left the tile origin', - }); - assert.equal(calls.length, 1, 'the redirect was not followed'); - assert.equal(inits[0].redirect, 'manual'); - assert.doesNotMatch(leakText(res), /MLY\|/); - }, - ); - }); - -test('an endless same-origin redirect stops after a few hops', async () => { - await withUpstream( - (n) => redirect(`/maps/vtp/mly1_public/2/14/15/15?hop=${n}`), - async ({ calls, call }) => { - const res = await call('/api/mapillary/tiles', '/coverage/14/15/15'); - assert.equal(res.statusCode, 502); - assert.deepEqual(json(res), { - error: 'Mapillary tile redirected too often', - }); - assert.equal(calls.length, tiles.TILE_MAX_REDIRECTS + 1); - }, - ); -}); - -test('cross-site requests are refused on both routes; the app itself passes', async () => { - await withUpstream( - () => new Response(tile([{ name: 'sequence' }]), { status: 200 }), - async ({ calls, call }) => { - const host = 'localhost:5173'; - const refused = [ - { host, 'sec-fetch-site': 'cross-site' }, // , navigation - { host, 'sec-fetch-site': 'same-site' }, - { host, origin: 'https://evil.example' }, // fetch from another page - { host, origin: 'null' }, // sandboxed frame - { host, 'x-forwarded-for': '203.0.113.9' }, // through a proxy - ]; - for (const headers of refused) - for (const [route, url] of [ - ['/api/mapillary/status', '/'], - ['/api/mapillary/tiles', '/coverage/14/13/13'], - ]) { - const res = await call(route, url, 'GET', undefined, headers); - assert.equal( - res.statusCode, - 403, - `${route} ${JSON.stringify(headers)}`, - ); - } - assert.equal(calls.length, 0, 'Mapillary is never asked'); - const app = { - host, - origin: `http://${host}`, - 'sec-fetch-site': 'same-origin', - }; - const status = await call( - '/api/mapillary/status', - '/', - 'GET', - undefined, - app, - ); - assert.equal(status.statusCode, 200); - assert.deepEqual(json(status), { configured: true }); - const res = await call( - '/api/mapillary/tiles', - '/coverage/14/13/14', - 'GET', - undefined, - { host, 'sec-fetch-site': 'same-origin' }, // a same-origin GET has no Origin - ); - assert.notEqual(res.statusCode, 403); - assert.equal(calls.length, 1); - }, - ); -}); - -test('the tile route answers 429 past its per-IP budget', async () => { - const saved = process.env.MAPILLARY_CLIENT_TOKEN; - delete process.env.MAPILLARY_CLIENT_TOKEN; - try { - const { call } = install(mapillaryProxy()); - for (let i = 0; i < TILE_ROUTE_MAX_PER_MIN; i++) { - const res = await call('/api/mapillary/tiles', '/coverage/14/1/2'); - assert.equal(res.statusCode, 503, `request ${i + 1} is within budget`); - } - const over = await call('/api/mapillary/tiles', '/coverage/14/1/2'); - assert.equal(over.statusCode, 429); - assert.equal(over.headers['retry-after'], '5'); - assert.deepEqual(json(over), { - error: 'Too many tile requests', - retryAfter: 5, - }); - const status = await call('/api/mapillary/status'); - assert.equal(status.statusCode, 200, 'the status route is not limited'); - } finally { - if (saved === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; - else process.env.MAPILLARY_CLIENT_TOKEN = saved; - } -}); - -// ── Tile cache and upstream bounds ── - -test('concurrent disk hits on an untrimmed tile rewrite it once, keeping its age', async (t) => { - const address = { layer: 'coverage', z: 14, x: 14, y: 14 }; - const file = tileFile(address); - await writeAged( - file, - tile([ - { name: 'sequence' }, - { name: 'image', payload: Buffer.alloc(4000, 1) }, - ]), - 23 * HOUR, - ); - const writtenAt = (await fsp.stat(file)).mtimeMs; - const writeFile = fsp.writeFile; - const writes = []; - t.mock.method(fsp, 'writeFile', (target, ...rest) => { - if (String(target).startsWith(file)) writes.push(String(target)); - return writeFile.call(fsp, target, ...rest); - }); - const warn = t.mock.method(console, 'warn', () => {}); - await withUpstream( - () => assert.fail('a disk hit never asks Mapillary'), - async () => { - const hits = await Promise.all([1, 2, 3].map(() => fetchTile(address))); - for (const hit of hits) { - assert.equal(hit.source, 'disk'); - assert.deepEqual(listTileLayers(hit.bytes), ['sequence']); - } - await tiles._settleTileWritesForTest(); - }, - ); - assert.equal(writes.length, 1, 'one rewrite for three hits'); - assert.equal(warn.mock.callCount(), 0); - const stat = await fsp.stat(file); - assert.equal(stat.size, tile([{ name: 'sequence' }]).length); - assert.ok(Math.abs(stat.mtimeMs - writtenAt) < 1000, 'no fresh 24 h of life'); -}); - -test('a rewrite overtaken by a fresh tile write steps aside cleanly', async (t) => { - const address = { layer: 'coverage', z: 14, x: 15, y: 15 }; - const file = tileFile(address); - await writeAged( - file, - tile([ - { name: 'sequence' }, - { name: 'image', payload: Buffer.alloc(4000, 1) }, - ]), - 23 * HOUR, - ); - const writtenAt = (await fsp.stat(file)).mtimeMs; - // Hold the retrim's re-dating step until the fresh write has landed. - const utimes = fsp.utimes; - let held = null; - let release; - const gate = new Promise((resolve) => (release = resolve)); - t.mock.method(fsp, 'utimes', async (target, ...rest) => { - if (String(target).startsWith(file)) { - held = String(target); - await gate; - } - return utimes.call(fsp, target, ...rest); - }); - const warn = t.mock.method(console, 'warn', () => {}); - const fresh = tile([{ name: 'sequence' }, { name: 'overview' }]); - await withUpstream( - () => new Response(fresh, { status: 200 }), - async ({ calls }) => { - assert.equal((await fetchTile(address)).source, 'disk'); - await waitFor(() => held, 'the retrim write'); - // Two hours on, the disk copy is past its 24 h: Mapillary is asked. - _resetTileMemoryForTest(); - const now = Date.now(); - t.mock.method(Date, 'now', () => now + 2 * HOUR); - assert.equal((await fetchTile(address)).source, 'upstream'); - assert.equal(calls.length, 1); - await waitFor( - async () => (await fsp.stat(file)).mtimeMs > writtenAt + HOUR, - 'the fresh tile write', - ); - release(); - await tiles._settleTileWritesForTest(); - }, - ); - assert.equal(warn.mock.callCount(), 0, 'no ENOENT from a shared temp file'); - assert.deepEqual(await fsp.readFile(file), fresh, 'the fresh tile stays'); - assert.ok((await fsp.stat(file)).mtimeMs > writtenAt + HOUR); - const left = await fsp.readdir(path.dirname(file)); - assert.deepEqual( - left.filter((name) => name.endsWith('.tmp')), - [], - 'no temporary file is left behind', - ); -}); - -test('at most a few upstream fetches run at once; a queued abort never fetches', async () => { - const addresses = Array.from( - { length: TILE_UPSTREAM_CONCURRENCY + 2 }, - (_, i) => ({ - layer: 'coverage', - z: 13, - x: 100 + i, - y: 100, - }), - ); - await withDeferredUpstream(addresses, async ({ upstream, settle }) => { - const controllers = addresses.map(() => new AbortController()); - const begin = (i) => - fetchTile(addresses[i], { signal: controllers[i].signal }); - // Fill every slot first, so the two extra misses are the ones that wait. - const results = []; - for (let i = 0; i < TILE_UPSTREAM_CONCURRENCY; i++) results.push(begin(i)); - await waitFor( - () => upstream.length === TILE_UPSTREAM_CONCURRENCY, - 'every slot in use', - ); - const queued = TILE_UPSTREAM_CONCURRENCY; - const aborted = queued + 1; - results.push(begin(queued), begin(aborted)); - await settle(); - assert.equal(upstream.length, TILE_UPSTREAM_CONCURRENCY, 'the rest wait'); - controllers[aborted].abort(); - await assert.rejects(results[aborted], { name: 'AbortError' }); - upstream[0].release(); - await waitFor( - () => upstream.length === TILE_UPSTREAM_CONCURRENCY + 1, - 'the queued miss taking the freed slot', - ); - assert.match(upstream.at(-1).url, new RegExp(`/13/${100 + queued}/100\\?`)); - for (const call of upstream) call.release(); - for (const result of results.slice(0, aborted)) - assert.equal((await result).source, 'upstream'); - await settle(); - assert.equal( - upstream.length, - TILE_UPSTREAM_CONCURRENCY + 1, - 'the aborted miss never reached Mapillary', - ); - }); -}); - -test('a caller gone while the disk cache is read starts no upstream fetch', async () => { - const address = { layer: 'coverage', z: 13, x: 120, y: 100 }; - await withDeferredUpstream([address], async ({ upstream, settle }) => { - const controller = new AbortController(); - const pending = fetchTile(address, { signal: controller.signal }); - controller.abort(); // fetchTile is still reading the disk cache - await assert.rejects(pending, { name: 'AbortError' }); - await settle(); - assert.equal(upstream.length, 0, 'Mapillary is not asked'); - }); -}); - -test('empty tiles are charged against the memory budget', async () => { - const addresses = [16, 17, 18].map((x) => ({ - layer: 'coverage', - z: 14, - x, - y: 16, - })); - for (const address of addresses) - await fsp.rm(tileFile(address), { force: true }); - await withUpstream( - () => new Response(null, { status: 204 }), - async () => { - for (const address of addresses) - assert.equal((await fetchTile(address)).bytes.length, 0); - assert.deepEqual(tiles._tileMemoryForTest(), { - entries: 3, - bytes: 3 * TILE_MEMORY_ENTRY_OVERHEAD_BYTES, - }); - }, - ); -}); - -/** A one-layer tile of exactly `size` bytes. */ -function tileOfSize(size) { - const probe = tile([{ name: 'sequence', payload: Buffer.alloc(size) }]); - const bytes = tile([ - { name: 'sequence', payload: Buffer.alloc(2 * size - probe.length) }, - ]); - assert.equal(bytes.length, size); - return bytes; -} - -/** Disk writes are off, so a tile memory dropped comes back from Mapillary. */ -async function withBigTiles(t, bodies, run) { - for (const address of bodies.keys()) - await fsp.rm(tileFile(address), { force: true }); - t.mock.method(fsp, 'writeFile', async () => { - throw new Error('disk writes are off in this test'); - }); - t.mock.method(console, 'warn', () => {}); - const byPath = new Map( - [...bodies].map(([{ z, x, y }, body]) => [`/2/${z}/${x}/${y}?`, body]), - ); - await withUpstream( - (_n, url) => { - const [, body] = [...byPath].find(([key]) => url.includes(key)); - return new Response(body, { status: 200 }); - }, - ({ calls }) => { - const fetched = (address) => - calls.filter((url) => - url.includes(`/2/${address.z}/${address.x}/${address.y}?`), - ).length; - return run({ fetched }); - }, - ); -} - -test('the memory cache evicts the least recently used tile, not the oldest fetched', async (t) => { - const MB = 1024 * 1024; - // Three fit in the budget, a fourth does not. - const size = 30 * MB; - const cost = size + TILE_MEMORY_ENTRY_OVERHEAD_BYTES; - assert.ok(3 * cost <= TILE_MEMORY_BUDGET_BYTES); - assert.ok(4 * cost > TILE_MEMORY_BUDGET_BYTES); - const body = tileOfSize(size); - const [a, b, c, d] = [20, 21, 22, 23].map((x) => ({ - layer: 'coverage', - z: 14, - x, - y: 20, - })); - await withBigTiles( - t, - new Map([a, b, c, d].map((address) => [address, body])), - async ({ fetched }) => { - for (const address of [a, b, c]) - assert.equal((await fetchTile(address)).source, 'upstream'); - assert.deepEqual(tiles._tileMemoryForTest(), { - entries: 3, - bytes: 3 * cost, - }); - // A is used again, so B is now the least recently used. - assert.equal((await fetchTile(a)).source, 'memory'); - assert.equal((await fetchTile(d)).source, 'upstream'); - assert.deepEqual(tiles._tileMemoryForTest(), { - entries: 3, - bytes: 3 * cost, - }); - for (const address of [a, c, d]) - assert.equal( - (await fetchTile(address)).source, - 'memory', - `${address.x} kept`, - ); - // B was evicted; asking for it again goes back to Mapillary. - assert.equal((await fetchTile(b)).source, 'upstream'); - assert.equal(fetched(b), 2); - assert.equal(fetched(a), 1); - }, - ); -}); - -test('a tile costing more than half the memory budget is served but not kept', async (t) => { - const half = TILE_MEMORY_BUDGET_BYTES / 2; - // At exactly half the budget a tile is kept; one byte more and it is not. - const fits = tileOfSize(half - TILE_MEMORY_ENTRY_OVERHEAD_BYTES); - const tooBig = tileOfSize(half - TILE_MEMORY_ENTRY_OVERHEAD_BYTES + 1); - assert.ok(tooBig.length <= TILE_MAX_BYTES, 'within the upstream size cap'); - const small = { layer: 'coverage', z: 14, x: 24, y: 20 }; - const kept = { layer: 'coverage', z: 14, x: 25, y: 20 }; - const big = { layer: 'coverage', z: 14, x: 26, y: 20 }; - await withBigTiles( - t, - new Map([ - [small, tile([{ name: 'sequence' }])], - [kept, fits], - [big, tooBig], - ]), - async ({ fetched }) => { - await fetchTile(small); - const before = tiles._tileMemoryForTest(); - const served = await fetchTile(big); - assert.equal(served.source, 'upstream'); - assert.equal(served.bytes.length, tooBig.length, 'served in full'); - assert.deepEqual( - tiles._tileMemoryForTest(), - before, - 'nothing evicted to make room', - ); - assert.equal((await fetchTile(small)).source, 'memory'); - assert.equal((await fetchTile(big)).source, 'upstream', 'not kept'); - assert.equal(fetched(big), 2); - assert.equal((await fetchTile(kept)).source, 'upstream'); - assert.equal((await fetchTile(kept)).source, 'memory', 'half fits'); - }, - ); -}); - -test('empty tile files count toward the disk cap', async () => { - await withCacheDir(async (dir) => { - const files = [3, 2, 1].map((age) => ({ - file: tileFile({ z: 13, x: age, y: age }, dir), - age, - })); - for (const { file, age } of files) - await writeAged(file, Buffer.alloc(0), age * HOUR); - const result = await tiles.sweepTileDisk({ - maxBytes: 2 * TILE_DISK_FILE_OVERHEAD_BYTES, - }); - assert.deepEqual(result, { - removed: 1, - bytes: 2 * TILE_DISK_FILE_OVERHEAD_BYTES, - }); - assert.equal(await exists(files[0].file), false, 'the oldest goes first'); - assert.equal(await exists(files[2].file), true); - }); -}); - -test('a body past the size cap aborts the upstream request as it streams', async () => { - const address = { layer: 'coverage', z: 14, x: 19, y: 19 }; - await fsp.rm(tileFile(address), { force: true }); - const MB = 1024 * 1024; - const chunk = new Uint8Array(MB); - let pulled = 0; - let cancelled = false; - let upstreamSignal = null; - const savedFetch = globalThis.fetch; - const savedToken = process.env.MAPILLARY_CLIENT_TOKEN; - process.env.MAPILLARY_CLIENT_TOKEN = 'MLY|test|token'; - _resetTileMemoryForTest(); - // Chunked, no Content-Length: only a running count can see it is too big. - globalThis.fetch = async (_url, { signal } = {}) => { - upstreamSignal = signal; - const body = new ReadableStream({ - pull(controller) { - controller.enqueue(chunk); - pulled += chunk.byteLength; - if (pulled >= TILE_MAX_BYTES + 8 * MB) controller.close(); - }, - cancel() { - cancelled = true; - }, - }); - return new Response(body, { status: 200 }); - }; - try { - await assert.rejects(fetchTile(address), { - name: 'TileUpstreamError', - status: 502, - message: 'Mapillary tile exceeds size cap', - }); - assert.equal(upstreamSignal.aborted, true, 'the request is aborted'); - assert.equal(cancelled, true, 'the body is cancelled'); - assert.ok( - pulled <= TILE_MAX_BYTES + 2 * MB, - `stopped after ${pulled} bytes`, - ); - } finally { - globalThis.fetch = savedFetch; - if (savedToken === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; - else process.env.MAPILLARY_CLIENT_TOKEN = savedToken; - _resetTileMemoryForTest(); - } -}); - -// ── Tile trimming ── /** Build a minimal MVT: layers with a name, a version and one opaque feature. */ function tile(layers) { const writer = new PbfWriter(); @@ -1284,29 +37,470 @@ function tile(layers) { return Buffer.from(writer.finish()); } +const SEQUENCE = tile([{ name: 'sequence' }]); +const UNTRIMMED = tile([ + { name: 'sequence' }, + { name: 'image', payload: Buffer.alloc(4000, 1) }, +]); + +/** Mount the plugin and return `call(route, url, method, headers)`. */ +function install(mode = 'configureServer') { + const routes = new Map(); + mapillaryProxy()[mode]({ + middlewares: { use: (route, handler) => routes.set(route, handler) }, + }); + const call = async (route, url = '/', method = 'GET', reqHeaders = {}) => { + const handler = routes.get(route); + assert.ok(handler, `route ${route} is mounted`); + const headers = {}; + const res = { + statusCode: 200, + headers, + setHeader: (name, value) => (headers[name.toLowerCase()] = value), + getHeader: (name) => headers[name.toLowerCase()], + writeHead(status, extra) { + this.statusCode = status; + Object.assign(headers, extra || {}); + }, + end(payload) { + this.body = payload; + this.writableEnded = true; + }, + }; + await handler({ url, method, headers: reqHeaders }, res); + return res; + }; + return { routes, call }; +} + +const json = (res) => JSON.parse(String(res.body)); +/** Every header value and the body, as text, for a token-leak check. */ +const leakText = (res) => + [ + ...Object.values(res.headers).map(String), + Buffer.isBuffer(res.body) ? res.body.toString('latin1') : String(res.body), + ].join('\n'); + +function withToken(token, run) { + const saved = process.env.MAPILLARY_CLIENT_TOKEN; + if (token === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; + else process.env.MAPILLARY_CLIENT_TOKEN = token; + return Promise.resolve() + .then(run) + .finally(() => { + if (saved === undefined) delete process.env.MAPILLARY_CLIENT_TOKEN; + else process.env.MAPILLARY_CLIENT_TOKEN = saved; + }); +} + +/** Drive the tile route against a scripted upstream `answer(n, url)`. */ +async function withUpstream(answer, run) { + const savedFetch = globalThis.fetch; + _resetTileCacheForTest(); + const calls = []; + const inits = []; + globalThis.fetch = async (url, init) => { + calls.push(String(url)); + inits.push(init); + return answer(calls.length, String(url)); + }; + try { + await withToken('MLY|test|token', () => + run({ calls, inits, call: install().call }), + ); + } finally { + globalThis.fetch = savedFetch; + _resetTileCacheForTest(); + } +} + +const ok = (body = SEQUENCE) => new Response(body, { status: 200 }); + +test('the plugin mounts the status and tile routes for dev and preview servers', () => { + for (const mode of ['configureServer', 'configurePreviewServer']) + assert.deepEqual([...install(mode).routes.keys()].sort(), [ + '/api/mapillary/status', + '/api/mapillary/tiles', + ]); +}); + +test('status reports whether a token exists, never its value', async () => { + const { call } = install(); + await withToken(undefined, async () => { + const res = await call('/api/mapillary/status'); + assert.equal(res.statusCode, 200); + assert.deepEqual(json(res), { configured: false }); + assert.equal( + (await call('/api/mapillary/status', '/', 'POST')).statusCode, + 405, + ); + }); + await withToken('MLY|secret', async () => { + const res = await call('/api/mapillary/status'); + assert.deepEqual(json(res), { configured: true }); + assert.doesNotMatch(leakText(res), /MLY\|/); + }); +}); + +test('the tile route validates the path, then needs a token', async () => { + await withUpstream( + () => assert.fail('Mapillary is not asked'), + async ({ call }) => { + for (const bad of [ + '/coverage/14/1/x', + '/signs/14/1/2', + '/coverage/10/0/0', + '/coverage/15/0/0', + '/coverage/11/2048/0', + '/coverage/11/0/2048', + '/coverage/14/1/2/3', + ]) { + const res = await call('/api/mapillary/tiles', bad); + assert.equal(res.statusCode, 400, bad); + } + const post = await call( + '/api/mapillary/tiles', + '/coverage/14/1/2', + 'POST', + ); + assert.equal(post.statusCode, 405); + }, + ); + await withToken(undefined, async () => { + const res = await install().call( + '/api/mapillary/tiles', + '/coverage/14/1/2', + ); + assert.equal(res.statusCode, 503); + assert.deepEqual(json(res), { error: 'no_key', keyRequired: true }); + }); +}); + +test('a tile is trimmed, served with browser caching, then from memory', async () => { + await withUpstream( + () => ok(UNTRIMMED), + async ({ calls, inits, call }) => { + const res = await call('/api/mapillary/tiles', '/coverage/14/9/9'); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['content-type'], 'application/x-protobuf'); + assert.equal(res.headers['cache-control'], 'public, max-age=3600'); + assert.equal(res.headers['x-gev-cache'], 'upstream'); + assert.deepEqual(res.body, SEQUENCE, 'the image layer is dropped'); + assert.doesNotMatch(leakText(res), /MLY\|/); + assert.equal( + calls[0], + 'https://tiles.mapillary.com/maps/vtp/mly1_public/2/14/9/9?access_token=MLY%7Ctest%7Ctoken', + ); + assert.equal(inits[0].redirect, 'manual'); + const again = await call('/api/mapillary/tiles', '/coverage/14/9/9?v=2'); + assert.equal(again.headers['x-gev-cache'], 'memory'); + assert.deepEqual(again.body, SEQUENCE); + assert.equal(calls.length, 1); + }, + ); +}); + +for (const status of [204, 404]) + test(`an empty upstream tile (${status}) is a 204 with no body, and is cached`, async () => { + await withUpstream( + () => new Response(null, { status }), + async ({ calls, call }) => { + for (const source of ['upstream', 'memory']) { + const res = await call('/api/mapillary/tiles', '/coverage/14/10/10'); + assert.equal(res.statusCode, 204); + assert.equal(res.body, undefined); + assert.equal(res.headers['x-gev-cache'], source); + } + assert.equal(calls.length, 1); + }, + ); + }); + +for (const status of [401, 403]) + test(`an upstream ${status} is a rejected key`, async () => { + await withUpstream( + () => new Response('{}', { status }), + async ({ call }) => { + const res = await call('/api/mapillary/tiles', '/coverage/14/5/5'); + assert.equal(res.statusCode, 403); + assert.deepEqual(json(res), { + error: 'Mapillary rejected the access token', + keyRejected: true, + }); + assert.doesNotMatch(leakText(res), /MLY\|/); + }, + ); + }); + +for (const [sent, expected] of [ + ['30', 30], + [null, 60], + ['Wed, 21 Oct 2026 07:28:00 GMT', 60], + ['999999', 600], +]) + test(`an upstream 429 (Retry-After ${sent}) is passed on as ${expected} s`, async () => { + await withUpstream( + () => + new Response('{}', { + status: 429, + headers: sent === null ? {} : { 'Retry-After': sent }, + }), + async ({ calls, call }) => { + const res = await call('/api/mapillary/tiles', '/coverage/14/5/5'); + assert.equal(res.statusCode, 429); + assert.equal(res.headers['retry-after'], String(expected)); + assert.deepEqual(json(res), { + error: 'Mapillary is rate-limiting tile requests', + retryAfter: expected, + }); + // Errors are not cached: the next request asks again. + await call('/api/mapillary/tiles', '/coverage/14/5/5'); + assert.equal(calls.length, 2); + }, + ); + }); + +for (const status of [400, 410, 500]) + test(`an upstream ${status} is a 502, not the client's fault`, async () => { + await withUpstream( + () => new Response('{}', { status }), + async ({ call }) => { + const res = await call('/api/mapillary/tiles', '/coverage/14/12/12'); + assert.equal(res.statusCode, 502); + assert.deepEqual(json(res), { + error: `Mapillary tiles HTTP ${status}`, + }); + }, + ); + }); + +const redirect = (location) => + new Response(null, { + status: 302, + headers: location === null ? {} : { location }, + }); + +test('a redirect within the tile origin is followed by hand; endless ones stop', async () => { + await withUpstream( + (n) => + n === 1 ? redirect('/maps/vtp/mly1_public/2/14/13/13?moved=1') : ok(), + async ({ calls, inits, call }) => { + const res = await call('/api/mapillary/tiles', '/coverage/14/13/13'); + assert.equal(res.statusCode, 200); + assert.equal( + calls[1], + 'https://tiles.mapillary.com/maps/vtp/mly1_public/2/14/13/13?moved=1', + ); + for (const init of inits) assert.equal(init.redirect, 'manual'); + }, + ); + await withUpstream( + (n) => redirect(`/maps/vtp/mly1_public/2/14/15/15?hop=${n}`), + async ({ calls, call }) => { + const res = await call('/api/mapillary/tiles', '/coverage/14/15/15'); + assert.equal(res.statusCode, 502); + assert.deepEqual(json(res), { + error: 'Mapillary tile redirected too often', + }); + assert.equal(calls.length, TILE_MAX_REDIRECTS + 1); + }, + ); +}); + +for (const [why, location] of [ + ['another host', 'https://evil.example/collect'], + ['a look-alike host', 'https://tiles.mapillary.com.evil.example/t'], + ['another port', 'https://tiles.mapillary.com:8443/t'], + ['plain HTTP', 'http://tiles.mapillary.com/maps/vtp/t'], + ['credentials in the URL', 'https://user:pw@tiles.mapillary.com/t'], + ['a scheme-relative host', '//evil.example/t'], + ['no Location', null], +]) + test(`a redirect off the tile origin (${why}) is a 502, and the token stays home`, async () => { + await withUpstream( + (n) => (n === 1 ? redirect(location) : ok()), + async ({ calls, call }) => { + const res = await call('/api/mapillary/tiles', '/coverage/14/14/14'); + assert.equal(res.statusCode, 502); + assert.deepEqual(json(res), { + error: 'Mapillary tile redirect left the tile origin', + }); + assert.equal(calls.length, 1, 'the redirect was not followed'); + }, + ); + }); + +test('a streamed body past the size cap is cancelled and answered 502', async () => { + const MB = 1024 * 1024; + let pulled = 0; + let cancelled = false; + // Chunked, no Content-Length: only a running count can see it is too big. + const body = () => + new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(MB)); + pulled += MB; + if (pulled >= TILE_MAX_BYTES + 8 * MB) controller.close(); + }, + cancel() { + cancelled = true; + }, + }); + await withUpstream( + () => new Response(body(), { status: 200 }), + async ({ call }) => { + const res = await call('/api/mapillary/tiles', '/coverage/14/19/19'); + assert.equal(res.statusCode, 502); + assert.deepEqual(json(res), { error: 'Mapillary tile exceeds size cap' }); + assert.equal(cancelled, true); + assert.ok(pulled <= TILE_MAX_BYTES + 2 * MB, `stopped after ${pulled}`); + }, + ); +}); + +test('cross-site requests are refused on both routes; the app itself passes', async () => { + await withUpstream( + () => ok(), + async ({ calls, call }) => { + const host = 'localhost:5173'; + for (const headers of [ + { host, 'sec-fetch-site': 'cross-site' }, // , navigation + { host, 'sec-fetch-site': 'same-site' }, + { host, origin: 'https://evil.example' }, // fetch from another page + { host, origin: 'null' }, // sandboxed frame + { host, 'x-forwarded-for': '203.0.113.9' }, // through a proxy + ]) + for (const [route, url] of [ + ['/api/mapillary/status', '/'], + ['/api/mapillary/tiles', '/coverage/14/13/13'], + ]) { + const res = await call(route, url, 'GET', headers); + assert.equal( + res.statusCode, + 403, + `${route} ${JSON.stringify(headers)}`, + ); + } + assert.equal(calls.length, 0, 'Mapillary is never asked'); + const app = { host, 'sec-fetch-site': 'same-origin' }; + const status = await call('/api/mapillary/status', '/', 'GET', { + ...app, + origin: `http://${host}`, + }); + assert.deepEqual(json(status), { configured: true }); + const res = await call( + '/api/mapillary/tiles', + '/coverage/14/13/14', + 'GET', + app, + ); + assert.equal(res.statusCode, 200); + }, + ); +}); + +test('the tile route answers 429 past its per-IP budget', async () => { + await withToken(undefined, async () => { + const { call } = install(); + for (let i = 0; i < TILE_ROUTE_MAX_PER_MIN; i++) + assert.equal( + (await call('/api/mapillary/tiles', '/coverage/14/1/2')).statusCode, + 503, + ); + const over = await call('/api/mapillary/tiles', '/coverage/14/1/2'); + assert.equal(over.statusCode, 429); + assert.equal(over.headers['retry-after'], '5'); + assert.deepEqual(json(over), { + error: 'Too many tile requests', + retryAfter: 5, + }); + const status = await call('/api/mapillary/status'); + assert.equal(status.statusCode, 200, 'the status route is not limited'); + }); +}); + +test('identical concurrent tile requests share one upstream fetch', async () => { + let release; + const gate = new Promise((resolve) => (release = resolve)); + await withUpstream( + async (n) => + n === 1 ? (await gate, ok()) : new Response('{}', { status: 500 }), + async ({ calls, call }) => { + const pending = [1, 2, 3].map(() => + call('/api/mapillary/tiles', '/coverage/14/1/1'), + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + assert.equal(calls.length, 1); + release(); + const sources = (await Promise.all(pending)).map((res) => { + assert.deepEqual(res.body, SEQUENCE); + return res.headers['x-gev-cache']; + }); + assert.deepEqual(sources, ['upstream', 'inflight', 'inflight']); + // A failed flight is not reused either. + const failed = await call('/api/mapillary/tiles', '/coverage/14/2/2'); + assert.equal(failed.statusCode, 502); + await call('/api/mapillary/tiles', '/coverage/14/2/2'); + assert.equal(calls.length, 3); + }, + ); +}); + +test('a tile held in memory past its TTL is fetched again', async (t) => { + await withUpstream( + () => ok(), + async ({ calls, call }) => { + await call('/api/mapillary/tiles', '/coverage/14/3/3'); + await call('/api/mapillary/tiles', '/coverage/14/3/3'); + assert.equal(calls.length, 1); + const now = Date.now(); + t.mock.method(Date, 'now', () => now + TILE_TTL_MS + 1000); + const res = await call('/api/mapillary/tiles', '/coverage/14/3/3'); + assert.equal(res.headers['x-gev-cache'], 'upstream'); + assert.equal(calls.length, 2); + }, + ); +}); + +test('the memory cache evicts the least recently used tile past its budget', async () => { + const MB = 1024 * 1024; + // Three 30 MB tiles fit in the budget, a fourth does not. + const big = tile([{ name: 'sequence', payload: Buffer.alloc(30 * MB) }]); + assert.ok(3 * big.length < TILE_MEMORY_BUDGET_BYTES); + assert.ok(4 * big.length > TILE_MEMORY_BUDGET_BYTES); + await withUpstream( + () => ok(big), + async ({ calls, call }) => { + const get = async (x) => + (await call('/api/mapillary/tiles', `/coverage/14/${x}/20`)).headers[ + 'x-gev-cache' + ]; + for (const x of [20, 21, 22]) assert.equal(await get(x), 'upstream'); + assert.equal(await get(20), 'memory', 'A is used again'); + assert.equal(await get(23), 'upstream'); + assert.equal(_tileMemoryForTest().entries, 3); + for (const x of [20, 22, 23]) assert.equal(await get(x), 'memory'); + assert.equal(await get(21), 'upstream', 'B was evicted'); + assert.equal(calls.length, 5); + }, + ); +}); + +// ── Tile trimming ── + test('dropping a layer keeps the others byte-for-byte', () => { - const big = Buffer.alloc(50_000, 7); const bytes = tile([ { name: 'sequence' }, - { name: 'image', payload: big }, + { name: 'image', payload: Buffer.alloc(50_000, 7) }, { name: 'overview' }, ]); const trimmed = stripTileLayers(bytes, ['image']); assert.deepEqual(listTileLayers(trimmed), ['sequence', 'overview']); - assert.ok(trimmed.length < 200, `trimmed to ${trimmed.length} bytes`); assert.deepEqual(trimmed, tile([{ name: 'sequence' }, { name: 'overview' }])); }); test('a tile without the layer is returned untouched', () => { - const bytes = tile([{ name: 'sequence' }]); - assert.equal(stripTileLayers(bytes, ['image']), bytes); - assert.equal(stripTileLayers(bytes, []), bytes); + assert.equal(stripTileLayers(SEQUENCE, ['image']), SEQUENCE); + assert.equal(stripTileLayers(SEQUENCE, []), SEQUENCE); assert.equal(stripTileLayers(Buffer.alloc(0), ['image']).length, 0); }); - -test('layer names are read without decoding features', () => { - assert.deepEqual(listTileLayers(tile([{ name: 'a' }, { name: 'b' }])), [ - 'a', - 'b', - ]); -});