refactor(street-level): slim Mapillary tile proxy — memory cache only, no server holds

Drop the disk tile cache (.gev-cache/mapillary/tiles, background sweep,
size cap, legacy retrim), the server-side 401/403/429 holds, the upstream
concurrency semaphore and per-waiter abort bookkeeping. The client already
pauses on 429 and stops on a rejected key.

Keep: server-side token, {configured}-only status, strict coverage path
validation (z11-14), image-layer trimming, same-site gate and per-IP limit,
HTTPS redirect pinning to the tile origin, the streamed size cap, the error
mapping the client reads, a 24 h byte-budget LRU memory cache, in-flight
coalescing via the shared coalesceProxyRequest, and browser Cache-Control.

Routes and constants fold into mapillary.js and mapillary/tiles.js.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
German KhokhlovandClaude Opus 5.5 committed 2026-10-07 22:27:34 +04:00
1 parent 37a96072fa
commit b5a78826f8
10 files changed
+717 -1938

No files matched your search

+4 -3
View File
@@ -10,9 +10,10 @@
the terrain and on Google 3D tiles; MapillaryJS loads on the first photo,
and opening one frames it with one camera flight.
Share links carry the Mapillary switch, the filters and the panel's collapsed
state. A server proxy caches Mapillary coverage tiles (24 h, 1 GiB on disk),
refuses cross-site requests, rate-limits per IP and backs off when Mapillary
rejects the token or rate-limits.
state. A server proxy adds the token to Mapillary coverage tiles, drops
their unused image layer and keeps them in memory (24 h); it refuses
cross-site requests and rate-limits per IP, and the panel stops asking when
Mapillary rejects the token or rate-limits.
- Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera
list is one keyless GeoJSON request to the agency's OGC API view of its
+1 -1
View File
@@ -28,7 +28,7 @@ These are designed to be used directly in the browser (like a Mapbox public toke
1. **Google Maps API key** — loads Photorealistic 3D Tiles directly and powers GEV place search. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you.
2. **Cesium ion token** (`CESIUM_ION_TOKEN`, optional — for ion-hosted Google Photorealistic 3D Tiles, Bing world imagery, and world terrain) — used as `Cesium.Ion.defaultAccessToken` client-side. Use a public **`assets:read`** token with **URL restrictions** for any hosted deployment. The Community plan has eligibility and usage limits; a public token is not a secret, but it can still consume the account's quota.
3. **Mapillary client token** (`MAPILLARY_CLIENT_TOKEN`, optional, Street Level) — used by the MapillaryJS viewer and Graph API lookups in the browser, and by the server's coverage-tile proxy. Mapillary client tokens are meant for browser use.
3. **Mapillary client token** (`MAPILLARY_CLIENT_TOKEN`, optional, Street Level) — used by the MapillaryJS viewer and Graph API lookups in the browser, and by the server's coverage-tile proxy, which sends it only to `tiles.mapillary.com` over HTTPS (redirects are followed by hand and pinned to that origin) and caches tiles in memory only. Mapillary client tokens are meant for browser use.
> The explicit browser `define` block in `build/vite.js` controls exactly what reaches the client: only these three keys. Everything else stays server-side.
+1 -1
View File
@@ -2928,7 +2928,7 @@ its criteria cannot be silently ignored.
| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static |
| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static |
| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) |
| Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, cached 24 h) |
| Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, server memory cache 24 h) |
| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) |
| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) |
-2
View File
@@ -884,8 +884,6 @@
"src/layers/streetLevel/providers/mapillary/source.js",
"src/layers/streetLevel/providers/mapillary/nearest.js",
"server/providers/mapillary.js",
"server/providers/mapillary/routes.js",
"server/providers/mapillary/constants.js",
"server/providers/mapillary/tiles.js",
"server/providers/mapillary/trim.js",
"src/layers/streetLevel/tileMath.test.mjs",
-2
View File
@@ -188,8 +188,6 @@
"exports": ["./server/providers/mapillary"],
"modules": [
"server/providers/mapillary.js",
"server/providers/mapillary/routes.js",
"server/providers/mapillary/constants.js",
"server/providers/mapillary/tiles.js",
"server/providers/mapillary/trim.js",
"server/providers/common/same-site.js",
+100 -14
View File
@@ -1,19 +1,105 @@
import { installMapillaryRoutes } from './mapillary/routes.js';
import { sameSiteGated } from './common/same-site.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import {
fetchTile,
mapillaryToken,
parseTilePath,
TileUpstreamError,
} from './mapillary/tiles.js';
/**
* Vite plugin: cached Mapillary coverage tiles (token added server-side) and a
* status route that says whether a token is configured.
* Tile requests per client IP per minute, cache hits included: 24 new 25-tile
* views a minute, more than a person flying the camera reaches.
*/
function mapillaryProxy() {
return {
name: 'mapillary-proxy',
configureServer(server) {
installMapillaryRoutes(server.middlewares);
},
configurePreviewServer(server) {
installMapillaryRoutes(server.middlewares);
},
};
export const TILE_ROUTE_MAX_PER_MIN = 600;
/** The longest upstream Retry-After passed on, in seconds. */
const MAX_RETRY_AFTER_SEC = 600;
function sendJson(res, status, payload) {
res.statusCode = status;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', 'no-store');
res.end(JSON.stringify(payload));
}
export { mapillaryProxy };
/** GET /api/mapillary/status — whether a token is configured, never its value. */
function handleStatus(req, res) {
if (req.method !== 'GET')
return sendJson(res, 405, { error: 'Method not allowed' });
sendJson(res, 200, { configured: Boolean(mapillaryToken()) });
}
/** GET /api/mapillary/tiles/coverage/{z}/{x}/{y} — trimmed protobuf tile. */
async function handleTile(req, res, allow) {
if (req.method !== 'GET')
return sendJson(res, 405, { error: 'Method not allowed' });
if (!allow(clientKey(req))) {
res.setHeader('Retry-After', '5');
return sendJson(res, 429, {
error: 'Too many tile requests',
retryAfter: 5,
});
}
const address = parseTilePath((req.url || '').split('?')[0]);
if (!address)
return sendJson(res, 400, {
error: 'Tile path must be /coverage/{z}/{x}/{y} with z 11–14',
});
if (!mapillaryToken())
return sendJson(res, 503, { error: 'no_key', keyRequired: true });
try {
const { bytes, source } = await fetchTile(address);
res.statusCode = bytes.length ? 200 : 204;
res.setHeader('Content-Type', 'application/x-protobuf');
res.setHeader('Cache-Control', 'public, max-age=3600');
res.setHeader('X-Gev-Cache', source);
res.end(bytes.length ? bytes : undefined);
} catch (error) {
const status = error instanceof TileUpstreamError ? error.status : 0;
// A rejected token is a key problem the panel can name, not a fault.
if (status === 401 || status === 403)
return sendJson(res, 403, {
error: 'Mapillary rejected the access token',
keyRejected: true,
});
if (status === 429) {
const sent = Number(error.retryAfter);
const retryAfter =
Number.isInteger(sent) && sent > 0
? Math.min(sent, MAX_RETRY_AFTER_SEC)
: 60;
res.setHeader('Retry-After', String(retryAfter));
return sendJson(res, 429, {
error: 'Mapillary is rate-limiting tile requests',
retryAfter,
});
}
// An upstream 400 must not read as a malformed request to this proxy.
sendJson(res, 502, { error: error?.message || 'Tile fetch failed' });
}
}
/**
* Vite plugin: Mapillary coverage tiles with the token added server-side, and
* a status route. Both refuse cross-site requests so another page cannot
* spend the token.
*/
export function mapillaryProxy() {
const install = (middlewares) => {
const allow = makeRateLimiter({
windowMs: 60_000,
max: TILE_ROUTE_MAX_PER_MIN,
globalMax: TILE_ROUTE_MAX_PER_MIN * 4,
});
middlewares.use('/api/mapillary/status', sameSiteGated(handleStatus));
middlewares.use(
'/api/mapillary/tiles',
sameSiteGated((req, res) => handleTile(req, res, allow)),
);
};
return {
name: 'mapillary-proxy',
configureServer: (server) => install(server.middlewares),
configurePreviewServer: (server) => install(server.middlewares),
};
}
-72
View File
@@ -1,72 +0,0 @@
import path from 'node:path';
/** Mapillary vector tile root; the path template is /{layer}/2/{z}/{x}/{y}. */
export const MAPILLARY_TILE_HOST = 'https://tiles.mapillary.com/maps/vtp';
/** Public tile layer names this proxy exposes, mapped to Mapillary's ids. */
export const TILE_LAYERS = Object.freeze({
// Only the street zooms the app draws (z11–14); the rest are refused. The
// z14 `image` layer is ~98% of a tile and unused (positions come from the
// graph API), so it is dropped in transit.
coverage: Object.freeze({
upstream: 'mly1_public',
zoomRanges: Object.freeze([Object.freeze([11, 14])]),
dropLayers: Object.freeze(['image']),
}),
});
/** Disk cache root, alongside the other providers' caches. */
export const MAPILLARY_CACHE_DIR = path.join(
process.cwd(),
'.gev-cache',
'mapillary',
);
export const TILE_DISK_DIR = path.join(MAPILLARY_CACHE_DIR, 'tiles');
/**
* Longer than the 15-minute upstream Cache-Control: tiles change only when new
* imagery is processed, and a day avoids re-downloading a city per camera move.
*/
export const TILE_DISK_TTL_MS = 24 * 60 * 60 * 1000;
/**
* A tile write starts a background sweep at most this often: expired files go
* first, then the oldest until the cache fits in TILE_DISK_MAX_BYTES.
*/
export const TILE_DISK_SWEEP_INTERVAL_MS = 15 * 60 * 1000;
export const TILE_DISK_MAX_BYTES = 1024 * 1024 * 1024;
/** Per-file cost on top of its size, so empty tiles still count. */
export const TILE_DISK_FILE_OVERHEAD_BYTES = 4096;
/** A z14 image tile over a dense city is ~11 MB; anything past this is wrong. */
export const TILE_MAX_BYTES = 48 * 1024 * 1024;
export const TILE_MEMORY_BUDGET_BYTES = 96 * 1024 * 1024;
/** Per-entry cost on top of its bytes, so empty tiles still count. */
export const TILE_MEMORY_ENTRY_OVERHEAD_BYTES = 1024;
export const TILE_FETCH_TIMEOUT_MS = 60_000;
/**
* Upstream fetches at once; further misses queue. A view needs at most 25
* tiles, and the cap stops a burst of misses opening thousands of requests.
*/
export const TILE_UPSTREAM_CONCURRENCY = 6;
/**
* Tile requests per client IP per minute, cache hits included. 600 is 24 new
* 25-tile views a minute, more than a person flying the camera reaches, but it
* stops one page draining the shared token into a Mapillary rate limit.
*/
export const TILE_ROUTE_MAX_PER_MIN = 600;
/**
* How long a 401/403 answers tile misses before asking Mapillary again. A
* changed token is tried at once.
*/
export const TILE_KEY_REJECTED_HOLD_MS = 5 * 60 * 1000;
/** After a 429 without a usable Retry-After, hold tile misses this long. */
export const TILE_RATE_LIMIT_HOLD_MS = 60_000;
/** The longest Retry-After honoured, so a bad header cannot stall coverage. */
export const TILE_RATE_LIMIT_MAX_HOLD_MS = 10 * 60 * 1000;
/** The client token lives in the browser by design; the server adds it to tile URLs too. */
export function mapillaryToken() {
return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim();
}
-95
View File
@@ -1,95 +0,0 @@
import { sameSiteGated } from '../common/same-site.js';
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
import { fetchTile, TileRequestError, TileUpstreamError } from './tiles.js';
import { mapillaryToken, TILE_ROUTE_MAX_PER_MIN } from './constants.js';
function sendJson(res, status, payload) {
res.statusCode = status;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', 'no-store');
res.end(JSON.stringify(payload));
}
/** GET /api/mapillary/status — whether a token is configured, never its value. */
function handleStatus(req, res) {
if (req.method !== 'GET')
return sendJson(res, 405, { error: 'Method not allowed' });
sendJson(res, 200, { configured: Boolean(mapillaryToken()) });
}
/** GET /api/mapillary/tiles/coverage/{z}/{x}/{y} — cached protobuf tile. */
async function handleTile(req, res, allow) {
if (req.method !== 'GET')
return sendJson(res, 405, { error: 'Method not allowed' });
if (!allow(clientKey(req))) {
res.setHeader('Retry-After', '5');
return sendJson(res, 429, {
error: 'Too many tile requests',
retryAfter: 5,
});
}
const match = /^\/(coverage)\/(\d{1,2})\/(\d{1,6})\/(\d{1,6})$/.exec(
(req.url || '').split('?')[0],
);
if (!match)
return sendJson(res, 400, {
error: 'Tile path must be /coverage/{z}/{x}/{y}',
});
if (!mapillaryToken())
return sendJson(res, 503, { error: 'no_key', keyRequired: true });
const abandoned = new AbortController();
req.on?.('aborted', () => abandoned.abort());
res.on?.('close', () => abandoned.abort());
try {
const { bytes, source } = await fetchTile(
{ layer: match[1], z: match[2], x: match[3], y: match[4] },
{ signal: abandoned.signal },
);
if (res.writableEnded) return;
res.statusCode = bytes.length ? 200 : 204;
res.setHeader('Content-Type', 'application/x-protobuf');
res.setHeader('Cache-Control', 'public, max-age=3600');
res.setHeader('X-Gev-Cache', source);
res.end(bytes.length ? bytes : undefined);
} catch (error) {
if (res.writableEnded || abandoned.signal.aborted) return;
if (error instanceof TileRequestError)
return sendJson(res, error.status, { error: error.message });
if (error instanceof TileUpstreamError) {
// A rejected token is a key problem the panel can name, not a fault.
if (error.keyRejected)
return sendJson(res, 403, {
error: 'Mapillary rejected the access token',
keyRejected: true,
});
if (error.status === 429) {
const retryAfter = error.retryAfterSec || 60;
res.setHeader('Retry-After', String(retryAfter));
return sendJson(res, 429, {
error: 'Mapillary is rate-limiting tile requests',
retryAfter,
});
}
// An upstream 400 must not read as a malformed request to this proxy.
return sendJson(res, 502, { error: error.message });
}
sendJson(res, 502, { error: error?.message || 'Tile fetch failed' });
}
}
/**
* Attach the Mapillary routes. Both refuse cross-site requests so another page
* cannot spend the token or provoke a 429 that holds every tile miss.
*/
export function installMapillaryRoutes(middlewares) {
const allow = makeRateLimiter({
windowMs: 60_000,
max: TILE_ROUTE_MAX_PER_MIN,
globalMax: TILE_ROUTE_MAX_PER_MIN * 4,
});
middlewares.use('/api/mapillary/status', sameSiteGated(handleStatus));
middlewares.use(
'/api/mapillary/tiles',
sameSiteGated((req, res) => handleTile(req, res, allow)),
);
}
+148 -479
View File
@@ -1,518 +1,187 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { readResponseBytesCapped } from '../common/http.js';
import { stripTileLayers } from './trim.js';
import {
MAPILLARY_TILE_HOST,
TILE_LAYERS,
TILE_DISK_DIR,
TILE_DISK_TTL_MS,
TILE_DISK_SWEEP_INTERVAL_MS,
TILE_DISK_MAX_BYTES,
TILE_DISK_FILE_OVERHEAD_BYTES,
TILE_MAX_BYTES,
TILE_MEMORY_BUDGET_BYTES,
TILE_MEMORY_ENTRY_OVERHEAD_BYTES,
TILE_FETCH_TIMEOUT_MS,
TILE_UPSTREAM_CONCURRENCY,
TILE_KEY_REJECTED_HOLD_MS,
TILE_RATE_LIMIT_HOLD_MS,
TILE_RATE_LIMIT_MAX_HOLD_MS,
mapillaryToken,
} from './constants.js';
coalesceProxyRequest,
readResponseBytesCapped,
} from '../common/http.js';
import { stripTileLayers } from './trim.js';
/** Log without the stack; the token is never in scope here. */
function warn(what, error) {
console.warn(`[Mapillary Proxy] ${what}:`, error?.message || error);
/** Mapillary vector tiles, /{layer}/2/{z}/{x}/{y}: the only host the token goes to. */
const TILE_HOST = 'https://tiles.mapillary.com/maps/vtp';
const TILE_ORIGIN = new URL(TILE_HOST).origin;
/** Only the street zooms the app draws are proxied. */
const MIN_ZOOM = 11;
const MAX_ZOOM = 14;
/**
* The z14 `image` layer is ~98% of a tile (12 MB over a dense city) and unused
* (positions come from the Graph API), so it is dropped in transit.
*/
const DROP_LAYERS = ['image'];
/** A z14 tile with its image layer is ~11 MB; anything past this is wrong. */
export const TILE_MAX_BYTES = 48 * 1024 * 1024;
export const TILE_MAX_REDIRECTS = 3;
const FETCH_TIMEOUT_MS = 60_000;
/** Tiles change only when new imagery is processed. */
export const TILE_TTL_MS = 24 * 60 * 60 * 1000;
export const TILE_MEMORY_BUDGET_BYTES = 96 * 1024 * 1024;
/** Per-entry cost on top of its bytes, so empty tiles still count. */
const ENTRY_OVERHEAD_BYTES = 1024;
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
/** The client token lives in the browser by design; the server adds it to tile URLs too. */
export function mapillaryToken() {
return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim();
}
/** Thrown for a request this proxy refuses before contacting Mapillary. */
export class TileRequestError extends Error {
constructor(message, status = 400) {
super(message);
this.name = 'TileRequestError';
this.status = status;
}
}
/** Mapillary answered with an error status; 401/403 set `keyRejected`. */
/** Mapillary answered with an error status (or a redirect/size this proxy refuses). */
export class TileUpstreamError extends Error {
constructor(status, message, { retryAfterSec = null } = {}) {
constructor(status, message, retryAfter = null) {
super(message || `Mapillary tiles HTTP ${status}`);
this.name = 'TileUpstreamError';
this.status = status;
this.keyRejected = status === 401 || status === 403;
this.retryAfterSec = retryAfterSec;
/** The upstream Retry-After header, as sent. */
this.retryAfter = retryAfter;
}
}
/**
* A rejected token or rate limit that answers every tile miss until it expires
* (or the token changes). Cached tiles are still served meanwhile.
* @type {{status: number, until: number, token: string}|null}
* Parse `/coverage/{z}/{x}/{y}`; null unless z is 11–14 and x/y are on the grid.
* @returns {{z: number, x: number, y: number, key: string}|null}
*/
let _upstreamHold = null;
/** Milliseconds to wait from a Retry-After header (seconds or an HTTP date). */
function retryAfterMs(header, now = Date.now()) {
const text = String(header ?? '').trim();
let ms = NaN;
if (/^\d+$/.test(text)) ms = Number(text) * 1000;
else if (text) ms = Date.parse(text) - now;
if (!Number.isFinite(ms) || ms <= 0) return TILE_RATE_LIMIT_HOLD_MS;
return Math.min(ms, TILE_RATE_LIMIT_MAX_HOLD_MS);
export function parseTilePath(pathname) {
const match = /^\/coverage\/(\d{1,2})\/(\d{1,6})\/(\d{1,6})$/.exec(pathname);
if (!match) return null;
const [z, x, y] = match.slice(1).map(Number);
if (z < MIN_ZOOM || z > MAX_ZOOM || x >= 2 ** z || y >= 2 ** z) return null;
return { z, x, y, key: `${z}/${x}/${y}` };
}
function heldRefusal(token) {
const hold = _upstreamHold;
if (!hold) return null;
const left = hold.until - Date.now();
if (left <= 0 || hold.token !== token) {
_upstreamHold = null;
return null;
}
return new TileUpstreamError(hold.status, undefined, {
retryAfterSec: Math.ceil(left / 1000),
});
}
/**
* Validate a tile address against the layer's zoom ranges.
* @returns {{layer:string, upstream:string, z:number, x:number, y:number, key:string}}
*/
export function normalizeTileAddress({ layer, z, x, y }) {
const spec = Object.hasOwn(TILE_LAYERS, layer) ? TILE_LAYERS[layer] : null;
if (!spec) throw new TileRequestError(`Unknown tile layer: ${layer}`);
const zi = Number(z);
const xi = Number(x);
const yi = Number(y);
if (
![zi, xi, yi].every((v) => Number.isInteger(v) && v >= 0) ||
!spec.zoomRanges.some(([min, max]) => zi >= min && zi <= max)
)
throw new TileRequestError(
`Tile zoom for ${layer} must be ${spec.zoomRanges
.map(([min, max]) => `${min}–${max}`)
.join(' or ')}`,
);
const n = 2 ** zi;
if (xi >= n || yi >= n)
throw new TileRequestError('Tile address out of range');
return {
layer,
upstream: spec.upstream,
dropLayers: spec.dropLayers || [],
z: zi,
x: xi,
y: yi,
key: `${layer}/${zi}/${xi}/${yi}`,
};
}
/** @type {Map<string, {bytes: Buffer, at: number}>} insertion-ordered LRU */
const _memory = new Map();
let _memoryBytes = 0;
const memoryCost = (bytes) => bytes.length + TILE_MEMORY_ENTRY_OVERHEAD_BYTES;
/**
* Upstream fetches shared by every request for the same tile; the last waiter
* to leave cancels it (see `joinFlight`).
* @type {Map<string, {controller: AbortController, promise: Promise<Buffer>, waiters: number}>}
*/
const _inFlight = new Map();
function memoryGet(key) {
const hit = _memory.get(key);
if (!hit) return null;
// Same life as the disk copy, so a long-running server never serves stale
// coverage.
if (Date.now() - hit.at > TILE_DISK_TTL_MS) {
_memory.delete(key);
_memoryBytes -= memoryCost(hit.bytes);
return null;
}
_memory.delete(key);
_memory.set(key, hit);
return hit.bytes;
}
/** `at` is when Mapillary served the bytes (a disk tile's mtime). */
function memoryPut(key, bytes, at = Date.now()) {
if (memoryCost(bytes) > TILE_MEMORY_BUDGET_BYTES / 2) return;
const existing = _memory.get(key);
if (existing) _memoryBytes -= memoryCost(existing.bytes);
_memory.set(key, { bytes, at });
_memoryBytes += memoryCost(bytes);
while (_memoryBytes > TILE_MEMORY_BUDGET_BYTES && _memory.size) {
const [oldest, entry] = _memory.entries().next().value;
_memory.delete(oldest);
_memoryBytes -= memoryCost(entry.bytes);
}
}
/** Disk cache root; tests point it at a temporary directory. */
let _diskDir = TILE_DISK_DIR;
function diskPath({ layer, z, x, y }, root = _diskDir) {
return path.join(root, layer, String(z), `${x}-${y}.pbf`);
}
async function readDisk(address) {
const file = diskPath(address);
/** The next hop, if it stays HTTPS on the tile origin (the token is in the query). */
function pinnedRedirect(location, base) {
let target = null;
try {
const stat = await fsp.stat(file);
if (Date.now() - stat.mtimeMs > TILE_DISK_TTL_MS) return null;
return { bytes: await fsp.readFile(file), at: stat.mtimeMs };
target = location ? new URL(location, base) : null;
} catch {
return null;
/* unparseable Location */
}
}
/** Unsettled background writes and sweeps; requests never wait on them. */
const _background = new Set();
function track(task) {
const settled = task
.catch(() => {})
.finally(() => _background.delete(settled));
_background.add(settled);
return settled;
}
/** @type {Map<string, number>} cache file to its newest write's sequence */
const _diskWrites = new Map();
let _diskWriteSeq = 0;
function diskWritePending(address) {
return _diskWrites.has(diskPath(address));
}
/**
* Write a tile in the background. `at` keeps a rewrite's original fetch time
* so it does not extend the tile's life. An overtaken write steps aside.
*/
function writeDisk(address, bytes, at = null) {
// Sweep this write's root even if the cache dir has moved since.
const root = _diskDir;
const file = diskPath(address, root);
const seq = ++_diskWriteSeq;
const tmp = `${file}.${process.pid}-${seq}.tmp`;
const newest = () => _diskWrites.get(file) === seq;
_diskWrites.set(file, seq);
return track(
fsp
.mkdir(path.dirname(file), { recursive: true })
.then(() => fsp.writeFile(tmp, bytes))
.then(() => at != null && fsp.utimes(tmp, new Date(), new Date(at)))
.then(() => {
if (!newest()) return fsp.rm(tmp, { force: true });
return fsp.rename(tmp, file).then(() => scheduleSweep(root));
})
.catch(async (error) => {
await fsp.rm(tmp, { force: true }).catch(() => {});
warn('tile cache write failed', error);
})
.finally(() => {
if (newest()) _diskWrites.delete(file);
}),
);
}
let _lastSweepAt = 0;
let _sweeping = null;
let _sweepWarned = false;
function scheduleSweep(root) {
if (_sweeping || Date.now() - _lastSweepAt < TILE_DISK_SWEEP_INTERVAL_MS)
return;
_lastSweepAt = Date.now();
const sweep = sweepTileDisk({ root })
.catch((error) => {
if (_sweepWarned) return;
_sweepWarned = true;
warn('tile cache sweep failed', error);
})
.finally(() => {
if (_sweeping === tracked) _sweeping = null;
});
const tracked = track(sweep);
_sweeping = tracked;
}
/** Every file under `dir` with size and mtime; a missing dir is empty. */
async function listCacheFiles(dir) {
let entries;
try {
entries = await fsp.readdir(dir, { withFileTypes: true });
} catch (error) {
if (error?.code === 'ENOENT') return [];
throw error;
}
const files = [];
for (const entry of entries) {
const file = path.join(dir, entry.name);
if (entry.isDirectory()) files.push(...(await listCacheFiles(file)));
else if (entry.isFile()) {
const stat = await fsp.stat(file).catch(() => null);
if (stat) files.push({ file, size: stat.size, at: stat.mtimeMs });
}
}
return files;
}
/**
* Delete expired tiles, then the oldest until the rest fit in `maxBytes`.
* @returns {Promise<{removed: number, bytes: number}>} files deleted, charged bytes kept
*/
export async function sweepTileDisk({
root = _diskDir,
maxBytes = TILE_DISK_MAX_BYTES,
} = {}) {
const now = Date.now();
const files = (await listCacheFiles(root)).sort((a, b) => a.at - b.at);
const cost = (size) => size + TILE_DISK_FILE_OVERHEAD_BYTES;
let bytes = files.reduce((sum, { size }) => sum + cost(size), 0);
let removed = 0;
for (const { file, size, at } of files) {
if (now - at <= TILE_DISK_TTL_MS && bytes <= maxBytes) break;
await fsp.rm(file, { force: true });
bytes -= cost(size);
removed++;
}
return { removed, bytes };
}
/** The one origin the tile token may be sent to, at every redirect hop. */
const TILE_ORIGIN = new URL(MAPILLARY_TILE_HOST).origin;
export const TILE_MAX_REDIRECTS = 3;
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
/**
* Follow redirects by hand so every hop stays HTTPS on the tile origin and the
* token in the query never reaches another host. A bad redirect is a 502.
*/
async function fetchPinned(url, init) {
let next = url;
for (let hop = 0; ; hop++) {
const response = await fetch(next, { ...init, redirect: 'manual' });
if (!REDIRECT_STATUSES.has(response.status)) return response;
await response.body?.cancel().catch(() => {});
if (hop >= TILE_MAX_REDIRECTS)
throw new TileUpstreamError(502, 'Mapillary tile redirected too often');
let target = null;
try {
target = new URL(response.headers.get('location') ?? '', next);
} catch {
/* unparseable Location */
}
if (
!response.headers.get('location') ||
target?.protocol !== 'https:' ||
target.origin !== TILE_ORIGIN ||
target.username ||
target.password
)
throw new TileUpstreamError(
502,
'Mapillary tile redirect left the tile origin',
);
next = target.href;
}
}
async function fetchUpstream(address, signal) {
const token = mapillaryToken();
if (!token) throw new TileRequestError('Mapillary token not configured', 503);
const held = heldRefusal(token);
if (held) throw held;
const url = `${MAPILLARY_TILE_HOST}/${address.upstream}/2/${address.z}/${address.x}/${address.y}?access_token=${encodeURIComponent(token)}`;
const timeout = AbortSignal.timeout(TILE_FETCH_TIMEOUT_MS);
const oversize = new AbortController();
const response = await fetchPinned(url, {
signal: AbortSignal.any([signal, timeout, oversize.signal].filter(Boolean)),
headers: { Accept: 'application/x-protobuf' },
});
if (response.status === 404 || response.status === 204) {
await response.body?.cancel().catch(() => {});
return Buffer.alloc(0);
}
if (!response.ok) {
await response.body?.cancel().catch(() => {});
const error = new TileUpstreamError(response.status);
if (error.keyRejected)
_upstreamHold = {
status: response.status,
until: Date.now() + TILE_KEY_REJECTED_HOLD_MS,
token,
};
else if (response.status === 429) {
const wait = retryAfterMs(response.headers.get('retry-after'));
_upstreamHold = { status: 429, until: Date.now() + wait, token };
error.retryAfterSec = Math.ceil(wait / 1000);
}
throw error;
}
// Chunked or compressed bodies have no Content-Length: the cap applies as
// the body streams, and an oversized one also aborts the request.
try {
const bytes = await readResponseBytesCapped(
response,
TILE_MAX_BYTES,
signal,
if (
target?.protocol !== 'https:' ||
target.origin !== TILE_ORIGIN ||
target.username ||
target.password
)
throw new TileUpstreamError(
502,
'Mapillary tile redirect left the tile origin',
);
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
} catch (error) {
if (error?.code !== 'RESPONSE_TOO_LARGE') throw error;
oversize.abort();
throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap');
}
return target.href;
}
/** @type {{active: number, queue: Array<() => void>}} upstream slots */
let _slots = { active: 0, queue: [] };
/**
* Resolve with a release function once a slot is free. An abort while
* queued leaves the queue.
* @returns {Promise<() => void>}
*/
function acquireUpstreamSlot(signal) {
const slots = _slots;
return new Promise((resolve, reject) => {
if (signal.aborted) return reject(signal.reason);
const onAbort = () => {
const index = slots.queue.indexOf(grant);
if (index !== -1) slots.queue.splice(index, 1);
reject(signal.reason);
};
function grant() {
signal.removeEventListener('abort', onAbort);
slots.active++;
let released = false;
resolve(() => {
if (released) return;
released = true;
slots.active--;
slots.queue.shift()?.();
});
}
if (slots.active < TILE_UPSTREAM_CONCURRENCY) return grant();
signal.addEventListener('abort', onAbort, { once: true });
slots.queue.push(grant);
});
}
function trim(address, bytes) {
if (!address.dropLayers.length || !bytes.length) return bytes;
function trim(bytes) {
try {
return stripTileLayers(bytes, address.dropLayers);
return stripTileLayers(bytes, DROP_LAYERS);
} catch (error) {
warn('tile trim failed, serving raw', error);
console.warn('[Mapillary Proxy] tile trim failed:', error?.message);
return bytes;
}
}
/**
* Fetch one tile from memory, disk, a shared flight, then Mapillary. Empty
* bytes mean no coverage.
* @param {{layer:string,z:number|string,x:number|string,y:number|string}} request
* @param {{signal?: AbortSignal}} [options]
* @returns {Promise<{bytes: Buffer, source: 'memory'|'disk'|'inflight'|'upstream', address: object}>}
*/
export async function fetchTile(request, { signal } = {}) {
const address = normalizeTileAddress(request);
const memory = memoryGet(address.key);
if (memory) return { bytes: memory, source: 'memory', address };
const disk = await readDisk(address);
if (disk) {
// Older cache files may be untrimmed: rewrite once, keeping the fetch time.
const bytes = trim(address, disk.bytes);
if (bytes !== disk.bytes && !diskWritePending(address))
writeDisk(address, bytes, disk.at);
memoryPut(address.key, bytes, disk.at);
return { bytes, source: 'disk', address };
}
// A caller gone while the disk was read must not start a fetch nobody joins.
signal?.throwIfAborted();
let flight = _inFlight.get(address.key);
const joined = Boolean(flight) && !flight.controller.signal.aborted;
if (!joined) flight = startFlight(address);
const bytes = await joinFlight(flight, signal);
return { bytes, source: joined ? 'inflight' : 'upstream', address };
}
/** Start the shared upstream fetch; it holds a slot until the body is read. */
function startFlight(address) {
const controller = new AbortController();
const flight = { controller, waiters: 0, promise: null };
flight.promise = acquireUpstreamSlot(controller.signal)
.then((release) =>
fetchUpstream(address, controller.signal).finally(release),
)
.then((raw) => trim(address, raw))
.then((bytes) => {
memoryPut(address.key, bytes);
writeDisk(address, bytes);
return bytes;
})
.finally(() => {
if (_inFlight.get(address.key) === flight) _inFlight.delete(address.key);
/** One tile from Mapillary, trimmed; empty bytes mean no coverage. */
async function fetchUpstream({ z, x, y }) {
const token = encodeURIComponent(mapillaryToken());
let url = `${TILE_HOST}/mly1_public/2/${z}/${x}/${y}?access_token=${token}`;
const signal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
for (let hop = 0; ; hop++) {
// Redirects are followed by hand so every hop is checked.
const response = await fetch(url, {
signal,
redirect: 'manual',
headers: { Accept: 'application/x-protobuf' },
});
// Every waiter may have left before it settles; nobody awaits it then.
flight.promise.catch(() => {});
_inFlight.set(address.key, flight);
return flight;
if (response.ok && response.status !== 204) {
try {
const bytes = await readResponseBytesCapped(response, TILE_MAX_BYTES);
return trim(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length));
} catch (error) {
if (error?.code !== 'RESPONSE_TOO_LARGE') throw error;
throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap');
}
}
await response.body?.cancel().catch(() => {});
if (response.status === 204 || response.status === 404)
return Buffer.alloc(0);
if (!REDIRECT_STATUSES.has(response.status))
throw new TileUpstreamError(
response.status,
undefined,
response.headers.get('retry-after'),
);
if (hop >= TILE_MAX_REDIRECTS)
throw new TileUpstreamError(502, 'Mapillary tile redirected too often');
url = pinnedRedirect(response.headers.get('location'), url);
}
}
/** @type {Map<string, {bytes: Buffer, at: number}>} insertion-ordered LRU */
const memory = new Map();
let memoryBytes = 0;
/** @type {Map<string, Promise<Buffer>>} upstream fetches shared per tile */
const inFlight = new Map();
const cost = (bytes) => bytes.length + ENTRY_OVERHEAD_BYTES;
function forget(key) {
const hit = memory.get(key);
if (!hit) return null;
memory.delete(key);
memoryBytes -= cost(hit.bytes);
return hit;
}
function remember(key, bytes, at = Date.now()) {
// One huge (untrimmed) tile must not flush everything else.
if (cost(bytes) > TILE_MEMORY_BUDGET_BYTES / 2) return;
forget(key);
memory.set(key, { bytes, at });
memoryBytes += cost(bytes);
for (const [oldest] of memory) {
if (memoryBytes <= TILE_MEMORY_BUDGET_BYTES) break;
forget(oldest);
}
}
function recall(key) {
const hit = forget(key);
if (!hit || Date.now() - hit.at > TILE_TTL_MS) return null;
remember(key, hit.bytes, hit.at); // most recently used again
return hit.bytes;
}
/**
* Wait on a shared flight. A caller's abort rejects only that caller; the
* fetch is cancelled when the last waiter leaves.
* One tile from memory, a shared in-flight fetch, or Mapillary.
* @param {{z: number, x: number, y: number, key: string}} address from parseTilePath
* @returns {Promise<{bytes: Buffer, source: 'memory'|'inflight'|'upstream'}>}
*/
function joinFlight(flight, signal) {
signal?.throwIfAborted();
flight.waiters++;
return new Promise((resolve, reject) => {
let done = false;
const leave = () => {
if (done) return false;
done = true;
signal?.removeEventListener('abort', onAbort);
flight.waiters--;
return true;
};
const onAbort = () => {
if (!leave()) return;
if (flight.waiters === 0) flight.controller.abort();
reject(signal.reason);
};
signal?.addEventListener('abort', onAbort, { once: true });
flight.promise.then(
(bytes) => leave() && resolve(bytes),
(error) => leave() && reject(error),
);
});
export async function fetchTile(address) {
const cached = recall(address.key);
if (cached) return { bytes: cached, source: 'memory' };
const { promise, shared } = coalesceProxyRequest(
inFlight,
address.key,
async () => {
const bytes = await fetchUpstream(address);
remember(address.key, bytes);
return bytes;
},
);
return { bytes: await promise, source: shared ? 'inflight' : 'upstream' };
}
/** Test seam: forget every cached tile held in memory. */
export function _resetTileMemoryForTest() {
_memory.clear();
_memoryBytes = 0;
_inFlight.clear();
_upstreamHold = null;
_slots = { active: 0, queue: [] };
/** Test seam: forget every cached and in-flight tile. */
export function _resetTileCacheForTest() {
memory.clear();
memoryBytes = 0;
inFlight.clear();
}
/** Test seam: how many tiles memory holds and the bytes charged for them. */
export function _tileMemoryForTest() {
return { entries: _memory.size, bytes: _memoryBytes };
}
/** Test seam: wait for every background write and the sweeps they start. */
export async function _settleTileWritesForTest() {
while (_background.size) await Promise.allSettled([..._background]);
}
/** Test seam: keep the disk cache in `dir` (null restores the default). */
export function _setTileCacheDirForTest(dir) {
_diskDir = dir || TILE_DISK_DIR;
_lastSweepAt = 0;
return { entries: memory.size, bytes: memoryBytes };
}
File diff suppressed because it is too large. Load diff