docs(street-level): shorten docs and comments before merge

Trim the doc text this PR adds (README, CHANGELOG, CURRENT-STATE,
KNOWN-ISSUES, SECURITY, DATA_SOURCES, .env.example, the CI job comment),
drop code comments that restate the code, cut design-note JSDoc to a line
or two, and drop review tags from test names. Comments only; no code
changes.

Also: README key count (seven keys, two metered), CURRENT-STATE's gate
paragraph (CI runs a production build with a dummy token), the stacked
doc above meshBoxInRange and the orphaned inMeshRange line in
coverage.js, and a duplicate ALPR row in DATA_SOURCES that a merge had
brought back.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Said TurkseverandClaude Opus 5.5 committed 2026-10-06 21:59:12 +01:00
1 parent c65feda4b1
commit 4ea6631425
85 files changed
+701 -1520

No files matched your search

+16 -34
View File
@@ -5,11 +5,9 @@ export const MAPILLARY_TILE_HOST = 'https://tiles.mapillary.com/maps/vtp';
/** Public tile layer names this proxy exposes, mapped to Mapillary's ids. */
export const TILE_LAYERS = Object.freeze({
// Overview points (z0–5), sequences (z6–14) and image points (z14 only).
// The app asks for overview z0–5 and sequences z11–14 only (see
// src/layers/streetLevel/tileMath.js); region-sized z6–10 tiles are refused.
// The z14 `image` point layer is ~98% of a 10 MB tile and unused here:
// image positions come from the graph API per sequence. Dropped in transit.
// Region-sized z6–10 tiles are refused; the app never asks for them. The
// z14 `image` layer is ~98% of a tile and unused (positions come from the
// graph API), so it is dropped in transit.
coverage: Object.freeze({
upstream: 'mly1_public',
zoomRanges: Object.freeze([Object.freeze([0, 5]), Object.freeze([11, 14])]),
@@ -26,57 +24,41 @@ export const MAPILLARY_CACHE_DIR = path.join(
export const TILE_DISK_DIR = path.join(MAPILLARY_CACHE_DIR, 'tiles');
/**
* Tiles change when new imagery is processed, which is far slower than the
* 15-minute upstream Cache-Control. A day keeps a session over one city from
* re-downloading its tiles on every camera move.
* Longer than the 15-minute upstream Cache-Control: tiles change only when new
* imagery is processed, and a day avoids re-downloading a city per camera move.
*/
export const TILE_DISK_TTL_MS = 24 * 60 * 60 * 1000;
/**
* Disk cache bound. A tile write starts a background sweep at most this often:
* it deletes files past TILE_DISK_TTL_MS, then the oldest files until the
* cache fits in TILE_DISK_MAX_BYTES (thousands of trimmed street-level tiles).
* A tile write starts a background sweep at most this often: expired files go
* first, then the oldest until the cache fits in TILE_DISK_MAX_BYTES.
*/
export const TILE_DISK_SWEEP_INTERVAL_MS = 15 * 60 * 1000;
export const TILE_DISK_MAX_BYTES = 1024 * 1024 * 1024;
/**
* What each cached file costs against TILE_DISK_MAX_BYTES on top of its size:
* one filesystem block and its directory entry. An empty (no coverage) tile is
* 0 bytes but still a file, so without this ocean tiles would never count.
*/
/** Per-file cost on top of its size, so empty tiles still count. */
export const TILE_DISK_FILE_OVERHEAD_BYTES = 4096;
/** A z14 image tile over a dense city is ~11 MB; anything past this is wrong. */
export const TILE_MAX_BYTES = 48 * 1024 * 1024;
/** In-memory tile cache budget (bytes) and upstream fetch timeout. */
export const TILE_MEMORY_BUDGET_BYTES = 96 * 1024 * 1024;
/**
* What each in-memory entry costs against the budget on top of its bytes (the
* key, the map entry and the Buffer object), so empty tiles count too.
*/
/** Per-entry cost on top of its bytes, so empty tiles still count. */
export const TILE_MEMORY_ENTRY_OVERHEAD_BYTES = 1024;
export const TILE_FETCH_TIMEOUT_MS = 60_000;
/**
* Upstream tile fetches running at once; further misses queue for a slot. A
* view asks for at most 25 tiles, so a few slots keep it quick while a burst
* of distinct misses cannot open thousands of buffered requests at once.
* Upstream fetches at once; further misses queue. A view needs at most 25
* tiles, and the cap stops a burst of misses opening thousands of requests.
*/
export const TILE_UPSTREAM_CONCURRENCY = 6;
/**
* Tile requests per client IP per minute (cache hits included). The app asks
* only for tiles it has not loaded yet, at most 9 street + 16 overview tiles
* per view after a 320 ms camera debounce (src/layers/streetLevel/providers/
* mapillary/policy.js), and the browser keeps each answer for an hour. 600 is
* 24 entirely new views a minute, one every 2.5 s, which a person flying the
* camera does not reach; it still stops one page from draining the shared
* token into Mapillary rate limits that would hold every miss for everyone.
* Tile requests per client IP per minute, cache hits included. 600 is 24 new
* 25-tile views a minute, more than a person flying the camera reaches, but it
* stops one page draining the shared token into a Mapillary rate limit.
*/
export const TILE_ROUTE_MAX_PER_MIN = 600;
/**
* After Mapillary rejects the token (401/403), tile misses are answered from
* that verdict for this long instead of asking again on every camera move.
* A changed token is asked at once.
* How long a 401/403 answers tile misses before asking Mapillary again. A
* changed token is tried at once.
*/
export const TILE_KEY_REJECTED_HOLD_MS = 5 * 60 * 1000;
/** After a 429 without a usable Retry-After, hold tile misses this long. */
+3 -7
View File
@@ -70,8 +70,7 @@ async function handleTile(req, res, allow) {
retryAfter,
});
}
// Any other refusal is Mapillary's fault, not the client's: an
// upstream 400 must not read as a malformed request to this proxy.
// An upstream 400 must not read as a malformed request to this proxy.
return sendJson(res, 502, { error: error.message });
}
sendJson(res, 502, { error: error?.message || 'Tile fetch failed' });
@@ -79,11 +78,8 @@ async function handleTile(req, res, allow) {
}
/**
* Attach every Mapillary route to a connect-style middleware stack. Both
* refuse cross-site browser requests (server/providers/common/same-site.js):
* another page could otherwise spend the user's token, and one Mapillary 429
* it provokes holds every tile miss. Tiles are also rate-limited per client
* IP (TILE_ROUTE_MAX_PER_MIN).
* Attach the Mapillary routes. Both refuse cross-site requests so another page
* cannot spend the token or provoke a 429 that holds every tile miss.
*/
export function installMapillaryRoutes(middlewares) {
const allow = makeRateLimiter({
+30 -79
View File
@@ -20,7 +20,7 @@ import {
mapillaryToken,
} from './constants.js';
/** Log a background failure without its stack (never the token: none is in scope). */
/** Log without the stack; the token is never in scope here. */
function warn(what, error) {
console.warn(`[Mapillary Proxy] ${what}:`, error?.message || error);
}
@@ -34,10 +34,7 @@ export class TileRequestError extends Error {
}
}
/**
* Thrown when Mapillary answered with an error status. A 401/403 means the
* token was rejected (`keyRejected`); a 429 carries how long to wait.
*/
/** Mapillary answered with an error status; 401/403 set `keyRejected`. */
export class TileUpstreamError extends Error {
constructor(status, message, { retryAfterSec = null } = {}) {
super(message || `Mapillary tiles HTTP ${status}`);
@@ -49,9 +46,8 @@ export class TileUpstreamError extends Error {
}
/**
* The last refusal from Mapillary that every tile miss shares: a rejected
* token (until the token changes or the hold ends) or a rate limit (until its
* Retry-After). Cached tiles are still served meanwhile.
* A rejected token or rate limit that answers every tile miss until it expires
* (or the token changes). Cached tiles are still served meanwhile.
* @type {{status: number, until: number, token: string}|null}
*/
let _upstreamHold = null;
@@ -66,7 +62,6 @@ function retryAfterMs(header, now = Date.now()) {
return Math.min(ms, TILE_RATE_LIMIT_MAX_HOLD_MS);
}
/** The refusal to answer a miss with, while one holds for this token. */
function heldRefusal(token) {
const hold = _upstreamHold;
if (!hold) return null;
@@ -81,8 +76,7 @@ function heldRefusal(token) {
}
/**
* Validate and normalize a tile address. Layer names are the proxy's public
* ids (coverage only); zoom must be inside one of the layer's zoom ranges.
* Validate a tile address against the layer's zoom ranges.
* @returns {{layer:string, upstream:string, z:number, x:number, y:number, key:string}}
*/
export function normalizeTileAddress({ layer, z, x, y }) {
@@ -116,13 +110,11 @@ export function normalizeTileAddress({ layer, z, x, y }) {
/** @type {Map<string, {bytes: Buffer, at: number}>} insertion-ordered LRU */
const _memory = new Map();
/** Charged bytes: every entry's size plus a fixed overhead (empty tiles too). */
let _memoryBytes = 0;
const memoryCost = (bytes) => bytes.length + TILE_MEMORY_ENTRY_OVERHEAD_BYTES;
/**
* Upstream fetches in progress, shared by every request for the same tile.
* A flight owns its controller; callers only count as waiters, and the last
* waiter to leave cancels it (see `joinFlight`).
* Upstream fetches shared by every request for the same tile; the last waiter
* to leave cancels it (see `joinFlight`).
* @type {Map<string, {controller: AbortController, promise: Promise<Buffer>, waiters: number}>}
*/
const _inFlight = new Map();
@@ -130,14 +122,13 @@ const _inFlight = new Map();
function memoryGet(key) {
const hit = _memory.get(key);
if (!hit) return null;
// Same 24 h life as the disk copy: a long-running server must not keep
// serving stale (or empty) coverage until eviction.
// Same life as the disk copy, so a long-running server never serves stale
// coverage.
if (Date.now() - hit.at > TILE_DISK_TTL_MS) {
_memory.delete(key);
_memoryBytes -= memoryCost(hit.bytes);
return null;
}
// Re-insert to mark as most recently used.
_memory.delete(key);
_memory.set(key, hit);
return hit.bytes;
@@ -164,7 +155,6 @@ function diskPath({ layer, z, x, y }, root = _diskDir) {
return path.join(root, layer, String(z), `${x}-${y}.pbf`);
}
/** Disk-cache read; the tile's age comes from the file's mtime. */
async function readDisk(address) {
const file = diskPath(address);
try {
@@ -176,14 +166,9 @@ async function readDisk(address) {
}
}
/**
* Background disk work (tile writes and sweeps) not yet settled. Requests
* never wait on it; tests settle it before removing a cache directory.
* @type {Set<Promise<void>>}
*/
/** Unsettled background writes and sweeps; requests never wait on them. */
const _background = new Set();
/** Track a background task until it settles; it never rejects. */
function track(task) {
const settled = task
.catch(() => {})
@@ -192,26 +177,20 @@ function track(task) {
return settled;
}
/**
* Disk writes in progress, by cache file: the sequence number of the newest.
* @type {Map<string, number>}
*/
/** @type {Map<string, number>} cache file to its newest write's sequence */
const _diskWrites = new Map();
let _diskWriteSeq = 0;
/** Whether a write of this tile's cache file is still in progress. */
function diskWritePending(address) {
return _diskWrites.has(diskPath(address));
}
/**
* Write a tile in the background. `at` dates a rewrite of an existing tile
* with its original fetch time, so the rewrite does not extend its life.
* Each write has its own temporary file, and a write that a newer one of the
* same tile overtook steps aside instead of renaming over it.
* Write a tile in the background. `at` keeps a rewrite's original fetch time
* so it does not extend the tile's life. An overtaken write steps aside.
*/
function writeDisk(address, bytes, at = null) {
// The sweep after it covers this write's cache root, even if it has moved.
// Sweep this write's root even if the cache dir has moved since.
const root = _diskDir;
const file = diskPath(address, root);
const seq = ++_diskWriteSeq;
@@ -241,7 +220,6 @@ let _lastSweepAt = 0;
let _sweeping = null;
let _sweepWarned = false;
/** Start a background sweep of `root` unless one ran within the interval. */
function scheduleSweep(root) {
if (_sweeping || Date.now() - _lastSweepAt < TILE_DISK_SWEEP_INTERVAL_MS)
return;
@@ -259,7 +237,7 @@ function scheduleSweep(root) {
_sweeping = tracked;
}
/** Every file under `dir` with its size and mtime; a missing dir is empty. */
/** Every file under `dir` with size and mtime; a missing dir is empty. */
async function listCacheFiles(dir) {
let entries;
try {
@@ -273,7 +251,6 @@ async function listCacheFiles(dir) {
const file = path.join(dir, entry.name);
if (entry.isDirectory()) files.push(...(await listCacheFiles(file)));
else if (entry.isFile()) {
// A file renamed or swept away meanwhile is simply skipped.
const stat = await fsp.stat(file).catch(() => null);
if (stat) files.push({ file, size: stat.size, at: stat.mtimeMs });
}
@@ -282,10 +259,7 @@ async function listCacheFiles(dir) {
}
/**
* Bound the disk cache: delete tiles past TILE_DISK_TTL_MS, then the oldest
* until the rest fit in `maxBytes`. Each file is charged its size plus
* TILE_DISK_FILE_OVERHEAD_BYTES, so empty tiles count toward the cap.
* Runs in the background after tile writes.
* Delete expired tiles, then the oldest until the rest fit in `maxBytes`.
* @returns {Promise<{removed: number, bytes: number}>} files deleted, charged bytes kept
*/
export async function sweepTileDisk({
@@ -308,15 +282,12 @@ export async function sweepTileDisk({
/** The one origin the tile token may be sent to, at every redirect hop. */
const TILE_ORIGIN = new URL(MAPILLARY_TILE_HOST).origin;
/** Redirect hops followed before the upstream counts as broken. */
export const TILE_MAX_REDIRECTS = 3;
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
/**
* Fetch a tile URL, following redirects by hand: each hop must stay HTTPS on
* the Mapillary tile origin (no credentials in the URL), so the access token
* in the query is never sent to another host. A refused or endless redirect
* is an upstream fault (502).
* Follow redirects by hand so every hop stays HTTPS on the tile origin and the
* token in the query never reaches another host. A bad redirect is a 502.
*/
async function fetchPinned(url, init) {
let next = url;
@@ -354,7 +325,6 @@ async function fetchUpstream(address, signal) {
if (held) throw held;
const url = `${MAPILLARY_TILE_HOST}/${address.upstream}/2/${address.z}/${address.x}/${address.y}?access_token=${encodeURIComponent(token)}`;
const timeout = AbortSignal.timeout(TILE_FETCH_TIMEOUT_MS);
// Aborts the request itself once its body runs past the size cap.
const oversize = new AbortController();
const response = await fetchPinned(url, {
signal: AbortSignal.any([signal, timeout, oversize.signal].filter(Boolean)),
@@ -389,9 +359,8 @@ async function fetchUpstream(address, signal) {
}
/**
* Read a tile body, counting as it arrives: a chunked or compressed response
* has no Content-Length to check first, so the request is aborted as soon as
* the running total passes TILE_MAX_BYTES rather than after buffering it all.
* Read a body, aborting once it passes TILE_MAX_BYTES: chunked or compressed
* responses have no Content-Length to check first.
*/
async function readCapped(response, abortRequest) {
if (!response.body) return Buffer.alloc(0);
@@ -412,17 +381,12 @@ async function readCapped(response, abortRequest) {
return Buffer.concat(chunks, total);
}
/**
* Upstream fetch slots: at most TILE_UPSTREAM_CONCURRENCY run at once, and
* further flights wait in order. Replaced whole by the test reset.
* @type {{active: number, queue: Array<() => void>}}
*/
/** @type {{active: number, queue: Array<() => void>}} upstream slots */
let _slots = { active: 0, queue: [] };
/**
* Wait for an upstream fetch slot. Resolves with its release function. A
* flight aborted while it waits leaves the queue and never fetches.
* @param {AbortSignal} signal
* Resolve with a release function once a slot is free. An abort while
* queued leaves the queue.
* @returns {Promise<() => void>}
*/
function acquireUpstreamSlot(signal) {
@@ -451,7 +415,6 @@ function acquireUpstreamSlot(signal) {
});
}
/** Strip the layers this proxy never serves for the address's layer spec. */
function trim(address, bytes) {
if (!address.dropLayers.length || !bytes.length) return bytes;
try {
@@ -463,8 +426,8 @@ function trim(address, bytes) {
}
/**
* Fetch one tile through memory, disk and in-flight coalescing, then
* Mapillary. Returns the raw protobuf bytes (empty for a tile with no data).
* Fetch one tile from memory, disk, a shared flight, then Mapillary. Empty
* bytes mean no coverage.
* @param {{layer:string,z:number|string,x:number|string,y:number|string}} request
* @param {{signal?: AbortSignal}} [options]
* @returns {Promise<{bytes: Buffer, source: 'memory'|'disk'|'inflight'|'upstream', address: object}>}
@@ -475,9 +438,7 @@ export async function fetchTile(request, { signal } = {}) {
if (memory) return { bytes: memory, source: 'memory', address };
const disk = await readDisk(address);
if (disk) {
// Older cache files may still hold the untrimmed tile: trim and replace,
// keeping the original fetch time so the tile still expires on schedule.
// Concurrent hits on one such tile rewrite it once.
// Older cache files may be untrimmed: rewrite once, keeping the fetch time.
const bytes = trim(address, disk.bytes);
if (bytes !== disk.bytes && !diskWritePending(address))
writeDisk(address, bytes, disk.at);
@@ -493,10 +454,7 @@ export async function fetchTile(request, { signal } = {}) {
return { bytes, source: joined ? 'inflight' : 'upstream', address };
}
/**
* Start the one upstream fetch for a tile, owned by the flight itself. It
* waits for an upstream slot first; the slot is held until the body is read.
*/
/** Start the shared upstream fetch; it holds a slot until the body is read. */
function startFlight(address) {
const controller = new AbortController();
const flight = { controller, waiters: 0, promise: null };
@@ -520,12 +478,8 @@ function startFlight(address) {
}
/**
* Wait on a shared flight as one caller. The caller's own abort rejects it at
* once without disturbing the others; the upstream fetch is cancelled only
* when the last waiter leaves.
* @param {{controller: AbortController, promise: Promise<Buffer>, waiters: number}} flight
* @param {AbortSignal} [signal]
* @returns {Promise<Buffer>}
* Wait on a shared flight. A caller's abort rejects only that caller; the
* fetch is cancelled when the last waiter leaves.
*/
function joinFlight(flight, signal) {
signal?.throwIfAborted();
@@ -566,10 +520,7 @@ export function _tileMemoryForTest() {
return { entries: _memory.size, bytes: _memoryBytes };
}
/**
* Test seam: wait until every background tile write and sweep has settled,
* including sweeps those writes start, so a test may remove its cache dir.
*/
/** Test seam: wait for every background write and the sweeps they start. */
export async function _settleTileWritesForTest() {
while (_background.size) await Promise.allSettled([..._background]);
}
+3 -5
View File
@@ -17,10 +17,8 @@ function layerName(layerBytes) {
}
/**
* Drop whole layers from a vector tile at the protobuf level. Nothing inside
* a kept layer is decoded or re-encoded, so a 12 MB tile whose bulk is an
* unused layer trims in a few milliseconds. Returns the input when nothing
* was dropped.
* Drop whole layers from a vector tile without decoding kept layers, so a
* 12 MB tile trims in milliseconds. Returns the input when nothing was dropped.
* @param {Buffer|Uint8Array} bytes
* @param {Iterable<string>} dropNames
*/
@@ -31,7 +29,7 @@ export function stripTileLayers(bytes, dropNames) {
const writer = new PbfWriter();
let dropped = false;
reader.readFields((tag, _result, pbf) => {
if (tag !== TILE_LAYER_FIELD) return; // pbf skips what we leave alone
if (tag !== TILE_LAYER_FIELD) return;
const end = pbf.readVarint() + pbf.pos;
const layerBytes = bytes.subarray(pbf.pos, end);
pbf.pos = end;