From 177b5951a00dc81493d15f24001066834c7caa20 Mon Sep 17 00:00:00 2001 From: manjunath22466 <301209272+manjunath22466@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:32:46 +0530 Subject: [PATCH] fix(outlines): harden shared cache and provider lifecycle --- CHANGELOG.md | 19 +- DATA_SOURCES.md | 4 +- scripts/build-wof-neighborhoods.py | 47 +- scripts/qa-admin-outlines.mjs | 44 +- scripts/qa-openfreemap-same-tile.mjs | 714 +++++++++++++++++++ scripts/wof-open-licenses.json | 43 ++ server/providers/regional/nominatimGate.js | 448 ++++++++++-- server/providers/regional/outlineSelect.js | 85 +++ src/annotations/annotationEngine.js | 13 + src/annotations/outlineRungs.test.mjs | 12 +- src/data/adminBoundaries.js | 48 +- src/data/adminBoundaries.test.mjs | 22 + src/data/placeBoundaries.test.mjs | 68 +- src/layers/traffic/source.test.mjs | 14 +- src/sources/openFreeMap.js | 179 ++++- src/sources/openFreeMap.test.mjs | 27 + src/sources/openFreeMapOutlines.js | 48 +- src/sources/openFreeMapOutlines.test.mjs | 208 +++++- src/sources/phaseTiming.js | 4 +- src/sources/vectorTiles.js | 263 +++++-- src/sources/vectorTiles.test.mjs | 240 ++++++- src/standalone/layerSources.js | 13 +- src/standalone/layerSources.test.mjs | 38 + src/tooling/nominatimGuardLifecycle.test.mjs | 296 +++++++- src/tooling/nominatimLockOwnership.test.mjs | 235 ++++++ src/tooling/nominatimOutlineRoute.test.mjs | 465 ++++++++++-- src/tooling/nominatimSearchRoute.test.mjs | 14 +- 27 files changed, 3363 insertions(+), 248 deletions(-) create mode 100644 scripts/qa-openfreemap-same-tile.mjs create mode 100644 scripts/wof-open-licenses.json create mode 100644 src/standalone/layerSources.test.mjs create mode 100644 src/tooling/nominatimLockOwnership.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d1bea0dc..d0889c01a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -187,8 +187,23 @@ - Voice outlines work again without Overpass. Bundled US Census places and Who's On First neighborhoods (lazy-loaded; `src/data/local_data/`) answer first, then streets, grounds and pointed-at buildings from OpenFreeMap - tiles, then Nominatim for one explicit ask at a time, cached and capped per - install. Anything else stays a point marked "outline unavailable". + tiles, then Nominatim for one explicit ask at a time. Traffic and outlines + share pending fetch/decode ownership and a projection-aware retained-byte + ceiling for the same immutable tile; consumer caches and request lifetimes + remain independent, and eviction or teardown retires queued idle projections. + Public-provider answers are cached and capped per + install. Public Nominatim requests now reserve their daily slot and paced + start atomically across processes sharing that installation's state file, + and fail closed when durable state is unavailable or invalid. Operator-selected + endpoints remain outside the public cap, spacing and state policy, and an + upstream cooldown is isolated to the endpoint that returned it. Anything else + stays a point marked "outline unavailable". Administrative Nominatim + outlines carry the selected record's verified country/admin-1/admin-2 + identity; ambiguous, missing, contradictory or retired evidence is not + promoted into a verified level. +- Who's On First rebuilds accept only exact raw registry licence values in the + reviewed open-licence allowlist. New, unknown, restricted or noncommercial + source licences stay excluded until they are reviewed explicitly. - New settings: `NOMINATIM_URL` (your own Nominatim instead of the public one; empty disables it), `NOMINATIM_DAILY_CAP` (public requests per day, default 50) and `PHOTON_URL` (browser place search; empty disables it). diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index eac027fb9..9a19586ff 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -96,8 +96,8 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Re:Earth Terrain.** Keyless (no API key). Used two ways: (1) `src/mapStackController.js` swaps in a `Cesium.CesiumTerrainProvider` pointed at Re:Earth's `cesium-mesh/ellipsoid` quantized-mesh endpoint for globe stacks without a Cesium ion token (e.g. OSM), replacing a flat `EllipsoidTerrainProvider`; falls back to the flat provider if the endpoint can't be reached. (2) The server-side `/api/terrain/heights` proxy (disk-cached, serve-stale) resolves per-point ellipsoidal ground height for entity placement. Both are best-effort with a keyless-safe fallback (bundled EGM96 geoid math) if Re:Earth is unreachable. - **OSRM on the FOSSGIS routing servers.** Keyless (no API key). Every route is requested by a person: clicking A and B in the Directions layer, or a voice "route from A to B". The server-side `/api/route` proxy sends an identifying `User-Agent`, pins the upstream host, accepts 2-12 coordinates, rejects any leg over 600 km or a total over 2,500 km, caps the buffered response at 8 MB, times the upstream out at 12 s, rate-limits per client, caches each route for 10 minutes, and coalesces identical requests that are in flight at the same time. Turn-by-turn maneuvers are asked for only when a caller wants them (`steps=1`), so the callers that do not read them send the smaller request. The "fix the map" link the policy asks for is in the Data attribution popover beside the OpenStreetMap credit. - **Mapped installations.** OpenFreeMap military polygons use a bundled Overture/OSM name index; wide views show named points. Optional operator-configured Overpass and explicit Google Places searches keep their own source paths. Mapped data does not establish current activity or complete coverage. -- **Place search.** Named-place fly-to uses Google Geocoding when a Maps key is configured, then Photon (`PHOTON_URL`; unset uses the public instance, empty disables it), then `/api/geocode` (Nominatim search) when neither answers. `/api/geocode` and the voice outline route `/api/geocode/outline` share one Nominatim queue (`NOMINATIM_URL`; unset uses the public instance, empty disables it): one request at a time at least 1.1 s apart, an identifying `User-Agent` and `Referer`, a short queue that refuses a burst, a pause after 429/403/5xx answers (honouring `Retry-After` in full), at most one retry after a network failure, and a per-install daily cap on public-instance requests (`NOMINATIM_DAILY_CAP`, default 50). The count and any pause are kept on disk, so restarts and several local processes share them. The public service's limit applies to all installs together; the per-install cap does not bound that total, so heavy or shared deployments should set their own `NOMINATIM_URL`. Answers are cached on disk (outlines 90 days, places 30 days, no-results 24 hours); failures are not cached. Outlines are looked up only for an explicit ask, after bundled data and any configured Overpass, and only results whose type suits the ask are used. It is not used for systematic or bulk queries. -- **Street and building outlines.** Voice street highlights, pointed-at buildings and grounds use the OpenFreeMap vector tiles the traffic layer already loads; no Overpass request is made unless `OVERPASS_UPSTREAMS` is configured. +- **Place search.** Named-place fly-to uses Google Geocoding when a Maps key is configured, then Photon (`PHOTON_URL`; unset uses the public instance, empty disables it), then `/api/geocode` (Nominatim search) when neither answers. `/api/geocode` and the voice outline route `/api/geocode/outline` share one Nominatim queue (`NOMINATIM_URL`; unset uses the public instance, empty disables it): one request at a time at least 1.1 s apart, an identifying `User-Agent` and `Referer`, a short queue that refuses a burst, a pause after 429/403/5xx answers (honouring `Retry-After` in full), at most one retry after a network failure, and a per-install daily cap on public-instance requests (`NOMINATIM_DAILY_CAP`, default 50). Processes using the same installation state file atomically reserve the next public start and daily slot; restarts share that count, spacing and any pause. Public access fails closed if the state file cannot be read, locked, written or semantically validated. This coordination does not span independent installations or state directories, and the public service's total limit still applies across all installs, so heavy or shared deployments should set their own `NOMINATIM_URL`. An operator-selected endpoint is explicitly outside the public cap, spacing and durable-state requirement; an upstream cooldown is kept with the endpoint that returned it rather than leaking across operator URLs. Answers are cached on disk (outlines 90 days, places 30 days, no-results 24 hours); failures are not cached. Outlines are looked up only for an explicit ask, after bundled data and any configured Overpass, and only results whose type suits the ask are used. Administrative outline responses expose country/admin-1/admin-2 identity only when the selected Nominatim row has a recognized, unambiguous address type consistent with its own name; numeric rank, ambiguous types, missing fields and contradictory names do not manufacture identity. It is not used for systematic or bulk queries. +- **Street and building outlines.** Voice street highlights, pointed-at buildings and grounds use the OpenFreeMap vector tiles the traffic layer already loads. Traffic and outlines share one bounded pending fetch/decode owner per immutable XYZ tile; materialized outline geometry is charged to the retained-byte ceiling and capped by coordinate count. Consumer result caches and request lifetimes remain independent, while eviction or explicit teardown cancels queued idle projections and releases their decoded owner. No Overpass request is made unless `OVERPASS_UPSTREAMS` is configured. - **Cockpit regional briefing.** `/api/regional-brief` rounds aircraft coordinates into 0.1° cache cells, caches results for five minutes, and resolves physical regions from bundled public-domain Natural Earth polygons without reverse-geocoding requests. Google News RSS is queried with the resolved locality/region first; GDELT is used only when that RSS query fails or is empty. Google's published Google News terms restrict that source to personal, noncommercial use, so commercial deployments must disable/replace it or obtain separate permission; GDELT permits commercial dataset use with citation. The Data attribution popover identifies the active headline sources; article links retain publisher attribution. Headlines are location-query matches, not verified incidents, risk rankings, or evidence that a location is safe. Empty, partial, stale, and unavailable source states remain distinct. Open-Meteo supplies current conditions independently of the news source. `WX OFF` disables cockpit weather rendering only; the Local Info briefing still fetches its source-backed weather values and displays the required linked Open-Meteo credit. - **Dynamic weather presentation.** While cockpit mode is active, `/api/weather-effects` requests current Open-Meteo observations for the aircraft/camera location, rounds coordinates into 0.1° cache cells, caches results for five minutes, and may retain a stale observation for up to 30 minutes during a transient outage. WMO condition code selects the visual family; observed cloud cover, precipitation, visibility, wind speed, and wind direction bound its strength and motion. Missing or expired weather renders no synthetic atmospheric effect, and normal globe view never renders the weather overlay. diff --git a/scripts/build-wof-neighborhoods.py b/scripts/build-wof-neighborhoods.py index 4cd92a675..74454c2f5 100644 --- a/scripts/build-wof-neighborhoods.py +++ b/scripts/build-wof-neighborhoods.py @@ -3,7 +3,8 @@ Reads the pinned Who's On First neighbourhood, macrohood and microhood archives and the WOF sources registry, keeps current polygon records whose -geometry source has a stated licence, simplifies them to about 15 m, and +geometry source matches the reviewed open-licence allowlist, simplifies them +to about 15 m, and writes src/data/local_data/wof_neighborhoods/: index.json format, licences per source, tile list with boxes @@ -25,7 +26,6 @@ import collections import hashlib import json import math -import re import subprocess import tarfile from pathlib import Path @@ -65,8 +65,9 @@ TOLERANCE_M = 15 PRECISION = 5 TILE_MAX_BYTES = 200_000 TILE_MAX_DEPTH = 16 -# A geometry source is kept only when the registry states its licence. -UNSTATED = re.compile(r'assumed|unknown|unresolved|n/a', re.I) +LICENSE_POLICY = json.loads((ROOT / 'scripts/wof-open-licenses.json').read_text()) +ALLOWED_LICENSES = frozenset(LICENSE_POLICY['licenses']) +REVIEWED_SOURCES = LICENSE_POLICY.get('reviewedSources', {}) FIELDS = ['id', 'name', 'country', 'type', 'bbox', 'aliases', 'source', 'rings', 'label'] @@ -149,9 +150,34 @@ def source_table(spec): return lookup -def stated(key, source): - licence = (source or {}).get('license_type') or '' - return key != 'unknown' and bool(source) and bool(licence) and not UNSTATED.search(licence) +def verify_source_reviews(lookup): + """Pin exceptional source decisions to the exact reviewed registry evidence.""" + registry_commit = SOURCES['url'].split('/')[-3] + for key, review in REVIEWED_SOURCES.items(): + source = lookup.get(key) + if not source: + raise ValueError(f'{key}: reviewed source is absent from the pinned registry') + if review.get('registryCommit') != registry_commit: + raise ValueError(f'{key}: review does not match the pinned registry commit') + if review.get('registrySha256') != SOURCES['sha256']: + raise ValueError(f'{key}: review does not match the pinned registry SHA-256') + if review.get('rawLicenseType') != source.get('license_type'): + raise ValueError(f'{key}: reviewed raw licence no longer matches the registry') + text = source.get('license_text') or '' + if review.get('licenseTextSha256') != hashlib.sha256(text.encode()).hexdigest(): + raise ValueError(f'{key}: reviewed licence text no longer matches the registry') + if review['rawLicenseType'] not in ALLOWED_LICENSES: + raise ValueError(f'{key}: reviewed raw licence is not admitted by policy') + + +def licence_allowed(value): + """Only exact raw registry strings reviewed in the policy are accepted.""" + return isinstance(value, str) and value in ALLOWED_LICENSES + + +def source_allowed(key, source): + return (key != 'unknown' and bool(source) and + licence_allowed(source.get('license_type'))) def build_records(archives, lookup): @@ -166,7 +192,7 @@ def build_records(archives, lookup): if not keep(properties): continue key = properties.get('src:geom', 'unknown') - if not stated(key, lookup.get(key)): + if not source_allowed(key, lookup.get(key)): excluded[key] += 1 continue name = properties['wof:name'] @@ -278,6 +304,7 @@ def main(): } spec = json.loads(fetch(args.cache, 'sources-spec-latest.json', SOURCES['url'], SOURCES['sha256']).read_text()) lookup = source_table(spec) + verify_source_reviews(lookup) features, excluded = build_records(archives, lookup) if len({f[0] for f in features}) != len(features): raise ValueError('duplicate WOF id') @@ -336,7 +363,7 @@ def main(): differ = sorted( name for name in shipped | set(out) if not (PACK / name).exists() or name not in out - or (PACK / name).read_text(encoding='utf-8') != out[name] + or (PACK / name).read_bytes() != out[name].encode('utf-8') ) if differ: raise SystemExit(f'rebuild differs from the shipped pack: {differ[:10]}') @@ -348,7 +375,7 @@ def main(): for name, text in out.items(): (PACK / name).write_text(text, encoding='utf-8') print(f'wof_neighborhoods: {len(features)} records in {len(tile_list)} tiles; ' - f'excluded {sum(excluded.values())} with an unstated licence: {dict(excluded)}', flush=True) + f'excluded {sum(excluded.values())} with an unapproved licence: {dict(excluded)}', flush=True) if __name__ == '__main__': diff --git a/scripts/qa-admin-outlines.mjs b/scripts/qa-admin-outlines.mjs index 114daacad..733172b70 100644 --- a/scripts/qa-admin-outlines.mjs +++ b/scripts/qa-admin-outlines.mjs @@ -3,7 +3,7 @@ * qa-admin-outlines.mjs — bundled state/province/county outlines, driven like a * user would drive them. * - * Run: node scripts/qa-admin-outlines.mjs http://localhost:4173 [--headful] [--no-video] + * Run: node scripts/qa-admin-outlines.mjs http://localhost:4173 [--headful] [--no-video] [--offline-imagery] * * Journey: frame Texas at state scale and outline "Texas"; pan and zoom toward * Austin in smooth 2–3 s moves; outline "Travis County, Texas"; pan north and @@ -33,7 +33,7 @@ import sharp from 'sharp'; const args = process.argv.slice(2); if (args.includes('--help')) { console.log( - 'Usage: node scripts/qa-admin-outlines.mjs [--headful] [--no-video]', + 'Usage: node scripts/qa-admin-outlines.mjs [--headful] [--no-video] [--offline-imagery]', ); process.exit(0); } @@ -41,6 +41,15 @@ const url = args.find((arg) => !arg.startsWith('--')); if (!url || !['http:', 'https:'].includes(new URL(url).protocol)) throw new Error('Supply a running dev server URL; see --help'); const recordVideo = !args.includes('--no-video'); +const offlineImagery = args.includes('--offline-imagery'); +const offlineImageryTile = offlineImagery + ? await fs.readFile( + new URL( + '../node_modules/cesium/Build/Cesium/Assets/Textures/NaturalEarthII/0/0/0.jpg', + import.meta.url, + ), + ) + : null; const shots = path.resolve('qa-shots'); const framesDir = path.join(shots, 'admin-outlines-frames'); await fs.mkdir(shots, { recursive: true }); @@ -61,6 +70,8 @@ const result = { pageErrors: [], screenshots: [], video: null, + offlineImagery, + offlineImageryRequests: 0, }; const browser = await puppeteer.launch({ @@ -87,6 +98,35 @@ try { sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed'); }); await page.setViewport({ width: 1280, height: 800 }); + await page.setRequestInterception(true); + page.on('request', (request) => { + let target; + try { + target = new URL(request.url()); + } catch { + request.continue(); + return; + } + const knownImageryTile = + request.method() === 'GET' && + ((target.origin === 'https://services.arcgisonline.com' && + /^\/ArcGIS\/rest\/services\/World_Imagery\/MapServer\/tile\/\d+\/\d+\/\d+$/.test( + target.pathname, + )) || + (target.origin === 'https://tile.openstreetmap.org' && + /^\/\d+\/\d+\/\d+\.png$/.test(target.pathname))); + if (offlineImagery && knownImageryTile) { + result.offlineImageryRequests += 1; + request.respond({ + status: 200, + contentType: 'image/jpeg', + headers: { 'Access-Control-Allow-Origin': '*' }, + body: offlineImageryTile, + }); + return; + } + request.continue(); + }); const client = await page.createCDPSession(); await client.send('Network.enable'); client.on('Network.requestWillBeSent', ({ request }) => { diff --git a/scripts/qa-openfreemap-same-tile.mjs b/scripts/qa-openfreemap-same-tile.mjs new file mode 100644 index 000000000..16a420b5b --- /dev/null +++ b/scripts/qa-openfreemap-same-tile.mjs @@ -0,0 +1,714 @@ +#!/usr/bin/env node +/** + * Traffic-warm OpenFreeMap outline acceptance. + * + * Each sample opens a fresh application page, loads the Texas Capitol z14 tile + * through Street Traffic, then circles a building from that same tile. It + * records application fetches separately from browser transfer bytes, + * body-read/decode/extraction/install measures, and Long Tasks. A second ask + * on the page proves the outline source's own consumer-cache path. A separate + * page holds traffic's target response while the outline consumer subscribes, + * proving that overlap still produces one application fetch and one decode. + * + * Acceptance is declared before measurement: cold outline decode, extraction, + * and install must fit one 60 Hz frame at p95 (16.7 ms), no sample may create a + * >=50 ms Long Task, and neither the first nor second outline ask may fetch or + * decode the traffic-warm tile again. The target tile's traffic-idle outline + * projection must also fit one frame; unrelated traffic activation Long Tasks + * are reported separately as baseline evidence. + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; + +const args = process.argv.slice(2); +const valueAfter = (flag, fallback) => { + const index = args.indexOf(flag); + return index >= 0 && args[index + 1] ? args[index + 1] : fallback; +}; +if (args.includes('--help') || args.includes('-h')) { + console.log( + 'Usage: node scripts/qa-openfreemap-same-tile.mjs [--url http://localhost:4173] [--runs 5] [--headful] [--out qa-shots/openfreemap-same-tile.json]', + ); + process.exit(0); +} + +const APP_URL = valueAfter('--url', 'http://localhost:4173'); +const RUNS = Number(valueAfter('--runs', '5')); +const OUTPUT = path.resolve( + valueAfter('--out', 'qa-shots/openfreemap-same-tile.json'), +); +const FRAME_BUDGET_MS = 16.7; +const LONG_TASK_MS = 50; +const TARGET = Object.freeze({ + latitude: 30.27472, + longitude: -97.74035, + z: 14, + x: 3743, + y: 6745, +}); +const TARGET_PATH = `/${TARGET.z}/${TARGET.x}/${TARGET.y}.pbf`; + +if (!Number.isInteger(RUNS) || RUNS < 3 || RUNS > 20) + throw new Error('--runs must be an integer from 3 through 20'); +const targetUrl = new URL(APP_URL); +if (!['http:', 'https:'].includes(targetUrl.protocol)) + throw new Error('--url must be HTTP(S)'); + +const percentile = (values, p) => { + if (!values.length) return null; + const sorted = values.slice().sort((a, b) => a - b); + return sorted[Math.ceil(p * sorted.length) - 1]; +}; +const round = (value) => + value == null ? null : Number(Number(value).toFixed(3)); +const measures = (entries, name) => + entries.filter((entry) => entry.name === name).map((entry) => entry.ms); +const sum = (values) => values.reduce((total, value) => total + value, 0); +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +await fs.mkdir(path.dirname(OUTPUT), { recursive: true }); + +const browser = await puppeteer.launch({ + headless: args.includes('--headful') ? false : 'new', + protocolTimeout: 240_000, + ...(process.env.PUPPETEER_EXECUTABLE_PATH + ? { executablePath: process.env.PUPPETEER_EXECUTABLE_PATH } + : {}), + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--use-gl=angle', + ...(args.includes('--headful') + ? [] + : ['--use-angle=swiftshader', '--enable-unsafe-swiftshader']), + '--disable-dev-shm-usage', + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + '--window-size=1440,1000', + ], +}); + +const report = { + appUrl: APP_URL, + target: TARGET, + targetPath: TARGET_PATH, + acceptance: { + coldCpuP95Ms: FRAME_BUDGET_MS, + maxLongTaskMs: LONG_TASK_MS, + trafficIdleProjectionMaxMs: FRAME_BUDGET_MS, + coldApplicationFetches: 0, + coldOutlineTransferBytes: 0, + coldDecodes: 0, + warmApplicationFetches: 0, + warmDecodes: 0, + overlapApplicationFetches: 1, + overlapDecodes: 1, + }, + renderer: null, + samples: [], + errors: [], +}; + +function relevantMeasures(raw) { + return raw + .filter( + (entry) => + entry.name.startsWith('roads:') || entry.name.startsWith('outlines:'), + ) + .map((entry) => ({ + name: entry.name, + ms: round(entry.duration), + at: round(entry.startTime), + detail: entry.detail || {}, + })); +} + +async function beginOutline(page, label) { + await page.evaluate( + ({ label, target }) => { + performance.clearMeasures(); + window.__qaOutlineLongTasks.length = 0; + window.__qaOutlineStarted = performance.now(); + window.__qaOutlinePromise = Promise.resolve( + window.__gevVoiceCommands.runner('annotate_map', { + annotations: [ + { + type: 'area', + target: 'this building', + label, + entityKind: 'building', + latitude: target.latitude, + longitude: target.longitude, + }, + ], + }), + ); + }, + { label, target: TARGET }, + ); +} + +async function settleOutline(page, label) { + return page.evaluate(async (label) => { + const result = await window.__qaOutlinePromise; + let annotation = null; + const deadline = performance.now() + 30_000; + while (performance.now() < deadline) { + annotation = (window.__gevAnnotations?.list?.() || []).find( + (item) => item.label === label, + ); + if (annotation && !annotation.pendingOutline) break; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + if (!annotation || annotation.pendingOutline) + throw new Error(`${label}: outline did not settle`); + const { scene } = window.__godsEyeView.viewer; + await new Promise((resolve) => { + const timer = setTimeout(resolve, 1500); + const remove = scene.postRender.addEventListener(() => { + clearTimeout(timer); + remove(); + resolve(); + }); + scene.requestRender(); + }); + await new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(resolve)), + ); + return { + commandToRenderMs: performance.now() - window.__qaOutlineStarted, + result, + annotation: { + pending: Boolean(annotation.pendingOutline), + unavailable: Boolean(annotation.outlineUnavailable), + synthesized: Boolean(annotation.synthesized), + footprintKind: annotation.footprintKind || null, + ringPoints: annotation.ring?.length || 0, + parts: annotation.polygons?.length || (annotation.ring ? 1 : 0), + }, + measures: performance.getEntriesByType('measure').map((entry) => ({ + name: entry.name, + duration: entry.duration, + startTime: entry.startTime, + detail: entry.detail, + })), + longTasks: window.__qaOutlineLongTasks.slice(), + }; + }, label); +} + +async function driveOutline(page, label) { + await beginOutline(page, label); + return settleOutline(page, label); +} + +try { + for (let run = 0; run <= RUNS; run += 1) { + const page = await browser.newPage(); + await page.setViewport({ width: 1440, height: 1000 }); + let overlapMode = false; + let overlapApplicationRequests = 0; + let heldTargetRequest = null; + let resolveHeldTarget; + const heldTarget = new Promise((resolve) => { + resolveHeldTarget = resolve; + }); + if (run === 0) { + await page.setRequestInterception(true); + page.on('request', (request) => { + let isTarget = false; + try { + isTarget = new URL(request.url()).pathname.endsWith(TARGET_PATH); + } catch { + // Invalid URLs are passed through for the browser to handle. + } + if (overlapMode && isTarget) { + overlapApplicationRequests += 1; + if (!heldTargetRequest) { + heldTargetRequest = request; + resolveHeldTarget(); + return; + } + } + void request.continue(); + }); + } + await page.evaluateOnNewDocument(() => { + sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed'); + window.__qaOutlineLongTasks = []; + try { + new PerformanceObserver((list) => { + for (const entry of list.getEntries()) + window.__qaOutlineLongTasks.push({ + at: entry.startTime, + ms: entry.duration, + }); + }).observe({ type: 'longtask', buffered: false }); + window.__qaLongTaskSupported = true; + } catch { + window.__qaLongTaskSupported = false; + } + }); + + let phase = 'startup'; + const network = []; + const pending = new Map(); + const client = await page.createCDPSession(); + await client.send('Network.enable'); + client.on('Network.requestWillBeSent', (event) => { + let url; + try { + url = new URL(event.request.url); + } catch { + return; + } + if ( + url.hostname !== 'tiles.openfreemap.org' || + !url.pathname.endsWith('.pbf') + ) + return; + const entry = { + requestId: event.requestId, + phase, + url: url.origin + url.pathname, + targetTile: url.pathname.endsWith(TARGET_PATH), + cached: false, + transferBytes: 0, + responseBytes: 0, + finished: false, + }; + pending.set(event.requestId, entry); + network.push(entry); + }); + client.on('Network.requestServedFromCache', ({ requestId }) => { + const entry = pending.get(requestId); + if (entry) entry.cached = true; + }); + client.on('Network.responseReceived', ({ requestId, response }) => { + const entry = pending.get(requestId); + if (!entry) return; + entry.status = response.status; + entry.cached ||= Boolean( + response.fromDiskCache || + response.fromServiceWorker || + response.fromPrefetchCache, + ); + }); + client.on('Network.dataReceived', ({ requestId, dataLength }) => { + const entry = pending.get(requestId); + if (entry) entry.responseBytes += dataLength; + }); + client.on('Network.loadingFinished', ({ requestId, encodedDataLength }) => { + const entry = pending.get(requestId); + if (!entry) return; + entry.transferBytes = entry.cached ? 0 : encodedDataLength; + entry.finished = true; + pending.delete(requestId); + }); + client.on('Network.loadingFailed', ({ requestId, errorText }) => { + const entry = pending.get(requestId); + if (!entry) return; + entry.failure = errorText; + entry.finished = true; + pending.delete(requestId); + }); + + const pageErrors = []; + page.on('pageerror', (error) => pageErrors.push(String(error.message))); + const navigation = new URL(APP_URL); + navigation.searchParams.set('welcome', '0'); + navigation.searchParams.set('trafficDebug', '1'); + navigation.searchParams.set('trafficRoads', 'osm'); + await page.goto(navigation.href, { + waitUntil: 'domcontentloaded', + timeout: 90_000, + }); + await page.waitForFunction( + () => + window.__godsEyeView?.viewer && + window.__godsEyeView?.dataManager && + window.__gevVoiceCommands?.runner && + window.__gevAnnotations && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 120_000, polling: 250 }, + ); + await page.evaluate( + () => window.__godsEyeView.styleManager.initialRestorePromise, + ); + await page.evaluate(() => + window.__godsEyeView.styleManager.setDetection({ enabled: false }), + ); + await page.evaluate(async (target) => { + const { viewer, dataManager } = window.__godsEyeView; + await dataManager.setEnabled('traffic', false); + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + target.longitude, + target.latitude, + 700, + ), + orientation: { + heading: 0, + pitch: (-65 * Math.PI) / 180, + roll: 0, + }, + }); + viewer.camera.moveEnd.raiseEvent(); + window.__gevAnnotations.clear(); + performance.clearMeasures(); + }, TARGET); + await sleep(750); + + if (run === 0) { + phase = 'overlap'; + overlapMode = true; + await page.evaluate(() => { + window.__qaOutlineLongTasks.length = 0; + performance.clearMeasures(); + }); + await page.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('traffic', true), + ); + await Promise.race([ + heldTarget, + sleep(30_000).then(() => { + throw new Error('overlap phase did not intercept the target tile'); + }), + ]); + await beginOutline(page, 'QA Capitol overlap'); + await sleep(100); + await heldTargetRequest.continue(); + const overlap = await settleOutline(page, 'QA Capitol overlap'); + await sleep(750); + const trafficStats = await page.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('traffic') + .module.getStats(), + ); + assert.equal(trafficStats.error, null, 'overlap traffic source error'); + overlap.measures = relevantMeasures(overlap.measures); + overlap.network = network.filter( + (entry) => entry.phase === 'overlap' && entry.targetTile, + ); + const overlapDecodes = overlap.measures + .filter( + (entry) => + entry.name === 'roads:decode' && + String(entry.detail?.key).endsWith( + `:${TARGET.z}/${TARGET.x}/${TARGET.y}`, + ), + ) + .map((entry) => entry.ms); + assert.equal( + overlapApplicationRequests, + 1, + 'overlapping traffic and outline consumers issue one application request', + ); + assert.equal( + overlap.network.length, + 1, + 'overlapping traffic and outline consumers emit one target network request', + ); + assert.equal( + overlapDecodes.length, + 1, + 'overlapping traffic and outline consumers decode once', + ); + assert.equal(overlap.annotation.unavailable, false); + assert.equal(overlap.annotation.footprintKind, 'building'); + report.overlap = { + applicationFetches: overlapApplicationRequests, + targetNetworkRequests: overlap.network.length, + transferBytes: sum( + overlap.network.map((entry) => entry.transferBytes), + ), + decodes: overlapDecodes.length, + decodeMs: round(sum(overlapDecodes)), + longTasks: overlap.longTasks.length, + }; + await page.close(); + continue; + } + + phase = 'traffic'; + await page.evaluate(() => { + window.__qaOutlineLongTasks.length = 0; + performance.clearMeasures(); + }); + await page.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('traffic', true), + ); + await page.waitForFunction( + () => { + const stats = window.__godsEyeView.dataManager.layers + .get('traffic') + .module.getStats(); + return stats.count > 0 && !stats.loading && !stats.error; + }, + { timeout: 180_000, polling: 250 }, + ); + await sleep(1500); + const trafficMeasures = relevantMeasures( + await page.evaluate(() => + performance.getEntriesByType('measure').map((entry) => ({ + name: entry.name, + duration: entry.duration, + startTime: entry.startTime, + detail: entry.detail, + })), + ), + ); + const trafficLongTasks = await page.evaluate(() => + window.__qaOutlineLongTasks.slice(), + ); + const trafficTarget = network.filter( + (entry) => entry.phase === 'traffic' && entry.targetTile, + ); + const targetTile = `${TARGET.z}/${TARGET.x}/${TARGET.y}`; + const targetProjectionMeasures = trafficMeasures.filter( + (entry) => + entry.detail?.tile === targetTile && + ['outlines:street-extract', 'outlines:polygon-extract'].includes( + entry.name, + ), + ); + const targetProjectionMs = sum( + targetProjectionMeasures.map((entry) => entry.ms), + ); + assert.ok( + trafficTarget.length >= 1, + `sample ${run}: traffic did not load ${TARGET_PATH}`, + ); + assert.equal( + targetProjectionMeasures.length, + 2, + `sample ${run}: traffic did not prepare the target outline projection`, + ); + assert.ok( + targetProjectionMs <= FRAME_BUDGET_MS, + `sample ${run}: traffic-idle target projection ${targetProjectionMs} ms exceeds ${FRAME_BUDGET_MS} ms`, + ); + + phase = 'outline-cold'; + const cold = await driveOutline(page, `QA Capitol cold ${run}`); + await sleep(750); + cold.measures = relevantMeasures(cold.measures); + cold.network = network.filter( + (entry) => entry.phase === 'outline-cold' && entry.targetTile, + ); + cold.longTasks = cold.longTasks.map((entry) => ({ + at: round(entry.at), + ms: round(entry.ms), + })); + + await page.evaluate(() => window.__gevAnnotations.clear()); + phase = 'outline-warm'; + const warm = await driveOutline(page, `QA Capitol warm ${run}`); + await sleep(750); + warm.measures = relevantMeasures(warm.measures); + warm.network = network.filter( + (entry) => entry.phase === 'outline-warm' && entry.targetTile, + ); + warm.longTasks = warm.longTasks.map((entry) => ({ + at: round(entry.at), + ms: round(entry.ms), + })); + + const supported = await page.evaluate(() => window.__qaLongTaskSupported); + assert.equal(supported, true, `sample ${run}: Long Tasks unsupported`); + assert.deepEqual(pageErrors, [], `sample ${run}: browser errors`); + assert.equal(cold.annotation.unavailable, false); + assert.equal(cold.annotation.synthesized, false); + assert.equal(cold.annotation.footprintKind, 'building'); + assert.ok(cold.annotation.ringPoints >= 4); + assert.equal(warm.annotation.footprintKind, 'building'); + assert.ok(warm.annotation.ringPoints >= 4); + + const coldDecodes = measures(cold.measures, 'roads:decode'); + const coldInstalls = measures(cold.measures, 'outlines:install'); + const warmDecodes = measures(warm.measures, 'roads:decode'); + assert.equal( + cold.network.length, + 0, + 'shared decoded tile performs no second application fetch', + ); + assert.equal(sum(cold.network.map((entry) => entry.transferBytes)), 0); + assert.equal(coldDecodes.length, 0, 'shared decoded tile is not parsed again'); + assert.equal( + measures(cold.measures, 'outlines:street-extract').length, + 0, + 'traffic idle preparation owns street extraction', + ); + assert.equal( + measures(cold.measures, 'outlines:polygon-extract').length, + 0, + 'traffic idle preparation owns polygon extraction', + ); + assert.ok(coldInstalls.length >= 1, 'cold outline installs geometry'); + assert.equal(warm.network.length, 0, 'decoded-cache hit performs no fetch'); + assert.equal(warmDecodes.length, 0, 'decoded-cache hit performs no decode'); + + report.renderer ||= await page.evaluate(() => { + const gl = window.__godsEyeView.viewer.scene.context._gl; + const debug = gl.getExtension('WEBGL_debug_renderer_info'); + return debug + ? gl.getParameter(debug.UNMASKED_RENDERER_WEBGL) + : gl.getParameter(gl.RENDERER); + }); + if (args.includes('--headful')) { + assert.doesNotMatch( + String(report.renderer), + /swiftshader|software rasterizer|llvmpipe/i, + `sample ${run}: headful evidence requires a hardware renderer`, + ); + } + report.samples.push({ + run, + pageErrors, + traffic: { + applicationFetches: trafficTarget.length, + transferBytes: sum(trafficTarget.map((entry) => entry.transferBytes)), + decodes: measures(trafficMeasures, 'roads:decode').length, + targetProjectionMs: round(targetProjectionMs), + longTasks: trafficLongTasks.map((entry) => ({ + at: round(entry.at), + ms: round(entry.ms), + })), + measures: trafficMeasures, + network: trafficTarget, + }, + cold: { + ...cold, + applicationFetches: cold.network.length, + transferBytes: sum(cold.network.map((entry) => entry.transferBytes)), + decodeMs: round(sum(coldDecodes)), + bodyReadMs: round(sum(measures(cold.measures, 'roads:tile-body-read'))), + streetExtractMs: round( + sum(measures(cold.measures, 'outlines:street-extract')), + ), + polygonExtractMs: round( + sum(measures(cold.measures, 'outlines:polygon-extract')), + ), + installMs: round(sum(coldInstalls)), + mainThreadCpuMs: round( + sum(coldDecodes) + + sum(measures(cold.measures, 'outlines:street-extract')) + + sum(measures(cold.measures, 'outlines:polygon-extract')) + + sum(coldInstalls), + ), + }, + warm: { + ...warm, + applicationFetches: warm.network.length, + transferBytes: sum(warm.network.map((entry) => entry.transferBytes)), + decodeMs: round(sum(warmDecodes)), + installMs: round(sum(measures(warm.measures, 'outlines:install'))), + }, + }); + await page.close(); + } + + const coldCpu = report.samples.map((sample) => sample.cold.mainThreadCpuMs); + const outlineLongTasks = report.samples.flatMap((sample) => [ + ...sample.cold.longTasks, + ...sample.warm.longTasks, + ]); + const trafficLongTasks = report.samples.flatMap( + (sample) => sample.traffic.longTasks, + ); + report.summary = { + samples: report.samples.length, + overlapApplicationFetches: report.overlap?.applicationFetches ?? null, + overlapDecodes: report.overlap?.decodes ?? null, + coldCpuP50Ms: round(percentile(coldCpu, 0.5)), + coldCpuP95Ms: round(percentile(coldCpu, 0.95)), + coldCpuMaxMs: round(Math.max(...coldCpu)), + coldCommandToRenderP95Ms: round( + percentile( + report.samples.map((sample) => sample.cold.commandToRenderMs), + 0.95, + ), + ), + coldApplicationFetches: sum( + report.samples.map((sample) => sample.cold.applicationFetches), + ), + coldTransferBytes: sum( + report.samples.map((sample) => sample.cold.transferBytes), + ), + coldDecodes: sum( + report.samples.map( + (sample) => measures(sample.cold.measures, 'roads:decode').length, + ), + ), + warmApplicationFetches: sum( + report.samples.map((sample) => sample.warm.applicationFetches), + ), + warmDecodes: sum( + report.samples.map( + (sample) => measures(sample.warm.measures, 'roads:decode').length, + ), + ), + trafficIdleProjectionMaxMs: round( + Math.max( + ...report.samples.map((sample) => sample.traffic.targetProjectionMs), + ), + ), + trafficBaselineLongTasks: trafficLongTasks.length, + trafficBaselineMaxLongTaskMs: round( + trafficLongTasks.length + ? Math.max(...trafficLongTasks.map((entry) => entry.ms)) + : 0, + ), + longTasks: outlineLongTasks.length, + maxLongTaskMs: round( + outlineLongTasks.length + ? Math.max(...outlineLongTasks.map((entry) => entry.ms)) + : 0, + ), + }; + assert.ok( + report.summary.coldCpuP95Ms <= FRAME_BUDGET_MS, + `cold decode+extract+install p95 ${report.summary.coldCpuP95Ms} ms exceeds ${FRAME_BUDGET_MS} ms`, + ); + assert.ok( + report.summary.trafficIdleProjectionMaxMs <= FRAME_BUDGET_MS, + 'traffic-idle outline projection exceeded one frame', + ); + assert.equal(report.summary.longTasks, 0, 'outline ask created a Long Task'); + assert.equal(report.summary.coldTransferBytes, 0); + assert.equal(report.summary.coldApplicationFetches, 0); + assert.equal(report.summary.coldDecodes, 0); + assert.equal(report.summary.warmApplicationFetches, 0); + assert.equal(report.summary.warmDecodes, 0); + assert.equal(report.summary.overlapApplicationFetches, 1); + assert.equal(report.summary.overlapDecodes, 1); + report.passed = true; + console.table( + report.samples.map((sample) => ({ + run: sample.run, + trafficPrepMs: sample.traffic.targetProjectionMs, + appFetches: sample.cold.applicationFetches, + transferBytes: sample.cold.transferBytes, + bodyReadMs: sample.cold.bodyReadMs, + decodeMs: sample.cold.decodeMs, + streetMs: sample.cold.streetExtractMs, + polygonMs: sample.cold.polygonExtractMs, + installMs: sample.cold.installMs, + cpuMs: sample.cold.mainThreadCpuMs, + renderMs: round(sample.cold.commandToRenderMs), + longTasks: sample.cold.longTasks.length, + })), + ); + console.log('OpenFreeMap same-tile acceptance passed:', report.summary); +} catch (error) { + report.passed = false; + report.failure = String(error?.stack || error); + console.error(error); + process.exitCode = 1; +} finally { + await fs.writeFile(OUTPUT, `${JSON.stringify(report, null, 2)}\n`); + await browser.close(); +} diff --git a/scripts/wof-open-licenses.json b/scripts/wof-open-licenses.json new file mode 100644 index 000000000..8ea94ae74 --- /dev/null +++ b/scripts/wof-open-licenses.json @@ -0,0 +1,43 @@ +{ + "reviewedSources": { + "zs": { + "name": "Zetashapes", + "registryCommit": "e17af153c144fe6e08aa8db2607b4bf8aecd448d", + "registrySha256": "d8a5dd873d26eee11583de4438185c046dec33848d5bde27047ca4bff44d9c39", + "rawLicenseType": "Public domain", + "licenseTextSha256": "91757f96f74f6dfb4a7206486b78586da0d039e9611e224fc793c7167dcfe615", + "decision": "Approved from the pinned WOF registry text: no added restrictions beyond public-domain US TIGER/Line data and Flickr-derived data; retain both upstream credits." + } + }, + "licenses": [ + "Attribution 3.0 Unported", + "CC BY", + "CC BY (equivalent)", + "CC BY 2.0", + "CC BY 3.0", + "CC BY 3.0 US", + "CC BY 4.0", + "CC-BY", + "CC0", + "CC0 (equivalent)", + "CC0 1.0 Universal", + "ODC-By, v1.0", + "ODC-By v1.0", + "OGL - Canada", + "OGL - City of New Westminster", + "OGL - City of Surrey", + "Open Government Licence v3.0", + "Open Government Licence - British Columbia, v2.0", + "Open Government Licence - Canada", + "Open Government Licence - City of Calgary", + "Open Government Licence - City of Regina", + "Open Government Licence - Edmonton", + "Open Government Licence - Ontario, v1.0", + "Open Government License - British Columbia, v2.0", + "Open Government License - Winnipeg", + "PDDL 1.0", + "Public domain", + "Public Domain", + "Singapore Open Data Licence v1" + ] +} diff --git a/server/providers/regional/nominatimGate.js b/server/providers/regional/nominatimGate.js index 6bde36d2d..288e663a0 100644 --- a/server/providers/regional/nominatimGate.js +++ b/server/providers/regional/nominatimGate.js @@ -1,13 +1,16 @@ import fs from 'node:fs'; import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import { readResponseJsonCapped } from '../common/http.js'; /** * One gate for every request this server sends to a Nominatim instance. * - * Place search and voice outlines share it, so the public instance sees one - * application: one request at a time, at least 1.1 s apart, a short queue, a - * daily ceiling per install, and a pause after the server says to stop. + * Place search and voice outlines share it. Public requests reserve one start + * at a time across processes sharing this installation's state file, remain at + * least 1.1 s apart, use a short queue and daily ceiling, and pause after the + * server says to stop. Independent installations do not share one app-wide + * limiter; each must identify itself and enforce the public policy locally. */ /** The public instance. Only this host is subject to the public-use limits. */ @@ -108,13 +111,110 @@ function gateError(code, message, extra = {}) { return Object.assign(new Error(message), { code, ...extra }); } +const HTTP_MONTHS = Object.freeze([ + 'Jan', + 'Feb', + 'Mar', + 'Apr', + 'May', + 'Jun', + 'Jul', + 'Aug', + 'Sep', + 'Oct', + 'Nov', + 'Dec', +]); +const HTTP_WEEKDAYS = Object.freeze([ + 'Sun', + 'Mon', + 'Tue', + 'Wed', + 'Thu', + 'Fri', + 'Sat', +]); +const HTTP_WEEKDAYS_LONG = Object.freeze([ + 'Sunday', + 'Monday', + 'Tuesday', + 'Wednesday', + 'Thursday', + 'Friday', + 'Saturday', +]); + +/** Strict RFC 9110 HTTP-date, including the two obsolete recipient formats. */ +function parseHttpDate(raw, now) { + let match = raw.match( + /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/, + ); + let weekday, day, month, year, hour, minute, second; + if (match) { + [, weekday, day, month, year, hour, minute, second] = match; + weekday = HTTP_WEEKDAYS.indexOf(weekday); + } else { + match = raw.match( + /^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), (\d{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{2}) (\d{2}):(\d{2}):(\d{2}) GMT$/, + ); + if (match) { + [, weekday, day, month, year, hour, minute, second] = match; + weekday = HTTP_WEEKDAYS_LONG.indexOf(weekday); + const currentYear = new Date(now).getUTCFullYear(); + if (!Number.isFinite(currentYear)) return null; + year = Math.floor(currentYear / 100) * 100 + Number(year); + if (year > currentYear + 50) year -= 100; + } else { + match = raw.match( + /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ([ \d]\d) (\d{2}):(\d{2}):(\d{2}) (\d{4})$/, + ); + if (!match) return null; + [, weekday, month, day, hour, minute, second, year] = match; + weekday = HTTP_WEEKDAYS.indexOf(weekday); + } + } + day = Number(day); + month = HTTP_MONTHS.indexOf(month); + year = Number(year); + hour = Number(hour); + minute = Number(minute); + second = Number(second); + if ( + weekday < 0 || + month < 0 || + year < 1601 || + hour > 23 || + minute > 59 || + second > 59 + ) + return null; + const at = Date.UTC(year, month, day, hour, minute, second); + const date = new Date(at); + return date.getUTCFullYear() === year && + date.getUTCMonth() === month && + date.getUTCDate() === day && + date.getUTCHours() === hour && + date.getUTCMinutes() === minute && + date.getUTCSeconds() === second && + date.getUTCDay() === weekday + ? at + : null; +} + /** Seconds or an HTTP date → milliseconds from now, or null. */ export function parseRetryAfter(value, now = Date.now()) { - if (value == null || value === '') return null; - const seconds = Number(value); - if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000); - const at = Date.parse(String(value)); - return Number.isFinite(at) ? Math.max(0, at - now) : null; + if (value == null) return null; + const raw = String(value).trim(); + if (!raw) return null; + if (/^\d+$/.test(raw)) { + const seconds = Number(raw); + const milliseconds = seconds * 1000; + return Number.isSafeInteger(milliseconds) ? milliseconds : null; + } + // Numeric-looking but invalid delay-seconds must not be reinterpreted as a date. + if (/^[+-]?(?:\d|\.\d)/.test(raw)) return null; + const at = parseHttpDate(raw, now); + return at === null ? null : Math.max(0, at - now); } /** UTC calendar day, the unit of the daily cap. */ @@ -122,30 +222,28 @@ function utcDay(ms) { return new Date(ms).toISOString().slice(0, 10); } -const sleepSync = (ms) => - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); - /** * Public-service state shared by every process using the same file: today's - * request count and any pause the server asked for. + * request count, the next reserved start, and any pause the server asked for. * - * Every change is a read-modify-write under an exclusive lock directory, and - * the file is replaced atomically, so two dev servers sharing a checkout add - * to one count instead of overwriting each other. Memory keeps the highest - * values this process has seen, so a failed or unreadable file never lowers - * the count or shortens a pause; a persistence failure is reported once and - * in `status()`. + * Every change is a read-modify-write under an OS-owned SQLite transaction, + * and the JSON file is replaced atomically, so two dev servers sharing a + * checkout add to one count instead of overwriting each other. A process exit + * releases only that process's transaction; there is no stale path to delete + * and therefore no stale reclaimer that can remove a successor's live lock. + * Memory keeps the highest values this process has seen, so a failed or + * unreadable file never lowers the count or shortens a pause; a persistence + * failure is reported once and in `status()`. */ export function createGateStateStore({ file = null, - lockStaleMs = 5000, lockWaitMs = 500, onError = (error) => console.warn( `[Nominatim] could not persist usage state: ${error?.message || error}`, ), } = {}) { - let memory = { day: '', count: 0, pausedUntil: 0 }; + let memory = { day: '', count: 0, pausedUntil: 0, nextStartAt: 0 }; let lastError = null; const report = (error) => { @@ -153,6 +251,64 @@ export function createGateStateStore({ lastError = error; }; + const pauseMarkerPrefix = file ? `${path.basename(file)}.pause.` : null; + + /** + * A refusal must remain visible even while another process owns the SQLite + * transaction. Immutable marker names make that write atomic and monotonic: + * concurrent writers cannot shorten one another's pause, and the next lock + * owner folds every marker into the normal JSON state. + */ + function pendingPauses({ strict = false } = {}) { + if (!file) return []; + let names; + try { + names = fs.readdirSync(path.dirname(file)); + } catch (error) { + if (error?.code === 'ENOENT') return []; + report(error); + if (strict) throw error; + return []; + } + const pauses = []; + for (const name of names) { + if (!name.startsWith(pauseMarkerPrefix)) continue; + const encoded = name.slice(pauseMarkerPrefix.length).split('.', 1)[0]; + if (!/^\d+$/.test(encoded)) continue; + const pausedUntil = Number(encoded); + if (!Number.isSafeInteger(pausedUntil)) { + const error = new Error('invalid durable Nominatim pause marker'); + report(error); + if (strict) throw error; + continue; + } + pauses.push({ + path: path.join(path.dirname(file), name), + pausedUntil, + }); + } + return pauses; + } + + function markPause(at) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const pausedUntil = Math.ceil(at); + const marker = `${file}.pause.${pausedUntil}.${process.pid}.${process.hrtime.bigint()}`; + fs.writeFileSync(marker, '', { flag: 'wx' }); + return pausedUntil; + } + + function clearPersistedPauses(pausedUntil) { + for (const marker of pendingPauses()) { + if (marker.pausedUntil > pausedUntil) continue; + try { + fs.unlinkSync(marker.path); + } catch (error) { + if (error?.code !== 'ENOENT') report(error); + } + } + } + const merge = (a, b) => ({ day: a.day >= b.day ? a.day : b.day, count: @@ -162,21 +318,58 @@ export function createGateStateStore({ ? a.count : b.count, pausedUntil: Math.max(a.pausedUntil || 0, b.pausedUntil || 0), + nextStartAt: Math.max(a.nextStartAt || 0, b.nextStartAt || 0), }); - function readDisk() { - if (!file) return null; + const parseState = (parsed) => { + const validDay = + parsed?.day === '' || + (typeof parsed?.day === 'string' && + /^\d{4}-\d{2}-\d{2}$/.test(parsed.day) && + new Date(`${parsed.day}T00:00:00.000Z`) + .toISOString() + .startsWith(parsed.day)); + const validCount = + Number.isInteger(parsed?.count) && + parsed.count >= 0 && + (parsed.day !== '' || parsed.count === 0); + const validPausedUntil = + Number.isFinite(parsed?.pausedUntil) && parsed.pausedUntil >= 0; + // nextStartAt was added after the original state format. Its absence is + // the one supported migration; a present value must still be trustworthy. + const validNextStartAt = + parsed?.nextStartAt === undefined || + (Number.isFinite(parsed.nextStartAt) && parsed.nextStartAt >= 0); + if ( + !parsed || + Array.isArray(parsed) || + typeof parsed !== 'object' || + !validDay || + !validCount || + !validPausedUntil || + !validNextStartAt + ) + throw new Error('invalid durable Nominatim usage state'); + return { + day: parsed.day, + count: parsed.count, + pausedUntil: parsed.pausedUntil, + nextStartAt: parsed.nextStartAt ?? 0, + }; + }; + + function readDisk({ strict = false } = {}) { + if (!file) { + if (strict) + throw new Error('no durable Nominatim usage state is configured'); + return null; + } try { - const parsed = JSON.parse(fs.readFileSync(file, 'utf8')); - return { - day: typeof parsed?.day === 'string' ? parsed.day : '', - count: Number.isFinite(parsed?.count) ? Math.max(0, parsed.count) : 0, - pausedUntil: Number.isFinite(parsed?.pausedUntil) - ? parsed.pausedUntil - : 0, - }; + return parseState(JSON.parse(fs.readFileSync(file, 'utf8'))); } catch (error) { - if (error?.code !== 'ENOENT') report(error); + if (error?.code === 'ENOENT') return null; + report(error); + if (strict) throw error; return null; } } @@ -184,34 +377,61 @@ export function createGateStateStore({ function current() { const disk = readDisk(); if (disk) memory = merge(memory, disk); + const pending = pendingPauses(); + if (pending.length) + memory = merge(memory, { + ...memory, + pausedUntil: Math.max(...pending.map((item) => item.pausedUntil)), + }); return memory; } function withLock(work) { - const lock = `${file}.lock`; - const deadline = Date.now() + lockWaitMs; fs.mkdirSync(path.dirname(file), { recursive: true }); - for (;;) { - try { - fs.mkdirSync(lock); - break; - } catch (error) { - if (error?.code !== 'EEXIST') throw error; + const waitMs = Number.isFinite(lockWaitMs) + ? Math.max(0, Math.floor(lockWaitMs)) + : 500; + const database = new DatabaseSync(`${file}.lock.sqlite`, { + timeout: waitMs, + }); + let transactionOpen = false; + try { + database.exec('BEGIN IMMEDIATE'); + transactionOpen = true; + const result = work(); + database.exec('COMMIT'); + transactionOpen = false; + return result; + } catch (error) { + if (transactionOpen) { try { - if (Date.now() - fs.statSync(lock).mtimeMs > lockStaleMs) - fs.rmSync(lock, { recursive: true, force: true }); + database.exec('ROLLBACK'); } catch { - // Another process released it meanwhile. + // Closing the connection below still releases this owner's lock. } - if (Date.now() > deadline) - throw new Error('usage state is locked by another process'); - sleepSync(5); + } + throw error; + } finally { + try { + database.close(); + } catch { + // The transaction has already been committed or rolled back. Closing + // is best effort during exceptional teardown and never removes a lock. } } + } + + function writeDisk(next) { + const temp = `${file}.${process.pid}.${process.hrtime.bigint()}.tmp`; try { - return work(); + fs.writeFileSync(temp, JSON.stringify(next)); + fs.renameSync(temp, file); } finally { - fs.rmSync(lock, { recursive: true, force: true }); + try { + fs.unlinkSync(temp); + } catch (error) { + if (error?.code !== 'ENOENT') report(error); + } } } @@ -224,10 +444,9 @@ export function createGateStateStore({ try { withLock(() => { const next = merge(memory, change(current())); - const temp = `${file}.${process.pid}.${Date.now()}.tmp`; - fs.writeFileSync(temp, JSON.stringify(next)); - fs.renameSync(temp, file); + writeDisk(next); memory = next; + clearPersistedPauses(next.pausedUntil); lastError = null; }); } catch (error) { @@ -247,16 +466,80 @@ export function createGateStateStore({ day, count: state.day === day ? state.count + 1 : 1, pausedUntil: state.pausedUntil, + nextStartAt: state.nextStartAt, })).count; }, + reserve(day, { now, dailyCap, minSpacingMs, latestStartAt = Infinity }) { + if (!file) { + const error = new Error( + 'no durable Nominatim usage state is configured', + ); + report(error); + return { status: 'unavailable', error }; + } + try { + let result; + withLock(() => { + const disk = readDisk({ strict: true }); + if (disk) memory = merge(memory, disk); + const pending = pendingPauses({ strict: true }); + if (pending.length) + memory = merge(memory, { + ...memory, + pausedUntil: Math.max(...pending.map((item) => item.pausedUntil)), + }); + const state = memory; + if ((state.pausedUntil || 0) > now) { + result = { + status: 'paused', + retryAfterMs: state.pausedUntil - now, + }; + return; + } + const count = state.day === day ? state.count : 0; + if (count >= dailyCap) { + result = { status: 'cap' }; + return; + } + const startAt = Math.max(now, state.nextStartAt || 0); + if (startAt > latestStartAt) { + result = { status: 'abandoned' }; + return; + } + const next = { + day, + count: count + 1, + pausedUntil: state.pausedUntil || 0, + nextStartAt: startAt + minSpacingMs, + }; + writeDisk(next); + memory = next; + clearPersistedPauses(next.pausedUntil); + lastError = null; + result = { status: 'reserved', startAt, count: next.count }; + }); + return result; + } catch (error) { + report(error); + return { status: 'unavailable', error }; + } + }, pausedUntil() { return current().pausedUntil || 0; }, pauseUntil(at) { + if (file) { + try { + at = markPause(at); + } catch (error) { + report(error); + } + } update((state) => ({ ...state, pausedUntil: at })); }, status: () => ({ file, + durable: Boolean(file), persisted: Boolean(file) && !lastError, error: lastError ? String(lastError.message || lastError) : null, }), @@ -289,9 +572,8 @@ export function createNominatimGate({ timeoutMs = 9000, } = {}) { let queue = Promise.resolve(); - let lastStartedAt = -Infinity; let pending = 0; - let pausedUntil = 0; + const pausedUntilByEndpoint = new Map(); let upstreamRequests = 0; let usageStore = usage; @@ -304,14 +586,18 @@ export function createNominatimGate({ */ function pause(ms, config) { const until = now() + Math.max(0, ms); - if (until > pausedUntil) pausedUntil = until; - if (config?.isPublic) usageStore.pauseUntil(pausedUntil); + const key = String(config?.endpoint || ''); + if (until > (pausedUntilByEndpoint.get(key) || 0)) + pausedUntilByEndpoint.set(key, until); + if (config?.isPublic) usageStore.pauseUntil(until); } function pauseEnd(config) { + const endpointPause = + pausedUntilByEndpoint.get(String(config?.endpoint || '')) || 0; return config?.isPublic - ? Math.max(pausedUntil, usageStore.pausedUntil()) - : pausedUntil; + ? Math.max(endpointPause, usageStore.pausedUntil()) + : endpointPause; } function pausedError(config) { @@ -324,19 +610,12 @@ export function createNominatimGate({ if (!config.endpoint) return gateError('NOMINATIM_DISABLED', 'Place service is not configured'); if (now() < pauseEnd(config)) return pausedError(config); - if (config.isPublic && usageStore.count(utcDay(now())) >= config.dailyCap) - return gateError( - 'NOMINATIM_DAILY_CAP', - 'Daily place lookup allowance is used', - ); return null; } /** One upstream request; classifies refusals and pauses the gate. */ async function send(url, config, maxBytes, signal) { - if (config.isPublic) usageStore.increment(utcDay(now())); upstreamRequests += 1; - lastStartedAt = now(); const controller = new AbortController(); const onAbort = () => controller.abort(); signal?.addEventListener('abort', onAbort, { once: true }); @@ -405,16 +684,45 @@ export function createNominatimGate({ } } - /** Wait for the spacing slot, then check the request is still wanted. */ + /** Atomically reserve the public slot, then wait until it may start. */ async function takeTurn(config, queuedAt, signal) { - const spacing = config.isPublic ? minSpacingMs : 0; - const waitMs = Math.max(0, spacing - (now() - lastStartedAt)); + if (signal?.aborted || now() - queuedAt > maxWaitMs) + throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned'); + const refused = admissionError(config); + if (refused) throw refused; + // An operator-selected endpoint is not governed by the public instance's + // shared budget, pacing, or durable-state requirement. + if (!config.isPublic) return; + const reserved = usageStore.reserve(utcDay(now()), { + now: now(), + dailyCap: config.dailyCap, + minSpacingMs, + latestStartAt: queuedAt + maxWaitMs, + }); + if (reserved.status === 'unavailable') + throw gateError( + 'NOMINATIM_STATE_UNAVAILABLE', + 'Public place service usage state is unavailable', + { cause: reserved.error }, + ); + if (reserved.status === 'cap') + throw gateError( + 'NOMINATIM_DAILY_CAP', + 'Daily place lookup allowance is used', + ); + if (reserved.status === 'paused') + throw gateError('NOMINATIM_PAUSED', 'Place service is pausing', { + retryAfterMs: reserved.retryAfterMs, + }); + if (reserved.status === 'abandoned') + throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned'); + const waitMs = Math.max(0, reserved.startAt - now()); if (waitMs) await sleep(waitMs); if (signal?.aborted || now() - queuedAt > maxWaitMs) throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned'); - // An outage or refusal that happened while this waited stops it here. - const refused = admissionError(config); - if (refused) throw refused; + // A refusal from another process after this reservation still stops it. + // The reserved daily slot remains spent; it cannot safely be reassigned. + if (now() < pauseEnd(config)) throw pausedError(config); } /** @@ -465,7 +773,7 @@ export function createNominatimGate({ return { requestJson, settings: current, - /** Share the daily count and pause through a file (composition chooses it). */ + /** Share public admission, daily count and pauses through one install's file. */ persistUsage(file) { usageStore = createGateStateStore({ file }); }, diff --git a/server/providers/regional/outlineSelect.js b/server/providers/regional/outlineSelect.js index f03e9cac7..738249e78 100644 --- a/server/providers/regional/outlineSelect.js +++ b/server/providers/regional/outlineSelect.js @@ -192,12 +192,95 @@ const ADMIN_EVIDENCE = Object.freeze({ }, }); +const KNOWN_ADMIN_TYPES = new Set( + Object.values(ADMIN_EVIDENCE).flatMap((evidence) => evidence.types), +); + +/** + * Nominatim address types whose meaning is stable enough for the area consumer's + * country/admin1/admin2 contract. `region`, `district` and numeric admin levels + * vary by country, so they remain unverified instead of being guessed. + */ +const VERIFIED_ADMIN_LEVEL = Object.freeze({ + country: 'country', + state: 'admin1', + province: 'admin1', + county: 'admin2', + state_district: 'admin2', +}); + +const EXPLICIT_NAME_LEVEL = Object.freeze({ + country: 'country', + nation: 'country', + province: 'admin1', + state: 'admin1', + region: 'admin1', + oblast: 'admin1', + prefecture: 'admin1', + canton: 'admin1', + governorate: 'admin1', + county: 'admin2', + district: 'admin2', + department: 'admin2', + zone: 'retired', +}); + +function explicitNameEvidence(value) { + const normalized = String(value || '') + .normalize('NFKC') + .toLocaleLowerCase() + .replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ') + .trim(); + const prefix = normalized.match( + /^(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)\s+of\b/, + )?.[1]; + const suffix = normalized.match( + /\b(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)$/, + )?.[1]; + const levels = new Set( + [prefix, suffix] + .filter(Boolean) + .map((type) => EXPLICIT_NAME_LEVEL[type]) + .filter(Boolean), + ); + if (levels.size > 1) return { contradictory: true, level: null }; + return { contradictory: false, level: levels.values().next().value || null }; +} + +/** + * Verified administrative identity carried by the selected result itself. + * Never infer it from the request, desired kind, rank, or a separate geocoder + * anchor. A retired name is preserved without a level so downstream policy can + * reject it explicitly; every other contradiction stays wholly unverified. + */ +function selectedAdminIdentity(kind, row) { + if (kind !== 'admin') return null; + const adminLevel = + VERIFIED_ADMIN_LEVEL[String(row?.addresstype || '').toLowerCase()]; + const authoritativeAdminArea = String(row?.name || '').trim(); + if (!adminLevel || !authoritativeAdminArea) return null; + // Validate the complete authoritative name. `adminArea` is truncated only + // for display after every prefix/suffix signal has been considered. + const { contradictory, level: nameLevel } = explicitNameEvidence( + authoritativeAdminArea, + ); + if (contradictory) return null; + const adminArea = authoritativeAdminArea.slice(0, 160); + if (nameLevel === 'retired') return { adminArea }; + if (nameLevel && nameLevel !== adminLevel) return null; + return { adminArea, adminLevel }; +} + /** Whether an administrative boundary is at the level the ask names. */ export function adminLevelFits(kind, row) { const evidence = ADMIN_EVIDENCE[kind]; if (!evidence) return true; const type = String(row?.addresstype || '').toLowerCase(); if (type && evidence.types.includes(type)) return true; + // A recognized type is direct evidence. Rank is only a fallback when the + // upstream answer omits the type or supplies a value we do not understand; + // the overlapping county/city rank bands must not override `county`. + if (KNOWN_ADMIN_TYPES.has(type)) return false; const rank = Number(row?.place_rank); if (Number.isFinite(rank)) return rank >= evidence.ranks[0] && rank <= evidence.ranks[1]; @@ -452,6 +535,7 @@ export function selectOutlineResult(rows, { kind, query }) { skipped.push(`${label}: too large`); continue; } + const adminIdentity = selectedAdminIdentity(kind, row); const lat = Number(row.lat); const lon = Number(row.lon); return { @@ -464,6 +548,7 @@ export function selectOutlineResult(rows, { kind, query }) { row.osm_type && row.osm_id != null ? `${row.osm_type}/${row.osm_id}` : null, + ...(adminIdentity || {}), center: finiteLat(lat) && finiteLon(lon) ? { lat, lon } : null, polygons, }, diff --git a/src/annotations/annotationEngine.js b/src/annotations/annotationEngine.js index e620d3526..fabc46fb1 100644 --- a/src/annotations/annotationEngine.js +++ b/src/annotations/annotationEngine.js @@ -11,6 +11,7 @@ import { sampleGroundHeight, } from './annotationResolver.js'; import { ringCentroid } from './drawMode.js'; +import { phaseTiming } from '../sources/phaseTiming.js'; // Dev convenience: expose the app's Cesium instance for console/preview probing // (single shared module instance — avoids dual-Cesium state bugs when testing). @@ -679,6 +680,7 @@ export function createAnnotationEngine({ if (!annotations.has(anno.id)) return; // mark replaced/removed while resolving anno.pendingOutline = false; anno.outlineUnavailable = isUnavailableCapability(fp); + const installStart = performance.now(); if (anno.outlineUnavailable) renderer.update(anno); if (fp && !anno.outlineUnavailable) { anno.ring = fp.ring; @@ -726,6 +728,17 @@ export function createAnnotationEngine({ } } renderer.sync(annotations); + if (fp && !anno.outlineUnavailable) + phaseTiming( + 'install', + installStart, + { + kind: anno.footprintKind || null, + parts: anno.polygons?.length || (anno.ring ? 1 : 0), + points: anno.ring?.length || 0, + }, + 'outlines', + ); // Final outcome → the voice layer (place names ride as structured DATA fields). // 'failed' covers both the definitive no-polygon and an exhausted transient // retry — either way the mark honestly stays a point. diff --git a/src/annotations/outlineRungs.test.mjs b/src/annotations/outlineRungs.test.mjs index 7dc9e8c68..92eb8d95c 100644 --- a/src/annotations/outlineRungs.test.mjs +++ b/src/annotations/outlineRungs.test.mjs @@ -6,7 +6,9 @@ // request reaches any Overpass endpoint. import test from 'node:test'; import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { Readable } from 'node:stream'; import { createOutlineRungs, @@ -20,7 +22,10 @@ import { createNominatimOutlineClient } from '../sources/nominatimOutlines.js'; import { createApplicationRequestServices } from '../services/requests.js'; import { overpassProxy } from '../../server/providers/overpass.js'; import { geocodeProxy } from '../../server/providers/regional/place.js'; -import { createNominatimGate } from '../../server/providers/regional/nominatimGate.js'; +import { + createGateStateStore, + createNominatimGate, +} from '../../server/providers/regional/nominatimGate.js'; const TILES = [6745, 6746].map((y) => decodeOpenFreeMapOutlineTile( @@ -332,6 +337,8 @@ test('default settings: outline, street and building asks send nothing to Overpa t.mock.method(globalThis, 'fetch', async (url) => assert.fail(`unexpected network request ${url}`)); const upstream = []; + const stateDir = mkdtempSync(path.join(os.tmpdir(), 'gev-outline-rungs-')); + t.after(() => rmSync(stateDir, { recursive: true, force: true })); const gate = createNominatimGate({ settings: { endpoint: 'https://nominatim.openstreetmap.org/search', isPublic: true, dailyCap: 50 }, fetchImpl: async (url) => { @@ -339,6 +346,7 @@ test('default settings: outline, street and building asks send nothing to Overpa const q = new URL(url).searchParams.get('q'); return Response.json(NOMINATIM_ROWS[q] || []); }, + usage: createGateStateStore({ file: path.join(stateDir, 'state.json') }), sleep: async () => {}, }); const handlers = routes(overpassProxy(), geocodeProxy({ gate })); diff --git a/src/data/adminBoundaries.js b/src/data/adminBoundaries.js index 3770ce466..f99527aff 100644 --- a/src/data/adminBoundaries.js +++ b/src/data/adminBoundaries.js @@ -420,8 +420,8 @@ const COUNTY_WORDS = [ ]; const COUNTY_WORD_RE = new RegExp(`\\b(${COUNTY_WORDS.join('|')})\\b`); const STATE_PREFIX_RE = - /^(?:free state|state|commonwealth|province|territory|prefecture|region|land|canton|department) of (.+)$/; -const STATE_SUFFIX_RE = /^(.+) (?:state|province|prefecture|territory)$/; + /^(free state|state|commonwealth|province|territory|prefecture|region|land|canton|department) of (.+)$/; +const STATE_SUFFIX_RE = /^(.+) (state|province|prefecture|territory)$/; /** * Split an ask into what it names and how. @@ -482,14 +482,17 @@ export function parseAdminQuery(text) { if (prefix) return { kind: 'state', - name: prefix[1], + name: prefix[2], qualifiers, postal: null, countyWord: null, full: head, form: 'prefix', + adminType: prefix[1], }; - const suffix = STATE_SUFFIX_RE.exec(head); + // South Africa's canonical province name is literally "Free State"; do not + // strip its final word as though the ask explicitly named an admin type. + const suffix = head === 'free state' ? null : STATE_SUFFIX_RE.exec(head); if (suffix) return { kind: 'state', @@ -499,6 +502,7 @@ export function parseAdminQuery(text) { countyWord: null, full: head, form: 'suffix', + adminType: suffix[2], }; return { kind: 'any', @@ -510,6 +514,30 @@ export function parseAdminQuery(text) { }; } +/** Explicit wording is stronger than a stale same-name geometry. */ +function explicitStateTypeFits(entry, parsed) { + if (!parsed.adminType) return true; + const words = new Set( + normalizeAdminName(entry.feature?.type) + .split(/[^a-z]+/) + .filter(Boolean), + ); + const accepted = + { + 'free state': ['state'], + state: ['state'], + commonwealth: ['state'], + province: ['province'], + territory: ['territory'], + prefecture: ['prefecture'], + region: ['region'], + land: ['state'], + canton: ['canton'], + department: ['department', 'departamento'], + }[parsed.adminType] || []; + return accepted.some((word) => words.has(word)); +} + /** A qualifier names the unit's country ("Bavaria, Germany") or the unit * itself ("Washington, DC"). */ function qualifierMatches(entry, qualifier) { @@ -647,6 +675,10 @@ async function stateCandidates(parsed, { allowAmbiguous = false } = {}) { list = list.filter((i) => parsed.qualifiers.every((q) => qualifierMatches(i.entry, q)), ); + // Preserve explicit identity at the selector seam. In particular, + // "Bagmati Province" must not select Natural Earth's historical NP-BA + // Bagmati Zone merely because the point and normalized name overlap. + list = list.filter((i) => explicitStateTypeFits(i.entry, parsed)); if (allowAmbiguous) return list; // A bare name answers alone only for a prominent unit — a US state, a // Canadian province, Bavaria — never a French département, an English @@ -759,6 +791,12 @@ export async function findAdminAreaAt(names, lat, lon, scope) { .map(parseAdminQuery) .filter((p) => p?.name); if (!parsedList.length) return null; + // If the ask explicitly names an admin type, do not let the geocoder's + // untyped alias reopen an incompatible historical same-name boundary. + const explicitStates = parsedList.filter( + (parsed) => parsed.kind === 'state' && parsed.adminType, + ); + const stateParses = explicitStates.length ? explicitStates : parsedList; const containing = (items) => { const out = []; for (const item of items) @@ -803,7 +841,7 @@ export async function findAdminAreaAt(names, lat, lon, scope) { return toResult(rank(found, { lat, lon })[0], found.length); } const states = []; - for (const parsed of parsedList) + for (const parsed of stateParses) states.push( ...(await stateCandidates( { ...parsed, qualifiers: [] }, diff --git a/src/data/adminBoundaries.test.mjs b/src/data/adminBoundaries.test.mjs index efc3ca259..f81e9adf2 100644 --- a/src/data/adminBoundaries.test.mjs +++ b/src/data/adminBoundaries.test.mjs @@ -225,6 +225,28 @@ test('geocoder-confirmed lookup: the named unit that contains the point', async assert.equal(await findAdminAreaAt(['Georgia'], 41.7, 44.8, 'state'), null); const newYork = await findAdminAreaAt(['New York'], 42.9, -75.5, 'state'); assert.equal(newYork?.id, 'US-NY'); + assert.equal( + ( + await findAdminAreaAt( + ['Ontario Province', 'Ontario'], + 43.6532, + -79.3832, + 'state', + ) + )?.id, + 'CA-ON', + ); + // Natural Earth still carries the historical NP-BA Bagmati Zone. An ask + // for today's province must not silently select that incompatible geometry. + assert.equal( + await findAdminAreaAt( + ['Bagmati Province', 'Bagmati'], + 27.7172, + 85.324, + 'state', + ), + null, + ); // A county the geocoder typed, named without the county word. const travis = await findAdminAreaAt( ['Travis', 'Travis County'], diff --git a/src/data/placeBoundaries.test.mjs b/src/data/placeBoundaries.test.mjs index d9c297e3b..21927a806 100644 --- a/src/data/placeBoundaries.test.mjs +++ b/src/data/placeBoundaries.test.mjs @@ -26,6 +26,10 @@ import { const PLACES = new URL('./local_data/us_census_places/', import.meta.url); const WOF = new URL('./local_data/wof_neighborhoods/', import.meta.url); +const WOF_LICENSES = new URL( + '../../scripts/wof-open-licenses.json', + import.meta.url, +); const readJson = (url) => JSON.parse(readFileSync(url, 'utf8')); const AUSTIN = { lat: 30.2672, lon: -97.7431 }; @@ -365,15 +369,71 @@ test('places pack: every state file, unique ids, sane and valid rings', () => { assert.equal(total, 32_629); }); -const UNSTATED = /assumed|unknown|unresolved|n\/a/i; +const WOF_LICENSE_POLICY = readJson(WOF_LICENSES); +const ALLOWED_WOF_LICENSES = new Set(WOF_LICENSE_POLICY.licenses); +const licenceAllowed = (value) => + typeof value === 'string' && ALLOWED_WOF_LICENSES.has(value); -test('neighborhood pack ships only sources whose licence is stated', () => { +test('the WOF licence policy accepts only exact reviewed registry strings', () => { + for (const accepted of WOF_LICENSE_POLICY.licenses) + assert.equal(licenceAllowed(accepted), true, accepted); + for (const rejected of [ + '', + null, + 0, + 'Unknown', + 'Restricted', + 'CC BY-NC 4.0', + 'CC BY-SA 4.0', + 'CC BY 5.0 (unreviewed)', + 'CC BY Restricted', + 'CC BY Unknown', + 'Creative Commons NonCommercial', + 'ODbL 1.0', + 'cc by 4.0', + 'CC BY 4.0 ', + ' CC BY 4.0', + 'CC BY 4.0', + 'CC\tBY 4.0', + ]) + assert.equal(licenceAllowed(rejected), false, JSON.stringify(rejected)); +}); + +test('the Zetashapes raw registry spelling is admitted by pinned provenance', () => { + const review = WOF_LICENSE_POLICY.reviewedSources.zs; + assert.deepEqual( + { + name: review.name, + registryCommit: review.registryCommit, + registrySha256: review.registrySha256, + rawLicenseType: review.rawLicenseType, + licenseTextSha256: review.licenseTextSha256, + }, + { + name: 'Zetashapes', + registryCommit: 'e17af153c144fe6e08aa8db2607b4bf8aecd448d', + registrySha256: + 'd8a5dd873d26eee11583de4438185c046dec33848d5bde27047ca4bff44d9c39', + rawLicenseType: 'Public domain', + licenseTextSha256: + '91757f96f74f6dfb4a7206486b78586da0d039e9611e224fc793c7167dcfe615', + }, + ); + assert.equal(licenceAllowed(review.rawLicenseType), true); + assert.match(review.decision, /TIGER\/Line/); + assert.match(review.decision, /Flickr/); +}); + +test('neighborhood pack ships only explicitly allowed and credited sources', () => { const index = readJson(new URL('index.json', WOF)); const sources = index.meta.sources; const credits = readFileSync(new URL('ATTRIBUTION.md', WOF), 'utf8'); for (const [key, source] of Object.entries(sources)) { - assert.ok(source.license, `${key} has a licence`); - assert.doesNotMatch(source.license, UNSTATED, key); + assert.equal( + licenceAllowed(source.license), + true, + `${key}: ${source.license}`, + ); assert.notEqual(key, 'unknown'); assert.ok(credits.includes(`\`${key}\``), `${key} credited`); } diff --git a/src/layers/traffic/source.test.mjs b/src/layers/traffic/source.test.mjs index affaff56d..08634a14f 100644 --- a/src/layers/traffic/source.test.mjs +++ b/src/layers/traffic/source.test.mjs @@ -3,7 +3,10 @@ import test from 'node:test'; import { readFileSync } from 'node:fs'; import { tilesForBounds, tileToBBox } from '../../data/tomtomTiles.js'; import { decodeFlowTile } from './flowDecode.js'; -import { decodeOpenFreeMapTile } from '../../sources/openFreeMap.js'; +import { + clearSharedOpenFreeMapTiles, + decodeOpenFreeMapTile, +} from '../../sources/openFreeMap.js'; import { RoadRequestError, roadRequestError, @@ -829,6 +832,8 @@ test('failed detail cores retain real coarse roads while successful cores replac }); test('traffic separates API and public tile transports, including the default tile fetch', async (t) => { + clearSharedOpenFreeMapTiles(); + t.after(clearSharedOpenFreeMapTiles); const publicUrls = [], apiUrls = []; const publicFetch = async (url) => { @@ -864,6 +869,11 @@ test('traffic separates API and public tile transports, including the default ti (await (await source.requestOsmRoads(bounds)).json()).roads.length, ); } - assert.ok(publicUrls.length >= 4); + assert.equal( + publicUrls.filter((url) => url.endsWith('.pbf')).length, + 1, + 'compatible sources reuse the shared decoded public tile', + ); + assert.equal(publicUrls.filter((url) => url.endsWith('/planet')).length, 2); assert.ok(apiUrls.length >= 4); }); diff --git a/src/sources/openFreeMap.js b/src/sources/openFreeMap.js index 3f40af1e8..76b4faa2b 100644 --- a/src/sources/openFreeMap.js +++ b/src/sources/openFreeMap.js @@ -1,6 +1,9 @@ import { PbfReader } from 'pbf'; import { VectorTile } from '@mapbox/vector-tile'; -import { createVectorTileSource } from './vectorTiles.js'; +import { + createSharedDecodedTileCache, + createVectorTileSource, +} from './vectorTiles.js'; import { clipTileRing, militaryOutlineLines } from './militaryTileGeometry.js'; import { tileToBBox } from '../data/tomtomTiles.js'; @@ -13,6 +16,65 @@ const ROAD_TYPES = Object.freeze({ minor: 'residential', }); +const SHARED_DECODED_TILES = createSharedDecodedTileCache({ + maxEntries: 192, + maxCacheBytes: 64 * 1024 * 1024, +}); +const TILE_PROJECTIONS = new Map(); +const MAX_PROJECTED_NUMBERS = 500_000; + +/** Conservative, deterministic retained-size estimate with a hard geometry cap. */ +export function estimateOpenFreeMapProjectionBytes(value) { + const seen = new Set(); + const stack = [value]; + let bytes = 0; + let numbers = 0; + while (stack.length) { + const item = stack.pop(); + if (item == null) continue; + if (typeof item === 'number') { + numbers += 1; + if (numbers > MAX_PROJECTED_NUMBERS) + throw new Error('Vector tile projection coordinate limit exceeded'); + bytes += 8; + } else if (typeof item === 'string') { + bytes += 16 + item.length * 2; + } else if (typeof item === 'boolean') { + bytes += 4; + } else if (typeof item === 'object') { + if (seen.has(item)) continue; + seen.add(item); + if (Array.isArray(item)) { + bytes += 24 + item.length * 8; + for (let i = 0; i < item.length; i += 1) stack.push(item[i]); + } else { + const entries = Object.entries(item); + bytes += 48 + entries.length * 16; + for (const [key, child] of entries) { + bytes += key.length * 2; + stack.push(child); + } + } + } + } + return bytes; +} + +/** Register one consumer view over the parsed OpenFreeMap tile. */ +export function registerOpenFreeMapProjection(name, project) { + if (!name || typeof project !== 'function') + throw new TypeError('Invalid OpenFreeMap tile projection'); + const current = TILE_PROJECTIONS.get(name); + if (current && current !== project) + throw new Error(`OpenFreeMap tile projection already registered: ${name}`); + TILE_PROJECTIONS.set(name, project); +} + +/** Explicit test/runtime reset; individual consumers never evict each other. */ +export function clearSharedOpenFreeMapTiles() { + SHARED_DECODED_TILES.clear(); +} + /** * Surface traffic uses public motor-road classes only. Untyped service ways * include car parks and grounds access, so none are used as through traffic. @@ -142,8 +204,8 @@ export function decodeOpenFreeMapMilitaryTile(bytes, z, x, y) { return decodeLayers(bytes, z, x, y, ['landuse']); } -function decodeLayers(bytes, z, x, y, layerNames) { - const tile = new VectorTile(new PbfReader(bytes)); +function decodeLayers(bytes, z, x, y, layerNames, parsedTile = null) { + const tile = parsedTile || new VectorTile(new PbfReader(bytes)); const roads = [], military = []; const box = tileToBBox(z, x, y); @@ -214,14 +276,121 @@ function decodeLayers(bytes, z, x, y, layerNames) { return { roads, military }; } +function decodeSharedOpenFreeMapTile(bytes, z, x, y) { + const tile = new VectorTile(new PbfReader(bytes)); + const base = decodeLayers( + bytes, + z, + x, + y, + ['transportation', 'landuse'], + tile, + ); + const projections = new Map(); + let baseBytes = 0; + let projectionBytes = 0; + let resizeCacheEntry = null; + let cacheBound = false; + const idleCallbacks = new Map(); + const cancelIdleProjection = (name) => { + if (!idleCallbacks.has(name)) return; + const handle = idleCallbacks.get(name); + idleCallbacks.delete(name); + globalThis.cancelIdleCallback?.(handle); + }; + const cancelIdleProjections = () => { + for (const name of [...idleCallbacks.keys()]) cancelIdleProjection(name); + }; + const project = (name) => { + if (projections.has(name)) return projections.get(name); + cancelIdleProjection(name); + const projector = TILE_PROJECTIONS.get(name); + if (!projector) + throw new Error(`Unknown OpenFreeMap tile projection: ${name}`); + const value = projector(bytes, z, x, y, tile); + const retainedBytes = estimateOpenFreeMapProjectionBytes(value); + projections.set(name, value); + projectionBytes += retainedBytes; + resizeCacheEntry?.(baseBytes + projectionBytes); + return value; + }; + const scheduleIdleProjections = () => { + // Scheduling is optional unless both lifecycle operations are available; + // otherwise eviction could not release the callback's decoded-tile closure. + if ( + !cacheBound || + typeof globalThis.requestIdleCallback !== 'function' || + typeof globalThis.cancelIdleCallback !== 'function' + ) + return; + for (const name of TILE_PROJECTIONS.keys()) + if (!projections.has(name) && !idleCallbacks.has(name)) { + const handle = globalThis.requestIdleCallback( + () => { + if (!cacheBound || idleCallbacks.get(name) !== handle) return; + idleCallbacks.delete(name); + try { + project(name); + } catch { + // The explicit consumer path retries and reports projection errors. + } + }, + { timeout: 1000 }, + ); + idleCallbacks.set(name, handle); + } + }; + return { + ...base, + project, + bindSharedCacheEntry(resize, initialBytes) { + cancelIdleProjections(); + resizeCacheEntry = resize; + baseBytes = initialBytes; + cacheBound = true; + resizeCacheEntry(baseBytes + projectionBytes); + scheduleIdleProjections(); + }, + releaseSharedCacheEntry() { + cacheBound = false; + cancelIdleProjections(); + projections.clear(); + baseBytes = 0; + projectionBytes = 0; + resizeCacheEntry = null; + }, + }; +} + /** Construct an immutable-version road/military tile source without starting I/O. */ export function createOpenFreeMapSource(options = {}) { + const { + projection = null, + decode, + sharedDecodedCache, + selectDecoded, + ...sourceOptions + } = options; + const shared = !decode && !selectDecoded; return createVectorTileSource({ tileJsonUrl: 'https://tiles.openfreemap.org/planet', allowedOrigin: 'https://tiles.openfreemap.org', - decode: decodeOpenFreeMapTile, + decode: shared + ? decodeSharedOpenFreeMapTile + : decode || decodeOpenFreeMapTile, + ...(shared + ? { + sharedDecodedCache: sharedDecodedCache || SHARED_DECODED_TILES, + selectDecoded: projection + ? (value) => value.project(projection) + : ({ roads, military }) => ({ roads, military }), + } + : { + ...(sharedDecodedCache ? { sharedDecodedCache } : {}), + ...(selectDecoded ? { selectDecoded } : {}), + }), maxEntries: 192, maxCacheBytes: 64 * 1024 * 1024, - ...options, + ...sourceOptions, }); } diff --git a/src/sources/openFreeMap.test.mjs b/src/sources/openFreeMap.test.mjs index f2a91098c..d99ba9251 100644 --- a/src/sources/openFreeMap.test.mjs +++ b/src/sources/openFreeMap.test.mjs @@ -7,11 +7,38 @@ import { polygonCentroid, clipTileLine, createOpenFreeMapSource, + estimateOpenFreeMapProjectionBytes, } from './openFreeMap.js'; import { createVectorTileSource } from './vectorTiles.js'; const fixture = (name) => readFileSync(new URL(`../data/fixtures/${name}`, import.meta.url)); +test('projected tile geometry has deterministic accounting and a coordinate cap', () => { + const small = { + streets: [ + { + name: 'A', + coordinates: [ + [0, 1], + [2, 3], + ], + }, + ], + }; + assert.equal( + estimateOpenFreeMapProjectionBytes(small), + estimateOpenFreeMapProjectionBytes(structuredClone(small)), + ); + assert.ok(estimateOpenFreeMapProjectionBytes(small) > 4 * 8); + assert.throws( + () => + estimateOpenFreeMapProjectionBytes({ + coordinates: Array.from({ length: 250_001 }, () => [0, 0]), + }), + /coordinate limit/, + ); +}); + test('real z14 and z12 Austin tiles decode only drivable roads', () => { for (const [z, x, y, name] of [ [14, 3743, 6745, 'ofm-austin-14-3743-6745.pbf'], diff --git a/src/sources/openFreeMapOutlines.js b/src/sources/openFreeMapOutlines.js index 66f9e3b3e..9acff8ce8 100644 --- a/src/sources/openFreeMapOutlines.js +++ b/src/sources/openFreeMapOutlines.js @@ -1,18 +1,24 @@ import { PbfReader } from 'pbf'; import { VectorTile } from '@mapbox/vector-tile'; -import { clipTileLine, createOpenFreeMapSource } from './openFreeMap.js'; +import { + clipTileLine, + createOpenFreeMapSource, + registerOpenFreeMapProjection, +} from './openFreeMap.js'; import { tileToBBox } from '../data/tomtomTiles.js'; import { approximateDistanceM, ringAreaM2, closeRing, } from './featureGeometry.js'; +import { phaseTiming } from './phaseTiming.js'; /** * Street and building outlines from the same OpenFreeMap z14 vector tiles the * traffic layer reads. No new tile provider: the source below points at the - * same immutable tile URLs, so tiles traffic already loaded come from the - * browser cache. Nothing here is fetched without an explicit ask. + * same immutable tile URLs and bounded decoded owner. Traffic prepares the + * outline projection in idle time; nothing here starts provider I/O without + * an explicit traffic or outline ask. */ /** Tile zoom used for outlines (the highest OpenMapTiles zoom with buildings). */ @@ -54,10 +60,17 @@ const GROUNDS_AREA_CLASSES = Object.freeze({ }); /** Lines of one tile's `transportation_name` layer and polygons of `building`. */ -export function decodeOpenFreeMapOutlineTile(bytes, z, x, y) { - const tile = new VectorTile(new PbfReader(bytes)); +export function decodeOpenFreeMapOutlineTile( + bytes, + z, + x, + y, + parsedTile = null, +) { + const tile = parsedTile || new VectorTile(new PbfReader(bytes)); const box = tileToBBox(z, x, y); const streets = []; + const streetsStart = performance.now(); const names = tile.layers.transportation_name; if (names) { if (names.length > LAYER_FEATURE_LIMIT) @@ -85,6 +98,17 @@ export function decodeOpenFreeMapOutlineTile(bytes, z, x, y) { }); } } + phaseTiming( + 'street-extract', + streetsStart, + { + tile: `${z}/${x}/${y}`, + features: names?.length || 0, + streets: streets.length, + }, + 'outlines', + ); + const polygonsStart = performance.now(); const buildings = decodePolygons( tile.layers.building, z, @@ -115,9 +139,21 @@ export function decodeOpenFreeMapOutlineTile(bytes, z, x, y) { ), ); } + phaseTiming( + 'polygon-extract', + polygonsStart, + { + tile: `${z}/${x}/${y}`, + buildings: buildings.length, + areas: areas.length, + }, + 'outlines', + ); return { streets, buildings, areas }; } +registerOpenFreeMapProjection('outlines', decodeOpenFreeMapOutlineTile); + /** Tile pixel -> longitude/latitude, exactly as vector-tile's toGeoJSON projects. */ function pixelToLonLat(px, py, extent, z, x, y) { const size = extent * 2 ** z; @@ -192,7 +228,7 @@ function decodePolygons(layer, z, x, y, keep, describe) { /** Construct the outline tile source: same OpenFreeMap tiles, outline decode. */ export function createOpenFreeMapOutlineSource(options = {}) { return createOpenFreeMapSource({ - decode: decodeOpenFreeMapOutlineTile, + projection: 'outlines', maxEntries: 32, maxCacheBytes: 16 * 1024 * 1024, ...options, diff --git a/src/sources/openFreeMapOutlines.test.mjs b/src/sources/openFreeMapOutlines.test.mjs index b8c156c0c..1f9431b6e 100644 --- a/src/sources/openFreeMapOutlines.test.mjs +++ b/src/sources/openFreeMapOutlines.test.mjs @@ -5,6 +5,7 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { buildingFromTiles, + createOpenFreeMapOutlineSource, decodeOpenFreeMapOutlineTile, enclosingAreaFromTiles, mergeLineSegments, @@ -12,6 +13,12 @@ import { streetFromTiles, streetNameMatches, } from './openFreeMapOutlines.js'; +import { + clearSharedOpenFreeMapTiles, + createOpenFreeMapSource, + registerOpenFreeMapProjection, +} from './openFreeMap.js'; +import { createSharedDecodedTileCache } from './vectorTiles.js'; import { approximateDistanceM } from './featureGeometry.js'; const fixture = (y) => @@ -43,7 +50,10 @@ test('tiles decode named street lines and building polygons', () => { test('street names compare with abbreviations and a leading direction only', () => { assert.equal(normalizeStreetName('Congress Ave.'), 'congress avenue'); - assert.equal(normalizeStreetName('Lombard St, San Francisco'), 'lombard street'); + assert.equal( + normalizeStreetName('Lombard St, San Francisco'), + 'lombard street', + ); assert.ok(streetNameMatches('Congress Avenue', 'congress ave')); assert.ok(streetNameMatches('North Congress Avenue', 'Congress Avenue')); assert.ok(!streetNameMatches('Congress Avenue', 'South Congress Avenue')); @@ -52,21 +62,33 @@ test('street names compare with abbreviations and a leading direction only', () }); test('same-name segments merge into polylines around the point (Congress Avenue)', () => { - const street = streetFromTiles(TILES, { name: 'Congress Avenue', ...CAPITOL }); + const street = streetFromTiles(TILES, { + name: 'Congress Avenue', + ...CAPITOL, + }); assert.equal(street.name, 'Congress Avenue'); assert.equal(street.lines.length, 1, 'the tile pieces join into one line'); const line = street.lines[0]; const [start, end] = [line[0], line.at(-1)]; const spanM = approximateDistanceM(start[1], start[0], end[1], end[0]); - assert.ok(spanM > 1200, `Congress runs ${spanM.toFixed(0)} m from the river to the Capitol`); + assert.ok( + spanM > 1200, + `Congress runs ${spanM.toFixed(0)} m from the river to the Capitol`, + ); // South Congress is a different street and stays out. - const south = streetFromTiles(TILES, { name: 'South Congress Avenue', ...CAPITOL }); + const south = streetFromTiles(TILES, { + name: 'South Congress Avenue', + ...CAPITOL, + }); assert.ok(south); assert.ok(south.lines.every((l) => l.every(([, lat]) => lat < 30.265))); }); test('street lookups are bounded by radius and answer null for unknown names', () => { - assert.equal(streetFromTiles(TILES, { name: 'Lombard Street', ...CAPITOL }), null); + assert.equal( + streetFromTiles(TILES, { name: 'Lombard Street', ...CAPITOL }), + null, + ); assert.equal( streetFromTiles(TILES, { name: 'Congress Avenue', lat: 30.4, lon: -97.74 }), null, @@ -75,9 +97,18 @@ test('street lookups are bounded by radius and answer null for unknown names', ( }); test('merge joins reversed and out-of-order pieces and keeps gaps apart', () => { - const a = [[0, 0], [0.001, 0]]; - const b = [[0.002, 0], [0.001, 0]]; // reversed, shares a's end - const c = [[0.01, 0], [0.011, 0]]; // separate + const a = [ + [0, 0], + [0.001, 0], + ]; + const b = [ + [0.002, 0], + [0.001, 0], + ]; // reversed, shares a's end + const c = [ + [0.01, 0], + [0.011, 0], + ]; // separate const chains = mergeLineSegments([b, c, a]); assert.equal(chains.length, 2); assert.equal(chains.find((ch) => ch.length === 3).length, 3); @@ -92,7 +123,13 @@ test('the building containing a point is picked (Texas Capitol)', () => { }); test('a point beside a building picks the nearest within 40 m, else none', () => { - const box = [[0, 0], [0.0002, 0], [0.0002, 0.0002], [0, 0.0002], [0, 0]]; + const box = [ + [0, 0], + [0.0002, 0], + [0.0002, 0.0002], + [0, 0.0002], + [0, 0], + ]; const tiles = [{ buildings: [{ rings: [box], heightM: null }] }]; const near = buildingFromTiles(tiles, { lat: 0.0001, lon: 0.00045 }); // ~28 m east assert.ok(near && !near.contains); @@ -102,7 +139,10 @@ test('a point beside a building picks the nearest within 40 m, else none', () => test('the open area enclosing the Capitol is its grounds, not a building', () => { const grounds = enclosingAreaFromTiles(TILES, CAPITOL); assert.ok(grounds, 'the park-like cover around the Capitol is found'); - assert.ok(grounds.areaM2 > 50_000 && grounds.areaM2 < 500_000, `${grounds.areaM2} m²`); + assert.ok( + grounds.areaM2 > 50_000 && grounds.areaM2 < 500_000, + `${grounds.areaM2} m²`, + ); assert.equal(grounds.class, 'park'); assert.equal( enclosingAreaFromTiles(TILES, { ...CAPITOL, minAreaM2: 1e6 }), @@ -110,3 +150,151 @@ test('the open area enclosing the Capitol is its grounds, not a building', () => 'nothing grounds-sized above the floor', ); }); + +test('traffic idle preparation gives outlines the same decoded tile without another request', async () => { + clearSharedOpenFreeMapTiles(); + const idle = []; + const previousIdle = globalThis.requestIdleCallback; + const previousCancelIdle = globalThis.cancelIdleCallback; + globalThis.requestIdleCallback = (callback) => { + idle.push(callback); + return idle.length; + }; + globalThis.cancelIdleCallback = () => {}; + let requests = 0; + const bytes = readFileSync( + new URL( + '../data/fixtures/ofm-outlines-austin-14-3743-6745.pbf', + import.meta.url, + ), + ); + const fetchImpl = async () => { + requests++; + return new Response(bytes); + }; + const tiles = [{ z: 14, x: 3743, y: 6745 }]; + const box = { south: 30.27, north: 30.28, west: -97.75, east: -97.73 }; + try { + const traffic = createOpenFreeMapSource({ + template: + 'https://tiles.openfreemap.org/planet/v/20250910_001001_pt/{z}/{x}/{y}.pbf', + fetchImpl, + }); + const outlines = createOpenFreeMapOutlineSource({ + template: + 'https://tiles.openfreemap.org/planet/v/20250910_001001_pt/{z}/{x}/{y}.pbf', + fetchImpl, + }); + await traffic.fetchBounds(box, { zoom: 14, tiles }); + assert.ok(idle.length > 0, 'traffic schedules the registered projection'); + for (const callback of idle) + callback({ didTimeout: false, timeRemaining: () => 50 }); + const result = await outlines.fetchBounds(box, { zoom: 14, tiles }); + assert.ok(result.tiles[0].buildings.length > 0); + assert.equal(requests, 1); + } finally { + globalThis.requestIdleCallback = previousIdle; + globalThis.cancelIdleCallback = previousCancelIdle; + clearSharedOpenFreeMapTiles(); + } +}); + +test('idle projections are retired on shared eviction, clear, and self-eviction', async () => { + const previousIdle = globalThis.requestIdleCallback; + const previousCancelIdle = globalThis.cancelIdleCallback; + const scheduled = new Map(); + const cancelled = new Set(); + let nextHandle = 1; + globalThis.requestIdleCallback = (callback) => { + const handle = nextHandle++; + scheduled.set(handle, callback); + return handle; + }; + globalThis.cancelIdleCallback = (handle) => cancelled.add(handle); + let projectionRuns = 0; + registerOpenFreeMapProjection('idle-lifecycle-test', () => { + projectionRuns++; + return { coordinates: Array.from({ length: 20_000 }, (_, i) => [i, i]) }; + }); + const firstBytes = readFileSync( + new URL( + '../data/fixtures/ofm-outlines-austin-14-3743-6745.pbf', + import.meta.url, + ), + ); + const secondBytes = readFileSync( + new URL( + '../data/fixtures/ofm-outlines-austin-14-3743-6746.pbf', + import.meta.url, + ), + ); + const sharedDecodedCache = createSharedDecodedTileCache({ + maxEntries: 1, + maxCacheBytes: 100_000, + }); + const source = createOpenFreeMapSource({ + template: 'https://tiles.openfreemap.org/planet/v/test/{z}/{x}/{y}.pbf', + sharedDecodedCache, + fetchImpl: async (url) => + new Response( + String(url).includes('/6746.pbf') ? secondBytes : firstBytes, + ), + }); + const box = { south: 30.24, north: 30.29, west: -97.75, east: -97.73 }; + try { + await source.fetchBounds(box, { + zoom: 14, + tiles: [{ z: 14, x: 3743, y: 6745 }], + }); + const firstHandles = [...scheduled.keys()]; + assert.ok(firstHandles.length >= 2); + + await source.fetchBounds(box, { + zoom: 14, + tiles: [{ z: 14, x: 3743, y: 6746 }], + }); + assert.ok(firstHandles.every((handle) => cancelled.has(handle))); + for (const handle of firstHandles) scheduled.get(handle)(); + assert.equal(projectionRuns, 0, 'late callbacks after eviction are inert'); + + const secondHandles = [...scheduled.keys()].filter( + (handle) => !firstHandles.includes(handle), + ); + sharedDecodedCache.clear(); + assert.ok(secondHandles.every((handle) => cancelled.has(handle))); + for (const handle of secondHandles) scheduled.get(handle)(); + assert.equal(projectionRuns, 0, 'late callbacks after clear are inert'); + + const selfEvictingCache = createSharedDecodedTileCache({ + maxEntries: 1, + maxCacheBytes: 100_000, + }); + const selfEvictingSource = createOpenFreeMapSource({ + template: 'https://tiles.openfreemap.org/planet/v/self/{z}/{x}/{y}.pbf', + sharedDecodedCache: selfEvictingCache, + fetchImpl: async () => new Response(firstBytes), + }); + const beforeSelfEviction = nextHandle; + await selfEvictingSource.fetchBounds(box, { + zoom: 14, + tiles: [{ z: 14, x: 3743, y: 6745 }], + }); + const selfEvictingHandles = [...scheduled.keys()].filter( + (handle) => handle >= beforeSelfEviction, + ); + scheduled.get(selfEvictingHandles.at(-1))(); + assert.equal(projectionRuns, 1); + assert.deepEqual(selfEvictingCache.stats(), { entries: 0, bytes: 0 }); + assert.ok( + selfEvictingHandles.slice(0, -1).every((handle) => cancelled.has(handle)), + ); + for (const handle of selfEvictingHandles) + if (!cancelled.has(handle)) scheduled.get(handle)(); + assert.equal(projectionRuns, 1, 'self-eviction retires sibling idle work'); + } finally { + source.clear(); + sharedDecodedCache.clear(); + globalThis.requestIdleCallback = previousIdle; + globalThis.cancelIdleCallback = previousCancelIdle; + } +}); diff --git a/src/sources/phaseTiming.js b/src/sources/phaseTiming.js index 79848a27b..a75c7ed42 100644 --- a/src/sources/phaseTiming.js +++ b/src/sources/phaseTiming.js @@ -1,7 +1,7 @@ /** Optional development phase measurements; no provider payloads or credentials. */ -export function phaseTiming(phase, start, detail = {}) { +export function phaseTiming(phase, start, detail = {}, scope = 'roads') { if (import.meta.env?.DEV) - performance.measure(`roads:${phase}`, { + performance.measure(`${scope}:${phase}`, { start, end: performance.now(), detail, diff --git a/src/sources/vectorTiles.js b/src/sources/vectorTiles.js index 7eef0d3e7..4d4e70678 100644 --- a/src/sources/vectorTiles.js +++ b/src/sources/vectorTiles.js @@ -16,6 +16,131 @@ export function validTileBounds(box) { ); } +/** + * Bounded decoded ownership shared by compatible vector-tile consumers. + * Entries are keyed by the immutable tile template plus XYZ identity; callers + * still keep their own consumer-shaped caches and request lifetimes. + */ +export function createSharedDecodedTileCache({ + maxEntries = 192, + maxCacheBytes = 64 * 1024 * 1024, +} = {}) { + const cache = new Map(); + const flights = new Map(); + let cacheBytes = 0; + let generation = 0; + const release = (entry) => { + if (!entry || entry.released) return; + entry.released = true; + entry.value?.releaseSharedCacheEntry?.(); + }; + const remove = (key, expected = null) => { + const entry = cache.get(key); + if (!entry || (expected && entry !== expected)) return false; + cache.delete(key); + cacheBytes -= entry.size; + release(entry); + return true; + }; + const evict = () => { + while (cache.size > maxEntries || cacheBytes > maxCacheBytes) { + const oldest = cache.keys().next().value; + remove(oldest); + } + }; + const resize = (key, value, entry, size) => { + if (!Number.isFinite(size) || size < 0) return; + const previousSize = entry.size; + entry.size = size; + if (cache.get(key) !== entry || entry.value !== value) return; + cacheBytes += size - previousSize; + evict(); + }; + const store = (key, value, size) => { + const entry = { value, size, released: false }; + if (!Number.isFinite(size) || size < 0 || size > maxCacheBytes) + return entry; + remove(key); + cache.set(key, entry); + cacheBytes += size; + try { + value?.bindSharedCacheEntry?.( + (nextSize) => resize(key, value, entry, nextSize), + size, + ); + } catch (error) { + remove(key, entry); + throw error; + } + evict(); + return entry; + }; + const subscribe = (flight, signal, key) => { + signal?.throwIfAborted(); + flight.users += 1; + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason); + signal?.addEventListener('abort', abort, { once: true }); + flight.promise + .then(resolve, reject) + .finally(() => signal?.removeEventListener('abort', abort)); + }).finally(() => { + flight.users -= 1; + if (!flight.users && flights.get(key) === flight) { + flights.delete(key); + flight.controller.abort(); + } + }); + }; + return { + get(key) { + const hit = cache.get(key); + if (!hit) return null; + cache.delete(key); + cache.set(key, hit); + return hit; + }, + set(key, value, size) { + return store(key, value, size); + }, + load(key, signal, loader) { + const hit = this.get(key); + if (hit) return Promise.resolve(hit); + let flight = flights.get(key); + if (!flight) { + flight = { + controller: new AbortController(), + users: 0, + promise: null, + generation, + }; + flights.set(key, flight); + flight.promise = Promise.resolve() + .then(() => loader(flight.controller.signal)) + .then(({ value, size }) => { + flight.controller.signal.throwIfAborted(); + if (flight.generation !== generation) + throw new DOMException('Shared tile cache cleared', 'AbortError'); + return store(key, value, size); + }) + .finally(() => { + if (flights.get(key) === flight) flights.delete(key); + }); + } + return subscribe(flight, signal, key); + }, + clear() { + generation += 1; + for (const flight of flights.values()) flight.controller.abort(); + flights.clear(); + for (const entry of cache.values()) release(entry); + cache.clear(); + cacheBytes = 0; + }, + stats: () => ({ entries: cache.size, bytes: cacheBytes }), + }; +} + /** A decoded XYZ cache with bounded workers, reads, retained bytes and request lifetimes. */ export function createVectorTileSource({ tileJsonUrl, @@ -31,6 +156,8 @@ export function createVectorTileSource({ maxResponseBytes = 4 * 1024 * 1024, metadataCooldownMs = 5000, now = Date.now, + sharedDecodedCache = null, + selectDecoded = (value) => value, }) { const cache = new Map(); const active = new Set(); @@ -44,13 +171,15 @@ export function createVectorTileSource({ let fetched = 0; let activeWorkers = 0; const waiters = []; + const consumerValue = (value) => + sharedDecodedCache ? structuredClone(value) : value; - async function request(url, parentSignal, read) { + async function request(url, parentSignal, read, trackAsSource = true) { const controller = new AbortController(); const abort = () => controller.abort(parentSignal?.reason); parentSignal?.throwIfAborted(); parentSignal?.addEventListener('abort', abort, { once: true }); - active.add(controller); + if (trackAsSource) active.add(controller); const timer = setTimeout( () => controller.abort( @@ -88,7 +217,7 @@ export function createVectorTileSource({ } finally { clearTimeout(timer); parentSignal?.removeEventListener('abort', abort); - active.delete(controller); + if (trackAsSource) active.delete(controller); } } @@ -232,7 +361,7 @@ export function createVectorTileSource({ const abort = () => reject(signal.reason); signal?.addEventListener('abort', abort, { once: true }); flight.promise - .then(resolve, reject) + .then((value) => resolve(consumerValue(value)), reject) .finally(() => signal?.removeEventListener('abort', abort)); }).finally(() => { if (--flight.users === 0 && flights.get(key) === flight) { @@ -249,54 +378,102 @@ export function createVectorTileSource({ if (hit && now() - hit.at < ttlMs) { cache.delete(key); cache.set(key, hit); - return hit.value; + return consumerValue(hit.value); } + if (hit) { + cache.delete(key); + cacheBytes -= hit.size; + } + const shared = sharedDecodedCache?.get(key); + if (shared) { + // A shared decode is authoritative cache state. Give each consumer its + // own value so traffic, outlines, or a future caller cannot mutate the + // cached object (including nested coordinate arrays) for everyone else. + const value = structuredClone(selectDecoded(shared.value, tile)); + const size = shared.size; + if (epoch === generation && size <= maxCacheBytes) { + cache.set(key, { at: now(), value, size }); + cacheBytes += size; + while (cache.size > maxEntries || cacheBytes > maxCacheBytes) { + const oldest = cache.keys().next().value; + cacheBytes -= cache.get(oldest).size; + cache.delete(oldest); + } + } + return consumerValue(value); + } + const loadDecoded = async (ownedSignal, sharedOwnership = false) => { + await acquire(ownedSignal); + try { + ownedSignal.throwIfAborted(); + fetched++; + const url = meta.template + .replace('{z}', tile.z) + .replace('{x}', tile.x) + .replace('{y}', tile.y); + const fetchStart = performance.now(); + const bytes = await request( + url, + ownedSignal, + async (res, requestSignal) => { + const bodyStart = performance.now(); + const value = await readResponseBytesCapped( + res, + maxResponseBytes, + requestSignal, + ); + phaseTiming('tile-body-read', bodyStart, { + source: allowedOrigin, + key, + bytes: value.byteLength, + }); + return value; + }, + !sharedOwnership, + ); + phaseTiming('tile-fetch', fetchStart, { + source: allowedOrigin, + key, + bytes: bytes.byteLength, + }); + const decodeStart = performance.now(); + const decoded = decode(bytes, tile.z, tile.x, tile.y); + phaseTiming('decode', decodeStart, { source: allowedOrigin, key }); + ownedSignal.throwIfAborted(); + // Body-based decoded-storage estimate; projections may increase it. + return { value: decoded, size: bytes.byteLength * 4 }; + } finally { + release(); + } + }; let flight = flights.get(key); if (!flight) { flight = { controller: new AbortController(), users: 0 }; const ownedSignal = flight.controller.signal; flights.set(key, flight); flight.promise = (async () => { - await acquire(ownedSignal); - try { - ownedSignal.throwIfAborted(); - if (epoch !== generation) - throw new DOMException('Source cleared', 'AbortError'); - fetched++; - const url = meta.template - .replace('{z}', tile.z) - .replace('{x}', tile.x) - .replace('{y}', tile.y); - const fetchStart = performance.now(); - const bytes = await request(url, ownedSignal, (res, requestSignal) => - readResponseBytesCapped(res, maxResponseBytes, requestSignal), - ); - phaseTiming('tile-fetch', fetchStart, { - source: allowedOrigin, - key, - bytes: bytes.byteLength, - }); - const decodeStart = performance.now(); - const value = decode(bytes, tile.z, tile.x, tile.y); - phaseTiming('decode', decodeStart, { source: allowedOrigin, key }); - ownedSignal.throwIfAborted(); - // Body-based decoded-storage estimate; never stringify geometry on the load path. - const size = bytes.byteLength * 4; - if (epoch === generation && size <= maxCacheBytes) { - if (cache.has(key)) cacheBytes -= cache.get(key).size; - cache.delete(key); - cache.set(key, { at: now(), value, size }); - cacheBytes += size; - while (cache.size > maxEntries || cacheBytes > maxCacheBytes) { - const oldest = cache.keys().next().value; - cacheBytes -= cache.get(oldest).size; - cache.delete(oldest); - } + const decodedEntry = sharedDecodedCache + ? await sharedDecodedCache.load(key, ownedSignal, (sharedSignal) => + loadDecoded(sharedSignal, true), + ) + : await loadDecoded(ownedSignal); + ownedSignal.throwIfAborted(); + const selected = selectDecoded(decodedEntry.value, tile); + const value = sharedDecodedCache ? structuredClone(selected) : selected; + const currentShared = sharedDecodedCache?.get(key); + const size = currentShared?.size ?? decodedEntry.size; + if (epoch === generation && size <= maxCacheBytes) { + if (cache.has(key)) cacheBytes -= cache.get(key).size; + cache.delete(key); + cache.set(key, { at: now(), value, size }); + cacheBytes += size; + while (cache.size > maxEntries || cacheBytes > maxCacheBytes) { + const oldest = cache.keys().next().value; + cacheBytes -= cache.get(oldest).size; + cache.delete(oldest); } - return value; - } finally { - release(); } + return value; })().finally(() => { if (flights.get(key) === flight) flights.delete(key); }); diff --git a/src/sources/vectorTiles.test.mjs b/src/sources/vectorTiles.test.mjs index bfc366964..c17661f35 100644 --- a/src/sources/vectorTiles.test.mjs +++ b/src/sources/vectorTiles.test.mjs @@ -1,6 +1,9 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { createVectorTileSource } from './vectorTiles.js'; +import { + createSharedDecodedTileCache, + createVectorTileSource, +} from './vectorTiles.js'; const meta = { tiles: ['https://tiles.example/{z}/{x}/{y}.pbf'] }; const box = { south: 30.267, north: 30.268, west: -97.744, east: -97.743 }; const options = { @@ -203,3 +206,238 @@ test('polygon-selected tiles keep nearest-first order and cached revisits issue 'over-budget selection is rejected before fetching', ); }); + +test('compatible consumers share bounded decoded ownership without sharing local lifetimes', async () => { + const sharedDecodedCache = createSharedDecodedTileCache({ + maxEntries: 2, + maxCacheBytes: 100, + }); + let decodes = 0, + requests = 0; + const common = { + ...options, + template: meta.tiles[0], + sharedDecodedCache, + decode: () => { + decodes++; + return { roads: ['road'], outlines: ['outline'] }; + }, + fetchImpl: async () => { + requests++; + return new Response(new Uint8Array([1, 2, 3])); + }, + }; + const roads = createVectorTileSource({ + ...common, + selectDecoded: (value) => value.roads, + }); + const outlines = createVectorTileSource({ + ...common, + selectDecoded: (value) => value.outlines, + }); + const tiles = [{ z: 14, x: 3743, y: 6745 }]; + assert.deepEqual((await roads.fetchBounds(box, { zoom: 14, tiles })).tiles, [ + ['road'], + ]); + roads.clear(); + assert.deepEqual( + (await outlines.fetchBounds(box, { zoom: 14, tiles })).tiles, + [['outline']], + ); + assert.equal(requests, 1); + assert.equal(decodes, 1); + assert.deepEqual(sharedDecodedCache.stats(), { entries: 1, bytes: 12 }); +}); + +test('shared decoded tiles are isolated from consumer mutation', async () => { + const sharedDecodedCache = createSharedDecodedTileCache(); + let decodes = 0, + requests = 0; + const common = { + ...options, + template: meta.tiles[0], + sharedDecodedCache, + decode: () => { + decodes++; + return { roads: [{ name: 'Congress Avenue', coordinates: [[-97, 30]] }] }; + }, + selectDecoded: (value) => value.roads, + fetchImpl: async () => { + requests++; + return new Response(new Uint8Array([1, 2, 3])); + }, + }; + const first = createVectorTileSource(common); + const second = createVectorTileSource(common); + const third = createVectorTileSource(common); + const tiles = [{ z: 14, x: 3743, y: 6745 }]; + const firstResult = await first.fetchBounds(box, { zoom: 14, tiles }); + firstResult.tiles[0][0].name = 'first mutation'; + firstResult.tiles[0][0].coordinates[0][0] = 0; + + const secondResult = await second.fetchBounds(box, { zoom: 14, tiles }); + assert.equal(secondResult.tiles[0][0].name, 'Congress Avenue'); + assert.equal(secondResult.tiles[0][0].coordinates[0][0], -97); + secondResult.tiles[0][0].name = 'second mutation'; + secondResult.tiles[0][0].coordinates[0][0] = 1; + + const thirdResult = await third.fetchBounds(box, { zoom: 14, tiles }); + assert.equal(thirdResult.tiles[0][0].name, 'Congress Avenue'); + assert.equal(thirdResult.tiles[0][0].coordinates[0][0], -97); + assert.equal(requests, 1); + assert.equal(decodes, 1); +}); + +test('one source isolates in-flight and completed cache results from consumer mutation', async () => { + const sharedDecodedCache = createSharedDecodedTileCache(); + let decodes = 0, + requests = 0, + releaseResponse; + const responseReady = new Promise((resolve) => { + releaseResponse = resolve; + }); + let requestStarted; + const started = new Promise((resolve) => { + requestStarted = resolve; + }); + const source = createVectorTileSource({ + ...options, + template: meta.tiles[0], + sharedDecodedCache, + decode: () => { + decodes++; + return { roads: [{ name: 'Congress Avenue', coordinates: [[-97, 30]] }] }; + }, + selectDecoded: (value) => value.roads, + fetchImpl: async () => { + requests++; + requestStarted(); + await responseReady; + return new Response(new Uint8Array([1, 2, 3])); + }, + }); + const tiles = [{ z: 14, x: 3743, y: 6745 }]; + const firstPending = source.fetchBounds(box, { zoom: 14, tiles }); + await started; + const concurrentPending = source.fetchBounds(box, { zoom: 14, tiles }); + releaseResponse(); + + const firstResult = await firstPending; + const concurrentResult = await concurrentPending; + firstResult.tiles[0][0].name = 'first mutation'; + firstResult.tiles[0][0].coordinates[0][0] = 0; + assert.equal(concurrentResult.tiles[0][0].name, 'Congress Avenue'); + assert.equal(concurrentResult.tiles[0][0].coordinates[0][0], -97); + + concurrentResult.tiles[0][0].name = 'concurrent mutation'; + concurrentResult.tiles[0][0].coordinates[0][0] = 1; + const cachedResult = await source.fetchBounds(box, { zoom: 14, tiles }); + assert.equal(cachedResult.tiles[0][0].name, 'Congress Avenue'); + assert.equal(cachedResult.tiles[0][0].coordinates[0][0], -97); + assert.equal(requests, 1); + assert.equal(decodes, 1); +}); + +test('compatible consumers share one in-flight fetch and survive one consumer clearing', async () => { + const sharedDecodedCache = createSharedDecodedTileCache(); + let requests = 0, + decodes = 0, + releaseResponse; + const responseReady = new Promise((resolve) => { + releaseResponse = resolve; + }); + let requestStarted; + const started = new Promise((resolve) => { + requestStarted = resolve; + }); + const common = { + ...options, + template: meta.tiles[0], + sharedDecodedCache, + decode: () => { + decodes++; + return { roads: ['road'], outlines: ['outline'] }; + }, + fetchImpl: async () => { + requests++; + requestStarted(); + await responseReady; + return new Response(new Uint8Array([1, 2, 3])); + }, + }; + const roads = createVectorTileSource({ + ...common, + selectDecoded: (value) => value.roads, + }); + const outlines = createVectorTileSource({ + ...common, + selectDecoded: (value) => value.outlines, + }); + const tiles = [{ z: 14, x: 3743, y: 6745 }]; + const roadResult = roads.fetchBounds(box, { zoom: 14, tiles }); + const roadRejected = assert.rejects(roadResult, { name: 'AbortError' }); + await started; + const outlineResult = outlines.fetchBounds(box, { zoom: 14, tiles }); + await new Promise((resolve) => setImmediate(resolve)); + roads.clear(); + releaseResponse(); + await roadRejected; + assert.deepEqual((await outlineResult).tiles, [['outline']]); + assert.equal(requests, 1); + assert.equal(decodes, 1); +}); + +test('shared ownership recharges materialized projections and evicts over budget', () => { + const shared = createSharedDecodedTileCache({ maxCacheBytes: 100 }); + let resize, + releases = 0; + const decoded = { + bindSharedCacheEntry(update) { + resize = update; + }, + releaseSharedCacheEntry() { + releases++; + }, + }; + const entry = shared.set('tile', decoded, 12); + assert.deepEqual(shared.stats(), { entries: 1, bytes: 12 }); + resize(80); + assert.deepEqual(shared.stats(), { entries: 1, bytes: 80 }); + resize(101); + assert.deepEqual(shared.stats(), { entries: 0, bytes: 0 }); + assert.equal(entry.size, 101, 'the returned entry retains its real size'); + assert.equal(releases, 1, 'self-eviction releases decoded ownership once'); + shared.clear(); + assert.equal(releases, 1, 'clearing after eviction does not release twice'); +}); + +test('shared clear releases entries and prevents a pending loader from refilling', async () => { + const shared = createSharedDecodedTileCache(); + let releases = 0; + const decoded = { + releaseSharedCacheEntry() { + releases++; + }, + }; + shared.set('resident', decoded, 12); + shared.clear(); + assert.deepEqual(shared.stats(), { entries: 0, bytes: 0 }); + assert.equal(releases, 1); + + let loaderStarted, resolveLoader; + const started = new Promise((resolve) => { + loaderStarted = resolve; + }); + const loaderResult = new Promise((resolve) => { + resolveLoader = resolve; + }); + const pending = shared.load('late', null, async () => { + loaderStarted(); + return loaderResult; + }); + await started; + resolveLoader({ value: {}, size: 16 }); + shared.clear(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.deepEqual(shared.stats(), { entries: 0, bytes: 0 }); +}); diff --git a/src/standalone/layerSources.js b/src/standalone/layerSources.js index 1856dfb69..f5b4b26b0 100644 --- a/src/standalone/layerSources.js +++ b/src/standalone/layerSources.js @@ -21,8 +21,13 @@ import { createReferenceSources } from '../sources/reference.js'; export { createReferenceSources as createStandaloneReferenceSources } from '../sources/reference.js'; /** Select standalone providers without starting their acquisition. */ -export function createStandaloneLayerSources() { - const mapTiles = createOpenFreeMapSource(); +export function createStandaloneLayerSources({ + createMapTiles = createOpenFreeMapSource, +} = {}) { + // Each subsystem owns its own request lifetime and local result cache. The + // sources still share the bounded decoded-tile owner inside OpenFreeMap. + const trafficTiles = createMapTiles(); + const installationTiles = createMapTiles(); return { ...createReferenceSources(), flights: createOpenSkySource(), @@ -32,10 +37,10 @@ export function createStandaloneLayerSources() { }), cctv: createCctvSource(), radio: createRadioSource(), - traffic: createTrafficSource({ mapTiles }), + traffic: createTrafficSource({ mapTiles: trafficTiles }), transit: createTransitSource(), bikeshare: createBikeshareSource(), - installations: createInstallationSource({ mapTiles }), + installations: createInstallationSource({ mapTiles: installationTiles }), satellites: createSatelliteSource(), launches: createLaunchSource(), alpr: createAlprTileSource(), diff --git a/src/standalone/layerSources.test.mjs b/src/standalone/layerSources.test.mjs new file mode 100644 index 000000000..72dd2d35c --- /dev/null +++ b/src/standalone/layerSources.test.mjs @@ -0,0 +1,38 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createStandaloneLayerSources } from './layerSources.js'; + +test('traffic and installations own separate local OpenFreeMap lifetimes', () => { + const mapTiles = []; + const sources = createStandaloneLayerSources({ + createMapTiles() { + const source = { + clears: 0, + clear() { + this.clears++; + }, + getMetadata: async () => ({ + template: 'https://tiles.openfreemap.org/planet/test/{z}/{x}/{y}.pbf', + }), + fetchBounds: async () => ({ tiles: [], loadedTiles: [] }), + }; + mapTiles.push(source); + return source; + }, + }); + + assert.equal(mapTiles.length, 2); + assert.notEqual(mapTiles[0], mapTiles[1]); + sources.traffic.resetFlowTileCache(); + assert.deepEqual( + mapTiles.map((source) => source.clears), + [1, 0], + 'traffic reset cannot clear installation work', + ); + sources.installations.destroy(); + assert.deepEqual( + mapTiles.map((source) => source.clears), + [1, 1], + 'installation destroy cannot clear traffic work', + ); +}); diff --git a/src/tooling/nominatimGuardLifecycle.test.mjs b/src/tooling/nominatimGuardLifecycle.test.mjs index f6f1b8c73..444a89ab0 100644 --- a/src/tooling/nominatimGuardLifecycle.test.mjs +++ b/src/tooling/nominatimGuardLifecycle.test.mjs @@ -25,6 +25,7 @@ import { } from '../../server/providers/regional/place.js'; const PUBLIC = { endpoint: PUBLIC_NOMINATIM_SEARCH, isPublic: true, dailyCap: 50 }; +const LOCAL = { ...PUBLIC, isPublic: false }; const tempDir = (t) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-')); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -117,21 +118,103 @@ test('a sweep removes expired and corrupt entries, never ones it could not read' assert.deepEqual([1, 2, 3].map((n) => fs.existsSync(file(n))), [true, false, false]); }); -test('processes sharing the state file add to one count', async (t) => { - const file = path.join(tempDir(t), 'state.json'); +test('processes sharing state reserve public spacing and cap atomically', async (t) => { + const dir = tempDir(t); const module = new URL('../../server/providers/regional/nominatimGate.js', import.meta.url).href; - const script = ` + const run = (file, dailyCap) => { + const script = ` const { createGateStateStore } = await import(${JSON.stringify(module)}); const store = createGateStateStore({ file: ${JSON.stringify(file)} }); - for (let i = 0; i < 25; i++) store.increment('2026-09-29'); + console.log(JSON.stringify(store.reserve('2026-09-29', { + now: 1000, + dailyCap: ${dailyCap}, + minSpacingMs: 1100, + latestStartAt: 10000, + }))); `; - const run = () => - new Promise((resolve, reject) => { - const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: 'inherit' }); - child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`exit ${code}`)))); + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script]); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => (stdout += chunk)); + child.stderr.on('data', (chunk) => (stderr += chunk)); + child.on('exit', (code) => + code === 0 + ? resolve(JSON.parse(stdout)) + : reject(new Error(`exit ${code}: ${stderr}`)), + ); }); - await Promise.all([run(), run(), run()]); - assert.equal(createGateStateStore({ file }).count('2026-09-29'), 75); + }; + + const spacedFile = path.join(dir, 'spaced.json'); + const reservations = await Promise.all([ + run(spacedFile, 3), + run(spacedFile, 3), + run(spacedFile, 3), + ]); + assert.deepEqual( + reservations.map((result) => result.status), + ['reserved', 'reserved', 'reserved'], + ); + assert.deepEqual( + reservations.map((result) => result.startAt).sort((a, b) => a - b), + [1000, 2100, 3200], + ); + assert.equal(createGateStateStore({ file: spacedFile }).count('2026-09-29'), 3); + + const cappedFile = path.join(dir, 'capped.json'); + const capped = await Promise.all([run(cappedFile, 1), run(cappedFile, 1)]); + assert.deepEqual( + capped.map((result) => result.status).sort(), + ['cap', 'reserved'], + ); + assert.equal(createGateStateStore({ file: cappedFile }).count('2026-09-29'), 1); +}); + +test('independent public gates sharing state enforce spacing and cap', async (t) => { + const dir = tempDir(t); + const starts = []; + const fetchImpl = async () => { + starts.push(Date.now()); + return json([]); + }; + const file = path.join(dir, 'spacing.json'); + const make = (options = {}) => + createNominatimGate({ + settings: PUBLIC, + fetchImpl, + usage: createGateStateStore({ file: options.file || file }), + minSpacingMs: options.minSpacingMs || 40, + }); + await Promise.all([ + createNominatimSearchProvider({ gate: make() })('one', null), + createNominatimSearchProvider({ gate: make() })('two', null), + ]); + starts.sort((a, b) => a - b); + assert.ok(starts[1] - starts[0] >= 35, `${starts[1] - starts[0]}ms apart`); + + const capFile = path.join(dir, 'cap.json'); + let sent = 0; + const capGate = () => + createNominatimGate({ + settings: { ...PUBLIC, dailyCap: 1 }, + fetchImpl: async () => { + sent += 1; + return json([]); + }, + usage: createGateStateStore({ file: capFile }), + minSpacingMs: 1, + }); + const results = await Promise.allSettled([ + createNominatimSearchProvider({ gate: capGate() })('three', null), + createNominatimSearchProvider({ gate: capGate() })('four', null), + ]); + assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1); + assert.equal( + results.find((result) => result.status === 'rejected').reason.code, + 'NOMINATIM_DAILY_CAP', + ); + assert.equal(sent, 1); }); test('a persistence failure is reported and never lowers the count', (t) => { @@ -151,6 +234,49 @@ test('a persistence failure is reported and never lowers the count', (t) => { assert.equal(errors.length, 1, 'reported once, not per request'); }); +test('parseable but invalid public usage state fails closed', async (t) => { + const dir = tempDir(t); + const file = path.join(dir, 'state.json'); + fs.writeFileSync( + file, + JSON.stringify({ + day: '2026-09-30', + count: '50', + pausedUntil: '9999999999999', + }), + ); + let sent = 0; + const gate = createNominatimGate({ + settings: PUBLIC, + fetchImpl: async () => { + sent += 1; + return json([]); + }, + usage: createGateStateStore({ file, onError: () => {} }), + ...quick, + }); + await assert.rejects( + gate.requestJson((endpoint) => `${endpoint}?q=x`), + { code: 'NOMINATIM_STATE_UNAVAILABLE' }, + ); + assert.equal(sent, 0); + + fs.writeFileSync( + file, + JSON.stringify({ day: '2026-09-30', count: 1, pausedUntil: 0 }), + ); + const migrated = createGateStateStore({ file }); + assert.equal( + migrated.reserve('2026-09-30', { + now: 1000, + dailyCap: 50, + minSpacingMs: 1100, + }).status, + 'reserved', + 'the original state format without nextStartAt remains supported', + ); +}); + test('a long Retry-After is honoured in full and survives a restart', async (t) => { const file = path.join(tempDir(t), 'state.json'); let clock = Date.UTC(2026, 8, 29, 12); @@ -189,6 +315,35 @@ test('a long Retry-After is honoured in full and survives a restart', async (t) assert.equal(calls.length, 1); }); +test('an operator refusal pauses only the endpoint that refused', async () => { + let active = { + endpoint: 'https://a.example.test/search', + isPublic: false, + dailyCap: 50, + }; + const calls = []; + const gate = createNominatimGate({ + settings: () => active, + fetchImpl: async (url) => { + calls.push(new URL(url).host); + return calls.length === 1 + ? new Response('', { + status: 429, + headers: { 'Retry-After': '60' }, + }) + : json([]); + }, + ...quick, + }); + const request = () => gate.requestJson((endpoint) => `${endpoint}?q=x`); + await assert.rejects(request(), { code: 'NOMINATIM_REFUSED' }); + active = { ...active, endpoint: 'https://b.example.test/search' }; + assert.deepEqual(await request(), []); + active = { ...active, endpoint: 'https://a.example.test/search' }; + await assert.rejects(request(), { code: 'NOMINATIM_PAUSED' }); + assert.deepEqual(calls, ['a.example.test', 'b.example.test']); +}); + test('a stalled body times out instead of holding the shared queue', async () => { let cancelled = 0; const stalled = (signal) => { @@ -217,7 +372,7 @@ test('a stalled body times out instead of holding the shared queue', async () => test('cache keys are request-equivalent: a miss for "Paris???" never answers "Paris"', async () => { const calls = []; const gate = createNominatimGate({ - settings: PUBLIC, + settings: LOCAL, fetchImpl: async (url) => { calls.push(new URL(url).searchParams.get('q')); return json(calls.length === 1 ? [] : [{ lat: '48.85', lon: '2.35', display_name: 'Paris, France', name: 'Paris', addresstype: 'city' }]); @@ -236,7 +391,7 @@ test('a malformed answer is a failure, not a cached "no result"', async () => { let shape = { error: 'Unexpected' }; let calls = 0; const gate = createNominatimGate({ - settings: PUBLIC, + settings: LOCAL, fetchImpl: async () => { calls += 1; return json(shape); @@ -301,9 +456,29 @@ test('administrative boundaries must be at the asked level', () => { const state = boundary({ name: 'Texas', addresstype: 'state', place_rank: 8 }); assert.equal(selectOutlineResult([state], { kind: 'city', query: 'Texas' }).outline, null); assert.equal(selectOutlineResult([city], { kind: 'admin', query: 'Austin' }).outline, null); + assert.equal( + selectOutlineResult( + [boundary({ name: 'Travis', addresstype: 'county', place_rank: 12 })], + { kind: 'city', query: 'Travis' }, + ).outline, + null, + ); + assert.equal( + selectOutlineResult( + [boundary({ name: 'Austin', addresstype: 'city', place_rank: 12 })], + { kind: 'admin', query: 'Austin' }, + ).outline, + null, + ); assert.equal(selectOutlineResult([state], { kind: 'neighborhood', query: 'Texas' }).outline, null); assert.ok(selectOutlineResult([county], { kind: 'admin', query: 'Travis County' }).outline); assert.ok(selectOutlineResult([boundary({ name: 'Somewhere', place_rank: 16 })], { kind: 'city', query: 'Somewhere' }).outline); + assert.ok( + selectOutlineResult( + [boundary({ name: 'Somewhere', addresstype: 'locality', place_rank: 16 })], + { kind: 'city', query: 'Somewhere' }, + ).outline, + ); assert.equal( selectOutlineResult([boundary({ name: 'Somewhere' })], { kind: 'city', query: 'Somewhere' }).outline, null, @@ -317,6 +492,103 @@ test('administrative boundaries must be at the asked level', () => { assert.equal(picked.outline.osm, 'relation/9'); }); +test('admin identity comes only from unambiguous evidence on the selected row', () => { + for (const [name, addresstype, adminLevel] of [ + ['Nepal', 'country', 'country'], + ['Bagmati Province', 'province', 'admin1'], + ['Province of Bagmati', 'province', 'admin1'], + ['Texas', 'state', 'admin1'], + ['Travis County', 'county', 'admin2'], + ['County of Travis', 'county', 'admin2'], + ['Kathmandu District', 'state_district', 'admin2'], + ]) { + const { outline } = selectOutlineResult( + [boundary({ name, addresstype })], + { kind: 'admin', query: name }, + ); + assert.equal(outline.adminArea, name); + assert.equal(outline.adminLevel, adminLevel); + } + + for (const row of [ + boundary({ name: 'Northern Region', addresstype: 'region', place_rank: 8 }), + boundary({ name: 'Central District', addresstype: 'district', place_rank: 10 }), + boundary({ name: 'Mystery Province', place_rank: 8 }), + boundary({ name: 'Example State', addresstype: 'county' }), + boundary({ name: 'County of Example', addresstype: 'state' }), + boundary({ name: 'Province of Example', addresstype: 'county' }), + boundary({ name: 'Example County', addresstype: 'state' }), + boundary({ name: 'Example Province', addresstype: 'county' }), + boundary({ name: 'County of Example Province', addresstype: 'county' }), + ]) { + const { outline } = selectOutlineResult([row], { + kind: 'admin', + query: row.name, + }); + assert.ok(outline, row.name); + assert.equal(outline.adminArea, undefined, row.name); + assert.equal(outline.adminLevel, undefined, row.name); + } + + const retired = selectOutlineResult( + [boundary({ name: 'Bagmati Zone', addresstype: 'state' })], + { kind: 'admin', query: 'Bagmati Zone' }, + ).outline; + assert.equal(retired.adminArea, 'Bagmati Zone'); + assert.equal(retired.adminLevel, undefined); + + const retiredPrefix = selectOutlineResult( + [boundary({ name: 'Zone of Bagmati', addresstype: 'state' })], + { kind: 'admin', query: 'Zone of Bagmati' }, + ).outline; + assert.equal(retiredPrefix.adminArea, 'Zone of Bagmati'); + assert.equal(retiredPrefix.adminLevel, undefined); +}); + +test('admin identity validates the full name before display truncation', () => { + const longValidName = `Province of ${'Example '.repeat(24)}`.trim(); + const valid = selectOutlineResult( + [boundary({ name: longValidName, addresstype: 'province' })], + { kind: 'admin', query: longValidName }, + ).outline; + assert.equal(valid.adminArea, longValidName.slice(0, 160)); + assert.equal(valid.adminLevel, 'admin1'); + + const longContradictoryName = `${'Very Long Administrative Name '.repeat(7)}County`; + assert.ok(longContradictoryName.length > 160); + const contradictory = selectOutlineResult( + [boundary({ name: longContradictoryName, addresstype: 'state' })], + { kind: 'admin', query: longContradictoryName }, + ).outline; + assert.ok(contradictory); + assert.equal(contradictory.adminArea, undefined); + assert.equal(contradictory.adminLevel, undefined); +}); + +test('non-administrative outline kinds never manufacture admin identity', () => { + const city = selectOutlineResult( + [boundary({ name: 'County of Austin', addresstype: 'city', place_rank: 16 })], + { kind: 'city', query: 'County of Austin' }, + ).outline; + const park = selectOutlineResult( + [ + { + category: 'leisure', + type: 'park', + name: 'Zilker Park', + lat: '1', + lon: '1', + geojson: { type: 'Polygon', coordinates: square(1, 1, 0.01) }, + }, + ], + { kind: 'landmark', query: 'Zilker Park' }, + ).outline; + for (const outline of [city, park]) { + assert.equal(outline.adminArea, undefined); + assert.equal(outline.adminLevel, undefined); + } +}); + test('building asks need building evidence', () => { const park = { category: 'tourism', type: 'theme_park', name: 'Big Top', lat: '1', lon: '1', geojson: { type: 'Polygon', coordinates: square(1, 1, 0.0001) } }; const camp = { ...park, type: 'camp_site' }; diff --git a/src/tooling/nominatimLockOwnership.test.mjs b/src/tooling/nominatimLockOwnership.test.mjs new file mode 100644 index 000000000..432c64440 --- /dev/null +++ b/src/tooling/nominatimLockOwnership.test.mjs @@ -0,0 +1,235 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { createGateStateStore } from '../../server/providers/regional/nominatimGate.js'; + +const moduleUrl = new URL( + '../../server/providers/regional/nominatimGate.js', + import.meta.url, +).href; + +function startModule(script) { + const child = spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['pipe', 'pipe', 'pipe'], + }); + const lines = []; + const waiters = []; + let stdout = ''; + let stderr = ''; + + const publish = (line) => { + lines.push(line); + for (const waiter of [...waiters]) { + if (!line.startsWith(waiter.prefix)) continue; + waiters.splice(waiters.indexOf(waiter), 1); + clearTimeout(waiter.timer); + waiter.resolve(line); + } + }; + + child.stdout.on('data', (chunk) => { + stdout += chunk; + const parts = stdout.split('\n'); + stdout = parts.pop(); + for (const line of parts) publish(line); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + + return { + child, + lines, + waitFor(prefix, timeoutMs = 3000) { + const existing = lines.find((line) => line.startsWith(prefix)); + if (existing) return Promise.resolve(existing); + return new Promise((resolve, reject) => { + const waiter = { prefix, resolve, reject, timer: null }; + waiter.timer = setTimeout(() => { + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + reject( + new Error( + `timed out waiting for ${prefix}; stderr: ${stderr.trim()}`, + ), + ); + }, timeoutMs); + waiters.push(waiter); + }); + }, + exited: new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => resolve({ code, signal })); + }), + }; +} + +function holderScript(lockFile) { + return ` + import { DatabaseSync } from 'node:sqlite'; + const database = new DatabaseSync(${JSON.stringify(lockFile)}); + database.exec('BEGIN IMMEDIATE'); + process.stdout.write('LOCKED\\n'); + process.stdin.once('data', () => { + database.exec('ROLLBACK'); + database.close(); + process.stdout.write('RELEASED\\n'); + }); + process.stdin.resume(); + `; +} + +function contenderScript(stateFile, lockWaitMs) { + return ` + const { createGateStateStore } = await import(${JSON.stringify(moduleUrl)}); + const store = createGateStateStore({ + file: ${JSON.stringify(stateFile)}, + lockWaitMs: ${lockWaitMs}, + onError: () => {}, + }); + process.stdout.write('ATTEMPTING\\n', () => { + const result = store.reserve('2026-10-01', { + now: 1000, + dailyCap: 50, + minSpacingMs: 1100, + }); + process.stdout.write('RESULT:' + JSON.stringify(result) + '\\n'); + }); + `; +} + +test('a crashed lock owner cannot delete or bypass a successor lock', async (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-lock-')); + const stateFile = path.join(dir, 'state.json'); + const lockFile = `${stateFile}.lock.sqlite`; + const children = []; + t.after(() => { + for (const process of children) { + if (process.exitCode == null && process.signalCode == null) + process.kill(); + } + fs.rmSync(dir, { recursive: true, force: true }); + }); + + const crashedOwner = startModule(holderScript(lockFile)); + children.push(crashedOwner.child); + await crashedOwner.waitFor('LOCKED'); + const originalLock = fs.statSync(lockFile); + + const blockedByCrashedOwner = startModule(contenderScript(stateFile, 0)); + children.push(blockedByCrashedOwner.child); + const blockedResult = JSON.parse( + (await blockedByCrashedOwner.waitFor('RESULT:')).slice('RESULT:'.length), + ); + assert.equal(blockedResult.status, 'unavailable'); + assert.equal((await blockedByCrashedOwner.exited).code, 0); + assert.equal(fs.existsSync(stateFile), false); + + crashedOwner.child.kill(); + await crashedOwner.exited; + + const successorOwner = startModule(holderScript(lockFile)); + children.push(successorOwner.child); + await successorOwner.waitFor('LOCKED'); + assert.equal(fs.statSync(lockFile).ino, originalLock.ino); + + const refused = startModule(contenderScript(stateFile, 0)); + children.push(refused.child); + const refusedResult = JSON.parse( + (await refused.waitFor('RESULT:')).slice('RESULT:'.length), + ); + assert.equal(refusedResult.status, 'unavailable'); + assert.equal((await refused.exited).code, 0); + assert.equal( + createGateStateStore({ file: stateFile }).count('2026-10-01'), + 0, + 'a timed-out contender fails closed without spending a public slot', + ); + assert.equal(fs.statSync(lockFile).ino, originalLock.ino); + + successorOwner.child.stdin.end('release\n'); + await successorOwner.waitFor('RELEASED'); + assert.equal((await successorOwner.exited).code, 0); + const afterRelease = createGateStateStore({ file: stateFile }).reserve( + '2026-10-01', + { + now: 2200, + dailyCap: 50, + minSpacingMs: 1100, + }, + ); + assert.equal(afterRelease.status, 'reserved'); + assert.equal(afterRelease.count, 1); + assert.equal(fs.statSync(lockFile).ino, originalLock.ino); +}); + +test('a public pause stays authoritative while another process owns the lock', async (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-pause-')); + const stateFile = path.join(dir, 'state.json'); + const lockFile = `${stateFile}.lock.sqlite`; + const children = []; + t.after(() => { + for (const process of children) { + if (process.exitCode == null && process.signalCode == null) + process.kill(); + } + fs.rmSync(dir, { recursive: true, force: true }); + }); + + fs.writeFileSync( + stateFile, + JSON.stringify({ + day: '2026-10-01', + count: 1, + pausedUntil: 0, + nextStartAt: 0, + }), + ); + const owner = startModule(holderScript(lockFile)); + children.push(owner.child); + await owner.waitFor('LOCKED'); + + const errors = []; + const blockedWriter = createGateStateStore({ + file: stateFile, + lockWaitMs: 0, + onError: (error) => errors.push(error), + }); + blockedWriter.pauseUntil(9000); + assert.equal(errors.length, 1, 'the blocked JSON consolidation is reported'); + assert.equal( + JSON.parse(fs.readFileSync(stateFile, 'utf8')).pausedUntil, + 0, + 'the lock owner prevents an unsafe JSON overwrite', + ); + + owner.child.stdin.end('release\n'); + await owner.waitFor('RELEASED'); + assert.equal((await owner.exited).code, 0); + + const successor = createGateStateStore({ file: stateFile }); + assert.deepEqual( + successor.reserve('2026-10-01', { + now: 1000, + dailyCap: 50, + minSpacingMs: 1100, + }), + { status: 'paused', retryAfterMs: 8000 }, + 'a fresh process observes the pending refusal pause', + ); + const resumed = successor.reserve('2026-10-01', { + now: 9001, + dailyCap: 50, + minSpacingMs: 1100, + }); + assert.equal(resumed.status, 'reserved'); + assert.equal(resumed.count, 2); + assert.equal( + fs.readdirSync(dir).some((name) => name.startsWith('state.json.pause.')), + false, + 'the next successful transaction consolidates and removes the marker', + ); +}); diff --git a/src/tooling/nominatimOutlineRoute.test.mjs b/src/tooling/nominatimOutlineRoute.test.mjs index 9fcab2dd7..c7633ea8b 100644 --- a/src/tooling/nominatimOutlineRoute.test.mjs +++ b/src/tooling/nominatimOutlineRoute.test.mjs @@ -100,6 +100,23 @@ function stubFetch(respond) { return { calls, fetchImpl }; } +function tempDir(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +function publicGate(t, options = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-gate-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return createNominatimGate({ + ...options, + settings: options.settings || PUBLIC, + usage: + options.usage || createUsageStore({ file: path.join(dir, 'state.json') }), + }); +} + // ---------------------------------------------------------------- settings test('endpoint settings: unset is public, empty disables, operator URLs are not public', () => { @@ -109,7 +126,10 @@ test('endpoint settings: unset is public, empty disables, operator URLs are not dailyCap: 50, }); for (const off of ['', ' ', 'off', 'none', 'disabled']) - assert.equal(resolveNominatimSettings({ NOMINATIM_URL: off }).endpoint, null); + assert.equal( + resolveNominatimSettings({ NOMINATIM_URL: off }).endpoint, + null, + ); const own = resolveNominatimSettings({ NOMINATIM_URL: 'https://geo.example.test', }); @@ -122,33 +142,84 @@ test('endpoint settings: unset is public, empty disables, operator URLs are not 'http://localhost:8080/search', ); // A mistyped value disables rather than silently falling back to public. - for (const bad of ['not a url', 'ftp://x.test', 'https://u:p@x.test', 'https://x.test/search?q=1']) - assert.equal(resolveNominatimSettings({ NOMINATIM_URL: bad }).endpoint, null, bad); + for (const bad of [ + 'not a url', + 'ftp://x.test', + 'https://u:p@x.test', + 'https://x.test/search?q=1', + ]) + assert.equal( + resolveNominatimSettings({ NOMINATIM_URL: bad }).endpoint, + null, + bad, + ); assert.equal( - resolveNominatimSettings({ NOMINATIM_URL: 'https://nominatim.openstreetmap.org/search' }).isPublic, + resolveNominatimSettings({ + NOMINATIM_URL: 'https://nominatim.openstreetmap.org/search', + }).isPublic, true, ); - assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '7' }).dailyCap, 7); - assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '0' }).dailyCap, 0); - assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: 'x' }).dailyCap, 50); + assert.equal( + resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '7' }).dailyCap, + 7, + ); + assert.equal( + resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '0' }).dailyCap, + 0, + ); + assert.equal( + resolveNominatimSettings({ NOMINATIM_DAILY_CAP: 'x' }).dailyCap, + 50, + ); }); test('Retry-After reads seconds and HTTP dates', () => { const now = Date.UTC(2026, 0, 1); + assert.equal(parseRetryAfter('0', now), 0); assert.equal(parseRetryAfter('120', now), 120_000); - assert.equal(parseRetryAfter(new Date(now + 30_000).toUTCString(), now), 30_000); + for (const httpDate of [ + 'Thu, 01 Jan 2026 00:00:30 GMT', + 'Thursday, 01-Jan-26 00:00:30 GMT', + 'Thu Jan 1 00:00:30 2026', + ]) + assert.equal(parseRetryAfter(httpDate, now), 30_000, httpDate); assert.equal(parseRetryAfter(null, now), null); assert.equal(parseRetryAfter('soon', now), null); + for (const malformed of [ + '-1', + ' ', + '1.5', + '1e3', + '9007199254741', + '999999999999999999999999', + '2099-01-01', + 'October 3, 2026', + '2026-10-03T00:00:00Z', + 'Fri, 01 Jan 2026 00:00:30 GMT', + 'Thu, 31 Feb 2026 00:00:30 GMT', + ]) + assert.equal(parseRetryAfter(malformed, now), null, malformed); }); // ---------------------------------------------------------------- the gate -test('search and outlines share one pacer: requests start at least 1.1 s apart', async () => { +test('search and outlines share one pacer: requests start at least 1.1 s apart', async (t) => { const time = clock(); const { calls, fetchImpl } = stubFetch(async (url) => - json(url.includes('polygon_geojson') ? [row({ category: 'boundary', type: 'administrative', addresstype: 'city', name: 'Austin' })] : []), + json( + url.includes('polygon_geojson') + ? [ + row({ + category: 'boundary', + type: 'administrative', + addresstype: 'city', + name: 'Austin', + }), + ] + : [], + ), ); - const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }); + const gate = publicGate(t, { fetchImpl, ...time }); const search = createNominatimSearchProvider({ gate }); const outline = createNominatimOutlineProvider({ gate }); await Promise.all([ @@ -162,10 +233,10 @@ test('search and outlines share one pacer: requests start at least 1.1 s apart', assert.equal(gate.stats().usedToday, 3, 'one daily count for both kinds'); }); -test('the daily cap stops public requests and is shared by search and outlines', async () => { +test('the daily cap stops public requests and is shared by search and outlines', async (t) => { const time = clock(); const { calls, fetchImpl } = stubFetch(async () => json([])); - const gate = createNominatimGate({ + const gate = publicGate(t, { settings: { ...PUBLIC, dailyCap: 2 }, fetchImpl, ...time, @@ -187,21 +258,58 @@ test('the daily cap stops public requests and is shared by search and outlines', assert.equal(calls.length, 3); }); -test('an operator endpoint is not subject to the public cap or spacing', async () => { +test('an operator endpoint is not subject to public cap, spacing, or durable state', async (t) => { const time = clock(); const { calls, fetchImpl } = stubFetch(async () => json([])); + const blocker = path.join(tempDir(t), 'not-a-directory'); + fs.writeFileSync(blocker, 'x'); const gate = createNominatimGate({ settings: { ...OPERATOR, dailyCap: 1 }, fetchImpl, + usage: createUsageStore({ file: path.join(blocker, 'state.json') }), ...time, }); const search = createNominatimSearchProvider({ gate }); await Promise.all([search('a', null), search('b', null), search('c', null)]); assert.equal(calls.length, 3); - assert.ok(calls.every((url) => url.startsWith('https://geo.example.test/search?'))); + assert.ok( + calls.every((url) => url.startsWith('https://geo.example.test/search?')), + ); assert.deepEqual(time.sleeps, []); }); +test('public access fails closed when durable state is missing or unwritable', async (t) => { + const { calls, fetchImpl } = stubFetch(async () => json([])); + const withoutState = createNominatimGate({ + settings: PUBLIC, + fetchImpl, + usage: createUsageStore({ onError: () => {} }), + }); + await assert.rejects( + createNominatimSearchProvider({ gate: withoutState })('missing', null), + { code: 'NOMINATIM_STATE_UNAVAILABLE' }, + ); + + const blocker = path.join(tempDir(t), 'not-a-directory'); + fs.writeFileSync(blocker, 'x'); + const invalidFile = path.join(blocker, 'state.json'); + const makeBrokenGate = () => + createNominatimGate({ + settings: PUBLIC, + fetchImpl, + usage: createUsageStore({ file: invalidFile, onError: () => {} }), + }); + await assert.rejects( + createNominatimSearchProvider({ gate: makeBrokenGate() })('broken', null), + { code: 'NOMINATIM_STATE_UNAVAILABLE' }, + ); + await assert.rejects( + createNominatimSearchProvider({ gate: makeBrokenGate() })('restart', null), + { code: 'NOMINATIM_STATE_UNAVAILABLE' }, + ); + assert.equal(calls.length, 0); +}); + test('a disabled endpoint sends nothing', async () => { const { calls, fetchImpl } = stubFetch(async () => json([])); const gate = createNominatimGate({ @@ -214,14 +322,17 @@ test('a disabled endpoint sends nothing', async () => { assert.equal(calls.length, 0); }); -test('429 with Retry-After pauses the gate and stops queued requests', async () => { +test('429 with Retry-After pauses the gate and stops queued requests', async (t) => { const time = clock(); const { calls, fetchImpl } = stubFetch(async (_url, _o, n) => n === 1 - ? new Response('slow down', { status: 429, headers: { 'Retry-After': '120' } }) + ? new Response('slow down', { + status: 429, + headers: { 'Retry-After': '120' }, + }) : json([]), ); - const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }); + const gate = publicGate(t, { fetchImpl, ...time }); const search = createNominatimSearchProvider({ gate }); const [first, queued] = await Promise.allSettled([ search('first', null), @@ -229,7 +340,11 @@ test('429 with Retry-After pauses the gate and stops queued requests', async () ]); assert.equal(first.reason.code, 'NOMINATIM_REFUSED'); assert.equal(first.reason.retryAfterMs, 120_000); - assert.equal(queued.reason.code, 'NOMINATIM_PAUSED', 'the queued request is not sent'); + assert.equal( + queued.reason.code, + 'NOMINATIM_PAUSED', + 'the queued request is not sent', + ); assert.equal(calls.length, 1); await assert.rejects(search('later', null), { code: 'NOMINATIM_PAUSED' }); assert.equal(calls.length, 1); @@ -241,13 +356,75 @@ test('429 with Retry-After pauses the gate and stops queued requests', async () assert.equal(calls.length, 3); }); -test('an outage stops queued requests; a server error is never cached as not found', async () => { +test('malformed Retry-After values retain the default refusal cooldown', async (t) => { + const malformedValues = [ + '-1', + ' ', + '1.5', + '1e3', + '2099-01-01', + 'October 3, 2026', + ]; + for (const [index, retryAfter] of malformedValues.entries()) { + const time = clock(); + const { fetchImpl } = stubFetch( + async () => + new Response('slow down', { + status: 429, + headers: { 'Retry-After': retryAfter }, + }), + ); + const gate = publicGate(t, { + fetchImpl, + ...time, + usage: createUsageStore({ + file: path.join(tempDir(t), `malformed-${index}.json`), + }), + }); + await assert.rejects( + createNominatimSearchProvider({ gate })(`bad-${index}`, null), + (error) => + error.code === 'NOMINATIM_REFUSED' && error.retryAfterMs === 60_000, + ); + } + + for (const [index, status, expected] of [ + [0, 503, 60_000], + [1, 403, 3_600_000], + [2, 418, 3_600_000], + [3, 502, 30_000], + ]) { + const time = clock(); + const file = path.join(tempDir(t), `malformed-status-${index}.json`); + const makeGate = () => + publicGate(t, { + fetchImpl: async () => + new Response('wait', { + status, + headers: { 'Retry-After': '2099-01-01' }, + }), + ...time, + usage: createUsageStore({ file }), + }); + await assert.rejects( + createNominatimSearchProvider({ gate: makeGate() })( + `status-${status}`, + null, + ), + ); + assert.equal(makeGate().stats().pausedForMs, expected, String(status)); + } +}); + +test('an outage stops queued requests; a server error is never cached as not found', async (t) => { const time = clock(); let down = true; const { calls, fetchImpl } = stubFetch(async () => - down ? new Response('bad gateway', { status: 502 }) : json([row({ category: 'leisure', type: 'park', name: 'Zilker Park' })]), + down + ? new Response('bad gateway', { status: 502 }) + : json([row({ category: 'leisure', type: 'park', name: 'Zilker Park' })]), ); - const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }); + const gate = publicGate(t, { fetchImpl, ...time }); const outline = createNominatimOutlineProvider({ gate }); const results = await Promise.allSettled([ outline({ query: 'Zilker Park', kind: 'landmark' }), @@ -263,14 +440,14 @@ test('an outage stops queued requests; a server error is never cached as not fou assert.equal(calls.length, 2); }); -test('a transport failure is retried once, then pauses and is not cached', async () => { +test('a transport failure is retried once, then pauses and is not cached', async (t) => { const time = clock(); let failures = 1; const { calls, fetchImpl } = stubFetch(async () => { if (failures-- > 0) throw new TypeError('fetch failed'); return json([]); }); - const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }); + const gate = publicGate(t, { fetchImpl, ...time }); const search = createNominatimSearchProvider({ gate }); assert.equal((await search('retry', null)).status, 'ZERO_RESULTS'); assert.equal(calls.length, 2, 'one retry after a transport failure'); @@ -280,24 +457,30 @@ test('a transport failure is retried once, then pauses and is not cached', async assert.equal(calls.length, 4, 'never more than one retry'); await assert.rejects(search('twice', null), { code: 'NOMINATIM_PAUSED' }); time.advance(31_000); - assert.equal((await search('twice', null)).cached, undefined, 'the failure was not cached'); + assert.equal( + (await search('twice', null)).cached, + undefined, + 'the failure was not cached', + ); assert.equal(calls.length, 5); }); -test('a 403 block pauses for an hour', async () => { +test('a 403 block pauses for an hour', async (t) => { const time = clock(); - const { fetchImpl } = stubFetch(async () => new Response('', { status: 403 })); - const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }); + const { fetchImpl } = stubFetch( + async () => new Response('', { status: 403 }), + ); + const gate = publicGate(t, { fetchImpl, ...time }); await assert.rejects(createNominatimSearchProvider({ gate })('x', null), { code: 'NOMINATIM_REFUSED', }); assert.ok(gate.stats().pausedForMs >= 3_599_000); }); -test('the queue is bounded', async () => { +test('the queue is bounded', async (t) => { const time = clock(); const { fetchImpl } = stubFetch(async () => json([])); - const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time, maxPending: 2 }); + const gate = publicGate(t, { fetchImpl, ...time, maxPending: 2 }); const search = createNominatimSearchProvider({ gate }); const results = await Promise.allSettled( ['a', 'b', 'c', 'd'].map((q) => search(q, null)), @@ -342,37 +525,70 @@ test('cache lifetimes: 90 days for outlines, 30 for places, 24 hours for no resu test('the disk cache survives a restart and stays within its bounds', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-cache-')); const time = clock(); - const first = createNominatimCache({ dir, now: time.now, maxDiskEntries: 3, sweepEvery: 1000 }); + const first = createNominatimCache({ + dir, + now: time.now, + maxDiskEntries: 3, + sweepEvery: 1000, + }); for (let i = 0; i < 5; i++) { time.advance(1000); - await first.set(`k${i}`, { i }, i === 0 ? 10 : NOMINATIM_CACHE_TTL_MS.outline); + await first.set( + `k${i}`, + { i }, + i === 0 ? 10 : NOMINATIM_CACHE_TTL_MS.outline, + ); } - const second = createNominatimCache({ dir, now: time.now, maxDiskEntries: 3 }); + const second = createNominatimCache({ + dir, + now: time.now, + maxDiskEntries: 3, + }); assert.deepEqual((await second.get('k4')).payload, { i: 4 }); await second.sweep(); const files = fs.readdirSync(dir).filter((name) => name.endsWith('.json')); assert.equal(files.length, 3, 'expired entry removed, then oldest trimmed'); - assert.equal(await createNominatimCache({ dir, now: time.now }).get('k0'), null); + assert.equal( + await createNominatimCache({ dir, now: time.now }).get('k0'), + null, + ); fs.rmSync(dir, { recursive: true, force: true }); }); test('names are normalized Unicode-aware so small differences share an entry', () => { - assert.equal(normalizePlaceName('São Paulo'), normalizePlaceName('SAO paulo!')); + assert.equal( + normalizePlaceName('São Paulo'), + normalizePlaceName('SAO paulo!'), + ); assert.equal(normalizePlaceName('The Pentagon'), 'pentagon'); assert.equal(normalizePlaceName('Zürich'), 'zurich'); - assert.equal(normalizePlaceName('東京都'), '東京都', 'non-Latin names survive'); + assert.equal( + normalizePlaceName('東京都'), + '東京都', + 'non-Latin names survive', + ); assert.equal(normalizePlaceName('Ｃｅｎｔｒａｌ　Ｐａｒｋ'), 'central park'); }); -test('small view changes reuse one cached outline', async () => { +test('small view changes reuse one cached outline', async (t) => { const time = clock(); const { calls, fetchImpl } = stubFetch(async () => json([row({ category: 'leisure', type: 'park', name: 'Central Park' })]), ); - const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }); + const gate = publicGate(t, { fetchImpl, ...time }); const outline = createNominatimOutlineProvider({ gate }); - await outline({ query: 'Central Park', kind: 'landmark', lat: 40.781, lon: -73.966 }); - const again = await outline({ query: ' Central Park ', kind: 'landmark', lat: 40.7822, lon: -73.9651 }); + await outline({ + query: 'Central Park', + kind: 'landmark', + lat: 40.781, + lon: -73.966, + }); + const again = await outline({ + query: ' Central Park ', + kind: 'landmark', + lat: 40.7822, + lon: -73.9651, + }); assert.equal(again.cached, true); assert.equal(calls.length, 1); assert.match(calls[0], /viewbox=/); @@ -390,13 +606,20 @@ test('wrong types are rejected: charging station for Mission District, station f }); for (const kind of ['neighborhood', 'landmark', 'city']) assert.equal( - selectOutlineResult([charging], { kind, query: 'Mission District, San Francisco' }).outline, + selectOutlineResult([charging], { + kind, + query: 'Mission District, San Francisco', + }).outline, null, kind, ); const station = row({ category: 'railway', type: 'station', name: 'Bandra' }); for (const kind of ['neighborhood', 'landmark', 'building', 'city']) - assert.equal(selectOutlineResult([station], { kind, query: 'Bandra' }).outline, null, kind); + assert.equal( + selectOutlineResult([station], { kind, query: 'Bandra' }).outline, + null, + kind, + ); assert.equal(acceptsOutlineClass('landmark', 'highway', 'primary'), false); assert.equal(acceptsOutlineClass('city', 'leisure', 'park'), false); assert.equal(acceptsOutlineClass('landmark', 'office', 'government'), true); @@ -411,10 +634,15 @@ test('the first polygon-bearing result of the right class is taken (Mumbai)', () name: 'Mumbai', geojson: { type: 'Point', coordinates: [72.8, 19.0] }, }); - const wrong = row({ category: 'railway', type: 'station', name: 'Mumbai Central' }); + const wrong = row({ + category: 'railway', + type: 'station', + name: 'Mumbai Central', + }); const boundary = row({ category: 'boundary', - type: 'administrative', addresstype: 'city', + type: 'administrative', + addresstype: 'city', name: 'Mumbai', osm_id: 7888990, geojson: { @@ -429,21 +657,38 @@ test('the first polygon-bearing result of the right class is taken (Mumbai)', () assert.equal(outline.osm, 'relation/7888990'); assert.equal(outline.polygons.length, 2); assert.equal(outline.polygons[0][0][1][0], 72.9 + 0.2, 'largest part first'); - assert.deepEqual(skipped, ['place=city: no polygon', 'railway=station: wrong type']); + assert.deepEqual(skipped, [ + 'place=city: no polygon', + 'railway=station: wrong type', + ]); }); test('a result whose name does not answer the ask is passed over', () => { const other = row({ category: 'leisure', type: 'park', name: 'Pease Park' }); - assert.equal(selectOutlineResult([other], { kind: 'landmark', query: 'Zilker Park' }).outline, null); + assert.equal( + selectOutlineResult([other], { kind: 'landmark', query: 'Zilker Park' }) + .outline, + null, + ); const english = row({ category: 'boundary', - type: 'administrative', addresstype: 'city', + type: 'administrative', + addresstype: 'city', name: '東京都', namedetails: { name: '東京都', 'name:en': 'Tokyo' }, }); - assert.ok(selectOutlineResult([english], { kind: 'city', query: 'Tokyo' }).outline); - const capitol = row({ category: 'office', type: 'government', name: 'Texas State Capitol' }); - assert.ok(selectOutlineResult([capitol], { kind: 'landmark', query: 'Texas Capitol' }).outline); + assert.ok( + selectOutlineResult([english], { kind: 'city', query: 'Tokyo' }).outline, + ); + const capitol = row({ + category: 'office', + type: 'government', + name: 'Texas State Capitol', + }); + assert.ok( + selectOutlineResult([capitol], { kind: 'landmark', query: 'Texas Capitol' }) + .outline, + ); }); test('outlines are held to a vertex cap', () => { @@ -478,12 +723,22 @@ function mount(options) { }, end(body) { res.writableEnded = true; - resolve({ status: res.status, headers: res.headers, body: JSON.parse(body) }); + resolve({ + status: res.status, + headers: res.headers, + body: JSON.parse(body), + }); }, }; Promise.resolve( handler( - { method: 'GET', url, headers, socket: { remoteAddress: `10.1.${Math.random()}` }, on() {} }, + { + method: 'GET', + url, + headers, + socket: { remoteAddress: `10.1.${Math.random()}` }, + on() {}, + }, res, () => resolve({ status: 'next' }), ), @@ -491,30 +746,114 @@ function mount(options) { }); } -test('the outline route answers OK with polygons and validates its input', async () => { +test('the outline route answers OK with polygons and validates its input', async (t) => { const time = clock(); const { calls, fetchImpl } = stubFetch(async () => - json([row({ category: 'boundary', type: 'administrative', addresstype: 'city', name: 'Paris' })]), + json([ + row({ + category: 'boundary', + type: 'administrative', + addresstype: 'city', + name: 'Paris', + }), + ]), ); - const request = mount({ gate: createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }) }); + const request = mount({ gate: publicGate(t, { fetchImpl, ...time }) }); const ok = await request('/outline?q=Paris&kind=city&lat=48.85&lon=2.35'); assert.equal(ok.status, 200); assert.equal(ok.body.status, 'OK'); assert.equal(ok.body.outline.polygons[0][0].length, 5); + assert.equal('adminArea' in ok.body.outline, false); + assert.equal('adminLevel' in ok.body.outline, false); assert.match(calls[0], /polygon_geojson=1/); assert.match(calls[0], /polygon_threshold=0\.0005/); assert.match(calls[0], /limit=3/); - for (const bad of ['/outline?q=Paris', '/outline?kind=city', '/outline?q=Paris&kind=planet']) + for (const bad of [ + '/outline?q=Paris', + '/outline?kind=city', + '/outline?q=Paris&kind=planet', + ]) assert.equal((await request(bad)).status, 400, bad); - const cross = await request('/outline?q=Paris&kind=city', { 'sec-fetch-site': 'cross-site' }); + const cross = await request('/outline?q=Paris&kind=city', { + 'sec-fetch-site': 'cross-site', + }); assert.equal(cross.status, 403); assert.equal(calls.length, 1); assert.equal((await request('/elsewhere')).status, 'next'); }); -test('the outline route reports disabled and capped as permanent, busy as retryable', async () => { +test('the outline route serializes only selected-row administrative identity', async (t) => { + const { fetchImpl } = stubFetch(async (url) => { + const query = new URL(url).searchParams.get('q'); + const fixtures = { + 'Bagmati Province': { addresstype: 'province' }, + 'Mystery Province': { place_rank: 8 }, + 'Missing Evidence': {}, + 'Northern Region': { addresstype: 'region', place_rank: 8 }, + 'Central District': { addresstype: 'district', place_rank: 10 }, + 'Example State': { addresstype: 'county' }, + 'Bagmati Zone': { addresstype: 'state' }, + Austin: { addresstype: 'city' }, + }; + return json([ + row({ + category: 'boundary', + type: 'administrative', + name: query, + ...fixtures[query], + }), + ]); + }); + const gate = createNominatimGate({ settings: OPERATOR, fetchImpl }); + const outline = createNominatimOutlineProvider({ + gate, + cache: createNominatimCache({ dir: tempDir(t) }), + }); + const request = mount({ outline }); + + const verified = await request('/outline?q=Bagmati%20Province&kind=admin'); + assert.equal(verified.status, 200); + assert.equal(verified.body.outline.adminArea, 'Bagmati Province'); + assert.equal(verified.body.outline.adminLevel, 'admin1'); + + for (const query of [ + 'Mystery Province', + 'Northern Region', + 'Central District', + 'Example State', + ]) { + const unverified = await request( + `/outline?q=${encodeURIComponent(query)}&kind=admin`, + ); + assert.equal(unverified.status, 200, query); + assert.equal(unverified.body.status, 'OK', query); + assert.equal('adminArea' in unverified.body.outline, false, query); + assert.equal('adminLevel' in unverified.body.outline, false, query); + } + + const missing = await request('/outline?q=Missing%20Evidence&kind=admin'); + assert.equal(missing.status, 200); + assert.equal(missing.body.status, 'ZERO_RESULTS'); + assert.equal(missing.body.outline, null); + + const retired = await request('/outline?q=Bagmati%20Zone&kind=admin'); + assert.equal(retired.status, 200); + assert.equal(retired.body.status, 'OK'); + assert.equal(retired.body.outline.adminArea, 'Bagmati Zone'); + assert.equal('adminLevel' in retired.body.outline, false); + + const city = await request('/outline?q=Austin&kind=city'); + assert.equal(city.status, 200); + assert.equal(city.body.status, 'OK'); + assert.equal('adminArea' in city.body.outline, false); + assert.equal('adminLevel' in city.body.outline, false); +}); + +test('the outline route reports disabled and capped as permanent, busy as retryable', async (t) => { const disabled = mount({ - gate: createNominatimGate({ settings: { endpoint: null, isPublic: false, dailyCap: 50 } }), + gate: createNominatimGate({ + settings: { endpoint: null, isPublic: false, dailyCap: 50 }, + }), }); const off = await disabled('/outline?q=Paris&kind=city'); assert.equal(off.status, 503); @@ -522,16 +861,16 @@ test('the outline route reports disabled and capped as permanent, busy as retrya assert.equal(off.body.retryable, false); const capped = mount({ - gate: createNominatimGate({ settings: { ...PUBLIC, dailyCap: 0 } }), + gate: publicGate(t, { settings: { ...PUBLIC, dailyCap: 0 } }), }); const cap = await capped('/outline?q=Paris&kind=city'); assert.equal(cap.status, 429); assert.equal(cap.body.retryable, false); const time = clock(); - const paused = createNominatimGate({ - settings: PUBLIC, - fetchImpl: async () => new Response('', { status: 429, headers: { 'Retry-After': '90' } }), + const paused = publicGate(t, { + fetchImpl: async () => + new Response('', { status: 429, headers: { 'Retry-After': '90' } }), ...time, }); const busy = mount({ gate: paused }); diff --git a/src/tooling/nominatimSearchRoute.test.mjs b/src/tooling/nominatimSearchRoute.test.mjs index 91fcb02e4..7586a2fa5 100644 --- a/src/tooling/nominatimSearchRoute.test.mjs +++ b/src/tooling/nominatimSearchRoute.test.mjs @@ -9,6 +9,9 @@ // Run with: npm test import test from 'node:test'; import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { fetchNominatimSearch, geocodeProxy, @@ -23,6 +26,8 @@ import { createPhotonGeocoder } from '../keylessGeocoder.js'; // Each case starts with a fresh shared gate: a pause after one case's // simulated outage must not decide the next case. test.beforeEach(() => setSharedNominatimGate(null)); +const storageRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-search-route-')); +test.after(() => fs.rmSync(storageRoot, { recursive: true, force: true })); const HIT = [ { @@ -38,13 +43,16 @@ const HIT = [ /** Mount the geocode route the way the dev server does. */ let clientCounter = 0; -function mountGeocode( - install = (server) => geocodeProxy().configureServer(server), -) { +function mountGeocode(install = null) { // A fresh client address per mount: the per-client rate limiter is module // state shared by every case in this file, and one case's burst must not // decide another case's outcome. clientCounter += 1; + if (!install) + install = (server) => + geocodeProxy({ + storageDir: path.join(storageRoot, `case-${clientCounter}`), + }).configureServer(server); const remoteAddress = `10.0.0.${clientCounter % 250}`; const routes = new Map(); install({