mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-06 03:14:00 +00:00
fix(outlines): harden shared cache and provider lifecycle
This commit is contained in:
1 parent
9559e7a282
commit
177b5951a0
27 files changed
+3363
-248
No files matched your search
@@ -11,6 +11,7 @@ import {
|
||||
sampleGroundHeight,
|
||||
} from './annotationResolver.js';
|
||||
import { ringCentroid } from './drawMode.js';
|
||||
import { phaseTiming } from '../sources/phaseTiming.js';
|
||||
|
||||
// Dev convenience: expose the app's Cesium instance for console/preview probing
|
||||
// (single shared module instance — avoids dual-Cesium state bugs when testing).
|
||||
@@ -679,6 +680,7 @@ export function createAnnotationEngine({
|
||||
if (!annotations.has(anno.id)) return; // mark replaced/removed while resolving
|
||||
anno.pendingOutline = false;
|
||||
anno.outlineUnavailable = isUnavailableCapability(fp);
|
||||
const installStart = performance.now();
|
||||
if (anno.outlineUnavailable) renderer.update(anno);
|
||||
if (fp && !anno.outlineUnavailable) {
|
||||
anno.ring = fp.ring;
|
||||
@@ -726,6 +728,17 @@ export function createAnnotationEngine({
|
||||
}
|
||||
}
|
||||
renderer.sync(annotations);
|
||||
if (fp && !anno.outlineUnavailable)
|
||||
phaseTiming(
|
||||
'install',
|
||||
installStart,
|
||||
{
|
||||
kind: anno.footprintKind || null,
|
||||
parts: anno.polygons?.length || (anno.ring ? 1 : 0),
|
||||
points: anno.ring?.length || 0,
|
||||
},
|
||||
'outlines',
|
||||
);
|
||||
// Final outcome → the voice layer (place names ride as structured DATA fields).
|
||||
// 'failed' covers both the definitive no-polygon and an exhausted transient
|
||||
// retry — either way the mark honestly stays a point.
|
||||
|
||||
@@ -6,7 +6,9 @@
|
||||
// request reaches any Overpass endpoint.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import {
|
||||
createOutlineRungs,
|
||||
@@ -20,7 +22,10 @@ import { createNominatimOutlineClient } from '../sources/nominatimOutlines.js';
|
||||
import { createApplicationRequestServices } from '../services/requests.js';
|
||||
import { overpassProxy } from '../../server/providers/overpass.js';
|
||||
import { geocodeProxy } from '../../server/providers/regional/place.js';
|
||||
import { createNominatimGate } from '../../server/providers/regional/nominatimGate.js';
|
||||
import {
|
||||
createGateStateStore,
|
||||
createNominatimGate,
|
||||
} from '../../server/providers/regional/nominatimGate.js';
|
||||
|
||||
const TILES = [6745, 6746].map((y) =>
|
||||
decodeOpenFreeMapOutlineTile(
|
||||
@@ -332,6 +337,8 @@ test('default settings: outline, street and building asks send nothing to Overpa
|
||||
t.mock.method(globalThis, 'fetch', async (url) => assert.fail(`unexpected network request ${url}`));
|
||||
|
||||
const upstream = [];
|
||||
const stateDir = mkdtempSync(path.join(os.tmpdir(), 'gev-outline-rungs-'));
|
||||
t.after(() => rmSync(stateDir, { recursive: true, force: true }));
|
||||
const gate = createNominatimGate({
|
||||
settings: { endpoint: 'https://nominatim.openstreetmap.org/search', isPublic: true, dailyCap: 50 },
|
||||
fetchImpl: async (url) => {
|
||||
@@ -339,6 +346,7 @@ test('default settings: outline, street and building asks send nothing to Overpa
|
||||
const q = new URL(url).searchParams.get('q');
|
||||
return Response.json(NOMINATIM_ROWS[q] || []);
|
||||
},
|
||||
usage: createGateStateStore({ file: path.join(stateDir, 'state.json') }),
|
||||
sleep: async () => {},
|
||||
});
|
||||
const handlers = routes(overpassProxy(), geocodeProxy({ gate }));
|
||||
|
||||
@@ -420,8 +420,8 @@ const COUNTY_WORDS = [
|
||||
];
|
||||
const COUNTY_WORD_RE = new RegExp(`\\b(${COUNTY_WORDS.join('|')})\\b`);
|
||||
const STATE_PREFIX_RE =
|
||||
/^(?:free state|state|commonwealth|province|territory|prefecture|region|land|canton|department) of (.+)$/;
|
||||
const STATE_SUFFIX_RE = /^(.+) (?:state|province|prefecture|territory)$/;
|
||||
/^(free state|state|commonwealth|province|territory|prefecture|region|land|canton|department) of (.+)$/;
|
||||
const STATE_SUFFIX_RE = /^(.+) (state|province|prefecture|territory)$/;
|
||||
|
||||
/**
|
||||
* Split an ask into what it names and how.
|
||||
@@ -482,14 +482,17 @@ export function parseAdminQuery(text) {
|
||||
if (prefix)
|
||||
return {
|
||||
kind: 'state',
|
||||
name: prefix[1],
|
||||
name: prefix[2],
|
||||
qualifiers,
|
||||
postal: null,
|
||||
countyWord: null,
|
||||
full: head,
|
||||
form: 'prefix',
|
||||
adminType: prefix[1],
|
||||
};
|
||||
const suffix = STATE_SUFFIX_RE.exec(head);
|
||||
// South Africa's canonical province name is literally "Free State"; do not
|
||||
// strip its final word as though the ask explicitly named an admin type.
|
||||
const suffix = head === 'free state' ? null : STATE_SUFFIX_RE.exec(head);
|
||||
if (suffix)
|
||||
return {
|
||||
kind: 'state',
|
||||
@@ -499,6 +502,7 @@ export function parseAdminQuery(text) {
|
||||
countyWord: null,
|
||||
full: head,
|
||||
form: 'suffix',
|
||||
adminType: suffix[2],
|
||||
};
|
||||
return {
|
||||
kind: 'any',
|
||||
@@ -510,6 +514,30 @@ export function parseAdminQuery(text) {
|
||||
};
|
||||
}
|
||||
|
||||
/** Explicit wording is stronger than a stale same-name geometry. */
|
||||
function explicitStateTypeFits(entry, parsed) {
|
||||
if (!parsed.adminType) return true;
|
||||
const words = new Set(
|
||||
normalizeAdminName(entry.feature?.type)
|
||||
.split(/[^a-z]+/)
|
||||
.filter(Boolean),
|
||||
);
|
||||
const accepted =
|
||||
{
|
||||
'free state': ['state'],
|
||||
state: ['state'],
|
||||
commonwealth: ['state'],
|
||||
province: ['province'],
|
||||
territory: ['territory'],
|
||||
prefecture: ['prefecture'],
|
||||
region: ['region'],
|
||||
land: ['state'],
|
||||
canton: ['canton'],
|
||||
department: ['department', 'departamento'],
|
||||
}[parsed.adminType] || [];
|
||||
return accepted.some((word) => words.has(word));
|
||||
}
|
||||
|
||||
/** A qualifier names the unit's country ("Bavaria, Germany") or the unit
|
||||
* itself ("Washington, DC"). */
|
||||
function qualifierMatches(entry, qualifier) {
|
||||
@@ -647,6 +675,10 @@ async function stateCandidates(parsed, { allowAmbiguous = false } = {}) {
|
||||
list = list.filter((i) =>
|
||||
parsed.qualifiers.every((q) => qualifierMatches(i.entry, q)),
|
||||
);
|
||||
// Preserve explicit identity at the selector seam. In particular,
|
||||
// "Bagmati Province" must not select Natural Earth's historical NP-BA
|
||||
// Bagmati Zone merely because the point and normalized name overlap.
|
||||
list = list.filter((i) => explicitStateTypeFits(i.entry, parsed));
|
||||
if (allowAmbiguous) return list;
|
||||
// A bare name answers alone only for a prominent unit — a US state, a
|
||||
// Canadian province, Bavaria — never a French département, an English
|
||||
@@ -759,6 +791,12 @@ export async function findAdminAreaAt(names, lat, lon, scope) {
|
||||
.map(parseAdminQuery)
|
||||
.filter((p) => p?.name);
|
||||
if (!parsedList.length) return null;
|
||||
// If the ask explicitly names an admin type, do not let the geocoder's
|
||||
// untyped alias reopen an incompatible historical same-name boundary.
|
||||
const explicitStates = parsedList.filter(
|
||||
(parsed) => parsed.kind === 'state' && parsed.adminType,
|
||||
);
|
||||
const stateParses = explicitStates.length ? explicitStates : parsedList;
|
||||
const containing = (items) => {
|
||||
const out = [];
|
||||
for (const item of items)
|
||||
@@ -803,7 +841,7 @@ export async function findAdminAreaAt(names, lat, lon, scope) {
|
||||
return toResult(rank(found, { lat, lon })[0], found.length);
|
||||
}
|
||||
const states = [];
|
||||
for (const parsed of parsedList)
|
||||
for (const parsed of stateParses)
|
||||
states.push(
|
||||
...(await stateCandidates(
|
||||
{ ...parsed, qualifiers: [] },
|
||||
|
||||
@@ -225,6 +225,28 @@ test('geocoder-confirmed lookup: the named unit that contains the point', async
|
||||
assert.equal(await findAdminAreaAt(['Georgia'], 41.7, 44.8, 'state'), null);
|
||||
const newYork = await findAdminAreaAt(['New York'], 42.9, -75.5, 'state');
|
||||
assert.equal(newYork?.id, 'US-NY');
|
||||
assert.equal(
|
||||
(
|
||||
await findAdminAreaAt(
|
||||
['Ontario Province', 'Ontario'],
|
||||
43.6532,
|
||||
-79.3832,
|
||||
'state',
|
||||
)
|
||||
)?.id,
|
||||
'CA-ON',
|
||||
);
|
||||
// Natural Earth still carries the historical NP-BA Bagmati Zone. An ask
|
||||
// for today's province must not silently select that incompatible geometry.
|
||||
assert.equal(
|
||||
await findAdminAreaAt(
|
||||
['Bagmati Province', 'Bagmati'],
|
||||
27.7172,
|
||||
85.324,
|
||||
'state',
|
||||
),
|
||||
null,
|
||||
);
|
||||
// A county the geocoder typed, named without the county word.
|
||||
const travis = await findAdminAreaAt(
|
||||
['Travis', 'Travis County'],
|
||||
|
||||
@@ -26,6 +26,10 @@ import {
|
||||
|
||||
const PLACES = new URL('./local_data/us_census_places/', import.meta.url);
|
||||
const WOF = new URL('./local_data/wof_neighborhoods/', import.meta.url);
|
||||
const WOF_LICENSES = new URL(
|
||||
'../../scripts/wof-open-licenses.json',
|
||||
import.meta.url,
|
||||
);
|
||||
const readJson = (url) => JSON.parse(readFileSync(url, 'utf8'));
|
||||
|
||||
const AUSTIN = { lat: 30.2672, lon: -97.7431 };
|
||||
@@ -365,15 +369,71 @@ test('places pack: every state file, unique ids, sane and valid rings', () => {
|
||||
assert.equal(total, 32_629);
|
||||
});
|
||||
|
||||
const UNSTATED = /assumed|unknown|unresolved|n\/a/i;
|
||||
const WOF_LICENSE_POLICY = readJson(WOF_LICENSES);
|
||||
const ALLOWED_WOF_LICENSES = new Set(WOF_LICENSE_POLICY.licenses);
|
||||
const licenceAllowed = (value) =>
|
||||
typeof value === 'string' && ALLOWED_WOF_LICENSES.has(value);
|
||||
|
||||
test('neighborhood pack ships only sources whose licence is stated', () => {
|
||||
test('the WOF licence policy accepts only exact reviewed registry strings', () => {
|
||||
for (const accepted of WOF_LICENSE_POLICY.licenses)
|
||||
assert.equal(licenceAllowed(accepted), true, accepted);
|
||||
for (const rejected of [
|
||||
'',
|
||||
null,
|
||||
0,
|
||||
'Unknown',
|
||||
'Restricted',
|
||||
'CC BY-NC 4.0',
|
||||
'CC BY-SA 4.0',
|
||||
'CC BY 5.0 (unreviewed)',
|
||||
'CC BY Restricted',
|
||||
'CC BY Unknown',
|
||||
'Creative Commons NonCommercial',
|
||||
'ODbL 1.0',
|
||||
'cc by 4.0',
|
||||
'CC BY 4.0 ',
|
||||
' CC BY 4.0',
|
||||
'CC BY 4.0',
|
||||
'CC\tBY 4.0',
|
||||
])
|
||||
assert.equal(licenceAllowed(rejected), false, JSON.stringify(rejected));
|
||||
});
|
||||
|
||||
test('the Zetashapes raw registry spelling is admitted by pinned provenance', () => {
|
||||
const review = WOF_LICENSE_POLICY.reviewedSources.zs;
|
||||
assert.deepEqual(
|
||||
{
|
||||
name: review.name,
|
||||
registryCommit: review.registryCommit,
|
||||
registrySha256: review.registrySha256,
|
||||
rawLicenseType: review.rawLicenseType,
|
||||
licenseTextSha256: review.licenseTextSha256,
|
||||
},
|
||||
{
|
||||
name: 'Zetashapes',
|
||||
registryCommit: 'e17af153c144fe6e08aa8db2607b4bf8aecd448d',
|
||||
registrySha256:
|
||||
'd8a5dd873d26eee11583de4438185c046dec33848d5bde27047ca4bff44d9c39',
|
||||
rawLicenseType: 'Public domain',
|
||||
licenseTextSha256:
|
||||
'91757f96f74f6dfb4a7206486b78586da0d039e9611e224fc793c7167dcfe615',
|
||||
},
|
||||
);
|
||||
assert.equal(licenceAllowed(review.rawLicenseType), true);
|
||||
assert.match(review.decision, /TIGER\/Line/);
|
||||
assert.match(review.decision, /Flickr/);
|
||||
});
|
||||
|
||||
test('neighborhood pack ships only explicitly allowed and credited sources', () => {
|
||||
const index = readJson(new URL('index.json', WOF));
|
||||
const sources = index.meta.sources;
|
||||
const credits = readFileSync(new URL('ATTRIBUTION.md', WOF), 'utf8');
|
||||
for (const [key, source] of Object.entries(sources)) {
|
||||
assert.ok(source.license, `${key} has a licence`);
|
||||
assert.doesNotMatch(source.license, UNSTATED, key);
|
||||
assert.equal(
|
||||
licenceAllowed(source.license),
|
||||
true,
|
||||
`${key}: ${source.license}`,
|
||||
);
|
||||
assert.notEqual(key, 'unknown');
|
||||
assert.ok(credits.includes(`\`${key}\``), `${key} credited`);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@ import test from 'node:test';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { tilesForBounds, tileToBBox } from '../../data/tomtomTiles.js';
|
||||
import { decodeFlowTile } from './flowDecode.js';
|
||||
import { decodeOpenFreeMapTile } from '../../sources/openFreeMap.js';
|
||||
import {
|
||||
clearSharedOpenFreeMapTiles,
|
||||
decodeOpenFreeMapTile,
|
||||
} from '../../sources/openFreeMap.js';
|
||||
import {
|
||||
RoadRequestError,
|
||||
roadRequestError,
|
||||
@@ -829,6 +832,8 @@ test('failed detail cores retain real coarse roads while successful cores replac
|
||||
});
|
||||
|
||||
test('traffic separates API and public tile transports, including the default tile fetch', async (t) => {
|
||||
clearSharedOpenFreeMapTiles();
|
||||
t.after(clearSharedOpenFreeMapTiles);
|
||||
const publicUrls = [],
|
||||
apiUrls = [];
|
||||
const publicFetch = async (url) => {
|
||||
@@ -864,6 +869,11 @@ test('traffic separates API and public tile transports, including the default ti
|
||||
(await (await source.requestOsmRoads(bounds)).json()).roads.length,
|
||||
);
|
||||
}
|
||||
assert.ok(publicUrls.length >= 4);
|
||||
assert.equal(
|
||||
publicUrls.filter((url) => url.endsWith('.pbf')).length,
|
||||
1,
|
||||
'compatible sources reuse the shared decoded public tile',
|
||||
);
|
||||
assert.equal(publicUrls.filter((url) => url.endsWith('/planet')).length, 2);
|
||||
assert.ok(apiUrls.length >= 4);
|
||||
});
|
||||
+174
-5
@@ -1,6 +1,9 @@
|
||||
import { PbfReader } from 'pbf';
|
||||
import { VectorTile } from '@mapbox/vector-tile';
|
||||
import { createVectorTileSource } from './vectorTiles.js';
|
||||
import {
|
||||
createSharedDecodedTileCache,
|
||||
createVectorTileSource,
|
||||
} from './vectorTiles.js';
|
||||
import { clipTileRing, militaryOutlineLines } from './militaryTileGeometry.js';
|
||||
import { tileToBBox } from '../data/tomtomTiles.js';
|
||||
|
||||
@@ -13,6 +16,65 @@ const ROAD_TYPES = Object.freeze({
|
||||
minor: 'residential',
|
||||
});
|
||||
|
||||
const SHARED_DECODED_TILES = createSharedDecodedTileCache({
|
||||
maxEntries: 192,
|
||||
maxCacheBytes: 64 * 1024 * 1024,
|
||||
});
|
||||
const TILE_PROJECTIONS = new Map();
|
||||
const MAX_PROJECTED_NUMBERS = 500_000;
|
||||
|
||||
/** Conservative, deterministic retained-size estimate with a hard geometry cap. */
|
||||
export function estimateOpenFreeMapProjectionBytes(value) {
|
||||
const seen = new Set();
|
||||
const stack = [value];
|
||||
let bytes = 0;
|
||||
let numbers = 0;
|
||||
while (stack.length) {
|
||||
const item = stack.pop();
|
||||
if (item == null) continue;
|
||||
if (typeof item === 'number') {
|
||||
numbers += 1;
|
||||
if (numbers > MAX_PROJECTED_NUMBERS)
|
||||
throw new Error('Vector tile projection coordinate limit exceeded');
|
||||
bytes += 8;
|
||||
} else if (typeof item === 'string') {
|
||||
bytes += 16 + item.length * 2;
|
||||
} else if (typeof item === 'boolean') {
|
||||
bytes += 4;
|
||||
} else if (typeof item === 'object') {
|
||||
if (seen.has(item)) continue;
|
||||
seen.add(item);
|
||||
if (Array.isArray(item)) {
|
||||
bytes += 24 + item.length * 8;
|
||||
for (let i = 0; i < item.length; i += 1) stack.push(item[i]);
|
||||
} else {
|
||||
const entries = Object.entries(item);
|
||||
bytes += 48 + entries.length * 16;
|
||||
for (const [key, child] of entries) {
|
||||
bytes += key.length * 2;
|
||||
stack.push(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/** Register one consumer view over the parsed OpenFreeMap tile. */
|
||||
export function registerOpenFreeMapProjection(name, project) {
|
||||
if (!name || typeof project !== 'function')
|
||||
throw new TypeError('Invalid OpenFreeMap tile projection');
|
||||
const current = TILE_PROJECTIONS.get(name);
|
||||
if (current && current !== project)
|
||||
throw new Error(`OpenFreeMap tile projection already registered: ${name}`);
|
||||
TILE_PROJECTIONS.set(name, project);
|
||||
}
|
||||
|
||||
/** Explicit test/runtime reset; individual consumers never evict each other. */
|
||||
export function clearSharedOpenFreeMapTiles() {
|
||||
SHARED_DECODED_TILES.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Surface traffic uses public motor-road classes only. Untyped service ways
|
||||
* include car parks and grounds access, so none are used as through traffic.
|
||||
@@ -142,8 +204,8 @@ export function decodeOpenFreeMapMilitaryTile(bytes, z, x, y) {
|
||||
return decodeLayers(bytes, z, x, y, ['landuse']);
|
||||
}
|
||||
|
||||
function decodeLayers(bytes, z, x, y, layerNames) {
|
||||
const tile = new VectorTile(new PbfReader(bytes));
|
||||
function decodeLayers(bytes, z, x, y, layerNames, parsedTile = null) {
|
||||
const tile = parsedTile || new VectorTile(new PbfReader(bytes));
|
||||
const roads = [],
|
||||
military = [];
|
||||
const box = tileToBBox(z, x, y);
|
||||
@@ -214,14 +276,121 @@ function decodeLayers(bytes, z, x, y, layerNames) {
|
||||
return { roads, military };
|
||||
}
|
||||
|
||||
function decodeSharedOpenFreeMapTile(bytes, z, x, y) {
|
||||
const tile = new VectorTile(new PbfReader(bytes));
|
||||
const base = decodeLayers(
|
||||
bytes,
|
||||
z,
|
||||
x,
|
||||
y,
|
||||
['transportation', 'landuse'],
|
||||
tile,
|
||||
);
|
||||
const projections = new Map();
|
||||
let baseBytes = 0;
|
||||
let projectionBytes = 0;
|
||||
let resizeCacheEntry = null;
|
||||
let cacheBound = false;
|
||||
const idleCallbacks = new Map();
|
||||
const cancelIdleProjection = (name) => {
|
||||
if (!idleCallbacks.has(name)) return;
|
||||
const handle = idleCallbacks.get(name);
|
||||
idleCallbacks.delete(name);
|
||||
globalThis.cancelIdleCallback?.(handle);
|
||||
};
|
||||
const cancelIdleProjections = () => {
|
||||
for (const name of [...idleCallbacks.keys()]) cancelIdleProjection(name);
|
||||
};
|
||||
const project = (name) => {
|
||||
if (projections.has(name)) return projections.get(name);
|
||||
cancelIdleProjection(name);
|
||||
const projector = TILE_PROJECTIONS.get(name);
|
||||
if (!projector)
|
||||
throw new Error(`Unknown OpenFreeMap tile projection: ${name}`);
|
||||
const value = projector(bytes, z, x, y, tile);
|
||||
const retainedBytes = estimateOpenFreeMapProjectionBytes(value);
|
||||
projections.set(name, value);
|
||||
projectionBytes += retainedBytes;
|
||||
resizeCacheEntry?.(baseBytes + projectionBytes);
|
||||
return value;
|
||||
};
|
||||
const scheduleIdleProjections = () => {
|
||||
// Scheduling is optional unless both lifecycle operations are available;
|
||||
// otherwise eviction could not release the callback's decoded-tile closure.
|
||||
if (
|
||||
!cacheBound ||
|
||||
typeof globalThis.requestIdleCallback !== 'function' ||
|
||||
typeof globalThis.cancelIdleCallback !== 'function'
|
||||
)
|
||||
return;
|
||||
for (const name of TILE_PROJECTIONS.keys())
|
||||
if (!projections.has(name) && !idleCallbacks.has(name)) {
|
||||
const handle = globalThis.requestIdleCallback(
|
||||
() => {
|
||||
if (!cacheBound || idleCallbacks.get(name) !== handle) return;
|
||||
idleCallbacks.delete(name);
|
||||
try {
|
||||
project(name);
|
||||
} catch {
|
||||
// The explicit consumer path retries and reports projection errors.
|
||||
}
|
||||
},
|
||||
{ timeout: 1000 },
|
||||
);
|
||||
idleCallbacks.set(name, handle);
|
||||
}
|
||||
};
|
||||
return {
|
||||
...base,
|
||||
project,
|
||||
bindSharedCacheEntry(resize, initialBytes) {
|
||||
cancelIdleProjections();
|
||||
resizeCacheEntry = resize;
|
||||
baseBytes = initialBytes;
|
||||
cacheBound = true;
|
||||
resizeCacheEntry(baseBytes + projectionBytes);
|
||||
scheduleIdleProjections();
|
||||
},
|
||||
releaseSharedCacheEntry() {
|
||||
cacheBound = false;
|
||||
cancelIdleProjections();
|
||||
projections.clear();
|
||||
baseBytes = 0;
|
||||
projectionBytes = 0;
|
||||
resizeCacheEntry = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Construct an immutable-version road/military tile source without starting I/O. */
|
||||
export function createOpenFreeMapSource(options = {}) {
|
||||
const {
|
||||
projection = null,
|
||||
decode,
|
||||
sharedDecodedCache,
|
||||
selectDecoded,
|
||||
...sourceOptions
|
||||
} = options;
|
||||
const shared = !decode && !selectDecoded;
|
||||
return createVectorTileSource({
|
||||
tileJsonUrl: 'https://tiles.openfreemap.org/planet',
|
||||
allowedOrigin: 'https://tiles.openfreemap.org',
|
||||
decode: decodeOpenFreeMapTile,
|
||||
decode: shared
|
||||
? decodeSharedOpenFreeMapTile
|
||||
: decode || decodeOpenFreeMapTile,
|
||||
...(shared
|
||||
? {
|
||||
sharedDecodedCache: sharedDecodedCache || SHARED_DECODED_TILES,
|
||||
selectDecoded: projection
|
||||
? (value) => value.project(projection)
|
||||
: ({ roads, military }) => ({ roads, military }),
|
||||
}
|
||||
: {
|
||||
...(sharedDecodedCache ? { sharedDecodedCache } : {}),
|
||||
...(selectDecoded ? { selectDecoded } : {}),
|
||||
}),
|
||||
maxEntries: 192,
|
||||
maxCacheBytes: 64 * 1024 * 1024,
|
||||
...options,
|
||||
...sourceOptions,
|
||||
});
|
||||
}
|
||||
@@ -7,11 +7,38 @@ import {
|
||||
polygonCentroid,
|
||||
clipTileLine,
|
||||
createOpenFreeMapSource,
|
||||
estimateOpenFreeMapProjectionBytes,
|
||||
} from './openFreeMap.js';
|
||||
import { createVectorTileSource } from './vectorTiles.js';
|
||||
const fixture = (name) =>
|
||||
readFileSync(new URL(`../data/fixtures/${name}`, import.meta.url));
|
||||
|
||||
test('projected tile geometry has deterministic accounting and a coordinate cap', () => {
|
||||
const small = {
|
||||
streets: [
|
||||
{
|
||||
name: 'A',
|
||||
coordinates: [
|
||||
[0, 1],
|
||||
[2, 3],
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
assert.equal(
|
||||
estimateOpenFreeMapProjectionBytes(small),
|
||||
estimateOpenFreeMapProjectionBytes(structuredClone(small)),
|
||||
);
|
||||
assert.ok(estimateOpenFreeMapProjectionBytes(small) > 4 * 8);
|
||||
assert.throws(
|
||||
() =>
|
||||
estimateOpenFreeMapProjectionBytes({
|
||||
coordinates: Array.from({ length: 250_001 }, () => [0, 0]),
|
||||
}),
|
||||
/coordinate limit/,
|
||||
);
|
||||
});
|
||||
|
||||
test('real z14 and z12 Austin tiles decode only drivable roads', () => {
|
||||
for (const [z, x, y, name] of [
|
||||
[14, 3743, 6745, 'ofm-austin-14-3743-6745.pbf'],
|
||||
|
||||
@@ -1,18 +1,24 @@
|
||||
import { PbfReader } from 'pbf';
|
||||
import { VectorTile } from '@mapbox/vector-tile';
|
||||
import { clipTileLine, createOpenFreeMapSource } from './openFreeMap.js';
|
||||
import {
|
||||
clipTileLine,
|
||||
createOpenFreeMapSource,
|
||||
registerOpenFreeMapProjection,
|
||||
} from './openFreeMap.js';
|
||||
import { tileToBBox } from '../data/tomtomTiles.js';
|
||||
import {
|
||||
approximateDistanceM,
|
||||
ringAreaM2,
|
||||
closeRing,
|
||||
} from './featureGeometry.js';
|
||||
import { phaseTiming } from './phaseTiming.js';
|
||||
|
||||
/**
|
||||
* Street and building outlines from the same OpenFreeMap z14 vector tiles the
|
||||
* traffic layer reads. No new tile provider: the source below points at the
|
||||
* same immutable tile URLs, so tiles traffic already loaded come from the
|
||||
* browser cache. Nothing here is fetched without an explicit ask.
|
||||
* same immutable tile URLs and bounded decoded owner. Traffic prepares the
|
||||
* outline projection in idle time; nothing here starts provider I/O without
|
||||
* an explicit traffic or outline ask.
|
||||
*/
|
||||
|
||||
/** Tile zoom used for outlines (the highest OpenMapTiles zoom with buildings). */
|
||||
@@ -54,10 +60,17 @@ const GROUNDS_AREA_CLASSES = Object.freeze({
|
||||
});
|
||||
|
||||
/** Lines of one tile's `transportation_name` layer and polygons of `building`. */
|
||||
export function decodeOpenFreeMapOutlineTile(bytes, z, x, y) {
|
||||
const tile = new VectorTile(new PbfReader(bytes));
|
||||
export function decodeOpenFreeMapOutlineTile(
|
||||
bytes,
|
||||
z,
|
||||
x,
|
||||
y,
|
||||
parsedTile = null,
|
||||
) {
|
||||
const tile = parsedTile || new VectorTile(new PbfReader(bytes));
|
||||
const box = tileToBBox(z, x, y);
|
||||
const streets = [];
|
||||
const streetsStart = performance.now();
|
||||
const names = tile.layers.transportation_name;
|
||||
if (names) {
|
||||
if (names.length > LAYER_FEATURE_LIMIT)
|
||||
@@ -85,6 +98,17 @@ export function decodeOpenFreeMapOutlineTile(bytes, z, x, y) {
|
||||
});
|
||||
}
|
||||
}
|
||||
phaseTiming(
|
||||
'street-extract',
|
||||
streetsStart,
|
||||
{
|
||||
tile: `${z}/${x}/${y}`,
|
||||
features: names?.length || 0,
|
||||
streets: streets.length,
|
||||
},
|
||||
'outlines',
|
||||
);
|
||||
const polygonsStart = performance.now();
|
||||
const buildings = decodePolygons(
|
||||
tile.layers.building,
|
||||
z,
|
||||
@@ -115,9 +139,21 @@ export function decodeOpenFreeMapOutlineTile(bytes, z, x, y) {
|
||||
),
|
||||
);
|
||||
}
|
||||
phaseTiming(
|
||||
'polygon-extract',
|
||||
polygonsStart,
|
||||
{
|
||||
tile: `${z}/${x}/${y}`,
|
||||
buildings: buildings.length,
|
||||
areas: areas.length,
|
||||
},
|
||||
'outlines',
|
||||
);
|
||||
return { streets, buildings, areas };
|
||||
}
|
||||
|
||||
registerOpenFreeMapProjection('outlines', decodeOpenFreeMapOutlineTile);
|
||||
|
||||
/** Tile pixel -> longitude/latitude, exactly as vector-tile's toGeoJSON projects. */
|
||||
function pixelToLonLat(px, py, extent, z, x, y) {
|
||||
const size = extent * 2 ** z;
|
||||
@@ -192,7 +228,7 @@ function decodePolygons(layer, z, x, y, keep, describe) {
|
||||
/** Construct the outline tile source: same OpenFreeMap tiles, outline decode. */
|
||||
export function createOpenFreeMapOutlineSource(options = {}) {
|
||||
return createOpenFreeMapSource({
|
||||
decode: decodeOpenFreeMapOutlineTile,
|
||||
projection: 'outlines',
|
||||
maxEntries: 32,
|
||||
maxCacheBytes: 16 * 1024 * 1024,
|
||||
...options,
|
||||
|
||||
@@ -5,6 +5,7 @@ import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import {
|
||||
buildingFromTiles,
|
||||
createOpenFreeMapOutlineSource,
|
||||
decodeOpenFreeMapOutlineTile,
|
||||
enclosingAreaFromTiles,
|
||||
mergeLineSegments,
|
||||
@@ -12,6 +13,12 @@ import {
|
||||
streetFromTiles,
|
||||
streetNameMatches,
|
||||
} from './openFreeMapOutlines.js';
|
||||
import {
|
||||
clearSharedOpenFreeMapTiles,
|
||||
createOpenFreeMapSource,
|
||||
registerOpenFreeMapProjection,
|
||||
} from './openFreeMap.js';
|
||||
import { createSharedDecodedTileCache } from './vectorTiles.js';
|
||||
import { approximateDistanceM } from './featureGeometry.js';
|
||||
|
||||
const fixture = (y) =>
|
||||
@@ -43,7 +50,10 @@ test('tiles decode named street lines and building polygons', () => {
|
||||
|
||||
test('street names compare with abbreviations and a leading direction only', () => {
|
||||
assert.equal(normalizeStreetName('Congress Ave.'), 'congress avenue');
|
||||
assert.equal(normalizeStreetName('Lombard St, San Francisco'), 'lombard street');
|
||||
assert.equal(
|
||||
normalizeStreetName('Lombard St, San Francisco'),
|
||||
'lombard street',
|
||||
);
|
||||
assert.ok(streetNameMatches('Congress Avenue', 'congress ave'));
|
||||
assert.ok(streetNameMatches('North Congress Avenue', 'Congress Avenue'));
|
||||
assert.ok(!streetNameMatches('Congress Avenue', 'South Congress Avenue'));
|
||||
@@ -52,21 +62,33 @@ test('street names compare with abbreviations and a leading direction only', ()
|
||||
});
|
||||
|
||||
test('same-name segments merge into polylines around the point (Congress Avenue)', () => {
|
||||
const street = streetFromTiles(TILES, { name: 'Congress Avenue', ...CAPITOL });
|
||||
const street = streetFromTiles(TILES, {
|
||||
name: 'Congress Avenue',
|
||||
...CAPITOL,
|
||||
});
|
||||
assert.equal(street.name, 'Congress Avenue');
|
||||
assert.equal(street.lines.length, 1, 'the tile pieces join into one line');
|
||||
const line = street.lines[0];
|
||||
const [start, end] = [line[0], line.at(-1)];
|
||||
const spanM = approximateDistanceM(start[1], start[0], end[1], end[0]);
|
||||
assert.ok(spanM > 1200, `Congress runs ${spanM.toFixed(0)} m from the river to the Capitol`);
|
||||
assert.ok(
|
||||
spanM > 1200,
|
||||
`Congress runs ${spanM.toFixed(0)} m from the river to the Capitol`,
|
||||
);
|
||||
// South Congress is a different street and stays out.
|
||||
const south = streetFromTiles(TILES, { name: 'South Congress Avenue', ...CAPITOL });
|
||||
const south = streetFromTiles(TILES, {
|
||||
name: 'South Congress Avenue',
|
||||
...CAPITOL,
|
||||
});
|
||||
assert.ok(south);
|
||||
assert.ok(south.lines.every((l) => l.every(([, lat]) => lat < 30.265)));
|
||||
});
|
||||
|
||||
test('street lookups are bounded by radius and answer null for unknown names', () => {
|
||||
assert.equal(streetFromTiles(TILES, { name: 'Lombard Street', ...CAPITOL }), null);
|
||||
assert.equal(
|
||||
streetFromTiles(TILES, { name: 'Lombard Street', ...CAPITOL }),
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
streetFromTiles(TILES, { name: 'Congress Avenue', lat: 30.4, lon: -97.74 }),
|
||||
null,
|
||||
@@ -75,9 +97,18 @@ test('street lookups are bounded by radius and answer null for unknown names', (
|
||||
});
|
||||
|
||||
test('merge joins reversed and out-of-order pieces and keeps gaps apart', () => {
|
||||
const a = [[0, 0], [0.001, 0]];
|
||||
const b = [[0.002, 0], [0.001, 0]]; // reversed, shares a's end
|
||||
const c = [[0.01, 0], [0.011, 0]]; // separate
|
||||
const a = [
|
||||
[0, 0],
|
||||
[0.001, 0],
|
||||
];
|
||||
const b = [
|
||||
[0.002, 0],
|
||||
[0.001, 0],
|
||||
]; // reversed, shares a's end
|
||||
const c = [
|
||||
[0.01, 0],
|
||||
[0.011, 0],
|
||||
]; // separate
|
||||
const chains = mergeLineSegments([b, c, a]);
|
||||
assert.equal(chains.length, 2);
|
||||
assert.equal(chains.find((ch) => ch.length === 3).length, 3);
|
||||
@@ -92,7 +123,13 @@ test('the building containing a point is picked (Texas Capitol)', () => {
|
||||
});
|
||||
|
||||
test('a point beside a building picks the nearest within 40 m, else none', () => {
|
||||
const box = [[0, 0], [0.0002, 0], [0.0002, 0.0002], [0, 0.0002], [0, 0]];
|
||||
const box = [
|
||||
[0, 0],
|
||||
[0.0002, 0],
|
||||
[0.0002, 0.0002],
|
||||
[0, 0.0002],
|
||||
[0, 0],
|
||||
];
|
||||
const tiles = [{ buildings: [{ rings: [box], heightM: null }] }];
|
||||
const near = buildingFromTiles(tiles, { lat: 0.0001, lon: 0.00045 }); // ~28 m east
|
||||
assert.ok(near && !near.contains);
|
||||
@@ -102,7 +139,10 @@ test('a point beside a building picks the nearest within 40 m, else none', () =>
|
||||
test('the open area enclosing the Capitol is its grounds, not a building', () => {
|
||||
const grounds = enclosingAreaFromTiles(TILES, CAPITOL);
|
||||
assert.ok(grounds, 'the park-like cover around the Capitol is found');
|
||||
assert.ok(grounds.areaM2 > 50_000 && grounds.areaM2 < 500_000, `${grounds.areaM2} m²`);
|
||||
assert.ok(
|
||||
grounds.areaM2 > 50_000 && grounds.areaM2 < 500_000,
|
||||
`${grounds.areaM2} m²`,
|
||||
);
|
||||
assert.equal(grounds.class, 'park');
|
||||
assert.equal(
|
||||
enclosingAreaFromTiles(TILES, { ...CAPITOL, minAreaM2: 1e6 }),
|
||||
@@ -110,3 +150,151 @@ test('the open area enclosing the Capitol is its grounds, not a building', () =>
|
||||
'nothing grounds-sized above the floor',
|
||||
);
|
||||
});
|
||||
|
||||
test('traffic idle preparation gives outlines the same decoded tile without another request', async () => {
|
||||
clearSharedOpenFreeMapTiles();
|
||||
const idle = [];
|
||||
const previousIdle = globalThis.requestIdleCallback;
|
||||
const previousCancelIdle = globalThis.cancelIdleCallback;
|
||||
globalThis.requestIdleCallback = (callback) => {
|
||||
idle.push(callback);
|
||||
return idle.length;
|
||||
};
|
||||
globalThis.cancelIdleCallback = () => {};
|
||||
let requests = 0;
|
||||
const bytes = readFileSync(
|
||||
new URL(
|
||||
'../data/fixtures/ofm-outlines-austin-14-3743-6745.pbf',
|
||||
import.meta.url,
|
||||
),
|
||||
);
|
||||
const fetchImpl = async () => {
|
||||
requests++;
|
||||
return new Response(bytes);
|
||||
};
|
||||
const tiles = [{ z: 14, x: 3743, y: 6745 }];
|
||||
const box = { south: 30.27, north: 30.28, west: -97.75, east: -97.73 };
|
||||
try {
|
||||
const traffic = createOpenFreeMapSource({
|
||||
template:
|
||||
'https://tiles.openfreemap.org/planet/v/20250910_001001_pt/{z}/{x}/{y}.pbf',
|
||||
fetchImpl,
|
||||
});
|
||||
const outlines = createOpenFreeMapOutlineSource({
|
||||
template:
|
||||
'https://tiles.openfreemap.org/planet/v/20250910_001001_pt/{z}/{x}/{y}.pbf',
|
||||
fetchImpl,
|
||||
});
|
||||
await traffic.fetchBounds(box, { zoom: 14, tiles });
|
||||
assert.ok(idle.length > 0, 'traffic schedules the registered projection');
|
||||
for (const callback of idle)
|
||||
callback({ didTimeout: false, timeRemaining: () => 50 });
|
||||
const result = await outlines.fetchBounds(box, { zoom: 14, tiles });
|
||||
assert.ok(result.tiles[0].buildings.length > 0);
|
||||
assert.equal(requests, 1);
|
||||
} finally {
|
||||
globalThis.requestIdleCallback = previousIdle;
|
||||
globalThis.cancelIdleCallback = previousCancelIdle;
|
||||
clearSharedOpenFreeMapTiles();
|
||||
}
|
||||
});
|
||||
|
||||
test('idle projections are retired on shared eviction, clear, and self-eviction', async () => {
|
||||
const previousIdle = globalThis.requestIdleCallback;
|
||||
const previousCancelIdle = globalThis.cancelIdleCallback;
|
||||
const scheduled = new Map();
|
||||
const cancelled = new Set();
|
||||
let nextHandle = 1;
|
||||
globalThis.requestIdleCallback = (callback) => {
|
||||
const handle = nextHandle++;
|
||||
scheduled.set(handle, callback);
|
||||
return handle;
|
||||
};
|
||||
globalThis.cancelIdleCallback = (handle) => cancelled.add(handle);
|
||||
let projectionRuns = 0;
|
||||
registerOpenFreeMapProjection('idle-lifecycle-test', () => {
|
||||
projectionRuns++;
|
||||
return { coordinates: Array.from({ length: 20_000 }, (_, i) => [i, i]) };
|
||||
});
|
||||
const firstBytes = readFileSync(
|
||||
new URL(
|
||||
'../data/fixtures/ofm-outlines-austin-14-3743-6745.pbf',
|
||||
import.meta.url,
|
||||
),
|
||||
);
|
||||
const secondBytes = readFileSync(
|
||||
new URL(
|
||||
'../data/fixtures/ofm-outlines-austin-14-3743-6746.pbf',
|
||||
import.meta.url,
|
||||
),
|
||||
);
|
||||
const sharedDecodedCache = createSharedDecodedTileCache({
|
||||
maxEntries: 1,
|
||||
maxCacheBytes: 100_000,
|
||||
});
|
||||
const source = createOpenFreeMapSource({
|
||||
template: 'https://tiles.openfreemap.org/planet/v/test/{z}/{x}/{y}.pbf',
|
||||
sharedDecodedCache,
|
||||
fetchImpl: async (url) =>
|
||||
new Response(
|
||||
String(url).includes('/6746.pbf') ? secondBytes : firstBytes,
|
||||
),
|
||||
});
|
||||
const box = { south: 30.24, north: 30.29, west: -97.75, east: -97.73 };
|
||||
try {
|
||||
await source.fetchBounds(box, {
|
||||
zoom: 14,
|
||||
tiles: [{ z: 14, x: 3743, y: 6745 }],
|
||||
});
|
||||
const firstHandles = [...scheduled.keys()];
|
||||
assert.ok(firstHandles.length >= 2);
|
||||
|
||||
await source.fetchBounds(box, {
|
||||
zoom: 14,
|
||||
tiles: [{ z: 14, x: 3743, y: 6746 }],
|
||||
});
|
||||
assert.ok(firstHandles.every((handle) => cancelled.has(handle)));
|
||||
for (const handle of firstHandles) scheduled.get(handle)();
|
||||
assert.equal(projectionRuns, 0, 'late callbacks after eviction are inert');
|
||||
|
||||
const secondHandles = [...scheduled.keys()].filter(
|
||||
(handle) => !firstHandles.includes(handle),
|
||||
);
|
||||
sharedDecodedCache.clear();
|
||||
assert.ok(secondHandles.every((handle) => cancelled.has(handle)));
|
||||
for (const handle of secondHandles) scheduled.get(handle)();
|
||||
assert.equal(projectionRuns, 0, 'late callbacks after clear are inert');
|
||||
|
||||
const selfEvictingCache = createSharedDecodedTileCache({
|
||||
maxEntries: 1,
|
||||
maxCacheBytes: 100_000,
|
||||
});
|
||||
const selfEvictingSource = createOpenFreeMapSource({
|
||||
template: 'https://tiles.openfreemap.org/planet/v/self/{z}/{x}/{y}.pbf',
|
||||
sharedDecodedCache: selfEvictingCache,
|
||||
fetchImpl: async () => new Response(firstBytes),
|
||||
});
|
||||
const beforeSelfEviction = nextHandle;
|
||||
await selfEvictingSource.fetchBounds(box, {
|
||||
zoom: 14,
|
||||
tiles: [{ z: 14, x: 3743, y: 6745 }],
|
||||
});
|
||||
const selfEvictingHandles = [...scheduled.keys()].filter(
|
||||
(handle) => handle >= beforeSelfEviction,
|
||||
);
|
||||
scheduled.get(selfEvictingHandles.at(-1))();
|
||||
assert.equal(projectionRuns, 1);
|
||||
assert.deepEqual(selfEvictingCache.stats(), { entries: 0, bytes: 0 });
|
||||
assert.ok(
|
||||
selfEvictingHandles.slice(0, -1).every((handle) => cancelled.has(handle)),
|
||||
);
|
||||
for (const handle of selfEvictingHandles)
|
||||
if (!cancelled.has(handle)) scheduled.get(handle)();
|
||||
assert.equal(projectionRuns, 1, 'self-eviction retires sibling idle work');
|
||||
} finally {
|
||||
source.clear();
|
||||
sharedDecodedCache.clear();
|
||||
globalThis.requestIdleCallback = previousIdle;
|
||||
globalThis.cancelIdleCallback = previousCancelIdle;
|
||||
}
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
/** Optional development phase measurements; no provider payloads or credentials. */
|
||||
export function phaseTiming(phase, start, detail = {}) {
|
||||
export function phaseTiming(phase, start, detail = {}, scope = 'roads') {
|
||||
if (import.meta.env?.DEV)
|
||||
performance.measure(`roads:${phase}`, {
|
||||
performance.measure(`${scope}:${phase}`, {
|
||||
start,
|
||||
end: performance.now(),
|
||||
detail,
|
||||
|
||||
+220
-43
@@ -16,6 +16,131 @@ export function validTileBounds(box) {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded decoded ownership shared by compatible vector-tile consumers.
|
||||
* Entries are keyed by the immutable tile template plus XYZ identity; callers
|
||||
* still keep their own consumer-shaped caches and request lifetimes.
|
||||
*/
|
||||
export function createSharedDecodedTileCache({
|
||||
maxEntries = 192,
|
||||
maxCacheBytes = 64 * 1024 * 1024,
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const flights = new Map();
|
||||
let cacheBytes = 0;
|
||||
let generation = 0;
|
||||
const release = (entry) => {
|
||||
if (!entry || entry.released) return;
|
||||
entry.released = true;
|
||||
entry.value?.releaseSharedCacheEntry?.();
|
||||
};
|
||||
const remove = (key, expected = null) => {
|
||||
const entry = cache.get(key);
|
||||
if (!entry || (expected && entry !== expected)) return false;
|
||||
cache.delete(key);
|
||||
cacheBytes -= entry.size;
|
||||
release(entry);
|
||||
return true;
|
||||
};
|
||||
const evict = () => {
|
||||
while (cache.size > maxEntries || cacheBytes > maxCacheBytes) {
|
||||
const oldest = cache.keys().next().value;
|
||||
remove(oldest);
|
||||
}
|
||||
};
|
||||
const resize = (key, value, entry, size) => {
|
||||
if (!Number.isFinite(size) || size < 0) return;
|
||||
const previousSize = entry.size;
|
||||
entry.size = size;
|
||||
if (cache.get(key) !== entry || entry.value !== value) return;
|
||||
cacheBytes += size - previousSize;
|
||||
evict();
|
||||
};
|
||||
const store = (key, value, size) => {
|
||||
const entry = { value, size, released: false };
|
||||
if (!Number.isFinite(size) || size < 0 || size > maxCacheBytes)
|
||||
return entry;
|
||||
remove(key);
|
||||
cache.set(key, entry);
|
||||
cacheBytes += size;
|
||||
try {
|
||||
value?.bindSharedCacheEntry?.(
|
||||
(nextSize) => resize(key, value, entry, nextSize),
|
||||
size,
|
||||
);
|
||||
} catch (error) {
|
||||
remove(key, entry);
|
||||
throw error;
|
||||
}
|
||||
evict();
|
||||
return entry;
|
||||
};
|
||||
const subscribe = (flight, signal, key) => {
|
||||
signal?.throwIfAborted();
|
||||
flight.users += 1;
|
||||
return new Promise((resolve, reject) => {
|
||||
const abort = () => reject(signal.reason);
|
||||
signal?.addEventListener('abort', abort, { once: true });
|
||||
flight.promise
|
||||
.then(resolve, reject)
|
||||
.finally(() => signal?.removeEventListener('abort', abort));
|
||||
}).finally(() => {
|
||||
flight.users -= 1;
|
||||
if (!flight.users && flights.get(key) === flight) {
|
||||
flights.delete(key);
|
||||
flight.controller.abort();
|
||||
}
|
||||
});
|
||||
};
|
||||
return {
|
||||
get(key) {
|
||||
const hit = cache.get(key);
|
||||
if (!hit) return null;
|
||||
cache.delete(key);
|
||||
cache.set(key, hit);
|
||||
return hit;
|
||||
},
|
||||
set(key, value, size) {
|
||||
return store(key, value, size);
|
||||
},
|
||||
load(key, signal, loader) {
|
||||
const hit = this.get(key);
|
||||
if (hit) return Promise.resolve(hit);
|
||||
let flight = flights.get(key);
|
||||
if (!flight) {
|
||||
flight = {
|
||||
controller: new AbortController(),
|
||||
users: 0,
|
||||
promise: null,
|
||||
generation,
|
||||
};
|
||||
flights.set(key, flight);
|
||||
flight.promise = Promise.resolve()
|
||||
.then(() => loader(flight.controller.signal))
|
||||
.then(({ value, size }) => {
|
||||
flight.controller.signal.throwIfAborted();
|
||||
if (flight.generation !== generation)
|
||||
throw new DOMException('Shared tile cache cleared', 'AbortError');
|
||||
return store(key, value, size);
|
||||
})
|
||||
.finally(() => {
|
||||
if (flights.get(key) === flight) flights.delete(key);
|
||||
});
|
||||
}
|
||||
return subscribe(flight, signal, key);
|
||||
},
|
||||
clear() {
|
||||
generation += 1;
|
||||
for (const flight of flights.values()) flight.controller.abort();
|
||||
flights.clear();
|
||||
for (const entry of cache.values()) release(entry);
|
||||
cache.clear();
|
||||
cacheBytes = 0;
|
||||
},
|
||||
stats: () => ({ entries: cache.size, bytes: cacheBytes }),
|
||||
};
|
||||
}
|
||||
|
||||
/** A decoded XYZ cache with bounded workers, reads, retained bytes and request lifetimes. */
|
||||
export function createVectorTileSource({
|
||||
tileJsonUrl,
|
||||
@@ -31,6 +156,8 @@ export function createVectorTileSource({
|
||||
maxResponseBytes = 4 * 1024 * 1024,
|
||||
metadataCooldownMs = 5000,
|
||||
now = Date.now,
|
||||
sharedDecodedCache = null,
|
||||
selectDecoded = (value) => value,
|
||||
}) {
|
||||
const cache = new Map();
|
||||
const active = new Set();
|
||||
@@ -44,13 +171,15 @@ export function createVectorTileSource({
|
||||
let fetched = 0;
|
||||
let activeWorkers = 0;
|
||||
const waiters = [];
|
||||
const consumerValue = (value) =>
|
||||
sharedDecodedCache ? structuredClone(value) : value;
|
||||
|
||||
async function request(url, parentSignal, read) {
|
||||
async function request(url, parentSignal, read, trackAsSource = true) {
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort(parentSignal?.reason);
|
||||
parentSignal?.throwIfAborted();
|
||||
parentSignal?.addEventListener('abort', abort, { once: true });
|
||||
active.add(controller);
|
||||
if (trackAsSource) active.add(controller);
|
||||
const timer = setTimeout(
|
||||
() =>
|
||||
controller.abort(
|
||||
@@ -88,7 +217,7 @@ export function createVectorTileSource({
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
parentSignal?.removeEventListener('abort', abort);
|
||||
active.delete(controller);
|
||||
if (trackAsSource) active.delete(controller);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -232,7 +361,7 @@ export function createVectorTileSource({
|
||||
const abort = () => reject(signal.reason);
|
||||
signal?.addEventListener('abort', abort, { once: true });
|
||||
flight.promise
|
||||
.then(resolve, reject)
|
||||
.then((value) => resolve(consumerValue(value)), reject)
|
||||
.finally(() => signal?.removeEventListener('abort', abort));
|
||||
}).finally(() => {
|
||||
if (--flight.users === 0 && flights.get(key) === flight) {
|
||||
@@ -249,54 +378,102 @@ export function createVectorTileSource({
|
||||
if (hit && now() - hit.at < ttlMs) {
|
||||
cache.delete(key);
|
||||
cache.set(key, hit);
|
||||
return hit.value;
|
||||
return consumerValue(hit.value);
|
||||
}
|
||||
if (hit) {
|
||||
cache.delete(key);
|
||||
cacheBytes -= hit.size;
|
||||
}
|
||||
const shared = sharedDecodedCache?.get(key);
|
||||
if (shared) {
|
||||
// A shared decode is authoritative cache state. Give each consumer its
|
||||
// own value so traffic, outlines, or a future caller cannot mutate the
|
||||
// cached object (including nested coordinate arrays) for everyone else.
|
||||
const value = structuredClone(selectDecoded(shared.value, tile));
|
||||
const size = shared.size;
|
||||
if (epoch === generation && size <= maxCacheBytes) {
|
||||
cache.set(key, { at: now(), value, size });
|
||||
cacheBytes += size;
|
||||
while (cache.size > maxEntries || cacheBytes > maxCacheBytes) {
|
||||
const oldest = cache.keys().next().value;
|
||||
cacheBytes -= cache.get(oldest).size;
|
||||
cache.delete(oldest);
|
||||
}
|
||||
}
|
||||
return consumerValue(value);
|
||||
}
|
||||
const loadDecoded = async (ownedSignal, sharedOwnership = false) => {
|
||||
await acquire(ownedSignal);
|
||||
try {
|
||||
ownedSignal.throwIfAborted();
|
||||
fetched++;
|
||||
const url = meta.template
|
||||
.replace('{z}', tile.z)
|
||||
.replace('{x}', tile.x)
|
||||
.replace('{y}', tile.y);
|
||||
const fetchStart = performance.now();
|
||||
const bytes = await request(
|
||||
url,
|
||||
ownedSignal,
|
||||
async (res, requestSignal) => {
|
||||
const bodyStart = performance.now();
|
||||
const value = await readResponseBytesCapped(
|
||||
res,
|
||||
maxResponseBytes,
|
||||
requestSignal,
|
||||
);
|
||||
phaseTiming('tile-body-read', bodyStart, {
|
||||
source: allowedOrigin,
|
||||
key,
|
||||
bytes: value.byteLength,
|
||||
});
|
||||
return value;
|
||||
},
|
||||
!sharedOwnership,
|
||||
);
|
||||
phaseTiming('tile-fetch', fetchStart, {
|
||||
source: allowedOrigin,
|
||||
key,
|
||||
bytes: bytes.byteLength,
|
||||
});
|
||||
const decodeStart = performance.now();
|
||||
const decoded = decode(bytes, tile.z, tile.x, tile.y);
|
||||
phaseTiming('decode', decodeStart, { source: allowedOrigin, key });
|
||||
ownedSignal.throwIfAborted();
|
||||
// Body-based decoded-storage estimate; projections may increase it.
|
||||
return { value: decoded, size: bytes.byteLength * 4 };
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
};
|
||||
let flight = flights.get(key);
|
||||
if (!flight) {
|
||||
flight = { controller: new AbortController(), users: 0 };
|
||||
const ownedSignal = flight.controller.signal;
|
||||
flights.set(key, flight);
|
||||
flight.promise = (async () => {
|
||||
await acquire(ownedSignal);
|
||||
try {
|
||||
ownedSignal.throwIfAborted();
|
||||
if (epoch !== generation)
|
||||
throw new DOMException('Source cleared', 'AbortError');
|
||||
fetched++;
|
||||
const url = meta.template
|
||||
.replace('{z}', tile.z)
|
||||
.replace('{x}', tile.x)
|
||||
.replace('{y}', tile.y);
|
||||
const fetchStart = performance.now();
|
||||
const bytes = await request(url, ownedSignal, (res, requestSignal) =>
|
||||
readResponseBytesCapped(res, maxResponseBytes, requestSignal),
|
||||
);
|
||||
phaseTiming('tile-fetch', fetchStart, {
|
||||
source: allowedOrigin,
|
||||
key,
|
||||
bytes: bytes.byteLength,
|
||||
});
|
||||
const decodeStart = performance.now();
|
||||
const value = decode(bytes, tile.z, tile.x, tile.y);
|
||||
phaseTiming('decode', decodeStart, { source: allowedOrigin, key });
|
||||
ownedSignal.throwIfAborted();
|
||||
// Body-based decoded-storage estimate; never stringify geometry on the load path.
|
||||
const size = bytes.byteLength * 4;
|
||||
if (epoch === generation && size <= maxCacheBytes) {
|
||||
if (cache.has(key)) cacheBytes -= cache.get(key).size;
|
||||
cache.delete(key);
|
||||
cache.set(key, { at: now(), value, size });
|
||||
cacheBytes += size;
|
||||
while (cache.size > maxEntries || cacheBytes > maxCacheBytes) {
|
||||
const oldest = cache.keys().next().value;
|
||||
cacheBytes -= cache.get(oldest).size;
|
||||
cache.delete(oldest);
|
||||
}
|
||||
const decodedEntry = sharedDecodedCache
|
||||
? await sharedDecodedCache.load(key, ownedSignal, (sharedSignal) =>
|
||||
loadDecoded(sharedSignal, true),
|
||||
)
|
||||
: await loadDecoded(ownedSignal);
|
||||
ownedSignal.throwIfAborted();
|
||||
const selected = selectDecoded(decodedEntry.value, tile);
|
||||
const value = sharedDecodedCache ? structuredClone(selected) : selected;
|
||||
const currentShared = sharedDecodedCache?.get(key);
|
||||
const size = currentShared?.size ?? decodedEntry.size;
|
||||
if (epoch === generation && size <= maxCacheBytes) {
|
||||
if (cache.has(key)) cacheBytes -= cache.get(key).size;
|
||||
cache.delete(key);
|
||||
cache.set(key, { at: now(), value, size });
|
||||
cacheBytes += size;
|
||||
while (cache.size > maxEntries || cacheBytes > maxCacheBytes) {
|
||||
const oldest = cache.keys().next().value;
|
||||
cacheBytes -= cache.get(oldest).size;
|
||||
cache.delete(oldest);
|
||||
}
|
||||
return value;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
return value;
|
||||
})().finally(() => {
|
||||
if (flights.get(key) === flight) flights.delete(key);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createVectorTileSource } from './vectorTiles.js';
|
||||
import {
|
||||
createSharedDecodedTileCache,
|
||||
createVectorTileSource,
|
||||
} from './vectorTiles.js';
|
||||
const meta = { tiles: ['https://tiles.example/{z}/{x}/{y}.pbf'] };
|
||||
const box = { south: 30.267, north: 30.268, west: -97.744, east: -97.743 };
|
||||
const options = {
|
||||
@@ -203,3 +206,238 @@ test('polygon-selected tiles keep nearest-first order and cached revisits issue
|
||||
'over-budget selection is rejected before fetching',
|
||||
);
|
||||
});
|
||||
|
||||
test('compatible consumers share bounded decoded ownership without sharing local lifetimes', async () => {
|
||||
const sharedDecodedCache = createSharedDecodedTileCache({
|
||||
maxEntries: 2,
|
||||
maxCacheBytes: 100,
|
||||
});
|
||||
let decodes = 0,
|
||||
requests = 0;
|
||||
const common = {
|
||||
...options,
|
||||
template: meta.tiles[0],
|
||||
sharedDecodedCache,
|
||||
decode: () => {
|
||||
decodes++;
|
||||
return { roads: ['road'], outlines: ['outline'] };
|
||||
},
|
||||
fetchImpl: async () => {
|
||||
requests++;
|
||||
return new Response(new Uint8Array([1, 2, 3]));
|
||||
},
|
||||
};
|
||||
const roads = createVectorTileSource({
|
||||
...common,
|
||||
selectDecoded: (value) => value.roads,
|
||||
});
|
||||
const outlines = createVectorTileSource({
|
||||
...common,
|
||||
selectDecoded: (value) => value.outlines,
|
||||
});
|
||||
const tiles = [{ z: 14, x: 3743, y: 6745 }];
|
||||
assert.deepEqual((await roads.fetchBounds(box, { zoom: 14, tiles })).tiles, [
|
||||
['road'],
|
||||
]);
|
||||
roads.clear();
|
||||
assert.deepEqual(
|
||||
(await outlines.fetchBounds(box, { zoom: 14, tiles })).tiles,
|
||||
[['outline']],
|
||||
);
|
||||
assert.equal(requests, 1);
|
||||
assert.equal(decodes, 1);
|
||||
assert.deepEqual(sharedDecodedCache.stats(), { entries: 1, bytes: 12 });
|
||||
});
|
||||
|
||||
test('shared decoded tiles are isolated from consumer mutation', async () => {
|
||||
const sharedDecodedCache = createSharedDecodedTileCache();
|
||||
let decodes = 0,
|
||||
requests = 0;
|
||||
const common = {
|
||||
...options,
|
||||
template: meta.tiles[0],
|
||||
sharedDecodedCache,
|
||||
decode: () => {
|
||||
decodes++;
|
||||
return { roads: [{ name: 'Congress Avenue', coordinates: [[-97, 30]] }] };
|
||||
},
|
||||
selectDecoded: (value) => value.roads,
|
||||
fetchImpl: async () => {
|
||||
requests++;
|
||||
return new Response(new Uint8Array([1, 2, 3]));
|
||||
},
|
||||
};
|
||||
const first = createVectorTileSource(common);
|
||||
const second = createVectorTileSource(common);
|
||||
const third = createVectorTileSource(common);
|
||||
const tiles = [{ z: 14, x: 3743, y: 6745 }];
|
||||
const firstResult = await first.fetchBounds(box, { zoom: 14, tiles });
|
||||
firstResult.tiles[0][0].name = 'first mutation';
|
||||
firstResult.tiles[0][0].coordinates[0][0] = 0;
|
||||
|
||||
const secondResult = await second.fetchBounds(box, { zoom: 14, tiles });
|
||||
assert.equal(secondResult.tiles[0][0].name, 'Congress Avenue');
|
||||
assert.equal(secondResult.tiles[0][0].coordinates[0][0], -97);
|
||||
secondResult.tiles[0][0].name = 'second mutation';
|
||||
secondResult.tiles[0][0].coordinates[0][0] = 1;
|
||||
|
||||
const thirdResult = await third.fetchBounds(box, { zoom: 14, tiles });
|
||||
assert.equal(thirdResult.tiles[0][0].name, 'Congress Avenue');
|
||||
assert.equal(thirdResult.tiles[0][0].coordinates[0][0], -97);
|
||||
assert.equal(requests, 1);
|
||||
assert.equal(decodes, 1);
|
||||
});
|
||||
|
||||
test('one source isolates in-flight and completed cache results from consumer mutation', async () => {
|
||||
const sharedDecodedCache = createSharedDecodedTileCache();
|
||||
let decodes = 0,
|
||||
requests = 0,
|
||||
releaseResponse;
|
||||
const responseReady = new Promise((resolve) => {
|
||||
releaseResponse = resolve;
|
||||
});
|
||||
let requestStarted;
|
||||
const started = new Promise((resolve) => {
|
||||
requestStarted = resolve;
|
||||
});
|
||||
const source = createVectorTileSource({
|
||||
...options,
|
||||
template: meta.tiles[0],
|
||||
sharedDecodedCache,
|
||||
decode: () => {
|
||||
decodes++;
|
||||
return { roads: [{ name: 'Congress Avenue', coordinates: [[-97, 30]] }] };
|
||||
},
|
||||
selectDecoded: (value) => value.roads,
|
||||
fetchImpl: async () => {
|
||||
requests++;
|
||||
requestStarted();
|
||||
await responseReady;
|
||||
return new Response(new Uint8Array([1, 2, 3]));
|
||||
},
|
||||
});
|
||||
const tiles = [{ z: 14, x: 3743, y: 6745 }];
|
||||
const firstPending = source.fetchBounds(box, { zoom: 14, tiles });
|
||||
await started;
|
||||
const concurrentPending = source.fetchBounds(box, { zoom: 14, tiles });
|
||||
releaseResponse();
|
||||
|
||||
const firstResult = await firstPending;
|
||||
const concurrentResult = await concurrentPending;
|
||||
firstResult.tiles[0][0].name = 'first mutation';
|
||||
firstResult.tiles[0][0].coordinates[0][0] = 0;
|
||||
assert.equal(concurrentResult.tiles[0][0].name, 'Congress Avenue');
|
||||
assert.equal(concurrentResult.tiles[0][0].coordinates[0][0], -97);
|
||||
|
||||
concurrentResult.tiles[0][0].name = 'concurrent mutation';
|
||||
concurrentResult.tiles[0][0].coordinates[0][0] = 1;
|
||||
const cachedResult = await source.fetchBounds(box, { zoom: 14, tiles });
|
||||
assert.equal(cachedResult.tiles[0][0].name, 'Congress Avenue');
|
||||
assert.equal(cachedResult.tiles[0][0].coordinates[0][0], -97);
|
||||
assert.equal(requests, 1);
|
||||
assert.equal(decodes, 1);
|
||||
});
|
||||
|
||||
test('compatible consumers share one in-flight fetch and survive one consumer clearing', async () => {
|
||||
const sharedDecodedCache = createSharedDecodedTileCache();
|
||||
let requests = 0,
|
||||
decodes = 0,
|
||||
releaseResponse;
|
||||
const responseReady = new Promise((resolve) => {
|
||||
releaseResponse = resolve;
|
||||
});
|
||||
let requestStarted;
|
||||
const started = new Promise((resolve) => {
|
||||
requestStarted = resolve;
|
||||
});
|
||||
const common = {
|
||||
...options,
|
||||
template: meta.tiles[0],
|
||||
sharedDecodedCache,
|
||||
decode: () => {
|
||||
decodes++;
|
||||
return { roads: ['road'], outlines: ['outline'] };
|
||||
},
|
||||
fetchImpl: async () => {
|
||||
requests++;
|
||||
requestStarted();
|
||||
await responseReady;
|
||||
return new Response(new Uint8Array([1, 2, 3]));
|
||||
},
|
||||
};
|
||||
const roads = createVectorTileSource({
|
||||
...common,
|
||||
selectDecoded: (value) => value.roads,
|
||||
});
|
||||
const outlines = createVectorTileSource({
|
||||
...common,
|
||||
selectDecoded: (value) => value.outlines,
|
||||
});
|
||||
const tiles = [{ z: 14, x: 3743, y: 6745 }];
|
||||
const roadResult = roads.fetchBounds(box, { zoom: 14, tiles });
|
||||
const roadRejected = assert.rejects(roadResult, { name: 'AbortError' });
|
||||
await started;
|
||||
const outlineResult = outlines.fetchBounds(box, { zoom: 14, tiles });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
roads.clear();
|
||||
releaseResponse();
|
||||
await roadRejected;
|
||||
assert.deepEqual((await outlineResult).tiles, [['outline']]);
|
||||
assert.equal(requests, 1);
|
||||
assert.equal(decodes, 1);
|
||||
});
|
||||
|
||||
test('shared ownership recharges materialized projections and evicts over budget', () => {
|
||||
const shared = createSharedDecodedTileCache({ maxCacheBytes: 100 });
|
||||
let resize,
|
||||
releases = 0;
|
||||
const decoded = {
|
||||
bindSharedCacheEntry(update) {
|
||||
resize = update;
|
||||
},
|
||||
releaseSharedCacheEntry() {
|
||||
releases++;
|
||||
},
|
||||
};
|
||||
const entry = shared.set('tile', decoded, 12);
|
||||
assert.deepEqual(shared.stats(), { entries: 1, bytes: 12 });
|
||||
resize(80);
|
||||
assert.deepEqual(shared.stats(), { entries: 1, bytes: 80 });
|
||||
resize(101);
|
||||
assert.deepEqual(shared.stats(), { entries: 0, bytes: 0 });
|
||||
assert.equal(entry.size, 101, 'the returned entry retains its real size');
|
||||
assert.equal(releases, 1, 'self-eviction releases decoded ownership once');
|
||||
shared.clear();
|
||||
assert.equal(releases, 1, 'clearing after eviction does not release twice');
|
||||
});
|
||||
|
||||
test('shared clear releases entries and prevents a pending loader from refilling', async () => {
|
||||
const shared = createSharedDecodedTileCache();
|
||||
let releases = 0;
|
||||
const decoded = {
|
||||
releaseSharedCacheEntry() {
|
||||
releases++;
|
||||
},
|
||||
};
|
||||
shared.set('resident', decoded, 12);
|
||||
shared.clear();
|
||||
assert.deepEqual(shared.stats(), { entries: 0, bytes: 0 });
|
||||
assert.equal(releases, 1);
|
||||
|
||||
let loaderStarted, resolveLoader;
|
||||
const started = new Promise((resolve) => {
|
||||
loaderStarted = resolve;
|
||||
});
|
||||
const loaderResult = new Promise((resolve) => {
|
||||
resolveLoader = resolve;
|
||||
});
|
||||
const pending = shared.load('late', null, async () => {
|
||||
loaderStarted();
|
||||
return loaderResult;
|
||||
});
|
||||
await started;
|
||||
resolveLoader({ value: {}, size: 16 });
|
||||
shared.clear();
|
||||
await assert.rejects(pending, { name: 'AbortError' });
|
||||
assert.deepEqual(shared.stats(), { entries: 0, bytes: 0 });
|
||||
});
|
||||
@@ -21,8 +21,13 @@ import { createReferenceSources } from '../sources/reference.js';
|
||||
export { createReferenceSources as createStandaloneReferenceSources } from '../sources/reference.js';
|
||||
|
||||
/** Select standalone providers without starting their acquisition. */
|
||||
export function createStandaloneLayerSources() {
|
||||
const mapTiles = createOpenFreeMapSource();
|
||||
export function createStandaloneLayerSources({
|
||||
createMapTiles = createOpenFreeMapSource,
|
||||
} = {}) {
|
||||
// Each subsystem owns its own request lifetime and local result cache. The
|
||||
// sources still share the bounded decoded-tile owner inside OpenFreeMap.
|
||||
const trafficTiles = createMapTiles();
|
||||
const installationTiles = createMapTiles();
|
||||
return {
|
||||
...createReferenceSources(),
|
||||
flights: createOpenSkySource(),
|
||||
@@ -32,10 +37,10 @@ export function createStandaloneLayerSources() {
|
||||
}),
|
||||
cctv: createCctvSource(),
|
||||
radio: createRadioSource(),
|
||||
traffic: createTrafficSource({ mapTiles }),
|
||||
traffic: createTrafficSource({ mapTiles: trafficTiles }),
|
||||
transit: createTransitSource(),
|
||||
bikeshare: createBikeshareSource(),
|
||||
installations: createInstallationSource({ mapTiles }),
|
||||
installations: createInstallationSource({ mapTiles: installationTiles }),
|
||||
satellites: createSatelliteSource(),
|
||||
launches: createLaunchSource(),
|
||||
alpr: createAlprTileSource(),
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createStandaloneLayerSources } from './layerSources.js';
|
||||
|
||||
test('traffic and installations own separate local OpenFreeMap lifetimes', () => {
|
||||
const mapTiles = [];
|
||||
const sources = createStandaloneLayerSources({
|
||||
createMapTiles() {
|
||||
const source = {
|
||||
clears: 0,
|
||||
clear() {
|
||||
this.clears++;
|
||||
},
|
||||
getMetadata: async () => ({
|
||||
template: 'https://tiles.openfreemap.org/planet/test/{z}/{x}/{y}.pbf',
|
||||
}),
|
||||
fetchBounds: async () => ({ tiles: [], loadedTiles: [] }),
|
||||
};
|
||||
mapTiles.push(source);
|
||||
return source;
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(mapTiles.length, 2);
|
||||
assert.notEqual(mapTiles[0], mapTiles[1]);
|
||||
sources.traffic.resetFlowTileCache();
|
||||
assert.deepEqual(
|
||||
mapTiles.map((source) => source.clears),
|
||||
[1, 0],
|
||||
'traffic reset cannot clear installation work',
|
||||
);
|
||||
sources.installations.destroy();
|
||||
assert.deepEqual(
|
||||
mapTiles.map((source) => source.clears),
|
||||
[1, 1],
|
||||
'installation destroy cannot clear traffic work',
|
||||
);
|
||||
});
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
} from '../../server/providers/regional/place.js';
|
||||
|
||||
const PUBLIC = { endpoint: PUBLIC_NOMINATIM_SEARCH, isPublic: true, dailyCap: 50 };
|
||||
const LOCAL = { ...PUBLIC, isPublic: false };
|
||||
const tempDir = (t) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-'));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
@@ -117,21 +118,103 @@ test('a sweep removes expired and corrupt entries, never ones it could not read'
|
||||
assert.deepEqual([1, 2, 3].map((n) => fs.existsSync(file(n))), [true, false, false]);
|
||||
});
|
||||
|
||||
test('processes sharing the state file add to one count', async (t) => {
|
||||
const file = path.join(tempDir(t), 'state.json');
|
||||
test('processes sharing state reserve public spacing and cap atomically', async (t) => {
|
||||
const dir = tempDir(t);
|
||||
const module = new URL('../../server/providers/regional/nominatimGate.js', import.meta.url).href;
|
||||
const script = `
|
||||
const run = (file, dailyCap) => {
|
||||
const script = `
|
||||
const { createGateStateStore } = await import(${JSON.stringify(module)});
|
||||
const store = createGateStateStore({ file: ${JSON.stringify(file)} });
|
||||
for (let i = 0; i < 25; i++) store.increment('2026-09-29');
|
||||
console.log(JSON.stringify(store.reserve('2026-09-29', {
|
||||
now: 1000,
|
||||
dailyCap: ${dailyCap},
|
||||
minSpacingMs: 1100,
|
||||
latestStartAt: 10000,
|
||||
})));
|
||||
`;
|
||||
const run = () =>
|
||||
new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: 'inherit' });
|
||||
child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`exit ${code}`))));
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ['--input-type=module', '-e', script]);
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', (chunk) => (stdout += chunk));
|
||||
child.stderr.on('data', (chunk) => (stderr += chunk));
|
||||
child.on('exit', (code) =>
|
||||
code === 0
|
||||
? resolve(JSON.parse(stdout))
|
||||
: reject(new Error(`exit ${code}: ${stderr}`)),
|
||||
);
|
||||
});
|
||||
await Promise.all([run(), run(), run()]);
|
||||
assert.equal(createGateStateStore({ file }).count('2026-09-29'), 75);
|
||||
};
|
||||
|
||||
const spacedFile = path.join(dir, 'spaced.json');
|
||||
const reservations = await Promise.all([
|
||||
run(spacedFile, 3),
|
||||
run(spacedFile, 3),
|
||||
run(spacedFile, 3),
|
||||
]);
|
||||
assert.deepEqual(
|
||||
reservations.map((result) => result.status),
|
||||
['reserved', 'reserved', 'reserved'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
reservations.map((result) => result.startAt).sort((a, b) => a - b),
|
||||
[1000, 2100, 3200],
|
||||
);
|
||||
assert.equal(createGateStateStore({ file: spacedFile }).count('2026-09-29'), 3);
|
||||
|
||||
const cappedFile = path.join(dir, 'capped.json');
|
||||
const capped = await Promise.all([run(cappedFile, 1), run(cappedFile, 1)]);
|
||||
assert.deepEqual(
|
||||
capped.map((result) => result.status).sort(),
|
||||
['cap', 'reserved'],
|
||||
);
|
||||
assert.equal(createGateStateStore({ file: cappedFile }).count('2026-09-29'), 1);
|
||||
});
|
||||
|
||||
test('independent public gates sharing state enforce spacing and cap', async (t) => {
|
||||
const dir = tempDir(t);
|
||||
const starts = [];
|
||||
const fetchImpl = async () => {
|
||||
starts.push(Date.now());
|
||||
return json([]);
|
||||
};
|
||||
const file = path.join(dir, 'spacing.json');
|
||||
const make = (options = {}) =>
|
||||
createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl,
|
||||
usage: createGateStateStore({ file: options.file || file }),
|
||||
minSpacingMs: options.minSpacingMs || 40,
|
||||
});
|
||||
await Promise.all([
|
||||
createNominatimSearchProvider({ gate: make() })('one', null),
|
||||
createNominatimSearchProvider({ gate: make() })('two', null),
|
||||
]);
|
||||
starts.sort((a, b) => a - b);
|
||||
assert.ok(starts[1] - starts[0] >= 35, `${starts[1] - starts[0]}ms apart`);
|
||||
|
||||
const capFile = path.join(dir, 'cap.json');
|
||||
let sent = 0;
|
||||
const capGate = () =>
|
||||
createNominatimGate({
|
||||
settings: { ...PUBLIC, dailyCap: 1 },
|
||||
fetchImpl: async () => {
|
||||
sent += 1;
|
||||
return json([]);
|
||||
},
|
||||
usage: createGateStateStore({ file: capFile }),
|
||||
minSpacingMs: 1,
|
||||
});
|
||||
const results = await Promise.allSettled([
|
||||
createNominatimSearchProvider({ gate: capGate() })('three', null),
|
||||
createNominatimSearchProvider({ gate: capGate() })('four', null),
|
||||
]);
|
||||
assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1);
|
||||
assert.equal(
|
||||
results.find((result) => result.status === 'rejected').reason.code,
|
||||
'NOMINATIM_DAILY_CAP',
|
||||
);
|
||||
assert.equal(sent, 1);
|
||||
});
|
||||
|
||||
test('a persistence failure is reported and never lowers the count', (t) => {
|
||||
@@ -151,6 +234,49 @@ test('a persistence failure is reported and never lowers the count', (t) => {
|
||||
assert.equal(errors.length, 1, 'reported once, not per request');
|
||||
});
|
||||
|
||||
test('parseable but invalid public usage state fails closed', async (t) => {
|
||||
const dir = tempDir(t);
|
||||
const file = path.join(dir, 'state.json');
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
JSON.stringify({
|
||||
day: '2026-09-30',
|
||||
count: '50',
|
||||
pausedUntil: '9999999999999',
|
||||
}),
|
||||
);
|
||||
let sent = 0;
|
||||
const gate = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl: async () => {
|
||||
sent += 1;
|
||||
return json([]);
|
||||
},
|
||||
usage: createGateStateStore({ file, onError: () => {} }),
|
||||
...quick,
|
||||
});
|
||||
await assert.rejects(
|
||||
gate.requestJson((endpoint) => `${endpoint}?q=x`),
|
||||
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
|
||||
);
|
||||
assert.equal(sent, 0);
|
||||
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
JSON.stringify({ day: '2026-09-30', count: 1, pausedUntil: 0 }),
|
||||
);
|
||||
const migrated = createGateStateStore({ file });
|
||||
assert.equal(
|
||||
migrated.reserve('2026-09-30', {
|
||||
now: 1000,
|
||||
dailyCap: 50,
|
||||
minSpacingMs: 1100,
|
||||
}).status,
|
||||
'reserved',
|
||||
'the original state format without nextStartAt remains supported',
|
||||
);
|
||||
});
|
||||
|
||||
test('a long Retry-After is honoured in full and survives a restart', async (t) => {
|
||||
const file = path.join(tempDir(t), 'state.json');
|
||||
let clock = Date.UTC(2026, 8, 29, 12);
|
||||
@@ -189,6 +315,35 @@ test('a long Retry-After is honoured in full and survives a restart', async (t)
|
||||
assert.equal(calls.length, 1);
|
||||
});
|
||||
|
||||
test('an operator refusal pauses only the endpoint that refused', async () => {
|
||||
let active = {
|
||||
endpoint: 'https://a.example.test/search',
|
||||
isPublic: false,
|
||||
dailyCap: 50,
|
||||
};
|
||||
const calls = [];
|
||||
const gate = createNominatimGate({
|
||||
settings: () => active,
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(new URL(url).host);
|
||||
return calls.length === 1
|
||||
? new Response('', {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '60' },
|
||||
})
|
||||
: json([]);
|
||||
},
|
||||
...quick,
|
||||
});
|
||||
const request = () => gate.requestJson((endpoint) => `${endpoint}?q=x`);
|
||||
await assert.rejects(request(), { code: 'NOMINATIM_REFUSED' });
|
||||
active = { ...active, endpoint: 'https://b.example.test/search' };
|
||||
assert.deepEqual(await request(), []);
|
||||
active = { ...active, endpoint: 'https://a.example.test/search' };
|
||||
await assert.rejects(request(), { code: 'NOMINATIM_PAUSED' });
|
||||
assert.deepEqual(calls, ['a.example.test', 'b.example.test']);
|
||||
});
|
||||
|
||||
test('a stalled body times out instead of holding the shared queue', async () => {
|
||||
let cancelled = 0;
|
||||
const stalled = (signal) => {
|
||||
@@ -217,7 +372,7 @@ test('a stalled body times out instead of holding the shared queue', async () =>
|
||||
test('cache keys are request-equivalent: a miss for "Paris???" never answers "Paris"', async () => {
|
||||
const calls = [];
|
||||
const gate = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
settings: LOCAL,
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(new URL(url).searchParams.get('q'));
|
||||
return json(calls.length === 1 ? [] : [{ lat: '48.85', lon: '2.35', display_name: 'Paris, France', name: 'Paris', addresstype: 'city' }]);
|
||||
@@ -236,7 +391,7 @@ test('a malformed answer is a failure, not a cached "no result"', async () => {
|
||||
let shape = { error: 'Unexpected' };
|
||||
let calls = 0;
|
||||
const gate = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
settings: LOCAL,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return json(shape);
|
||||
@@ -301,9 +456,29 @@ test('administrative boundaries must be at the asked level', () => {
|
||||
const state = boundary({ name: 'Texas', addresstype: 'state', place_rank: 8 });
|
||||
assert.equal(selectOutlineResult([state], { kind: 'city', query: 'Texas' }).outline, null);
|
||||
assert.equal(selectOutlineResult([city], { kind: 'admin', query: 'Austin' }).outline, null);
|
||||
assert.equal(
|
||||
selectOutlineResult(
|
||||
[boundary({ name: 'Travis', addresstype: 'county', place_rank: 12 })],
|
||||
{ kind: 'city', query: 'Travis' },
|
||||
).outline,
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
selectOutlineResult(
|
||||
[boundary({ name: 'Austin', addresstype: 'city', place_rank: 12 })],
|
||||
{ kind: 'admin', query: 'Austin' },
|
||||
).outline,
|
||||
null,
|
||||
);
|
||||
assert.equal(selectOutlineResult([state], { kind: 'neighborhood', query: 'Texas' }).outline, null);
|
||||
assert.ok(selectOutlineResult([county], { kind: 'admin', query: 'Travis County' }).outline);
|
||||
assert.ok(selectOutlineResult([boundary({ name: 'Somewhere', place_rank: 16 })], { kind: 'city', query: 'Somewhere' }).outline);
|
||||
assert.ok(
|
||||
selectOutlineResult(
|
||||
[boundary({ name: 'Somewhere', addresstype: 'locality', place_rank: 16 })],
|
||||
{ kind: 'city', query: 'Somewhere' },
|
||||
).outline,
|
||||
);
|
||||
assert.equal(
|
||||
selectOutlineResult([boundary({ name: 'Somewhere' })], { kind: 'city', query: 'Somewhere' }).outline,
|
||||
null,
|
||||
@@ -317,6 +492,103 @@ test('administrative boundaries must be at the asked level', () => {
|
||||
assert.equal(picked.outline.osm, 'relation/9');
|
||||
});
|
||||
|
||||
test('admin identity comes only from unambiguous evidence on the selected row', () => {
|
||||
for (const [name, addresstype, adminLevel] of [
|
||||
['Nepal', 'country', 'country'],
|
||||
['Bagmati Province', 'province', 'admin1'],
|
||||
['Province of Bagmati', 'province', 'admin1'],
|
||||
['Texas', 'state', 'admin1'],
|
||||
['Travis County', 'county', 'admin2'],
|
||||
['County of Travis', 'county', 'admin2'],
|
||||
['Kathmandu District', 'state_district', 'admin2'],
|
||||
]) {
|
||||
const { outline } = selectOutlineResult(
|
||||
[boundary({ name, addresstype })],
|
||||
{ kind: 'admin', query: name },
|
||||
);
|
||||
assert.equal(outline.adminArea, name);
|
||||
assert.equal(outline.adminLevel, adminLevel);
|
||||
}
|
||||
|
||||
for (const row of [
|
||||
boundary({ name: 'Northern Region', addresstype: 'region', place_rank: 8 }),
|
||||
boundary({ name: 'Central District', addresstype: 'district', place_rank: 10 }),
|
||||
boundary({ name: 'Mystery Province', place_rank: 8 }),
|
||||
boundary({ name: 'Example State', addresstype: 'county' }),
|
||||
boundary({ name: 'County of Example', addresstype: 'state' }),
|
||||
boundary({ name: 'Province of Example', addresstype: 'county' }),
|
||||
boundary({ name: 'Example County', addresstype: 'state' }),
|
||||
boundary({ name: 'Example Province', addresstype: 'county' }),
|
||||
boundary({ name: 'County of Example Province', addresstype: 'county' }),
|
||||
]) {
|
||||
const { outline } = selectOutlineResult([row], {
|
||||
kind: 'admin',
|
||||
query: row.name,
|
||||
});
|
||||
assert.ok(outline, row.name);
|
||||
assert.equal(outline.adminArea, undefined, row.name);
|
||||
assert.equal(outline.adminLevel, undefined, row.name);
|
||||
}
|
||||
|
||||
const retired = selectOutlineResult(
|
||||
[boundary({ name: 'Bagmati Zone', addresstype: 'state' })],
|
||||
{ kind: 'admin', query: 'Bagmati Zone' },
|
||||
).outline;
|
||||
assert.equal(retired.adminArea, 'Bagmati Zone');
|
||||
assert.equal(retired.adminLevel, undefined);
|
||||
|
||||
const retiredPrefix = selectOutlineResult(
|
||||
[boundary({ name: 'Zone of Bagmati', addresstype: 'state' })],
|
||||
{ kind: 'admin', query: 'Zone of Bagmati' },
|
||||
).outline;
|
||||
assert.equal(retiredPrefix.adminArea, 'Zone of Bagmati');
|
||||
assert.equal(retiredPrefix.adminLevel, undefined);
|
||||
});
|
||||
|
||||
test('admin identity validates the full name before display truncation', () => {
|
||||
const longValidName = `Province of ${'Example '.repeat(24)}`.trim();
|
||||
const valid = selectOutlineResult(
|
||||
[boundary({ name: longValidName, addresstype: 'province' })],
|
||||
{ kind: 'admin', query: longValidName },
|
||||
).outline;
|
||||
assert.equal(valid.adminArea, longValidName.slice(0, 160));
|
||||
assert.equal(valid.adminLevel, 'admin1');
|
||||
|
||||
const longContradictoryName = `${'Very Long Administrative Name '.repeat(7)}County`;
|
||||
assert.ok(longContradictoryName.length > 160);
|
||||
const contradictory = selectOutlineResult(
|
||||
[boundary({ name: longContradictoryName, addresstype: 'state' })],
|
||||
{ kind: 'admin', query: longContradictoryName },
|
||||
).outline;
|
||||
assert.ok(contradictory);
|
||||
assert.equal(contradictory.adminArea, undefined);
|
||||
assert.equal(contradictory.adminLevel, undefined);
|
||||
});
|
||||
|
||||
test('non-administrative outline kinds never manufacture admin identity', () => {
|
||||
const city = selectOutlineResult(
|
||||
[boundary({ name: 'County of Austin', addresstype: 'city', place_rank: 16 })],
|
||||
{ kind: 'city', query: 'County of Austin' },
|
||||
).outline;
|
||||
const park = selectOutlineResult(
|
||||
[
|
||||
{
|
||||
category: 'leisure',
|
||||
type: 'park',
|
||||
name: 'Zilker Park',
|
||||
lat: '1',
|
||||
lon: '1',
|
||||
geojson: { type: 'Polygon', coordinates: square(1, 1, 0.01) },
|
||||
},
|
||||
],
|
||||
{ kind: 'landmark', query: 'Zilker Park' },
|
||||
).outline;
|
||||
for (const outline of [city, park]) {
|
||||
assert.equal(outline.adminArea, undefined);
|
||||
assert.equal(outline.adminLevel, undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test('building asks need building evidence', () => {
|
||||
const park = { category: 'tourism', type: 'theme_park', name: 'Big Top', lat: '1', lon: '1', geojson: { type: 'Polygon', coordinates: square(1, 1, 0.0001) } };
|
||||
const camp = { ...park, type: 'camp_site' };
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createGateStateStore } from '../../server/providers/regional/nominatimGate.js';
|
||||
|
||||
const moduleUrl = new URL(
|
||||
'../../server/providers/regional/nominatimGate.js',
|
||||
import.meta.url,
|
||||
).href;
|
||||
|
||||
function startModule(script) {
|
||||
const child = spawn(process.execPath, ['--input-type=module', '-e', script], {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
const lines = [];
|
||||
const waiters = [];
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
const publish = (line) => {
|
||||
lines.push(line);
|
||||
for (const waiter of [...waiters]) {
|
||||
if (!line.startsWith(waiter.prefix)) continue;
|
||||
waiters.splice(waiters.indexOf(waiter), 1);
|
||||
clearTimeout(waiter.timer);
|
||||
waiter.resolve(line);
|
||||
}
|
||||
};
|
||||
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout += chunk;
|
||||
const parts = stdout.split('\n');
|
||||
stdout = parts.pop();
|
||||
for (const line of parts) publish(line);
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
|
||||
return {
|
||||
child,
|
||||
lines,
|
||||
waitFor(prefix, timeoutMs = 3000) {
|
||||
const existing = lines.find((line) => line.startsWith(prefix));
|
||||
if (existing) return Promise.resolve(existing);
|
||||
return new Promise((resolve, reject) => {
|
||||
const waiter = { prefix, resolve, reject, timer: null };
|
||||
waiter.timer = setTimeout(() => {
|
||||
const index = waiters.indexOf(waiter);
|
||||
if (index >= 0) waiters.splice(index, 1);
|
||||
reject(
|
||||
new Error(
|
||||
`timed out waiting for ${prefix}; stderr: ${stderr.trim()}`,
|
||||
),
|
||||
);
|
||||
}, timeoutMs);
|
||||
waiters.push(waiter);
|
||||
});
|
||||
},
|
||||
exited: new Promise((resolve, reject) => {
|
||||
child.once('error', reject);
|
||||
child.once('exit', (code, signal) => resolve({ code, signal }));
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function holderScript(lockFile) {
|
||||
return `
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
const database = new DatabaseSync(${JSON.stringify(lockFile)});
|
||||
database.exec('BEGIN IMMEDIATE');
|
||||
process.stdout.write('LOCKED\\n');
|
||||
process.stdin.once('data', () => {
|
||||
database.exec('ROLLBACK');
|
||||
database.close();
|
||||
process.stdout.write('RELEASED\\n');
|
||||
});
|
||||
process.stdin.resume();
|
||||
`;
|
||||
}
|
||||
|
||||
function contenderScript(stateFile, lockWaitMs) {
|
||||
return `
|
||||
const { createGateStateStore } = await import(${JSON.stringify(moduleUrl)});
|
||||
const store = createGateStateStore({
|
||||
file: ${JSON.stringify(stateFile)},
|
||||
lockWaitMs: ${lockWaitMs},
|
||||
onError: () => {},
|
||||
});
|
||||
process.stdout.write('ATTEMPTING\\n', () => {
|
||||
const result = store.reserve('2026-10-01', {
|
||||
now: 1000,
|
||||
dailyCap: 50,
|
||||
minSpacingMs: 1100,
|
||||
});
|
||||
process.stdout.write('RESULT:' + JSON.stringify(result) + '\\n');
|
||||
});
|
||||
`;
|
||||
}
|
||||
|
||||
test('a crashed lock owner cannot delete or bypass a successor lock', async (t) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-lock-'));
|
||||
const stateFile = path.join(dir, 'state.json');
|
||||
const lockFile = `${stateFile}.lock.sqlite`;
|
||||
const children = [];
|
||||
t.after(() => {
|
||||
for (const process of children) {
|
||||
if (process.exitCode == null && process.signalCode == null)
|
||||
process.kill();
|
||||
}
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const crashedOwner = startModule(holderScript(lockFile));
|
||||
children.push(crashedOwner.child);
|
||||
await crashedOwner.waitFor('LOCKED');
|
||||
const originalLock = fs.statSync(lockFile);
|
||||
|
||||
const blockedByCrashedOwner = startModule(contenderScript(stateFile, 0));
|
||||
children.push(blockedByCrashedOwner.child);
|
||||
const blockedResult = JSON.parse(
|
||||
(await blockedByCrashedOwner.waitFor('RESULT:')).slice('RESULT:'.length),
|
||||
);
|
||||
assert.equal(blockedResult.status, 'unavailable');
|
||||
assert.equal((await blockedByCrashedOwner.exited).code, 0);
|
||||
assert.equal(fs.existsSync(stateFile), false);
|
||||
|
||||
crashedOwner.child.kill();
|
||||
await crashedOwner.exited;
|
||||
|
||||
const successorOwner = startModule(holderScript(lockFile));
|
||||
children.push(successorOwner.child);
|
||||
await successorOwner.waitFor('LOCKED');
|
||||
assert.equal(fs.statSync(lockFile).ino, originalLock.ino);
|
||||
|
||||
const refused = startModule(contenderScript(stateFile, 0));
|
||||
children.push(refused.child);
|
||||
const refusedResult = JSON.parse(
|
||||
(await refused.waitFor('RESULT:')).slice('RESULT:'.length),
|
||||
);
|
||||
assert.equal(refusedResult.status, 'unavailable');
|
||||
assert.equal((await refused.exited).code, 0);
|
||||
assert.equal(
|
||||
createGateStateStore({ file: stateFile }).count('2026-10-01'),
|
||||
0,
|
||||
'a timed-out contender fails closed without spending a public slot',
|
||||
);
|
||||
assert.equal(fs.statSync(lockFile).ino, originalLock.ino);
|
||||
|
||||
successorOwner.child.stdin.end('release\n');
|
||||
await successorOwner.waitFor('RELEASED');
|
||||
assert.equal((await successorOwner.exited).code, 0);
|
||||
const afterRelease = createGateStateStore({ file: stateFile }).reserve(
|
||||
'2026-10-01',
|
||||
{
|
||||
now: 2200,
|
||||
dailyCap: 50,
|
||||
minSpacingMs: 1100,
|
||||
},
|
||||
);
|
||||
assert.equal(afterRelease.status, 'reserved');
|
||||
assert.equal(afterRelease.count, 1);
|
||||
assert.equal(fs.statSync(lockFile).ino, originalLock.ino);
|
||||
});
|
||||
|
||||
test('a public pause stays authoritative while another process owns the lock', async (t) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-pause-'));
|
||||
const stateFile = path.join(dir, 'state.json');
|
||||
const lockFile = `${stateFile}.lock.sqlite`;
|
||||
const children = [];
|
||||
t.after(() => {
|
||||
for (const process of children) {
|
||||
if (process.exitCode == null && process.signalCode == null)
|
||||
process.kill();
|
||||
}
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
fs.writeFileSync(
|
||||
stateFile,
|
||||
JSON.stringify({
|
||||
day: '2026-10-01',
|
||||
count: 1,
|
||||
pausedUntil: 0,
|
||||
nextStartAt: 0,
|
||||
}),
|
||||
);
|
||||
const owner = startModule(holderScript(lockFile));
|
||||
children.push(owner.child);
|
||||
await owner.waitFor('LOCKED');
|
||||
|
||||
const errors = [];
|
||||
const blockedWriter = createGateStateStore({
|
||||
file: stateFile,
|
||||
lockWaitMs: 0,
|
||||
onError: (error) => errors.push(error),
|
||||
});
|
||||
blockedWriter.pauseUntil(9000);
|
||||
assert.equal(errors.length, 1, 'the blocked JSON consolidation is reported');
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(stateFile, 'utf8')).pausedUntil,
|
||||
0,
|
||||
'the lock owner prevents an unsafe JSON overwrite',
|
||||
);
|
||||
|
||||
owner.child.stdin.end('release\n');
|
||||
await owner.waitFor('RELEASED');
|
||||
assert.equal((await owner.exited).code, 0);
|
||||
|
||||
const successor = createGateStateStore({ file: stateFile });
|
||||
assert.deepEqual(
|
||||
successor.reserve('2026-10-01', {
|
||||
now: 1000,
|
||||
dailyCap: 50,
|
||||
minSpacingMs: 1100,
|
||||
}),
|
||||
{ status: 'paused', retryAfterMs: 8000 },
|
||||
'a fresh process observes the pending refusal pause',
|
||||
);
|
||||
const resumed = successor.reserve('2026-10-01', {
|
||||
now: 9001,
|
||||
dailyCap: 50,
|
||||
minSpacingMs: 1100,
|
||||
});
|
||||
assert.equal(resumed.status, 'reserved');
|
||||
assert.equal(resumed.count, 2);
|
||||
assert.equal(
|
||||
fs.readdirSync(dir).some((name) => name.startsWith('state.json.pause.')),
|
||||
false,
|
||||
'the next successful transaction consolidates and removes the marker',
|
||||
);
|
||||
});
|
||||
@@ -100,6 +100,23 @@ function stubFetch(respond) {
|
||||
return { calls, fetchImpl };
|
||||
}
|
||||
|
||||
function tempDir(t) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-'));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
function publicGate(t, options = {}) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-gate-'));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
return createNominatimGate({
|
||||
...options,
|
||||
settings: options.settings || PUBLIC,
|
||||
usage:
|
||||
options.usage || createUsageStore({ file: path.join(dir, 'state.json') }),
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- settings
|
||||
|
||||
test('endpoint settings: unset is public, empty disables, operator URLs are not public', () => {
|
||||
@@ -109,7 +126,10 @@ test('endpoint settings: unset is public, empty disables, operator URLs are not
|
||||
dailyCap: 50,
|
||||
});
|
||||
for (const off of ['', ' ', 'off', 'none', 'disabled'])
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_URL: off }).endpoint, null);
|
||||
assert.equal(
|
||||
resolveNominatimSettings({ NOMINATIM_URL: off }).endpoint,
|
||||
null,
|
||||
);
|
||||
const own = resolveNominatimSettings({
|
||||
NOMINATIM_URL: 'https://geo.example.test',
|
||||
});
|
||||
@@ -122,33 +142,84 @@ test('endpoint settings: unset is public, empty disables, operator URLs are not
|
||||
'http://localhost:8080/search',
|
||||
);
|
||||
// A mistyped value disables rather than silently falling back to public.
|
||||
for (const bad of ['not a url', 'ftp://x.test', 'https://u:[email protected]', 'https://x.test/search?q=1'])
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_URL: bad }).endpoint, null, bad);
|
||||
for (const bad of [
|
||||
'not a url',
|
||||
'ftp://x.test',
|
||||
'https://u:[email protected]',
|
||||
'https://x.test/search?q=1',
|
||||
])
|
||||
assert.equal(
|
||||
resolveNominatimSettings({ NOMINATIM_URL: bad }).endpoint,
|
||||
null,
|
||||
bad,
|
||||
);
|
||||
assert.equal(
|
||||
resolveNominatimSettings({ NOMINATIM_URL: 'https://nominatim.openstreetmap.org/search' }).isPublic,
|
||||
resolveNominatimSettings({
|
||||
NOMINATIM_URL: 'https://nominatim.openstreetmap.org/search',
|
||||
}).isPublic,
|
||||
true,
|
||||
);
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '7' }).dailyCap, 7);
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '0' }).dailyCap, 0);
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: 'x' }).dailyCap, 50);
|
||||
assert.equal(
|
||||
resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '7' }).dailyCap,
|
||||
7,
|
||||
);
|
||||
assert.equal(
|
||||
resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '0' }).dailyCap,
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
resolveNominatimSettings({ NOMINATIM_DAILY_CAP: 'x' }).dailyCap,
|
||||
50,
|
||||
);
|
||||
});
|
||||
|
||||
test('Retry-After reads seconds and HTTP dates', () => {
|
||||
const now = Date.UTC(2026, 0, 1);
|
||||
assert.equal(parseRetryAfter('0', now), 0);
|
||||
assert.equal(parseRetryAfter('120', now), 120_000);
|
||||
assert.equal(parseRetryAfter(new Date(now + 30_000).toUTCString(), now), 30_000);
|
||||
for (const httpDate of [
|
||||
'Thu, 01 Jan 2026 00:00:30 GMT',
|
||||
'Thursday, 01-Jan-26 00:00:30 GMT',
|
||||
'Thu Jan 1 00:00:30 2026',
|
||||
])
|
||||
assert.equal(parseRetryAfter(httpDate, now), 30_000, httpDate);
|
||||
assert.equal(parseRetryAfter(null, now), null);
|
||||
assert.equal(parseRetryAfter('soon', now), null);
|
||||
for (const malformed of [
|
||||
'-1',
|
||||
' ',
|
||||
'1.5',
|
||||
'1e3',
|
||||
'9007199254741',
|
||||
'999999999999999999999999',
|
||||
'2099-01-01',
|
||||
'October 3, 2026',
|
||||
'2026-10-03T00:00:00Z',
|
||||
'Fri, 01 Jan 2026 00:00:30 GMT',
|
||||
'Thu, 31 Feb 2026 00:00:30 GMT',
|
||||
])
|
||||
assert.equal(parseRetryAfter(malformed, now), null, malformed);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- the gate
|
||||
|
||||
test('search and outlines share one pacer: requests start at least 1.1 s apart', async () => {
|
||||
test('search and outlines share one pacer: requests start at least 1.1 s apart', async (t) => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async (url) =>
|
||||
json(url.includes('polygon_geojson') ? [row({ category: 'boundary', type: 'administrative', addresstype: 'city', name: 'Austin' })] : []),
|
||||
json(
|
||||
url.includes('polygon_geojson')
|
||||
? [
|
||||
row({
|
||||
category: 'boundary',
|
||||
type: 'administrative',
|
||||
addresstype: 'city',
|
||||
name: 'Austin',
|
||||
}),
|
||||
]
|
||||
: [],
|
||||
),
|
||||
);
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const gate = publicGate(t, { fetchImpl, ...time });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
const outline = createNominatimOutlineProvider({ gate });
|
||||
await Promise.all([
|
||||
@@ -162,10 +233,10 @@ test('search and outlines share one pacer: requests start at least 1.1 s apart',
|
||||
assert.equal(gate.stats().usedToday, 3, 'one daily count for both kinds');
|
||||
});
|
||||
|
||||
test('the daily cap stops public requests and is shared by search and outlines', async () => {
|
||||
test('the daily cap stops public requests and is shared by search and outlines', async (t) => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async () => json([]));
|
||||
const gate = createNominatimGate({
|
||||
const gate = publicGate(t, {
|
||||
settings: { ...PUBLIC, dailyCap: 2 },
|
||||
fetchImpl,
|
||||
...time,
|
||||
@@ -187,21 +258,58 @@ test('the daily cap stops public requests and is shared by search and outlines',
|
||||
assert.equal(calls.length, 3);
|
||||
});
|
||||
|
||||
test('an operator endpoint is not subject to the public cap or spacing', async () => {
|
||||
test('an operator endpoint is not subject to public cap, spacing, or durable state', async (t) => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async () => json([]));
|
||||
const blocker = path.join(tempDir(t), 'not-a-directory');
|
||||
fs.writeFileSync(blocker, 'x');
|
||||
const gate = createNominatimGate({
|
||||
settings: { ...OPERATOR, dailyCap: 1 },
|
||||
fetchImpl,
|
||||
usage: createUsageStore({ file: path.join(blocker, 'state.json') }),
|
||||
...time,
|
||||
});
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
await Promise.all([search('a', null), search('b', null), search('c', null)]);
|
||||
assert.equal(calls.length, 3);
|
||||
assert.ok(calls.every((url) => url.startsWith('https://geo.example.test/search?')));
|
||||
assert.ok(
|
||||
calls.every((url) => url.startsWith('https://geo.example.test/search?')),
|
||||
);
|
||||
assert.deepEqual(time.sleeps, []);
|
||||
});
|
||||
|
||||
test('public access fails closed when durable state is missing or unwritable', async (t) => {
|
||||
const { calls, fetchImpl } = stubFetch(async () => json([]));
|
||||
const withoutState = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl,
|
||||
usage: createUsageStore({ onError: () => {} }),
|
||||
});
|
||||
await assert.rejects(
|
||||
createNominatimSearchProvider({ gate: withoutState })('missing', null),
|
||||
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
|
||||
);
|
||||
|
||||
const blocker = path.join(tempDir(t), 'not-a-directory');
|
||||
fs.writeFileSync(blocker, 'x');
|
||||
const invalidFile = path.join(blocker, 'state.json');
|
||||
const makeBrokenGate = () =>
|
||||
createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl,
|
||||
usage: createUsageStore({ file: invalidFile, onError: () => {} }),
|
||||
});
|
||||
await assert.rejects(
|
||||
createNominatimSearchProvider({ gate: makeBrokenGate() })('broken', null),
|
||||
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
|
||||
);
|
||||
await assert.rejects(
|
||||
createNominatimSearchProvider({ gate: makeBrokenGate() })('restart', null),
|
||||
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
|
||||
);
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
|
||||
test('a disabled endpoint sends nothing', async () => {
|
||||
const { calls, fetchImpl } = stubFetch(async () => json([]));
|
||||
const gate = createNominatimGate({
|
||||
@@ -214,14 +322,17 @@ test('a disabled endpoint sends nothing', async () => {
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
|
||||
test('429 with Retry-After pauses the gate and stops queued requests', async () => {
|
||||
test('429 with Retry-After pauses the gate and stops queued requests', async (t) => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async (_url, _o, n) =>
|
||||
n === 1
|
||||
? new Response('slow down', { status: 429, headers: { 'Retry-After': '120' } })
|
||||
? new Response('slow down', {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '120' },
|
||||
})
|
||||
: json([]),
|
||||
);
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const gate = publicGate(t, { fetchImpl, ...time });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
const [first, queued] = await Promise.allSettled([
|
||||
search('first', null),
|
||||
@@ -229,7 +340,11 @@ test('429 with Retry-After pauses the gate and stops queued requests', async ()
|
||||
]);
|
||||
assert.equal(first.reason.code, 'NOMINATIM_REFUSED');
|
||||
assert.equal(first.reason.retryAfterMs, 120_000);
|
||||
assert.equal(queued.reason.code, 'NOMINATIM_PAUSED', 'the queued request is not sent');
|
||||
assert.equal(
|
||||
queued.reason.code,
|
||||
'NOMINATIM_PAUSED',
|
||||
'the queued request is not sent',
|
||||
);
|
||||
assert.equal(calls.length, 1);
|
||||
await assert.rejects(search('later', null), { code: 'NOMINATIM_PAUSED' });
|
||||
assert.equal(calls.length, 1);
|
||||
@@ -241,13 +356,75 @@ test('429 with Retry-After pauses the gate and stops queued requests', async ()
|
||||
assert.equal(calls.length, 3);
|
||||
});
|
||||
|
||||
test('an outage stops queued requests; a server error is never cached as not found', async () => {
|
||||
test('malformed Retry-After values retain the default refusal cooldown', async (t) => {
|
||||
const malformedValues = [
|
||||
'-1',
|
||||
' ',
|
||||
'1.5',
|
||||
'1e3',
|
||||
'2099-01-01',
|
||||
'October 3, 2026',
|
||||
];
|
||||
for (const [index, retryAfter] of malformedValues.entries()) {
|
||||
const time = clock();
|
||||
const { fetchImpl } = stubFetch(
|
||||
async () =>
|
||||
new Response('slow down', {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': retryAfter },
|
||||
}),
|
||||
);
|
||||
const gate = publicGate(t, {
|
||||
fetchImpl,
|
||||
...time,
|
||||
usage: createUsageStore({
|
||||
file: path.join(tempDir(t), `malformed-${index}.json`),
|
||||
}),
|
||||
});
|
||||
await assert.rejects(
|
||||
createNominatimSearchProvider({ gate })(`bad-${index}`, null),
|
||||
(error) =>
|
||||
error.code === 'NOMINATIM_REFUSED' && error.retryAfterMs === 60_000,
|
||||
);
|
||||
}
|
||||
|
||||
for (const [index, status, expected] of [
|
||||
[0, 503, 60_000],
|
||||
[1, 403, 3_600_000],
|
||||
[2, 418, 3_600_000],
|
||||
[3, 502, 30_000],
|
||||
]) {
|
||||
const time = clock();
|
||||
const file = path.join(tempDir(t), `malformed-status-${index}.json`);
|
||||
const makeGate = () =>
|
||||
publicGate(t, {
|
||||
fetchImpl: async () =>
|
||||
new Response('wait', {
|
||||
status,
|
||||
headers: { 'Retry-After': '2099-01-01' },
|
||||
}),
|
||||
...time,
|
||||
usage: createUsageStore({ file }),
|
||||
});
|
||||
await assert.rejects(
|
||||
createNominatimSearchProvider({ gate: makeGate() })(
|
||||
`status-${status}`,
|
||||
null,
|
||||
),
|
||||
);
|
||||
assert.equal(makeGate().stats().pausedForMs, expected, String(status));
|
||||
}
|
||||
});
|
||||
|
||||
test('an outage stops queued requests; a server error is never cached as not found', async (t) => {
|
||||
const time = clock();
|
||||
let down = true;
|
||||
const { calls, fetchImpl } = stubFetch(async () =>
|
||||
down ? new Response('bad gateway', { status: 502 }) : json([row({ category: 'leisure', type: 'park', name: 'Zilker Park' })]),
|
||||
down
|
||||
? new Response('bad gateway', { status: 502 })
|
||||
: json([row({ category: 'leisure', type: 'park', name: 'Zilker Park' })]),
|
||||
);
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const gate = publicGate(t, { fetchImpl, ...time });
|
||||
const outline = createNominatimOutlineProvider({ gate });
|
||||
const results = await Promise.allSettled([
|
||||
outline({ query: 'Zilker Park', kind: 'landmark' }),
|
||||
@@ -263,14 +440,14 @@ test('an outage stops queued requests; a server error is never cached as not fou
|
||||
assert.equal(calls.length, 2);
|
||||
});
|
||||
|
||||
test('a transport failure is retried once, then pauses and is not cached', async () => {
|
||||
test('a transport failure is retried once, then pauses and is not cached', async (t) => {
|
||||
const time = clock();
|
||||
let failures = 1;
|
||||
const { calls, fetchImpl } = stubFetch(async () => {
|
||||
if (failures-- > 0) throw new TypeError('fetch failed');
|
||||
return json([]);
|
||||
});
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const gate = publicGate(t, { fetchImpl, ...time });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
assert.equal((await search('retry', null)).status, 'ZERO_RESULTS');
|
||||
assert.equal(calls.length, 2, 'one retry after a transport failure');
|
||||
@@ -280,24 +457,30 @@ test('a transport failure is retried once, then pauses and is not cached', async
|
||||
assert.equal(calls.length, 4, 'never more than one retry');
|
||||
await assert.rejects(search('twice', null), { code: 'NOMINATIM_PAUSED' });
|
||||
time.advance(31_000);
|
||||
assert.equal((await search('twice', null)).cached, undefined, 'the failure was not cached');
|
||||
assert.equal(
|
||||
(await search('twice', null)).cached,
|
||||
undefined,
|
||||
'the failure was not cached',
|
||||
);
|
||||
assert.equal(calls.length, 5);
|
||||
});
|
||||
|
||||
test('a 403 block pauses for an hour', async () => {
|
||||
test('a 403 block pauses for an hour', async (t) => {
|
||||
const time = clock();
|
||||
const { fetchImpl } = stubFetch(async () => new Response('', { status: 403 }));
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const { fetchImpl } = stubFetch(
|
||||
async () => new Response('', { status: 403 }),
|
||||
);
|
||||
const gate = publicGate(t, { fetchImpl, ...time });
|
||||
await assert.rejects(createNominatimSearchProvider({ gate })('x', null), {
|
||||
code: 'NOMINATIM_REFUSED',
|
||||
});
|
||||
assert.ok(gate.stats().pausedForMs >= 3_599_000);
|
||||
});
|
||||
|
||||
test('the queue is bounded', async () => {
|
||||
test('the queue is bounded', async (t) => {
|
||||
const time = clock();
|
||||
const { fetchImpl } = stubFetch(async () => json([]));
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time, maxPending: 2 });
|
||||
const gate = publicGate(t, { fetchImpl, ...time, maxPending: 2 });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
const results = await Promise.allSettled(
|
||||
['a', 'b', 'c', 'd'].map((q) => search(q, null)),
|
||||
@@ -342,37 +525,70 @@ test('cache lifetimes: 90 days for outlines, 30 for places, 24 hours for no resu
|
||||
test('the disk cache survives a restart and stays within its bounds', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-cache-'));
|
||||
const time = clock();
|
||||
const first = createNominatimCache({ dir, now: time.now, maxDiskEntries: 3, sweepEvery: 1000 });
|
||||
const first = createNominatimCache({
|
||||
dir,
|
||||
now: time.now,
|
||||
maxDiskEntries: 3,
|
||||
sweepEvery: 1000,
|
||||
});
|
||||
for (let i = 0; i < 5; i++) {
|
||||
time.advance(1000);
|
||||
await first.set(`k${i}`, { i }, i === 0 ? 10 : NOMINATIM_CACHE_TTL_MS.outline);
|
||||
await first.set(
|
||||
`k${i}`,
|
||||
{ i },
|
||||
i === 0 ? 10 : NOMINATIM_CACHE_TTL_MS.outline,
|
||||
);
|
||||
}
|
||||
const second = createNominatimCache({ dir, now: time.now, maxDiskEntries: 3 });
|
||||
const second = createNominatimCache({
|
||||
dir,
|
||||
now: time.now,
|
||||
maxDiskEntries: 3,
|
||||
});
|
||||
assert.deepEqual((await second.get('k4')).payload, { i: 4 });
|
||||
await second.sweep();
|
||||
const files = fs.readdirSync(dir).filter((name) => name.endsWith('.json'));
|
||||
assert.equal(files.length, 3, 'expired entry removed, then oldest trimmed');
|
||||
assert.equal(await createNominatimCache({ dir, now: time.now }).get('k0'), null);
|
||||
assert.equal(
|
||||
await createNominatimCache({ dir, now: time.now }).get('k0'),
|
||||
null,
|
||||
);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('names are normalized Unicode-aware so small differences share an entry', () => {
|
||||
assert.equal(normalizePlaceName('São Paulo'), normalizePlaceName('SAO paulo!'));
|
||||
assert.equal(
|
||||
normalizePlaceName('São Paulo'),
|
||||
normalizePlaceName('SAO paulo!'),
|
||||
);
|
||||
assert.equal(normalizePlaceName('The Pentagon'), 'pentagon');
|
||||
assert.equal(normalizePlaceName('Zürich'), 'zurich');
|
||||
assert.equal(normalizePlaceName('東京都'), '東京都', 'non-Latin names survive');
|
||||
assert.equal(
|
||||
normalizePlaceName('東京都'),
|
||||
'東京都',
|
||||
'non-Latin names survive',
|
||||
);
|
||||
assert.equal(normalizePlaceName('Central Park'), 'central park');
|
||||
});
|
||||
|
||||
test('small view changes reuse one cached outline', async () => {
|
||||
test('small view changes reuse one cached outline', async (t) => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async () =>
|
||||
json([row({ category: 'leisure', type: 'park', name: 'Central Park' })]),
|
||||
);
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const gate = publicGate(t, { fetchImpl, ...time });
|
||||
const outline = createNominatimOutlineProvider({ gate });
|
||||
await outline({ query: 'Central Park', kind: 'landmark', lat: 40.781, lon: -73.966 });
|
||||
const again = await outline({ query: ' Central Park ', kind: 'landmark', lat: 40.7822, lon: -73.9651 });
|
||||
await outline({
|
||||
query: 'Central Park',
|
||||
kind: 'landmark',
|
||||
lat: 40.781,
|
||||
lon: -73.966,
|
||||
});
|
||||
const again = await outline({
|
||||
query: ' Central Park ',
|
||||
kind: 'landmark',
|
||||
lat: 40.7822,
|
||||
lon: -73.9651,
|
||||
});
|
||||
assert.equal(again.cached, true);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.match(calls[0], /viewbox=/);
|
||||
@@ -390,13 +606,20 @@ test('wrong types are rejected: charging station for Mission District, station f
|
||||
});
|
||||
for (const kind of ['neighborhood', 'landmark', 'city'])
|
||||
assert.equal(
|
||||
selectOutlineResult([charging], { kind, query: 'Mission District, San Francisco' }).outline,
|
||||
selectOutlineResult([charging], {
|
||||
kind,
|
||||
query: 'Mission District, San Francisco',
|
||||
}).outline,
|
||||
null,
|
||||
kind,
|
||||
);
|
||||
const station = row({ category: 'railway', type: 'station', name: 'Bandra' });
|
||||
for (const kind of ['neighborhood', 'landmark', 'building', 'city'])
|
||||
assert.equal(selectOutlineResult([station], { kind, query: 'Bandra' }).outline, null, kind);
|
||||
assert.equal(
|
||||
selectOutlineResult([station], { kind, query: 'Bandra' }).outline,
|
||||
null,
|
||||
kind,
|
||||
);
|
||||
assert.equal(acceptsOutlineClass('landmark', 'highway', 'primary'), false);
|
||||
assert.equal(acceptsOutlineClass('city', 'leisure', 'park'), false);
|
||||
assert.equal(acceptsOutlineClass('landmark', 'office', 'government'), true);
|
||||
@@ -411,10 +634,15 @@ test('the first polygon-bearing result of the right class is taken (Mumbai)', ()
|
||||
name: 'Mumbai',
|
||||
geojson: { type: 'Point', coordinates: [72.8, 19.0] },
|
||||
});
|
||||
const wrong = row({ category: 'railway', type: 'station', name: 'Mumbai Central' });
|
||||
const wrong = row({
|
||||
category: 'railway',
|
||||
type: 'station',
|
||||
name: 'Mumbai Central',
|
||||
});
|
||||
const boundary = row({
|
||||
category: 'boundary',
|
||||
type: 'administrative', addresstype: 'city',
|
||||
type: 'administrative',
|
||||
addresstype: 'city',
|
||||
name: 'Mumbai',
|
||||
osm_id: 7888990,
|
||||
geojson: {
|
||||
@@ -429,21 +657,38 @@ test('the first polygon-bearing result of the right class is taken (Mumbai)', ()
|
||||
assert.equal(outline.osm, 'relation/7888990');
|
||||
assert.equal(outline.polygons.length, 2);
|
||||
assert.equal(outline.polygons[0][0][1][0], 72.9 + 0.2, 'largest part first');
|
||||
assert.deepEqual(skipped, ['place=city: no polygon', 'railway=station: wrong type']);
|
||||
assert.deepEqual(skipped, [
|
||||
'place=city: no polygon',
|
||||
'railway=station: wrong type',
|
||||
]);
|
||||
});
|
||||
|
||||
test('a result whose name does not answer the ask is passed over', () => {
|
||||
const other = row({ category: 'leisure', type: 'park', name: 'Pease Park' });
|
||||
assert.equal(selectOutlineResult([other], { kind: 'landmark', query: 'Zilker Park' }).outline, null);
|
||||
assert.equal(
|
||||
selectOutlineResult([other], { kind: 'landmark', query: 'Zilker Park' })
|
||||
.outline,
|
||||
null,
|
||||
);
|
||||
const english = row({
|
||||
category: 'boundary',
|
||||
type: 'administrative', addresstype: 'city',
|
||||
type: 'administrative',
|
||||
addresstype: 'city',
|
||||
name: '東京都',
|
||||
namedetails: { name: '東京都', 'name:en': 'Tokyo' },
|
||||
});
|
||||
assert.ok(selectOutlineResult([english], { kind: 'city', query: 'Tokyo' }).outline);
|
||||
const capitol = row({ category: 'office', type: 'government', name: 'Texas State Capitol' });
|
||||
assert.ok(selectOutlineResult([capitol], { kind: 'landmark', query: 'Texas Capitol' }).outline);
|
||||
assert.ok(
|
||||
selectOutlineResult([english], { kind: 'city', query: 'Tokyo' }).outline,
|
||||
);
|
||||
const capitol = row({
|
||||
category: 'office',
|
||||
type: 'government',
|
||||
name: 'Texas State Capitol',
|
||||
});
|
||||
assert.ok(
|
||||
selectOutlineResult([capitol], { kind: 'landmark', query: 'Texas Capitol' })
|
||||
.outline,
|
||||
);
|
||||
});
|
||||
|
||||
test('outlines are held to a vertex cap', () => {
|
||||
@@ -478,12 +723,22 @@ function mount(options) {
|
||||
},
|
||||
end(body) {
|
||||
res.writableEnded = true;
|
||||
resolve({ status: res.status, headers: res.headers, body: JSON.parse(body) });
|
||||
resolve({
|
||||
status: res.status,
|
||||
headers: res.headers,
|
||||
body: JSON.parse(body),
|
||||
});
|
||||
},
|
||||
};
|
||||
Promise.resolve(
|
||||
handler(
|
||||
{ method: 'GET', url, headers, socket: { remoteAddress: `10.1.${Math.random()}` }, on() {} },
|
||||
{
|
||||
method: 'GET',
|
||||
url,
|
||||
headers,
|
||||
socket: { remoteAddress: `10.1.${Math.random()}` },
|
||||
on() {},
|
||||
},
|
||||
res,
|
||||
() => resolve({ status: 'next' }),
|
||||
),
|
||||
@@ -491,30 +746,114 @@ function mount(options) {
|
||||
});
|
||||
}
|
||||
|
||||
test('the outline route answers OK with polygons and validates its input', async () => {
|
||||
test('the outline route answers OK with polygons and validates its input', async (t) => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async () =>
|
||||
json([row({ category: 'boundary', type: 'administrative', addresstype: 'city', name: 'Paris' })]),
|
||||
json([
|
||||
row({
|
||||
category: 'boundary',
|
||||
type: 'administrative',
|
||||
addresstype: 'city',
|
||||
name: 'Paris',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
const request = mount({ gate: createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }) });
|
||||
const request = mount({ gate: publicGate(t, { fetchImpl, ...time }) });
|
||||
const ok = await request('/outline?q=Paris&kind=city&lat=48.85&lon=2.35');
|
||||
assert.equal(ok.status, 200);
|
||||
assert.equal(ok.body.status, 'OK');
|
||||
assert.equal(ok.body.outline.polygons[0][0].length, 5);
|
||||
assert.equal('adminArea' in ok.body.outline, false);
|
||||
assert.equal('adminLevel' in ok.body.outline, false);
|
||||
assert.match(calls[0], /polygon_geojson=1/);
|
||||
assert.match(calls[0], /polygon_threshold=0\.0005/);
|
||||
assert.match(calls[0], /limit=3/);
|
||||
for (const bad of ['/outline?q=Paris', '/outline?kind=city', '/outline?q=Paris&kind=planet'])
|
||||
for (const bad of [
|
||||
'/outline?q=Paris',
|
||||
'/outline?kind=city',
|
||||
'/outline?q=Paris&kind=planet',
|
||||
])
|
||||
assert.equal((await request(bad)).status, 400, bad);
|
||||
const cross = await request('/outline?q=Paris&kind=city', { 'sec-fetch-site': 'cross-site' });
|
||||
const cross = await request('/outline?q=Paris&kind=city', {
|
||||
'sec-fetch-site': 'cross-site',
|
||||
});
|
||||
assert.equal(cross.status, 403);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal((await request('/elsewhere')).status, 'next');
|
||||
});
|
||||
|
||||
test('the outline route reports disabled and capped as permanent, busy as retryable', async () => {
|
||||
test('the outline route serializes only selected-row administrative identity', async (t) => {
|
||||
const { fetchImpl } = stubFetch(async (url) => {
|
||||
const query = new URL(url).searchParams.get('q');
|
||||
const fixtures = {
|
||||
'Bagmati Province': { addresstype: 'province' },
|
||||
'Mystery Province': { place_rank: 8 },
|
||||
'Missing Evidence': {},
|
||||
'Northern Region': { addresstype: 'region', place_rank: 8 },
|
||||
'Central District': { addresstype: 'district', place_rank: 10 },
|
||||
'Example State': { addresstype: 'county' },
|
||||
'Bagmati Zone': { addresstype: 'state' },
|
||||
Austin: { addresstype: 'city' },
|
||||
};
|
||||
return json([
|
||||
row({
|
||||
category: 'boundary',
|
||||
type: 'administrative',
|
||||
name: query,
|
||||
...fixtures[query],
|
||||
}),
|
||||
]);
|
||||
});
|
||||
const gate = createNominatimGate({ settings: OPERATOR, fetchImpl });
|
||||
const outline = createNominatimOutlineProvider({
|
||||
gate,
|
||||
cache: createNominatimCache({ dir: tempDir(t) }),
|
||||
});
|
||||
const request = mount({ outline });
|
||||
|
||||
const verified = await request('/outline?q=Bagmati%20Province&kind=admin');
|
||||
assert.equal(verified.status, 200);
|
||||
assert.equal(verified.body.outline.adminArea, 'Bagmati Province');
|
||||
assert.equal(verified.body.outline.adminLevel, 'admin1');
|
||||
|
||||
for (const query of [
|
||||
'Mystery Province',
|
||||
'Northern Region',
|
||||
'Central District',
|
||||
'Example State',
|
||||
]) {
|
||||
const unverified = await request(
|
||||
`/outline?q=${encodeURIComponent(query)}&kind=admin`,
|
||||
);
|
||||
assert.equal(unverified.status, 200, query);
|
||||
assert.equal(unverified.body.status, 'OK', query);
|
||||
assert.equal('adminArea' in unverified.body.outline, false, query);
|
||||
assert.equal('adminLevel' in unverified.body.outline, false, query);
|
||||
}
|
||||
|
||||
const missing = await request('/outline?q=Missing%20Evidence&kind=admin');
|
||||
assert.equal(missing.status, 200);
|
||||
assert.equal(missing.body.status, 'ZERO_RESULTS');
|
||||
assert.equal(missing.body.outline, null);
|
||||
|
||||
const retired = await request('/outline?q=Bagmati%20Zone&kind=admin');
|
||||
assert.equal(retired.status, 200);
|
||||
assert.equal(retired.body.status, 'OK');
|
||||
assert.equal(retired.body.outline.adminArea, 'Bagmati Zone');
|
||||
assert.equal('adminLevel' in retired.body.outline, false);
|
||||
|
||||
const city = await request('/outline?q=Austin&kind=city');
|
||||
assert.equal(city.status, 200);
|
||||
assert.equal(city.body.status, 'OK');
|
||||
assert.equal('adminArea' in city.body.outline, false);
|
||||
assert.equal('adminLevel' in city.body.outline, false);
|
||||
});
|
||||
|
||||
test('the outline route reports disabled and capped as permanent, busy as retryable', async (t) => {
|
||||
const disabled = mount({
|
||||
gate: createNominatimGate({ settings: { endpoint: null, isPublic: false, dailyCap: 50 } }),
|
||||
gate: createNominatimGate({
|
||||
settings: { endpoint: null, isPublic: false, dailyCap: 50 },
|
||||
}),
|
||||
});
|
||||
const off = await disabled('/outline?q=Paris&kind=city');
|
||||
assert.equal(off.status, 503);
|
||||
@@ -522,16 +861,16 @@ test('the outline route reports disabled and capped as permanent, busy as retrya
|
||||
assert.equal(off.body.retryable, false);
|
||||
|
||||
const capped = mount({
|
||||
gate: createNominatimGate({ settings: { ...PUBLIC, dailyCap: 0 } }),
|
||||
gate: publicGate(t, { settings: { ...PUBLIC, dailyCap: 0 } }),
|
||||
});
|
||||
const cap = await capped('/outline?q=Paris&kind=city');
|
||||
assert.equal(cap.status, 429);
|
||||
assert.equal(cap.body.retryable, false);
|
||||
|
||||
const time = clock();
|
||||
const paused = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl: async () => new Response('', { status: 429, headers: { 'Retry-After': '90' } }),
|
||||
const paused = publicGate(t, {
|
||||
fetchImpl: async () =>
|
||||
new Response('', { status: 429, headers: { 'Retry-After': '90' } }),
|
||||
...time,
|
||||
});
|
||||
const busy = mount({ gate: paused });
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
// Run with: npm test
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
fetchNominatimSearch,
|
||||
geocodeProxy,
|
||||
@@ -23,6 +26,8 @@ import { createPhotonGeocoder } from '../keylessGeocoder.js';
|
||||
// Each case starts with a fresh shared gate: a pause after one case's
|
||||
// simulated outage must not decide the next case.
|
||||
test.beforeEach(() => setSharedNominatimGate(null));
|
||||
const storageRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-search-route-'));
|
||||
test.after(() => fs.rmSync(storageRoot, { recursive: true, force: true }));
|
||||
|
||||
const HIT = [
|
||||
{
|
||||
@@ -38,13 +43,16 @@ const HIT = [
|
||||
/** Mount the geocode route the way the dev server does. */
|
||||
let clientCounter = 0;
|
||||
|
||||
function mountGeocode(
|
||||
install = (server) => geocodeProxy().configureServer(server),
|
||||
) {
|
||||
function mountGeocode(install = null) {
|
||||
// A fresh client address per mount: the per-client rate limiter is module
|
||||
// state shared by every case in this file, and one case's burst must not
|
||||
// decide another case's outcome.
|
||||
clientCounter += 1;
|
||||
if (!install)
|
||||
install = (server) =>
|
||||
geocodeProxy({
|
||||
storageDir: path.join(storageRoot, `case-${clientCounter}`),
|
||||
}).configureServer(server);
|
||||
const remoteAddress = `10.0.0.${clientCounter % 250}`;
|
||||
const routes = new Map();
|
||||
install({
|
||||
|
||||
Reference in new issue
Block a user