fix(outlines): harden shared cache and provider lifecycle

This commit is contained in:
manjunath22466 committed 2026-10-02 22:32:46 +05:30
1 parent 9559e7a282
commit 177b5951a0
27 files changed
+3363 -248

No files matched your search

+284 -12
View File
@@ -25,6 +25,7 @@ import {
} from '../../server/providers/regional/place.js';
const PUBLIC = { endpoint: PUBLIC_NOMINATIM_SEARCH, isPublic: true, dailyCap: 50 };
const LOCAL = { ...PUBLIC, isPublic: false };
const tempDir = (t) => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-'));
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
@@ -117,21 +118,103 @@ test('a sweep removes expired and corrupt entries, never ones it could not read'
assert.deepEqual([1, 2, 3].map((n) => fs.existsSync(file(n))), [true, false, false]);
});
test('processes sharing the state file add to one count', async (t) => {
const file = path.join(tempDir(t), 'state.json');
test('processes sharing state reserve public spacing and cap atomically', async (t) => {
const dir = tempDir(t);
const module = new URL('../../server/providers/regional/nominatimGate.js', import.meta.url).href;
const script = `
const run = (file, dailyCap) => {
const script = `
const { createGateStateStore } = await import(${JSON.stringify(module)});
const store = createGateStateStore({ file: ${JSON.stringify(file)} });
for (let i = 0; i < 25; i++) store.increment('2026-09-29');
console.log(JSON.stringify(store.reserve('2026-09-29', {
now: 1000,
dailyCap: ${dailyCap},
minSpacingMs: 1100,
latestStartAt: 10000,
})));
`;
const run = () =>
new Promise((resolve, reject) => {
const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: 'inherit' });
child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`exit ${code}`))));
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, ['--input-type=module', '-e', script]);
let stdout = '';
let stderr = '';
child.stdout.on('data', (chunk) => (stdout += chunk));
child.stderr.on('data', (chunk) => (stderr += chunk));
child.on('exit', (code) =>
code === 0
? resolve(JSON.parse(stdout))
: reject(new Error(`exit ${code}: ${stderr}`)),
);
});
await Promise.all([run(), run(), run()]);
assert.equal(createGateStateStore({ file }).count('2026-09-29'), 75);
};
const spacedFile = path.join(dir, 'spaced.json');
const reservations = await Promise.all([
run(spacedFile, 3),
run(spacedFile, 3),
run(spacedFile, 3),
]);
assert.deepEqual(
reservations.map((result) => result.status),
['reserved', 'reserved', 'reserved'],
);
assert.deepEqual(
reservations.map((result) => result.startAt).sort((a, b) => a - b),
[1000, 2100, 3200],
);
assert.equal(createGateStateStore({ file: spacedFile }).count('2026-09-29'), 3);
const cappedFile = path.join(dir, 'capped.json');
const capped = await Promise.all([run(cappedFile, 1), run(cappedFile, 1)]);
assert.deepEqual(
capped.map((result) => result.status).sort(),
['cap', 'reserved'],
);
assert.equal(createGateStateStore({ file: cappedFile }).count('2026-09-29'), 1);
});
test('independent public gates sharing state enforce spacing and cap', async (t) => {
const dir = tempDir(t);
const starts = [];
const fetchImpl = async () => {
starts.push(Date.now());
return json([]);
};
const file = path.join(dir, 'spacing.json');
const make = (options = {}) =>
createNominatimGate({
settings: PUBLIC,
fetchImpl,
usage: createGateStateStore({ file: options.file || file }),
minSpacingMs: options.minSpacingMs || 40,
});
await Promise.all([
createNominatimSearchProvider({ gate: make() })('one', null),
createNominatimSearchProvider({ gate: make() })('two', null),
]);
starts.sort((a, b) => a - b);
assert.ok(starts[1] - starts[0] >= 35, `${starts[1] - starts[0]}ms apart`);
const capFile = path.join(dir, 'cap.json');
let sent = 0;
const capGate = () =>
createNominatimGate({
settings: { ...PUBLIC, dailyCap: 1 },
fetchImpl: async () => {
sent += 1;
return json([]);
},
usage: createGateStateStore({ file: capFile }),
minSpacingMs: 1,
});
const results = await Promise.allSettled([
createNominatimSearchProvider({ gate: capGate() })('three', null),
createNominatimSearchProvider({ gate: capGate() })('four', null),
]);
assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1);
assert.equal(
results.find((result) => result.status === 'rejected').reason.code,
'NOMINATIM_DAILY_CAP',
);
assert.equal(sent, 1);
});
test('a persistence failure is reported and never lowers the count', (t) => {
@@ -151,6 +234,49 @@ test('a persistence failure is reported and never lowers the count', (t) => {
assert.equal(errors.length, 1, 'reported once, not per request');
});
test('parseable but invalid public usage state fails closed', async (t) => {
const dir = tempDir(t);
const file = path.join(dir, 'state.json');
fs.writeFileSync(
file,
JSON.stringify({
day: '2026-09-30',
count: '50',
pausedUntil: '9999999999999',
}),
);
let sent = 0;
const gate = createNominatimGate({
settings: PUBLIC,
fetchImpl: async () => {
sent += 1;
return json([]);
},
usage: createGateStateStore({ file, onError: () => {} }),
...quick,
});
await assert.rejects(
gate.requestJson((endpoint) => `${endpoint}?q=x`),
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
);
assert.equal(sent, 0);
fs.writeFileSync(
file,
JSON.stringify({ day: '2026-09-30', count: 1, pausedUntil: 0 }),
);
const migrated = createGateStateStore({ file });
assert.equal(
migrated.reserve('2026-09-30', {
now: 1000,
dailyCap: 50,
minSpacingMs: 1100,
}).status,
'reserved',
'the original state format without nextStartAt remains supported',
);
});
test('a long Retry-After is honoured in full and survives a restart', async (t) => {
const file = path.join(tempDir(t), 'state.json');
let clock = Date.UTC(2026, 8, 29, 12);
@@ -189,6 +315,35 @@ test('a long Retry-After is honoured in full and survives a restart', async (t)
assert.equal(calls.length, 1);
});
test('an operator refusal pauses only the endpoint that refused', async () => {
let active = {
endpoint: 'https://a.example.test/search',
isPublic: false,
dailyCap: 50,
};
const calls = [];
const gate = createNominatimGate({
settings: () => active,
fetchImpl: async (url) => {
calls.push(new URL(url).host);
return calls.length === 1
? new Response('', {
status: 429,
headers: { 'Retry-After': '60' },
})
: json([]);
},
...quick,
});
const request = () => gate.requestJson((endpoint) => `${endpoint}?q=x`);
await assert.rejects(request(), { code: 'NOMINATIM_REFUSED' });
active = { ...active, endpoint: 'https://b.example.test/search' };
assert.deepEqual(await request(), []);
active = { ...active, endpoint: 'https://a.example.test/search' };
await assert.rejects(request(), { code: 'NOMINATIM_PAUSED' });
assert.deepEqual(calls, ['a.example.test', 'b.example.test']);
});
test('a stalled body times out instead of holding the shared queue', async () => {
let cancelled = 0;
const stalled = (signal) => {
@@ -217,7 +372,7 @@ test('a stalled body times out instead of holding the shared queue', async () =>
test('cache keys are request-equivalent: a miss for "Paris???" never answers "Paris"', async () => {
const calls = [];
const gate = createNominatimGate({
settings: PUBLIC,
settings: LOCAL,
fetchImpl: async (url) => {
calls.push(new URL(url).searchParams.get('q'));
return json(calls.length === 1 ? [] : [{ lat: '48.85', lon: '2.35', display_name: 'Paris, France', name: 'Paris', addresstype: 'city' }]);
@@ -236,7 +391,7 @@ test('a malformed answer is a failure, not a cached "no result"', async () => {
let shape = { error: 'Unexpected' };
let calls = 0;
const gate = createNominatimGate({
settings: PUBLIC,
settings: LOCAL,
fetchImpl: async () => {
calls += 1;
return json(shape);
@@ -301,9 +456,29 @@ test('administrative boundaries must be at the asked level', () => {
const state = boundary({ name: 'Texas', addresstype: 'state', place_rank: 8 });
assert.equal(selectOutlineResult([state], { kind: 'city', query: 'Texas' }).outline, null);
assert.equal(selectOutlineResult([city], { kind: 'admin', query: 'Austin' }).outline, null);
assert.equal(
selectOutlineResult(
[boundary({ name: 'Travis', addresstype: 'county', place_rank: 12 })],
{ kind: 'city', query: 'Travis' },
).outline,
null,
);
assert.equal(
selectOutlineResult(
[boundary({ name: 'Austin', addresstype: 'city', place_rank: 12 })],
{ kind: 'admin', query: 'Austin' },
).outline,
null,
);
assert.equal(selectOutlineResult([state], { kind: 'neighborhood', query: 'Texas' }).outline, null);
assert.ok(selectOutlineResult([county], { kind: 'admin', query: 'Travis County' }).outline);
assert.ok(selectOutlineResult([boundary({ name: 'Somewhere', place_rank: 16 })], { kind: 'city', query: 'Somewhere' }).outline);
assert.ok(
selectOutlineResult(
[boundary({ name: 'Somewhere', addresstype: 'locality', place_rank: 16 })],
{ kind: 'city', query: 'Somewhere' },
).outline,
);
assert.equal(
selectOutlineResult([boundary({ name: 'Somewhere' })], { kind: 'city', query: 'Somewhere' }).outline,
null,
@@ -317,6 +492,103 @@ test('administrative boundaries must be at the asked level', () => {
assert.equal(picked.outline.osm, 'relation/9');
});
test('admin identity comes only from unambiguous evidence on the selected row', () => {
for (const [name, addresstype, adminLevel] of [
['Nepal', 'country', 'country'],
['Bagmati Province', 'province', 'admin1'],
['Province of Bagmati', 'province', 'admin1'],
['Texas', 'state', 'admin1'],
['Travis County', 'county', 'admin2'],
['County of Travis', 'county', 'admin2'],
['Kathmandu District', 'state_district', 'admin2'],
]) {
const { outline } = selectOutlineResult(
[boundary({ name, addresstype })],
{ kind: 'admin', query: name },
);
assert.equal(outline.adminArea, name);
assert.equal(outline.adminLevel, adminLevel);
}
for (const row of [
boundary({ name: 'Northern Region', addresstype: 'region', place_rank: 8 }),
boundary({ name: 'Central District', addresstype: 'district', place_rank: 10 }),
boundary({ name: 'Mystery Province', place_rank: 8 }),
boundary({ name: 'Example State', addresstype: 'county' }),
boundary({ name: 'County of Example', addresstype: 'state' }),
boundary({ name: 'Province of Example', addresstype: 'county' }),
boundary({ name: 'Example County', addresstype: 'state' }),
boundary({ name: 'Example Province', addresstype: 'county' }),
boundary({ name: 'County of Example Province', addresstype: 'county' }),
]) {
const { outline } = selectOutlineResult([row], {
kind: 'admin',
query: row.name,
});
assert.ok(outline, row.name);
assert.equal(outline.adminArea, undefined, row.name);
assert.equal(outline.adminLevel, undefined, row.name);
}
const retired = selectOutlineResult(
[boundary({ name: 'Bagmati Zone', addresstype: 'state' })],
{ kind: 'admin', query: 'Bagmati Zone' },
).outline;
assert.equal(retired.adminArea, 'Bagmati Zone');
assert.equal(retired.adminLevel, undefined);
const retiredPrefix = selectOutlineResult(
[boundary({ name: 'Zone of Bagmati', addresstype: 'state' })],
{ kind: 'admin', query: 'Zone of Bagmati' },
).outline;
assert.equal(retiredPrefix.adminArea, 'Zone of Bagmati');
assert.equal(retiredPrefix.adminLevel, undefined);
});
test('admin identity validates the full name before display truncation', () => {
const longValidName = `Province of ${'Example '.repeat(24)}`.trim();
const valid = selectOutlineResult(
[boundary({ name: longValidName, addresstype: 'province' })],
{ kind: 'admin', query: longValidName },
).outline;
assert.equal(valid.adminArea, longValidName.slice(0, 160));
assert.equal(valid.adminLevel, 'admin1');
const longContradictoryName = `${'Very Long Administrative Name '.repeat(7)}County`;
assert.ok(longContradictoryName.length > 160);
const contradictory = selectOutlineResult(
[boundary({ name: longContradictoryName, addresstype: 'state' })],
{ kind: 'admin', query: longContradictoryName },
).outline;
assert.ok(contradictory);
assert.equal(contradictory.adminArea, undefined);
assert.equal(contradictory.adminLevel, undefined);
});
test('non-administrative outline kinds never manufacture admin identity', () => {
const city = selectOutlineResult(
[boundary({ name: 'County of Austin', addresstype: 'city', place_rank: 16 })],
{ kind: 'city', query: 'County of Austin' },
).outline;
const park = selectOutlineResult(
[
{
category: 'leisure',
type: 'park',
name: 'Zilker Park',
lat: '1',
lon: '1',
geojson: { type: 'Polygon', coordinates: square(1, 1, 0.01) },
},
],
{ kind: 'landmark', query: 'Zilker Park' },
).outline;
for (const outline of [city, park]) {
assert.equal(outline.adminArea, undefined);
assert.equal(outline.adminLevel, undefined);
}
});
test('building asks need building evidence', () => {
const park = { category: 'tourism', type: 'theme_park', name: 'Big Top', lat: '1', lon: '1', geojson: { type: 'Polygon', coordinates: square(1, 1, 0.0001) } };
const camp = { ...park, type: 'camp_site' };
+235
View File
@@ -0,0 +1,235 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { createGateStateStore } from '../../server/providers/regional/nominatimGate.js';
const moduleUrl = new URL(
'../../server/providers/regional/nominatimGate.js',
import.meta.url,
).href;
function startModule(script) {
const child = spawn(process.execPath, ['--input-type=module', '-e', script], {
stdio: ['pipe', 'pipe', 'pipe'],
});
const lines = [];
const waiters = [];
let stdout = '';
let stderr = '';
const publish = (line) => {
lines.push(line);
for (const waiter of [...waiters]) {
if (!line.startsWith(waiter.prefix)) continue;
waiters.splice(waiters.indexOf(waiter), 1);
clearTimeout(waiter.timer);
waiter.resolve(line);
}
};
child.stdout.on('data', (chunk) => {
stdout += chunk;
const parts = stdout.split('\n');
stdout = parts.pop();
for (const line of parts) publish(line);
});
child.stderr.on('data', (chunk) => {
stderr += chunk;
});
return {
child,
lines,
waitFor(prefix, timeoutMs = 3000) {
const existing = lines.find((line) => line.startsWith(prefix));
if (existing) return Promise.resolve(existing);
return new Promise((resolve, reject) => {
const waiter = { prefix, resolve, reject, timer: null };
waiter.timer = setTimeout(() => {
const index = waiters.indexOf(waiter);
if (index >= 0) waiters.splice(index, 1);
reject(
new Error(
`timed out waiting for ${prefix}; stderr: ${stderr.trim()}`,
),
);
}, timeoutMs);
waiters.push(waiter);
});
},
exited: new Promise((resolve, reject) => {
child.once('error', reject);
child.once('exit', (code, signal) => resolve({ code, signal }));
}),
};
}
function holderScript(lockFile) {
return `
import { DatabaseSync } from 'node:sqlite';
const database = new DatabaseSync(${JSON.stringify(lockFile)});
database.exec('BEGIN IMMEDIATE');
process.stdout.write('LOCKED\\n');
process.stdin.once('data', () => {
database.exec('ROLLBACK');
database.close();
process.stdout.write('RELEASED\\n');
});
process.stdin.resume();
`;
}
function contenderScript(stateFile, lockWaitMs) {
return `
const { createGateStateStore } = await import(${JSON.stringify(moduleUrl)});
const store = createGateStateStore({
file: ${JSON.stringify(stateFile)},
lockWaitMs: ${lockWaitMs},
onError: () => {},
});
process.stdout.write('ATTEMPTING\\n', () => {
const result = store.reserve('2026-10-01', {
now: 1000,
dailyCap: 50,
minSpacingMs: 1100,
});
process.stdout.write('RESULT:' + JSON.stringify(result) + '\\n');
});
`;
}
test('a crashed lock owner cannot delete or bypass a successor lock', async (t) => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-lock-'));
const stateFile = path.join(dir, 'state.json');
const lockFile = `${stateFile}.lock.sqlite`;
const children = [];
t.after(() => {
for (const process of children) {
if (process.exitCode == null && process.signalCode == null)
process.kill();
}
fs.rmSync(dir, { recursive: true, force: true });
});
const crashedOwner = startModule(holderScript(lockFile));
children.push(crashedOwner.child);
await crashedOwner.waitFor('LOCKED');
const originalLock = fs.statSync(lockFile);
const blockedByCrashedOwner = startModule(contenderScript(stateFile, 0));
children.push(blockedByCrashedOwner.child);
const blockedResult = JSON.parse(
(await blockedByCrashedOwner.waitFor('RESULT:')).slice('RESULT:'.length),
);
assert.equal(blockedResult.status, 'unavailable');
assert.equal((await blockedByCrashedOwner.exited).code, 0);
assert.equal(fs.existsSync(stateFile), false);
crashedOwner.child.kill();
await crashedOwner.exited;
const successorOwner = startModule(holderScript(lockFile));
children.push(successorOwner.child);
await successorOwner.waitFor('LOCKED');
assert.equal(fs.statSync(lockFile).ino, originalLock.ino);
const refused = startModule(contenderScript(stateFile, 0));
children.push(refused.child);
const refusedResult = JSON.parse(
(await refused.waitFor('RESULT:')).slice('RESULT:'.length),
);
assert.equal(refusedResult.status, 'unavailable');
assert.equal((await refused.exited).code, 0);
assert.equal(
createGateStateStore({ file: stateFile }).count('2026-10-01'),
0,
'a timed-out contender fails closed without spending a public slot',
);
assert.equal(fs.statSync(lockFile).ino, originalLock.ino);
successorOwner.child.stdin.end('release\n');
await successorOwner.waitFor('RELEASED');
assert.equal((await successorOwner.exited).code, 0);
const afterRelease = createGateStateStore({ file: stateFile }).reserve(
'2026-10-01',
{
now: 2200,
dailyCap: 50,
minSpacingMs: 1100,
},
);
assert.equal(afterRelease.status, 'reserved');
assert.equal(afterRelease.count, 1);
assert.equal(fs.statSync(lockFile).ino, originalLock.ino);
});
test('a public pause stays authoritative while another process owns the lock', async (t) => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-pause-'));
const stateFile = path.join(dir, 'state.json');
const lockFile = `${stateFile}.lock.sqlite`;
const children = [];
t.after(() => {
for (const process of children) {
if (process.exitCode == null && process.signalCode == null)
process.kill();
}
fs.rmSync(dir, { recursive: true, force: true });
});
fs.writeFileSync(
stateFile,
JSON.stringify({
day: '2026-10-01',
count: 1,
pausedUntil: 0,
nextStartAt: 0,
}),
);
const owner = startModule(holderScript(lockFile));
children.push(owner.child);
await owner.waitFor('LOCKED');
const errors = [];
const blockedWriter = createGateStateStore({
file: stateFile,
lockWaitMs: 0,
onError: (error) => errors.push(error),
});
blockedWriter.pauseUntil(9000);
assert.equal(errors.length, 1, 'the blocked JSON consolidation is reported');
assert.equal(
JSON.parse(fs.readFileSync(stateFile, 'utf8')).pausedUntil,
0,
'the lock owner prevents an unsafe JSON overwrite',
);
owner.child.stdin.end('release\n');
await owner.waitFor('RELEASED');
assert.equal((await owner.exited).code, 0);
const successor = createGateStateStore({ file: stateFile });
assert.deepEqual(
successor.reserve('2026-10-01', {
now: 1000,
dailyCap: 50,
minSpacingMs: 1100,
}),
{ status: 'paused', retryAfterMs: 8000 },
'a fresh process observes the pending refusal pause',
);
const resumed = successor.reserve('2026-10-01', {
now: 9001,
dailyCap: 50,
minSpacingMs: 1100,
});
assert.equal(resumed.status, 'reserved');
assert.equal(resumed.count, 2);
assert.equal(
fs.readdirSync(dir).some((name) => name.startsWith('state.json.pause.')),
false,
'the next successful transaction consolidates and removes the marker',
);
});
+402 -63
View File
@@ -100,6 +100,23 @@ function stubFetch(respond) {
return { calls, fetchImpl };
}
function tempDir(t) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-'));
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
return dir;
}
function publicGate(t, options = {}) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-gate-'));
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
return createNominatimGate({
...options,
settings: options.settings || PUBLIC,
usage:
options.usage || createUsageStore({ file: path.join(dir, 'state.json') }),
});
}
// ---------------------------------------------------------------- settings
test('endpoint settings: unset is public, empty disables, operator URLs are not public', () => {
@@ -109,7 +126,10 @@ test('endpoint settings: unset is public, empty disables, operator URLs are not
dailyCap: 50,
});
for (const off of ['', ' ', 'off', 'none', 'disabled'])
assert.equal(resolveNominatimSettings({ NOMINATIM_URL: off }).endpoint, null);
assert.equal(
resolveNominatimSettings({ NOMINATIM_URL: off }).endpoint,
null,
);
const own = resolveNominatimSettings({
NOMINATIM_URL: 'https://geo.example.test',
});
@@ -122,33 +142,84 @@ test('endpoint settings: unset is public, empty disables, operator URLs are not
'http://localhost:8080/search',
);
// A mistyped value disables rather than silently falling back to public.
for (const bad of ['not a url', 'ftp://x.test', 'https://u:[email protected]', 'https://x.test/search?q=1'])
assert.equal(resolveNominatimSettings({ NOMINATIM_URL: bad }).endpoint, null, bad);
for (const bad of [
'not a url',
'ftp://x.test',
'https://u:[email protected]',
'https://x.test/search?q=1',
])
assert.equal(
resolveNominatimSettings({ NOMINATIM_URL: bad }).endpoint,
null,
bad,
);
assert.equal(
resolveNominatimSettings({ NOMINATIM_URL: 'https://nominatim.openstreetmap.org/search' }).isPublic,
resolveNominatimSettings({
NOMINATIM_URL: 'https://nominatim.openstreetmap.org/search',
}).isPublic,
true,
);
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '7' }).dailyCap, 7);
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '0' }).dailyCap, 0);
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: 'x' }).dailyCap, 50);
assert.equal(
resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '7' }).dailyCap,
7,
);
assert.equal(
resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '0' }).dailyCap,
0,
);
assert.equal(
resolveNominatimSettings({ NOMINATIM_DAILY_CAP: 'x' }).dailyCap,
50,
);
});
test('Retry-After reads seconds and HTTP dates', () => {
const now = Date.UTC(2026, 0, 1);
assert.equal(parseRetryAfter('0', now), 0);
assert.equal(parseRetryAfter('120', now), 120_000);
assert.equal(parseRetryAfter(new Date(now + 30_000).toUTCString(), now), 30_000);
for (const httpDate of [
'Thu, 01 Jan 2026 00:00:30 GMT',
'Thursday, 01-Jan-26 00:00:30 GMT',
'Thu Jan 1 00:00:30 2026',
])
assert.equal(parseRetryAfter(httpDate, now), 30_000, httpDate);
assert.equal(parseRetryAfter(null, now), null);
assert.equal(parseRetryAfter('soon', now), null);
for (const malformed of [
'-1',
' ',
'1.5',
'1e3',
'9007199254741',
'999999999999999999999999',
'2099-01-01',
'October 3, 2026',
'2026-10-03T00:00:00Z',
'Fri, 01 Jan 2026 00:00:30 GMT',
'Thu, 31 Feb 2026 00:00:30 GMT',
])
assert.equal(parseRetryAfter(malformed, now), null, malformed);
});
// ---------------------------------------------------------------- the gate
test('search and outlines share one pacer: requests start at least 1.1 s apart', async () => {
test('search and outlines share one pacer: requests start at least 1.1 s apart', async (t) => {
const time = clock();
const { calls, fetchImpl } = stubFetch(async (url) =>
json(url.includes('polygon_geojson') ? [row({ category: 'boundary', type: 'administrative', addresstype: 'city', name: 'Austin' })] : []),
json(
url.includes('polygon_geojson')
? [
row({
category: 'boundary',
type: 'administrative',
addresstype: 'city',
name: 'Austin',
}),
]
: [],
),
);
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
const gate = publicGate(t, { fetchImpl, ...time });
const search = createNominatimSearchProvider({ gate });
const outline = createNominatimOutlineProvider({ gate });
await Promise.all([
@@ -162,10 +233,10 @@ test('search and outlines share one pacer: requests start at least 1.1 s apart',
assert.equal(gate.stats().usedToday, 3, 'one daily count for both kinds');
});
test('the daily cap stops public requests and is shared by search and outlines', async () => {
test('the daily cap stops public requests and is shared by search and outlines', async (t) => {
const time = clock();
const { calls, fetchImpl } = stubFetch(async () => json([]));
const gate = createNominatimGate({
const gate = publicGate(t, {
settings: { ...PUBLIC, dailyCap: 2 },
fetchImpl,
...time,
@@ -187,21 +258,58 @@ test('the daily cap stops public requests and is shared by search and outlines',
assert.equal(calls.length, 3);
});
test('an operator endpoint is not subject to the public cap or spacing', async () => {
test('an operator endpoint is not subject to public cap, spacing, or durable state', async (t) => {
const time = clock();
const { calls, fetchImpl } = stubFetch(async () => json([]));
const blocker = path.join(tempDir(t), 'not-a-directory');
fs.writeFileSync(blocker, 'x');
const gate = createNominatimGate({
settings: { ...OPERATOR, dailyCap: 1 },
fetchImpl,
usage: createUsageStore({ file: path.join(blocker, 'state.json') }),
...time,
});
const search = createNominatimSearchProvider({ gate });
await Promise.all([search('a', null), search('b', null), search('c', null)]);
assert.equal(calls.length, 3);
assert.ok(calls.every((url) => url.startsWith('https://geo.example.test/search?')));
assert.ok(
calls.every((url) => url.startsWith('https://geo.example.test/search?')),
);
assert.deepEqual(time.sleeps, []);
});
test('public access fails closed when durable state is missing or unwritable', async (t) => {
const { calls, fetchImpl } = stubFetch(async () => json([]));
const withoutState = createNominatimGate({
settings: PUBLIC,
fetchImpl,
usage: createUsageStore({ onError: () => {} }),
});
await assert.rejects(
createNominatimSearchProvider({ gate: withoutState })('missing', null),
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
);
const blocker = path.join(tempDir(t), 'not-a-directory');
fs.writeFileSync(blocker, 'x');
const invalidFile = path.join(blocker, 'state.json');
const makeBrokenGate = () =>
createNominatimGate({
settings: PUBLIC,
fetchImpl,
usage: createUsageStore({ file: invalidFile, onError: () => {} }),
});
await assert.rejects(
createNominatimSearchProvider({ gate: makeBrokenGate() })('broken', null),
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
);
await assert.rejects(
createNominatimSearchProvider({ gate: makeBrokenGate() })('restart', null),
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
);
assert.equal(calls.length, 0);
});
test('a disabled endpoint sends nothing', async () => {
const { calls, fetchImpl } = stubFetch(async () => json([]));
const gate = createNominatimGate({
@@ -214,14 +322,17 @@ test('a disabled endpoint sends nothing', async () => {
assert.equal(calls.length, 0);
});
test('429 with Retry-After pauses the gate and stops queued requests', async () => {
test('429 with Retry-After pauses the gate and stops queued requests', async (t) => {
const time = clock();
const { calls, fetchImpl } = stubFetch(async (_url, _o, n) =>
n === 1
? new Response('slow down', { status: 429, headers: { 'Retry-After': '120' } })
? new Response('slow down', {
status: 429,
headers: { 'Retry-After': '120' },
})
: json([]),
);
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
const gate = publicGate(t, { fetchImpl, ...time });
const search = createNominatimSearchProvider({ gate });
const [first, queued] = await Promise.allSettled([
search('first', null),
@@ -229,7 +340,11 @@ test('429 with Retry-After pauses the gate and stops queued requests', async ()
]);
assert.equal(first.reason.code, 'NOMINATIM_REFUSED');
assert.equal(first.reason.retryAfterMs, 120_000);
assert.equal(queued.reason.code, 'NOMINATIM_PAUSED', 'the queued request is not sent');
assert.equal(
queued.reason.code,
'NOMINATIM_PAUSED',
'the queued request is not sent',
);
assert.equal(calls.length, 1);
await assert.rejects(search('later', null), { code: 'NOMINATIM_PAUSED' });
assert.equal(calls.length, 1);
@@ -241,13 +356,75 @@ test('429 with Retry-After pauses the gate and stops queued requests', async ()
assert.equal(calls.length, 3);
});
test('an outage stops queued requests; a server error is never cached as not found', async () => {
test('malformed Retry-After values retain the default refusal cooldown', async (t) => {
const malformedValues = [
'-1',
' ',
'1.5',
'1e3',
'2099-01-01',
'October 3, 2026',
];
for (const [index, retryAfter] of malformedValues.entries()) {
const time = clock();
const { fetchImpl } = stubFetch(
async () =>
new Response('slow down', {
status: 429,
headers: { 'Retry-After': retryAfter },
}),
);
const gate = publicGate(t, {
fetchImpl,
...time,
usage: createUsageStore({
file: path.join(tempDir(t), `malformed-${index}.json`),
}),
});
await assert.rejects(
createNominatimSearchProvider({ gate })(`bad-${index}`, null),
(error) =>
error.code === 'NOMINATIM_REFUSED' && error.retryAfterMs === 60_000,
);
}
for (const [index, status, expected] of [
[0, 503, 60_000],
[1, 403, 3_600_000],
[2, 418, 3_600_000],
[3, 502, 30_000],
]) {
const time = clock();
const file = path.join(tempDir(t), `malformed-status-${index}.json`);
const makeGate = () =>
publicGate(t, {
fetchImpl: async () =>
new Response('wait', {
status,
headers: { 'Retry-After': '2099-01-01' },
}),
...time,
usage: createUsageStore({ file }),
});
await assert.rejects(
createNominatimSearchProvider({ gate: makeGate() })(
`status-${status}`,
null,
),
);
assert.equal(makeGate().stats().pausedForMs, expected, String(status));
}
});
test('an outage stops queued requests; a server error is never cached as not found', async (t) => {
const time = clock();
let down = true;
const { calls, fetchImpl } = stubFetch(async () =>
down ? new Response('bad gateway', { status: 502 }) : json([row({ category: 'leisure', type: 'park', name: 'Zilker Park' })]),
down
? new Response('bad gateway', { status: 502 })
: json([row({ category: 'leisure', type: 'park', name: 'Zilker Park' })]),
);
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
const gate = publicGate(t, { fetchImpl, ...time });
const outline = createNominatimOutlineProvider({ gate });
const results = await Promise.allSettled([
outline({ query: 'Zilker Park', kind: 'landmark' }),
@@ -263,14 +440,14 @@ test('an outage stops queued requests; a server error is never cached as not fou
assert.equal(calls.length, 2);
});
test('a transport failure is retried once, then pauses and is not cached', async () => {
test('a transport failure is retried once, then pauses and is not cached', async (t) => {
const time = clock();
let failures = 1;
const { calls, fetchImpl } = stubFetch(async () => {
if (failures-- > 0) throw new TypeError('fetch failed');
return json([]);
});
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
const gate = publicGate(t, { fetchImpl, ...time });
const search = createNominatimSearchProvider({ gate });
assert.equal((await search('retry', null)).status, 'ZERO_RESULTS');
assert.equal(calls.length, 2, 'one retry after a transport failure');
@@ -280,24 +457,30 @@ test('a transport failure is retried once, then pauses and is not cached', async
assert.equal(calls.length, 4, 'never more than one retry');
await assert.rejects(search('twice', null), { code: 'NOMINATIM_PAUSED' });
time.advance(31_000);
assert.equal((await search('twice', null)).cached, undefined, 'the failure was not cached');
assert.equal(
(await search('twice', null)).cached,
undefined,
'the failure was not cached',
);
assert.equal(calls.length, 5);
});
test('a 403 block pauses for an hour', async () => {
test('a 403 block pauses for an hour', async (t) => {
const time = clock();
const { fetchImpl } = stubFetch(async () => new Response('', { status: 403 }));
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
const { fetchImpl } = stubFetch(
async () => new Response('', { status: 403 }),
);
const gate = publicGate(t, { fetchImpl, ...time });
await assert.rejects(createNominatimSearchProvider({ gate })('x', null), {
code: 'NOMINATIM_REFUSED',
});
assert.ok(gate.stats().pausedForMs >= 3_599_000);
});
test('the queue is bounded', async () => {
test('the queue is bounded', async (t) => {
const time = clock();
const { fetchImpl } = stubFetch(async () => json([]));
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time, maxPending: 2 });
const gate = publicGate(t, { fetchImpl, ...time, maxPending: 2 });
const search = createNominatimSearchProvider({ gate });
const results = await Promise.allSettled(
['a', 'b', 'c', 'd'].map((q) => search(q, null)),
@@ -342,37 +525,70 @@ test('cache lifetimes: 90 days for outlines, 30 for places, 24 hours for no resu
test('the disk cache survives a restart and stays within its bounds', async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-cache-'));
const time = clock();
const first = createNominatimCache({ dir, now: time.now, maxDiskEntries: 3, sweepEvery: 1000 });
const first = createNominatimCache({
dir,
now: time.now,
maxDiskEntries: 3,
sweepEvery: 1000,
});
for (let i = 0; i < 5; i++) {
time.advance(1000);
await first.set(`k${i}`, { i }, i === 0 ? 10 : NOMINATIM_CACHE_TTL_MS.outline);
await first.set(
`k${i}`,
{ i },
i === 0 ? 10 : NOMINATIM_CACHE_TTL_MS.outline,
);
}
const second = createNominatimCache({ dir, now: time.now, maxDiskEntries: 3 });
const second = createNominatimCache({
dir,
now: time.now,
maxDiskEntries: 3,
});
assert.deepEqual((await second.get('k4')).payload, { i: 4 });
await second.sweep();
const files = fs.readdirSync(dir).filter((name) => name.endsWith('.json'));
assert.equal(files.length, 3, 'expired entry removed, then oldest trimmed');
assert.equal(await createNominatimCache({ dir, now: time.now }).get('k0'), null);
assert.equal(
await createNominatimCache({ dir, now: time.now }).get('k0'),
null,
);
fs.rmSync(dir, { recursive: true, force: true });
});
test('names are normalized Unicode-aware so small differences share an entry', () => {
assert.equal(normalizePlaceName('São Paulo'), normalizePlaceName('SAO paulo!'));
assert.equal(
normalizePlaceName('São Paulo'),
normalizePlaceName('SAO paulo!'),
);
assert.equal(normalizePlaceName('The Pentagon'), 'pentagon');
assert.equal(normalizePlaceName('Zürich'), 'zurich');
assert.equal(normalizePlaceName('東京都'), '東京都', 'non-Latin names survive');
assert.equal(
normalizePlaceName('東京都'),
'東京都',
'non-Latin names survive',
);
assert.equal(normalizePlaceName('Central Park'), 'central park');
});
test('small view changes reuse one cached outline', async () => {
test('small view changes reuse one cached outline', async (t) => {
const time = clock();
const { calls, fetchImpl } = stubFetch(async () =>
json([row({ category: 'leisure', type: 'park', name: 'Central Park' })]),
);
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
const gate = publicGate(t, { fetchImpl, ...time });
const outline = createNominatimOutlineProvider({ gate });
await outline({ query: 'Central Park', kind: 'landmark', lat: 40.781, lon: -73.966 });
const again = await outline({ query: ' Central Park ', kind: 'landmark', lat: 40.7822, lon: -73.9651 });
await outline({
query: 'Central Park',
kind: 'landmark',
lat: 40.781,
lon: -73.966,
});
const again = await outline({
query: ' Central Park ',
kind: 'landmark',
lat: 40.7822,
lon: -73.9651,
});
assert.equal(again.cached, true);
assert.equal(calls.length, 1);
assert.match(calls[0], /viewbox=/);
@@ -390,13 +606,20 @@ test('wrong types are rejected: charging station for Mission District, station f
});
for (const kind of ['neighborhood', 'landmark', 'city'])
assert.equal(
selectOutlineResult([charging], { kind, query: 'Mission District, San Francisco' }).outline,
selectOutlineResult([charging], {
kind,
query: 'Mission District, San Francisco',
}).outline,
null,
kind,
);
const station = row({ category: 'railway', type: 'station', name: 'Bandra' });
for (const kind of ['neighborhood', 'landmark', 'building', 'city'])
assert.equal(selectOutlineResult([station], { kind, query: 'Bandra' }).outline, null, kind);
assert.equal(
selectOutlineResult([station], { kind, query: 'Bandra' }).outline,
null,
kind,
);
assert.equal(acceptsOutlineClass('landmark', 'highway', 'primary'), false);
assert.equal(acceptsOutlineClass('city', 'leisure', 'park'), false);
assert.equal(acceptsOutlineClass('landmark', 'office', 'government'), true);
@@ -411,10 +634,15 @@ test('the first polygon-bearing result of the right class is taken (Mumbai)', ()
name: 'Mumbai',
geojson: { type: 'Point', coordinates: [72.8, 19.0] },
});
const wrong = row({ category: 'railway', type: 'station', name: 'Mumbai Central' });
const wrong = row({
category: 'railway',
type: 'station',
name: 'Mumbai Central',
});
const boundary = row({
category: 'boundary',
type: 'administrative', addresstype: 'city',
type: 'administrative',
addresstype: 'city',
name: 'Mumbai',
osm_id: 7888990,
geojson: {
@@ -429,21 +657,38 @@ test('the first polygon-bearing result of the right class is taken (Mumbai)', ()
assert.equal(outline.osm, 'relation/7888990');
assert.equal(outline.polygons.length, 2);
assert.equal(outline.polygons[0][0][1][0], 72.9 + 0.2, 'largest part first');
assert.deepEqual(skipped, ['place=city: no polygon', 'railway=station: wrong type']);
assert.deepEqual(skipped, [
'place=city: no polygon',
'railway=station: wrong type',
]);
});
test('a result whose name does not answer the ask is passed over', () => {
const other = row({ category: 'leisure', type: 'park', name: 'Pease Park' });
assert.equal(selectOutlineResult([other], { kind: 'landmark', query: 'Zilker Park' }).outline, null);
assert.equal(
selectOutlineResult([other], { kind: 'landmark', query: 'Zilker Park' })
.outline,
null,
);
const english = row({
category: 'boundary',
type: 'administrative', addresstype: 'city',
type: 'administrative',
addresstype: 'city',
name: '東京都',
namedetails: { name: '東京都', 'name:en': 'Tokyo' },
});
assert.ok(selectOutlineResult([english], { kind: 'city', query: 'Tokyo' }).outline);
const capitol = row({ category: 'office', type: 'government', name: 'Texas State Capitol' });
assert.ok(selectOutlineResult([capitol], { kind: 'landmark', query: 'Texas Capitol' }).outline);
assert.ok(
selectOutlineResult([english], { kind: 'city', query: 'Tokyo' }).outline,
);
const capitol = row({
category: 'office',
type: 'government',
name: 'Texas State Capitol',
});
assert.ok(
selectOutlineResult([capitol], { kind: 'landmark', query: 'Texas Capitol' })
.outline,
);
});
test('outlines are held to a vertex cap', () => {
@@ -478,12 +723,22 @@ function mount(options) {
},
end(body) {
res.writableEnded = true;
resolve({ status: res.status, headers: res.headers, body: JSON.parse(body) });
resolve({
status: res.status,
headers: res.headers,
body: JSON.parse(body),
});
},
};
Promise.resolve(
handler(
{ method: 'GET', url, headers, socket: { remoteAddress: `10.1.${Math.random()}` }, on() {} },
{
method: 'GET',
url,
headers,
socket: { remoteAddress: `10.1.${Math.random()}` },
on() {},
},
res,
() => resolve({ status: 'next' }),
),
@@ -491,30 +746,114 @@ function mount(options) {
});
}
test('the outline route answers OK with polygons and validates its input', async () => {
test('the outline route answers OK with polygons and validates its input', async (t) => {
const time = clock();
const { calls, fetchImpl } = stubFetch(async () =>
json([row({ category: 'boundary', type: 'administrative', addresstype: 'city', name: 'Paris' })]),
json([
row({
category: 'boundary',
type: 'administrative',
addresstype: 'city',
name: 'Paris',
}),
]),
);
const request = mount({ gate: createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }) });
const request = mount({ gate: publicGate(t, { fetchImpl, ...time }) });
const ok = await request('/outline?q=Paris&kind=city&lat=48.85&lon=2.35');
assert.equal(ok.status, 200);
assert.equal(ok.body.status, 'OK');
assert.equal(ok.body.outline.polygons[0][0].length, 5);
assert.equal('adminArea' in ok.body.outline, false);
assert.equal('adminLevel' in ok.body.outline, false);
assert.match(calls[0], /polygon_geojson=1/);
assert.match(calls[0], /polygon_threshold=0\.0005/);
assert.match(calls[0], /limit=3/);
for (const bad of ['/outline?q=Paris', '/outline?kind=city', '/outline?q=Paris&kind=planet'])
for (const bad of [
'/outline?q=Paris',
'/outline?kind=city',
'/outline?q=Paris&kind=planet',
])
assert.equal((await request(bad)).status, 400, bad);
const cross = await request('/outline?q=Paris&kind=city', { 'sec-fetch-site': 'cross-site' });
const cross = await request('/outline?q=Paris&kind=city', {
'sec-fetch-site': 'cross-site',
});
assert.equal(cross.status, 403);
assert.equal(calls.length, 1);
assert.equal((await request('/elsewhere')).status, 'next');
});
test('the outline route reports disabled and capped as permanent, busy as retryable', async () => {
test('the outline route serializes only selected-row administrative identity', async (t) => {
const { fetchImpl } = stubFetch(async (url) => {
const query = new URL(url).searchParams.get('q');
const fixtures = {
'Bagmati Province': { addresstype: 'province' },
'Mystery Province': { place_rank: 8 },
'Missing Evidence': {},
'Northern Region': { addresstype: 'region', place_rank: 8 },
'Central District': { addresstype: 'district', place_rank: 10 },
'Example State': { addresstype: 'county' },
'Bagmati Zone': { addresstype: 'state' },
Austin: { addresstype: 'city' },
};
return json([
row({
category: 'boundary',
type: 'administrative',
name: query,
...fixtures[query],
}),
]);
});
const gate = createNominatimGate({ settings: OPERATOR, fetchImpl });
const outline = createNominatimOutlineProvider({
gate,
cache: createNominatimCache({ dir: tempDir(t) }),
});
const request = mount({ outline });
const verified = await request('/outline?q=Bagmati%20Province&kind=admin');
assert.equal(verified.status, 200);
assert.equal(verified.body.outline.adminArea, 'Bagmati Province');
assert.equal(verified.body.outline.adminLevel, 'admin1');
for (const query of [
'Mystery Province',
'Northern Region',
'Central District',
'Example State',
]) {
const unverified = await request(
`/outline?q=${encodeURIComponent(query)}&kind=admin`,
);
assert.equal(unverified.status, 200, query);
assert.equal(unverified.body.status, 'OK', query);
assert.equal('adminArea' in unverified.body.outline, false, query);
assert.equal('adminLevel' in unverified.body.outline, false, query);
}
const missing = await request('/outline?q=Missing%20Evidence&kind=admin');
assert.equal(missing.status, 200);
assert.equal(missing.body.status, 'ZERO_RESULTS');
assert.equal(missing.body.outline, null);
const retired = await request('/outline?q=Bagmati%20Zone&kind=admin');
assert.equal(retired.status, 200);
assert.equal(retired.body.status, 'OK');
assert.equal(retired.body.outline.adminArea, 'Bagmati Zone');
assert.equal('adminLevel' in retired.body.outline, false);
const city = await request('/outline?q=Austin&kind=city');
assert.equal(city.status, 200);
assert.equal(city.body.status, 'OK');
assert.equal('adminArea' in city.body.outline, false);
assert.equal('adminLevel' in city.body.outline, false);
});
test('the outline route reports disabled and capped as permanent, busy as retryable', async (t) => {
const disabled = mount({
gate: createNominatimGate({ settings: { endpoint: null, isPublic: false, dailyCap: 50 } }),
gate: createNominatimGate({
settings: { endpoint: null, isPublic: false, dailyCap: 50 },
}),
});
const off = await disabled('/outline?q=Paris&kind=city');
assert.equal(off.status, 503);
@@ -522,16 +861,16 @@ test('the outline route reports disabled and capped as permanent, busy as retrya
assert.equal(off.body.retryable, false);
const capped = mount({
gate: createNominatimGate({ settings: { ...PUBLIC, dailyCap: 0 } }),
gate: publicGate(t, { settings: { ...PUBLIC, dailyCap: 0 } }),
});
const cap = await capped('/outline?q=Paris&kind=city');
assert.equal(cap.status, 429);
assert.equal(cap.body.retryable, false);
const time = clock();
const paused = createNominatimGate({
settings: PUBLIC,
fetchImpl: async () => new Response('', { status: 429, headers: { 'Retry-After': '90' } }),
const paused = publicGate(t, {
fetchImpl: async () =>
new Response('', { status: 429, headers: { 'Retry-After': '90' } }),
...time,
});
const busy = mount({ gate: paused });
+11 -3
View File
@@ -9,6 +9,9 @@
// Run with: npm test
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import {
fetchNominatimSearch,
geocodeProxy,
@@ -23,6 +26,8 @@ import { createPhotonGeocoder } from '../keylessGeocoder.js';
// Each case starts with a fresh shared gate: a pause after one case's
// simulated outage must not decide the next case.
test.beforeEach(() => setSharedNominatimGate(null));
const storageRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-search-route-'));
test.after(() => fs.rmSync(storageRoot, { recursive: true, force: true }));
const HIT = [
{
@@ -38,13 +43,16 @@ const HIT = [
/** Mount the geocode route the way the dev server does. */
let clientCounter = 0;
function mountGeocode(
install = (server) => geocodeProxy().configureServer(server),
) {
function mountGeocode(install = null) {
// A fresh client address per mount: the per-client rate limiter is module
// state shared by every case in this file, and one case's burst must not
// decide another case's outcome.
clientCounter += 1;
if (!install)
install = (server) =>
geocodeProxy({
storageDir: path.join(storageRoot, `case-${clientCounter}`),
}).configureServer(server);
const remoteAddress = `10.0.0.${clientCounter % 250}`;
const routes = new Map();
install({