mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
feat(outlines): voice outlines without Overpass — bundled US places and neighborhoods, guarded Nominatim, OpenFreeMap streets and buildings
This commit is contained in:
1 parent
e7707d9a0f
commit
9559e7a282
333 files changed
+13203
-271
No files matched your search
@@ -0,0 +1,342 @@
|
||||
// Review regressions for the Nominatim gate: state that survives restarts and
|
||||
// sweeps, several processes sharing one count, long refusals, stalled bodies,
|
||||
// request-equivalent cache keys, and stricter outline selection.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
import {
|
||||
createGateStateStore,
|
||||
createNominatimGate,
|
||||
resolveNominatimSettings,
|
||||
PUBLIC_NOMINATIM_SEARCH,
|
||||
} from '../../server/providers/regional/nominatimGate.js';
|
||||
import { createNominatimCache } from '../../server/providers/regional/nominatimCache.js';
|
||||
import {
|
||||
outlineBias,
|
||||
selectOutlineResult,
|
||||
} from '../../server/providers/regional/outlineSelect.js';
|
||||
import {
|
||||
createNominatimOutlineProvider,
|
||||
createNominatimSearchProvider,
|
||||
geocodeProxy,
|
||||
} from '../../server/providers/regional/place.js';
|
||||
|
||||
const PUBLIC = { endpoint: PUBLIC_NOMINATIM_SEARCH, isPublic: true, dailyCap: 50 };
|
||||
const tempDir = (t) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-'));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
};
|
||||
const quick = { sleep: async () => {} };
|
||||
const json = (value) => Response.json(value);
|
||||
const square = (lon, lat, d) => [
|
||||
[
|
||||
[lon, lat],
|
||||
[lon + d, lat],
|
||||
[lon + d, lat + d],
|
||||
[lon, lat + d],
|
||||
[lon, lat],
|
||||
],
|
||||
];
|
||||
|
||||
function mount(options) {
|
||||
let handler;
|
||||
geocodeProxy(options).configureServer({
|
||||
middlewares: { use: (_route, fn) => (handler = fn) },
|
||||
});
|
||||
return (url) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const res = {
|
||||
writableEnded: false,
|
||||
on() {},
|
||||
writeHead(status) {
|
||||
res.status = status;
|
||||
},
|
||||
end(body) {
|
||||
res.writableEnded = true;
|
||||
resolve({ status: res.status, body: JSON.parse(body) });
|
||||
},
|
||||
};
|
||||
Promise.resolve(
|
||||
handler(
|
||||
{ method: 'GET', url, headers: {}, socket: { remoteAddress: `10.9.${Math.random()}` }, on() {} },
|
||||
res,
|
||||
),
|
||||
).catch(reject);
|
||||
});
|
||||
}
|
||||
|
||||
test('the daily count survives cache sweeps and a restart of the real composition', async (t) => {
|
||||
const storageDir = tempDir(t);
|
||||
const fetchImpl = async () => json([]);
|
||||
const first = createNominatimGate({ settings: PUBLIC, fetchImpl, ...quick });
|
||||
const request = mount({ storageDir, gate: first });
|
||||
for (let i = 0; i < 3; i++)
|
||||
assert.equal((await request(`/?q=place-${i}`)).status, 200);
|
||||
assert.equal(first.stats().usedToday, 3);
|
||||
|
||||
// A foreign file in the answers directory and a full sweep leave the
|
||||
// state alone and the stranger untouched.
|
||||
const answers = path.join(storageDir, 'answers');
|
||||
fs.writeFileSync(path.join(answers, 'notes.json'), '{}');
|
||||
await createNominatimCache({ dir: answers, sweepEvery: 1 }).sweep();
|
||||
assert.ok(fs.existsSync(path.join(answers, 'notes.json')));
|
||||
assert.ok(fs.existsSync(path.join(storageDir, 'state.json')));
|
||||
|
||||
// Restart: a new gate composed on the same directory, which sweeps at start.
|
||||
const second = createNominatimGate({ settings: { ...PUBLIC, dailyCap: 4 }, fetchImpl, ...quick });
|
||||
const again = mount({ storageDir, gate: second });
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
assert.equal(second.stats().usedToday, 3, 'the count came back');
|
||||
assert.equal((await again('/?q=place-new')).status, 200);
|
||||
const capped = await again('/?q=place-over');
|
||||
assert.equal(capped.status, 429, 'the cap still counts yesterday-restart usage');
|
||||
assert.equal(capped.body.code, 'NOMINATIM_DAILY_CAP');
|
||||
});
|
||||
|
||||
test('a sweep removes expired and corrupt entries, never ones it could not read', async (t) => {
|
||||
const dir = tempDir(t);
|
||||
const file = (n) => path.join(dir, `${String(n).repeat(40)}.json`);
|
||||
const live = { key: 'k', payload: null, storedAt: 1, expiresAt: Date.now() + 60_000 };
|
||||
fs.writeFileSync(file(1), JSON.stringify(live));
|
||||
fs.writeFileSync(file(2), JSON.stringify({ ...live, expiresAt: 1 }));
|
||||
fs.writeFileSync(file(3), '{not json');
|
||||
const cache = createNominatimCache({ dir });
|
||||
// Reads failing (a busy disk, a test double) must not empty the cache.
|
||||
const fsp = (await import('node:fs/promises')).default;
|
||||
const readFile = t.mock.method(fsp, 'readFile', async () => {
|
||||
throw Object.assign(new Error('EMFILE'), { code: 'EMFILE' });
|
||||
});
|
||||
await cache.sweep();
|
||||
assert.deepEqual([1, 2, 3].map((n) => fs.existsSync(file(n))), [true, true, true]);
|
||||
readFile.mock.restore();
|
||||
await cache.sweep();
|
||||
assert.deepEqual([1, 2, 3].map((n) => fs.existsSync(file(n))), [true, false, false]);
|
||||
});
|
||||
|
||||
test('processes sharing the state file add to one count', async (t) => {
|
||||
const file = path.join(tempDir(t), 'state.json');
|
||||
const module = new URL('../../server/providers/regional/nominatimGate.js', import.meta.url).href;
|
||||
const script = `
|
||||
const { createGateStateStore } = await import(${JSON.stringify(module)});
|
||||
const store = createGateStateStore({ file: ${JSON.stringify(file)} });
|
||||
for (let i = 0; i < 25; i++) store.increment('2026-09-29');
|
||||
`;
|
||||
const run = () =>
|
||||
new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: 'inherit' });
|
||||
child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`exit ${code}`))));
|
||||
});
|
||||
await Promise.all([run(), run(), run()]);
|
||||
assert.equal(createGateStateStore({ file }).count('2026-09-29'), 75);
|
||||
});
|
||||
|
||||
test('a persistence failure is reported and never lowers the count', (t) => {
|
||||
const dir = tempDir(t);
|
||||
const blocker = path.join(dir, 'not-a-dir');
|
||||
fs.writeFileSync(blocker, 'x');
|
||||
const errors = [];
|
||||
const store = createGateStateStore({
|
||||
file: path.join(blocker, 'state.json'),
|
||||
onError: (error) => errors.push(error),
|
||||
});
|
||||
store.increment('2026-09-29');
|
||||
store.increment('2026-09-29');
|
||||
assert.equal(store.count('2026-09-29'), 2);
|
||||
assert.equal(store.status().persisted, false);
|
||||
assert.ok(store.status().error);
|
||||
assert.equal(errors.length, 1, 'reported once, not per request');
|
||||
});
|
||||
|
||||
test('a long Retry-After is honoured in full and survives a restart', async (t) => {
|
||||
const file = path.join(tempDir(t), 'state.json');
|
||||
let clock = Date.UTC(2026, 8, 29, 12);
|
||||
const now = () => clock;
|
||||
const refused = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl: async () => new Response('', { status: 429, headers: { 'Retry-After': '86400' } }),
|
||||
usage: createGateStateStore({ file }),
|
||||
now,
|
||||
...quick,
|
||||
});
|
||||
await assert.rejects(createNominatimSearchProvider({ gate: refused })('x', null), {
|
||||
code: 'NOMINATIM_REFUSED',
|
||||
});
|
||||
const calls = [];
|
||||
const restarted = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(url);
|
||||
return json([]);
|
||||
},
|
||||
usage: createGateStateStore({ file }),
|
||||
now,
|
||||
...quick,
|
||||
});
|
||||
const search = createNominatimSearchProvider({ gate: restarted });
|
||||
clock += 7 * 3_600_000; // past the old six-hour ceiling
|
||||
await assert.rejects(search('y', null), (error) => {
|
||||
assert.equal(error.code, 'NOMINATIM_PAUSED');
|
||||
assert.equal(error.retryAfterMs, 17 * 3_600_000);
|
||||
return true;
|
||||
});
|
||||
assert.equal(calls.length, 0);
|
||||
clock += 17 * 3_600_000 + 1000;
|
||||
await search('y', null);
|
||||
assert.equal(calls.length, 1);
|
||||
});
|
||||
|
||||
test('a stalled body times out instead of holding the shared queue', async () => {
|
||||
let cancelled = 0;
|
||||
const stalled = (signal) => {
|
||||
const body = new ReadableStream({
|
||||
pull: () => new Promise(() => {}),
|
||||
cancel: () => {
|
||||
cancelled += 1;
|
||||
},
|
||||
});
|
||||
signal.addEventListener('abort', () => body.cancel().catch(() => {}));
|
||||
return new Response(body, { status: 200 });
|
||||
};
|
||||
let n = 0;
|
||||
const gate = createNominatimGate({
|
||||
settings: { ...PUBLIC, isPublic: false },
|
||||
fetchImpl: async (_url, { signal }) => (++n <= 2 ? stalled(signal) : json([])),
|
||||
timeoutMs: 150,
|
||||
});
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
const started = Date.now();
|
||||
await assert.rejects(search('stall', null), { code: 'NOMINATIM_TRANSPORT' });
|
||||
assert.ok(Date.now() - started < 3000);
|
||||
assert.ok(cancelled >= 1, 'the body read was cancelled');
|
||||
});
|
||||
|
||||
test('cache keys are request-equivalent: a miss for "Paris???" never answers "Paris"', async () => {
|
||||
const calls = [];
|
||||
const gate = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(new URL(url).searchParams.get('q'));
|
||||
return json(calls.length === 1 ? [] : [{ lat: '48.85', lon: '2.35', display_name: 'Paris, France', name: 'Paris', addresstype: 'city' }]);
|
||||
},
|
||||
...quick,
|
||||
});
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
assert.equal((await search('Paris???', null)).status, 'ZERO_RESULTS');
|
||||
assert.equal((await search('Paris', null)).status, 'OK');
|
||||
assert.deepEqual(calls, ['Paris???', 'Paris']);
|
||||
// Whitespace-only differences are the same upstream request.
|
||||
assert.equal((await search(' Paris ', null)).cached, true);
|
||||
});
|
||||
|
||||
test('a malformed answer is a failure, not a cached "no result"', async () => {
|
||||
let shape = { error: 'Unexpected' };
|
||||
let calls = 0;
|
||||
const gate = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return json(shape);
|
||||
},
|
||||
...quick,
|
||||
});
|
||||
const outline = createNominatimOutlineProvider({ gate });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
await assert.rejects(outline({ query: 'Zilker Park', kind: 'landmark' }), { code: 'NOMINATIM_MALFORMED' });
|
||||
await assert.rejects(search('Zilker Park', null), { code: 'NOMINATIM_MALFORMED' });
|
||||
shape = [];
|
||||
assert.equal((await outline({ query: 'Zilker Park', kind: 'landmark' })).status, 'ZERO_RESULTS');
|
||||
assert.equal(calls, 3, 'the malformed answers were not cached');
|
||||
});
|
||||
|
||||
test('cache entries are namespaced by endpoint', async () => {
|
||||
const cache = createNominatimCache();
|
||||
const hits = [];
|
||||
const gateFor = (endpoint) =>
|
||||
createNominatimGate({
|
||||
settings: { endpoint, isPublic: false, dailyCap: 50 },
|
||||
fetchImpl: async (url) => {
|
||||
hits.push(new URL(url).host);
|
||||
return json([]);
|
||||
},
|
||||
});
|
||||
await createNominatimSearchProvider({ gate: gateFor('https://a.example.test/search'), cache })('x', null);
|
||||
await createNominatimSearchProvider({ gate: gateFor('https://b.example.test/search'), cache })('x', null);
|
||||
assert.deepEqual(hits, ['a.example.test', 'b.example.test']);
|
||||
});
|
||||
|
||||
test('public-host detection ignores a trailing DNS dot', () => {
|
||||
assert.equal(
|
||||
resolveNominatimSettings({ NOMINATIM_URL: 'https://nominatim.openstreetmap.org./search' }).isPublic,
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('missing coordinates are no bias, never (0, 0)', () => {
|
||||
for (const [lat, lon] of [
|
||||
[null, null],
|
||||
['', ''],
|
||||
[undefined, 2],
|
||||
[48.8, null],
|
||||
])
|
||||
assert.equal(outlineBias('city', lat, lon), null, `${lat},${lon}`);
|
||||
assert.ok(outlineBias('city', 0, 0), 'a real (0, 0) is still a position');
|
||||
});
|
||||
|
||||
const boundary = (extra) => ({
|
||||
category: 'boundary',
|
||||
type: 'administrative',
|
||||
lat: '1',
|
||||
lon: '1',
|
||||
geojson: { type: 'Polygon', coordinates: square(1, 1, 0.1) },
|
||||
...extra,
|
||||
});
|
||||
|
||||
test('administrative boundaries must be at the asked level', () => {
|
||||
const county = boundary({ name: 'Travis County', addresstype: 'county', place_rank: 12 });
|
||||
const city = boundary({ name: 'Austin', addresstype: 'city', place_rank: 16 });
|
||||
const state = boundary({ name: 'Texas', addresstype: 'state', place_rank: 8 });
|
||||
assert.equal(selectOutlineResult([state], { kind: 'city', query: 'Texas' }).outline, null);
|
||||
assert.equal(selectOutlineResult([city], { kind: 'admin', query: 'Austin' }).outline, null);
|
||||
assert.equal(selectOutlineResult([state], { kind: 'neighborhood', query: 'Texas' }).outline, null);
|
||||
assert.ok(selectOutlineResult([county], { kind: 'admin', query: 'Travis County' }).outline);
|
||||
assert.ok(selectOutlineResult([boundary({ name: 'Somewhere', place_rank: 16 })], { kind: 'city', query: 'Somewhere' }).outline);
|
||||
assert.equal(
|
||||
selectOutlineResult([boundary({ name: 'Somewhere' })], { kind: 'city', query: 'Somewhere' }).outline,
|
||||
null,
|
||||
'no level evidence at all',
|
||||
);
|
||||
// An unsuitable early boundary does not block a suitable later one.
|
||||
const picked = selectOutlineResult([state, boundary({ name: 'Texas', addresstype: 'city', osm_type: 'relation', osm_id: 9 })], {
|
||||
kind: 'city',
|
||||
query: 'Texas',
|
||||
});
|
||||
assert.equal(picked.outline.osm, 'relation/9');
|
||||
});
|
||||
|
||||
test('building asks need building evidence', () => {
|
||||
const park = { category: 'tourism', type: 'theme_park', name: 'Big Top', lat: '1', lon: '1', geojson: { type: 'Polygon', coordinates: square(1, 1, 0.0001) } };
|
||||
const camp = { ...park, type: 'camp_site' };
|
||||
const campus = { ...park, category: 'amenity', type: 'university' };
|
||||
for (const row of [park, camp, campus])
|
||||
assert.equal(selectOutlineResult([row], { kind: 'building', query: 'Big Top' }).outline, null, row.type);
|
||||
const hugeOffice = { ...park, category: 'office', type: 'government', geojson: { type: 'Polygon', coordinates: square(1, 1, 0.01) } };
|
||||
assert.equal(selectOutlineResult([hugeOffice], { kind: 'building', query: 'Big Top' }).outline, null);
|
||||
const office = { ...hugeOffice, geojson: { type: 'Polygon', coordinates: square(1, 1, 0.001) } };
|
||||
assert.ok(selectOutlineResult([office], { kind: 'building', query: 'Big Top' }).outline);
|
||||
const hall = { ...hugeOffice, category: 'building', type: 'yes' };
|
||||
assert.ok(selectOutlineResult([hall], { kind: 'building', query: 'Big Top' }).outline);
|
||||
});
|
||||
|
||||
test('names match on whole words', () => {
|
||||
const row = (name) => ({ category: 'leisure', type: 'park', name, lat: '1', lon: '1', geojson: { type: 'Polygon', coordinates: square(1, 1, 0.01) } });
|
||||
assert.equal(selectOutlineResult([row('Yorkshire Park')], { kind: 'landmark', query: 'York' }).outline, null);
|
||||
assert.ok(selectOutlineResult([row('York Park')], { kind: 'landmark', query: 'York' }).outline);
|
||||
assert.ok(selectOutlineResult([row('Zilker Metropolitan Park')], { kind: 'landmark', query: 'Zilker Park' }).outline);
|
||||
assert.ok(
|
||||
selectOutlineResult([{ ...row('新宿御苑'), namedetails: { name: '新宿御苑' } }], { kind: 'landmark', query: '新宿御苑' }).outline,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,541 @@
|
||||
// THE GUARDED NOMINATIM GATE AND OUTLINE ROUTE.
|
||||
//
|
||||
// Search and voice outlines share one gate to the public instance: 1.1 s
|
||||
// spacing, a short queue, a daily cap per install, a pause when the server
|
||||
// says stop, one retry only after a transport failure, and caches that never
|
||||
// record a failure as "no such place". Outline answers are accepted only when
|
||||
// the result's class suits the ask.
|
||||
//
|
||||
// Run with: npm test
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
createNominatimGate,
|
||||
createUsageStore,
|
||||
parseRetryAfter,
|
||||
resolveNominatimSettings,
|
||||
PUBLIC_NOMINATIM_SEARCH,
|
||||
} from '../../server/providers/regional/nominatimGate.js';
|
||||
import {
|
||||
createNominatimCache,
|
||||
normalizePlaceName,
|
||||
NOMINATIM_CACHE_TTL_MS,
|
||||
} from '../../server/providers/regional/nominatimCache.js';
|
||||
import {
|
||||
acceptsOutlineClass,
|
||||
capOutlineVertices,
|
||||
outlineBias,
|
||||
selectOutlineResult,
|
||||
} from '../../server/providers/regional/outlineSelect.js';
|
||||
import {
|
||||
createNominatimOutlineProvider,
|
||||
createNominatimSearchProvider,
|
||||
geocodeProxy,
|
||||
} from '../../server/providers/regional/place.js';
|
||||
|
||||
const PUBLIC = {
|
||||
endpoint: PUBLIC_NOMINATIM_SEARCH,
|
||||
isPublic: true,
|
||||
dailyCap: 50,
|
||||
};
|
||||
const OPERATOR = {
|
||||
endpoint: 'https://geo.example.test/search',
|
||||
isPublic: false,
|
||||
dailyCap: 50,
|
||||
};
|
||||
|
||||
const square = (lon, lat, d = 0.001) => [
|
||||
[
|
||||
[lon, lat],
|
||||
[lon + d, lat],
|
||||
[lon + d, lat + d],
|
||||
[lon, lat + d],
|
||||
[lon, lat],
|
||||
],
|
||||
];
|
||||
|
||||
const row = (overrides) => ({
|
||||
lat: '30.27',
|
||||
lon: '-97.74',
|
||||
display_name: 'Somewhere',
|
||||
osm_type: 'relation',
|
||||
osm_id: 1,
|
||||
geojson: { type: 'Polygon', coordinates: square(-97.74, 30.27) },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const json = (value, init = {}) =>
|
||||
new Response(JSON.stringify(value), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
...init,
|
||||
});
|
||||
|
||||
/** A fake clock whose sleeps advance time instantly and are recorded. */
|
||||
function clock(start = Date.UTC(2026, 8, 28, 12)) {
|
||||
let t = start;
|
||||
const sleeps = [];
|
||||
return {
|
||||
now: () => t,
|
||||
advance: (ms) => {
|
||||
t += ms;
|
||||
},
|
||||
sleep: async (ms) => {
|
||||
sleeps.push(ms);
|
||||
t += ms;
|
||||
},
|
||||
sleeps,
|
||||
};
|
||||
}
|
||||
|
||||
function stubFetch(respond) {
|
||||
const calls = [];
|
||||
const fetchImpl = async (url, options) => {
|
||||
calls.push(String(url));
|
||||
return await respond(String(url), options, calls.length);
|
||||
};
|
||||
return { calls, fetchImpl };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- settings
|
||||
|
||||
test('endpoint settings: unset is public, empty disables, operator URLs are not public', () => {
|
||||
assert.deepEqual(resolveNominatimSettings({}), {
|
||||
endpoint: PUBLIC_NOMINATIM_SEARCH,
|
||||
isPublic: true,
|
||||
dailyCap: 50,
|
||||
});
|
||||
for (const off of ['', ' ', 'off', 'none', 'disabled'])
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_URL: off }).endpoint, null);
|
||||
const own = resolveNominatimSettings({
|
||||
NOMINATIM_URL: 'https://geo.example.test',
|
||||
});
|
||||
assert.equal(own.endpoint, 'https://geo.example.test/search');
|
||||
assert.equal(own.isPublic, false);
|
||||
assert.equal(
|
||||
resolveNominatimSettings({
|
||||
NOMINATIM_URL: 'http://localhost:8080/search',
|
||||
}).endpoint,
|
||||
'http://localhost:8080/search',
|
||||
);
|
||||
// A mistyped value disables rather than silently falling back to public.
|
||||
for (const bad of ['not a url', 'ftp://x.test', 'https://u:[email protected]', 'https://x.test/search?q=1'])
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_URL: bad }).endpoint, null, bad);
|
||||
assert.equal(
|
||||
resolveNominatimSettings({ NOMINATIM_URL: 'https://nominatim.openstreetmap.org/search' }).isPublic,
|
||||
true,
|
||||
);
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '7' }).dailyCap, 7);
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: '0' }).dailyCap, 0);
|
||||
assert.equal(resolveNominatimSettings({ NOMINATIM_DAILY_CAP: 'x' }).dailyCap, 50);
|
||||
});
|
||||
|
||||
test('Retry-After reads seconds and HTTP dates', () => {
|
||||
const now = Date.UTC(2026, 0, 1);
|
||||
assert.equal(parseRetryAfter('120', now), 120_000);
|
||||
assert.equal(parseRetryAfter(new Date(now + 30_000).toUTCString(), now), 30_000);
|
||||
assert.equal(parseRetryAfter(null, now), null);
|
||||
assert.equal(parseRetryAfter('soon', now), null);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- the gate
|
||||
|
||||
test('search and outlines share one pacer: requests start at least 1.1 s apart', async () => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async (url) =>
|
||||
json(url.includes('polygon_geojson') ? [row({ category: 'boundary', type: 'administrative', addresstype: 'city', name: 'Austin' })] : []),
|
||||
);
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
const outline = createNominatimOutlineProvider({ gate });
|
||||
await Promise.all([
|
||||
search('austin', null),
|
||||
outline({ query: 'Austin', kind: 'city' }),
|
||||
search('dallas', null),
|
||||
]);
|
||||
assert.equal(calls.length, 3);
|
||||
// The first starts at once; each later one waits out the full spacing.
|
||||
assert.deepEqual(time.sleeps, [1100, 1100]);
|
||||
assert.equal(gate.stats().usedToday, 3, 'one daily count for both kinds');
|
||||
});
|
||||
|
||||
test('the daily cap stops public requests and is shared by search and outlines', async () => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async () => json([]));
|
||||
const gate = createNominatimGate({
|
||||
settings: { ...PUBLIC, dailyCap: 2 },
|
||||
fetchImpl,
|
||||
...time,
|
||||
});
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
const outline = createNominatimOutlineProvider({ gate });
|
||||
await search('one', null);
|
||||
await outline({ query: 'two', kind: 'landmark' });
|
||||
await assert.rejects(search('three', null), { code: 'NOMINATIM_DAILY_CAP' });
|
||||
await assert.rejects(outline({ query: 'four', kind: 'city' }), {
|
||||
code: 'NOMINATIM_DAILY_CAP',
|
||||
});
|
||||
assert.equal(calls.length, 2);
|
||||
// Cached answers still work after the cap.
|
||||
assert.equal((await search('one', null)).cached, true);
|
||||
// A new UTC day restores the allowance.
|
||||
time.advance(86_400_000);
|
||||
await search('five', null);
|
||||
assert.equal(calls.length, 3);
|
||||
});
|
||||
|
||||
test('an operator endpoint is not subject to the public cap or spacing', async () => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async () => json([]));
|
||||
const gate = createNominatimGate({
|
||||
settings: { ...OPERATOR, dailyCap: 1 },
|
||||
fetchImpl,
|
||||
...time,
|
||||
});
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
await Promise.all([search('a', null), search('b', null), search('c', null)]);
|
||||
assert.equal(calls.length, 3);
|
||||
assert.ok(calls.every((url) => url.startsWith('https://geo.example.test/search?')));
|
||||
assert.deepEqual(time.sleeps, []);
|
||||
});
|
||||
|
||||
test('a disabled endpoint sends nothing', async () => {
|
||||
const { calls, fetchImpl } = stubFetch(async () => json([]));
|
||||
const gate = createNominatimGate({
|
||||
settings: { endpoint: null, isPublic: false, dailyCap: 50 },
|
||||
fetchImpl,
|
||||
});
|
||||
await assert.rejects(createNominatimSearchProvider({ gate })('x', null), {
|
||||
code: 'NOMINATIM_DISABLED',
|
||||
});
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
|
||||
test('429 with Retry-After pauses the gate and stops queued requests', async () => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async (_url, _o, n) =>
|
||||
n === 1
|
||||
? new Response('slow down', { status: 429, headers: { 'Retry-After': '120' } })
|
||||
: json([]),
|
||||
);
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
const [first, queued] = await Promise.allSettled([
|
||||
search('first', null),
|
||||
search('queued', null),
|
||||
]);
|
||||
assert.equal(first.reason.code, 'NOMINATIM_REFUSED');
|
||||
assert.equal(first.reason.retryAfterMs, 120_000);
|
||||
assert.equal(queued.reason.code, 'NOMINATIM_PAUSED', 'the queued request is not sent');
|
||||
assert.equal(calls.length, 1);
|
||||
await assert.rejects(search('later', null), { code: 'NOMINATIM_PAUSED' });
|
||||
assert.equal(calls.length, 1);
|
||||
time.advance(121_000);
|
||||
await search('after', null);
|
||||
assert.equal(calls.length, 2, 'requests resume once the pause has passed');
|
||||
// The refusal was not remembered as "no such place".
|
||||
await search('first', null);
|
||||
assert.equal(calls.length, 3);
|
||||
});
|
||||
|
||||
test('an outage stops queued requests; a server error is never cached as not found', async () => {
|
||||
const time = clock();
|
||||
let down = true;
|
||||
const { calls, fetchImpl } = stubFetch(async () =>
|
||||
down ? new Response('bad gateway', { status: 502 }) : json([row({ category: 'leisure', type: 'park', name: 'Zilker Park' })]),
|
||||
);
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const outline = createNominatimOutlineProvider({ gate });
|
||||
const results = await Promise.allSettled([
|
||||
outline({ query: 'Zilker Park', kind: 'landmark' }),
|
||||
outline({ query: 'Barton Springs', kind: 'landmark' }),
|
||||
]);
|
||||
assert.equal(results[0].reason.code, 'NOMINATIM_UPSTREAM');
|
||||
assert.equal(results[1].reason.code, 'NOMINATIM_PAUSED');
|
||||
assert.equal(calls.length, 1);
|
||||
down = false;
|
||||
time.advance(31_000);
|
||||
const answer = await outline({ query: 'Zilker Park', kind: 'landmark' });
|
||||
assert.equal(answer.status, 'OK');
|
||||
assert.equal(calls.length, 2);
|
||||
});
|
||||
|
||||
test('a transport failure is retried once, then pauses and is not cached', async () => {
|
||||
const time = clock();
|
||||
let failures = 1;
|
||||
const { calls, fetchImpl } = stubFetch(async () => {
|
||||
if (failures-- > 0) throw new TypeError('fetch failed');
|
||||
return json([]);
|
||||
});
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
assert.equal((await search('retry', null)).status, 'ZERO_RESULTS');
|
||||
assert.equal(calls.length, 2, 'one retry after a transport failure');
|
||||
|
||||
failures = 2;
|
||||
await assert.rejects(search('twice', null), { code: 'NOMINATIM_TRANSPORT' });
|
||||
assert.equal(calls.length, 4, 'never more than one retry');
|
||||
await assert.rejects(search('twice', null), { code: 'NOMINATIM_PAUSED' });
|
||||
time.advance(31_000);
|
||||
assert.equal((await search('twice', null)).cached, undefined, 'the failure was not cached');
|
||||
assert.equal(calls.length, 5);
|
||||
});
|
||||
|
||||
test('a 403 block pauses for an hour', async () => {
|
||||
const time = clock();
|
||||
const { fetchImpl } = stubFetch(async () => new Response('', { status: 403 }));
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
await assert.rejects(createNominatimSearchProvider({ gate })('x', null), {
|
||||
code: 'NOMINATIM_REFUSED',
|
||||
});
|
||||
assert.ok(gate.stats().pausedForMs >= 3_599_000);
|
||||
});
|
||||
|
||||
test('the queue is bounded', async () => {
|
||||
const time = clock();
|
||||
const { fetchImpl } = stubFetch(async () => json([]));
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time, maxPending: 2 });
|
||||
const search = createNominatimSearchProvider({ gate });
|
||||
const results = await Promise.allSettled(
|
||||
['a', 'b', 'c', 'd'].map((q) => search(q, null)),
|
||||
);
|
||||
assert.deepEqual(
|
||||
results.map((r) => r.status === 'fulfilled' || r.reason.code),
|
||||
[true, true, 'NOMINATIM_QUEUE_FULL', 'NOMINATIM_QUEUE_FULL'],
|
||||
);
|
||||
});
|
||||
|
||||
test('the daily count survives a restart when persisted', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-usage-'));
|
||||
const file = path.join(dir, 'usage.json');
|
||||
createUsageStore({ file }).increment('2026-09-28');
|
||||
createUsageStore({ file }).increment('2026-09-28');
|
||||
assert.equal(createUsageStore({ file }).count('2026-09-28'), 2);
|
||||
assert.equal(createUsageStore({ file }).count('2026-09-29'), 0);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- caches
|
||||
|
||||
test('cache lifetimes: 90 days for outlines, 30 for places, 24 hours for no result', async () => {
|
||||
const time = clock();
|
||||
const cache = createNominatimCache({ now: time.now });
|
||||
await cache.set('o', 1, NOMINATIM_CACHE_TTL_MS.outline);
|
||||
await cache.set('p', 2, NOMINATIM_CACHE_TTL_MS.place);
|
||||
await cache.set('n', 3, NOMINATIM_CACHE_TTL_MS.notFound);
|
||||
time.advance(23 * 3_600_000);
|
||||
assert.ok(await cache.get('n'));
|
||||
time.advance(2 * 3_600_000);
|
||||
assert.equal(await cache.get('n'), null);
|
||||
time.advance(28 * 86_400_000);
|
||||
assert.ok(await cache.get('p'));
|
||||
time.advance(2 * 86_400_000);
|
||||
assert.equal(await cache.get('p'), null);
|
||||
assert.ok(await cache.get('o'));
|
||||
time.advance(60 * 86_400_000);
|
||||
assert.equal(await cache.get('o'), null);
|
||||
});
|
||||
|
||||
test('the disk cache survives a restart and stays within its bounds', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-cache-'));
|
||||
const time = clock();
|
||||
const first = createNominatimCache({ dir, now: time.now, maxDiskEntries: 3, sweepEvery: 1000 });
|
||||
for (let i = 0; i < 5; i++) {
|
||||
time.advance(1000);
|
||||
await first.set(`k${i}`, { i }, i === 0 ? 10 : NOMINATIM_CACHE_TTL_MS.outline);
|
||||
}
|
||||
const second = createNominatimCache({ dir, now: time.now, maxDiskEntries: 3 });
|
||||
assert.deepEqual((await second.get('k4')).payload, { i: 4 });
|
||||
await second.sweep();
|
||||
const files = fs.readdirSync(dir).filter((name) => name.endsWith('.json'));
|
||||
assert.equal(files.length, 3, 'expired entry removed, then oldest trimmed');
|
||||
assert.equal(await createNominatimCache({ dir, now: time.now }).get('k0'), null);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('names are normalized Unicode-aware so small differences share an entry', () => {
|
||||
assert.equal(normalizePlaceName('São Paulo'), normalizePlaceName('SAO paulo!'));
|
||||
assert.equal(normalizePlaceName('The Pentagon'), 'pentagon');
|
||||
assert.equal(normalizePlaceName('Zürich'), 'zurich');
|
||||
assert.equal(normalizePlaceName('東京都'), '東京都', 'non-Latin names survive');
|
||||
assert.equal(normalizePlaceName('Central Park'), 'central park');
|
||||
});
|
||||
|
||||
test('small view changes reuse one cached outline', async () => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async () =>
|
||||
json([row({ category: 'leisure', type: 'park', name: 'Central Park' })]),
|
||||
);
|
||||
const gate = createNominatimGate({ settings: PUBLIC, fetchImpl, ...time });
|
||||
const outline = createNominatimOutlineProvider({ gate });
|
||||
await outline({ query: 'Central Park', kind: 'landmark', lat: 40.781, lon: -73.966 });
|
||||
const again = await outline({ query: ' Central Park ', kind: 'landmark', lat: 40.7822, lon: -73.9651 });
|
||||
assert.equal(again.cached, true);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.match(calls[0], /viewbox=/);
|
||||
assert.equal(outlineBias('city', 48.85, 2.35).key, '49,2');
|
||||
assert.equal(outlineBias('landmark', 30.2747, -97.7404).key, '30.25,-97.75');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- selection
|
||||
|
||||
test('wrong types are rejected: charging station for Mission District, station for Bandra', () => {
|
||||
const charging = row({
|
||||
category: 'amenity',
|
||||
type: 'charging_station',
|
||||
name: 'Revel - Mission District',
|
||||
});
|
||||
for (const kind of ['neighborhood', 'landmark', 'city'])
|
||||
assert.equal(
|
||||
selectOutlineResult([charging], { kind, query: 'Mission District, San Francisco' }).outline,
|
||||
null,
|
||||
kind,
|
||||
);
|
||||
const station = row({ category: 'railway', type: 'station', name: 'Bandra' });
|
||||
for (const kind of ['neighborhood', 'landmark', 'building', 'city'])
|
||||
assert.equal(selectOutlineResult([station], { kind, query: 'Bandra' }).outline, null, kind);
|
||||
assert.equal(acceptsOutlineClass('landmark', 'highway', 'primary'), false);
|
||||
assert.equal(acceptsOutlineClass('city', 'leisure', 'park'), false);
|
||||
assert.equal(acceptsOutlineClass('landmark', 'office', 'government'), true);
|
||||
assert.equal(acceptsOutlineClass('landmark', 'man_made', 'tower'), true);
|
||||
assert.equal(acceptsOutlineClass('landmark', 'water', 'reservoir'), true);
|
||||
});
|
||||
|
||||
test('the first polygon-bearing result of the right class is taken (Mumbai)', () => {
|
||||
const node = row({
|
||||
category: 'place',
|
||||
type: 'city',
|
||||
name: 'Mumbai',
|
||||
geojson: { type: 'Point', coordinates: [72.8, 19.0] },
|
||||
});
|
||||
const wrong = row({ category: 'railway', type: 'station', name: 'Mumbai Central' });
|
||||
const boundary = row({
|
||||
category: 'boundary',
|
||||
type: 'administrative', addresstype: 'city',
|
||||
name: 'Mumbai',
|
||||
osm_id: 7888990,
|
||||
geojson: {
|
||||
type: 'MultiPolygon',
|
||||
coordinates: [square(72.8, 19.0, 0.01), square(72.9, 19.1, 0.2)],
|
||||
},
|
||||
});
|
||||
const { outline, skipped } = selectOutlineResult([node, wrong, boundary], {
|
||||
kind: 'city',
|
||||
query: 'Mumbai',
|
||||
});
|
||||
assert.equal(outline.osm, 'relation/7888990');
|
||||
assert.equal(outline.polygons.length, 2);
|
||||
assert.equal(outline.polygons[0][0][1][0], 72.9 + 0.2, 'largest part first');
|
||||
assert.deepEqual(skipped, ['place=city: no polygon', 'railway=station: wrong type']);
|
||||
});
|
||||
|
||||
test('a result whose name does not answer the ask is passed over', () => {
|
||||
const other = row({ category: 'leisure', type: 'park', name: 'Pease Park' });
|
||||
assert.equal(selectOutlineResult([other], { kind: 'landmark', query: 'Zilker Park' }).outline, null);
|
||||
const english = row({
|
||||
category: 'boundary',
|
||||
type: 'administrative', addresstype: 'city',
|
||||
name: '東京都',
|
||||
namedetails: { name: '東京都', 'name:en': 'Tokyo' },
|
||||
});
|
||||
assert.ok(selectOutlineResult([english], { kind: 'city', query: 'Tokyo' }).outline);
|
||||
const capitol = row({ category: 'office', type: 'government', name: 'Texas State Capitol' });
|
||||
assert.ok(selectOutlineResult([capitol], { kind: 'landmark', query: 'Texas Capitol' }).outline);
|
||||
});
|
||||
|
||||
test('outlines are held to a vertex cap', () => {
|
||||
const ring = [];
|
||||
for (let i = 0; i < 20_000; i++) {
|
||||
const a = (i / 20_000) * Math.PI * 2;
|
||||
ring.push([Math.cos(a) + Math.sin(a * 40) * 0.001, Math.sin(a)]);
|
||||
}
|
||||
ring.push([...ring[0]]);
|
||||
const capped = capOutlineVertices([[ring]], 6000);
|
||||
const total = capped.flat().reduce((n, r) => n + r.length, 0);
|
||||
assert.ok(total <= 6000, `${total} vertices`);
|
||||
const first = capped[0][0];
|
||||
assert.deepEqual(first[0], first.at(-1), 'rings stay closed');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- the route
|
||||
|
||||
function mount(options) {
|
||||
let handler;
|
||||
geocodeProxy(options).configureServer({
|
||||
middlewares: { use: (_route, fn) => (handler = fn) },
|
||||
});
|
||||
return (url, headers = {}) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const res = {
|
||||
writableEnded: false,
|
||||
on() {},
|
||||
writeHead(status, sent) {
|
||||
res.status = status;
|
||||
res.headers = sent;
|
||||
},
|
||||
end(body) {
|
||||
res.writableEnded = true;
|
||||
resolve({ status: res.status, headers: res.headers, body: JSON.parse(body) });
|
||||
},
|
||||
};
|
||||
Promise.resolve(
|
||||
handler(
|
||||
{ method: 'GET', url, headers, socket: { remoteAddress: `10.1.${Math.random()}` }, on() {} },
|
||||
res,
|
||||
() => resolve({ status: 'next' }),
|
||||
),
|
||||
).catch(reject);
|
||||
});
|
||||
}
|
||||
|
||||
test('the outline route answers OK with polygons and validates its input', async () => {
|
||||
const time = clock();
|
||||
const { calls, fetchImpl } = stubFetch(async () =>
|
||||
json([row({ category: 'boundary', type: 'administrative', addresstype: 'city', name: 'Paris' })]),
|
||||
);
|
||||
const request = mount({ gate: createNominatimGate({ settings: PUBLIC, fetchImpl, ...time }) });
|
||||
const ok = await request('/outline?q=Paris&kind=city&lat=48.85&lon=2.35');
|
||||
assert.equal(ok.status, 200);
|
||||
assert.equal(ok.body.status, 'OK');
|
||||
assert.equal(ok.body.outline.polygons[0][0].length, 5);
|
||||
assert.match(calls[0], /polygon_geojson=1/);
|
||||
assert.match(calls[0], /polygon_threshold=0\.0005/);
|
||||
assert.match(calls[0], /limit=3/);
|
||||
for (const bad of ['/outline?q=Paris', '/outline?kind=city', '/outline?q=Paris&kind=planet'])
|
||||
assert.equal((await request(bad)).status, 400, bad);
|
||||
const cross = await request('/outline?q=Paris&kind=city', { 'sec-fetch-site': 'cross-site' });
|
||||
assert.equal(cross.status, 403);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal((await request('/elsewhere')).status, 'next');
|
||||
});
|
||||
|
||||
test('the outline route reports disabled and capped as permanent, busy as retryable', async () => {
|
||||
const disabled = mount({
|
||||
gate: createNominatimGate({ settings: { endpoint: null, isPublic: false, dailyCap: 50 } }),
|
||||
});
|
||||
const off = await disabled('/outline?q=Paris&kind=city');
|
||||
assert.equal(off.status, 503);
|
||||
assert.equal(off.body.code, 'NOMINATIM_DISABLED');
|
||||
assert.equal(off.body.retryable, false);
|
||||
|
||||
const capped = mount({
|
||||
gate: createNominatimGate({ settings: { ...PUBLIC, dailyCap: 0 } }),
|
||||
});
|
||||
const cap = await capped('/outline?q=Paris&kind=city');
|
||||
assert.equal(cap.status, 429);
|
||||
assert.equal(cap.body.retryable, false);
|
||||
|
||||
const time = clock();
|
||||
const paused = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl: async () => new Response('', { status: 429, headers: { 'Retry-After': '90' } }),
|
||||
...time,
|
||||
});
|
||||
const busy = mount({ gate: paused });
|
||||
const refused = await busy('/outline?q=Paris&kind=city');
|
||||
assert.equal(refused.status, 429);
|
||||
assert.equal(refused.headers['Retry-After'], '90');
|
||||
});
|
||||
@@ -15,10 +15,15 @@ import {
|
||||
NOMINATIM_MAX_PENDING,
|
||||
} from '../../server/providers/regional/place.js';
|
||||
import { fetchRegionalJson } from '../../server/providers/regional/http.js';
|
||||
import { setSharedNominatimGate } from '../../server/providers/regional/nominatimGate.js';
|
||||
import { localProviderPlugins } from '../../server/providers/local.js';
|
||||
import { createPlaceSearch, createGoogleGeocoder } from '../search/index.js';
|
||||
import { createPhotonGeocoder } from '../keylessGeocoder.js';
|
||||
|
||||
// Each case starts with a fresh shared gate: a pause after one case's
|
||||
// simulated outage must not decide the next case.
|
||||
test.beforeEach(() => setSharedNominatimGate(null));
|
||||
|
||||
const HIT = [
|
||||
{
|
||||
lat: '43.1731',
|
||||
|
||||
@@ -31,6 +31,13 @@ test('explicit build inputs preserve browser-only defines, plugin order and loop
|
||||
'import.meta.env.GOOGLE_MAPS_API_KEY': '"browser-fixture"',
|
||||
'import.meta.env.CESIUM_ION_TOKEN': '"ion-fixture"',
|
||||
});
|
||||
assert.equal(
|
||||
createBrowserViteConfig({ photonUrl: '' }).define[
|
||||
'import.meta.env.PHOTON_URL'
|
||||
],
|
||||
'""',
|
||||
'an empty PHOTON_URL reaches the browser as "disabled"',
|
||||
);
|
||||
assert.equal(
|
||||
createBrowserViteConfig({ host: '0.0.0.0', port: '4800' }).server
|
||||
.allowedHosts,
|
||||
|
||||
Reference in new issue
Block a user