fix(outlines): harden shared cache and provider lifecycle

This commit is contained in:
manjunath22466 committed 2026-10-02 22:32:46 +05:30
1 parent 9559e7a282
commit 177b5951a0
27 files changed
+3363 -248

No files matched your search

+284 -12
View File
@@ -25,6 +25,7 @@ import {
} from '../../server/providers/regional/place.js';
const PUBLIC = { endpoint: PUBLIC_NOMINATIM_SEARCH, isPublic: true, dailyCap: 50 };
const LOCAL = { ...PUBLIC, isPublic: false };
const tempDir = (t) => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-'));
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
@@ -117,21 +118,103 @@ test('a sweep removes expired and corrupt entries, never ones it could not read'
assert.deepEqual([1, 2, 3].map((n) => fs.existsSync(file(n))), [true, false, false]);
});
test('processes sharing the state file add to one count', async (t) => {
const file = path.join(tempDir(t), 'state.json');
test('processes sharing state reserve public spacing and cap atomically', async (t) => {
const dir = tempDir(t);
const module = new URL('../../server/providers/regional/nominatimGate.js', import.meta.url).href;
const script = `
const run = (file, dailyCap) => {
const script = `
const { createGateStateStore } = await import(${JSON.stringify(module)});
const store = createGateStateStore({ file: ${JSON.stringify(file)} });
for (let i = 0; i < 25; i++) store.increment('2026-09-29');
console.log(JSON.stringify(store.reserve('2026-09-29', {
now: 1000,
dailyCap: ${dailyCap},
minSpacingMs: 1100,
latestStartAt: 10000,
})));
`;
const run = () =>
new Promise((resolve, reject) => {
const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: 'inherit' });
child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`exit ${code}`))));
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, ['--input-type=module', '-e', script]);
let stdout = '';
let stderr = '';
child.stdout.on('data', (chunk) => (stdout += chunk));
child.stderr.on('data', (chunk) => (stderr += chunk));
child.on('exit', (code) =>
code === 0
? resolve(JSON.parse(stdout))
: reject(new Error(`exit ${code}: ${stderr}`)),
);
});
await Promise.all([run(), run(), run()]);
assert.equal(createGateStateStore({ file }).count('2026-09-29'), 75);
};
const spacedFile = path.join(dir, 'spaced.json');
const reservations = await Promise.all([
run(spacedFile, 3),
run(spacedFile, 3),
run(spacedFile, 3),
]);
assert.deepEqual(
reservations.map((result) => result.status),
['reserved', 'reserved', 'reserved'],
);
assert.deepEqual(
reservations.map((result) => result.startAt).sort((a, b) => a - b),
[1000, 2100, 3200],
);
assert.equal(createGateStateStore({ file: spacedFile }).count('2026-09-29'), 3);
const cappedFile = path.join(dir, 'capped.json');
const capped = await Promise.all([run(cappedFile, 1), run(cappedFile, 1)]);
assert.deepEqual(
capped.map((result) => result.status).sort(),
['cap', 'reserved'],
);
assert.equal(createGateStateStore({ file: cappedFile }).count('2026-09-29'), 1);
});
test('independent public gates sharing state enforce spacing and cap', async (t) => {
const dir = tempDir(t);
const starts = [];
const fetchImpl = async () => {
starts.push(Date.now());
return json([]);
};
const file = path.join(dir, 'spacing.json');
const make = (options = {}) =>
createNominatimGate({
settings: PUBLIC,
fetchImpl,
usage: createGateStateStore({ file: options.file || file }),
minSpacingMs: options.minSpacingMs || 40,
});
await Promise.all([
createNominatimSearchProvider({ gate: make() })('one', null),
createNominatimSearchProvider({ gate: make() })('two', null),
]);
starts.sort((a, b) => a - b);
assert.ok(starts[1] - starts[0] >= 35, `${starts[1] - starts[0]}ms apart`);
const capFile = path.join(dir, 'cap.json');
let sent = 0;
const capGate = () =>
createNominatimGate({
settings: { ...PUBLIC, dailyCap: 1 },
fetchImpl: async () => {
sent += 1;
return json([]);
},
usage: createGateStateStore({ file: capFile }),
minSpacingMs: 1,
});
const results = await Promise.allSettled([
createNominatimSearchProvider({ gate: capGate() })('three', null),
createNominatimSearchProvider({ gate: capGate() })('four', null),
]);
assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1);
assert.equal(
results.find((result) => result.status === 'rejected').reason.code,
'NOMINATIM_DAILY_CAP',
);
assert.equal(sent, 1);
});
test('a persistence failure is reported and never lowers the count', (t) => {
@@ -151,6 +234,49 @@ test('a persistence failure is reported and never lowers the count', (t) => {
assert.equal(errors.length, 1, 'reported once, not per request');
});
test('parseable but invalid public usage state fails closed', async (t) => {
const dir = tempDir(t);
const file = path.join(dir, 'state.json');
fs.writeFileSync(
file,
JSON.stringify({
day: '2026-09-30',
count: '50',
pausedUntil: '9999999999999',
}),
);
let sent = 0;
const gate = createNominatimGate({
settings: PUBLIC,
fetchImpl: async () => {
sent += 1;
return json([]);
},
usage: createGateStateStore({ file, onError: () => {} }),
...quick,
});
await assert.rejects(
gate.requestJson((endpoint) => `${endpoint}?q=x`),
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
);
assert.equal(sent, 0);
fs.writeFileSync(
file,
JSON.stringify({ day: '2026-09-30', count: 1, pausedUntil: 0 }),
);
const migrated = createGateStateStore({ file });
assert.equal(
migrated.reserve('2026-09-30', {
now: 1000,
dailyCap: 50,
minSpacingMs: 1100,
}).status,
'reserved',
'the original state format without nextStartAt remains supported',
);
});
test('a long Retry-After is honoured in full and survives a restart', async (t) => {
const file = path.join(tempDir(t), 'state.json');
let clock = Date.UTC(2026, 8, 29, 12);
@@ -189,6 +315,35 @@ test('a long Retry-After is honoured in full and survives a restart', async (t)
assert.equal(calls.length, 1);
});
test('an operator refusal pauses only the endpoint that refused', async () => {
let active = {
endpoint: 'https://a.example.test/search',
isPublic: false,
dailyCap: 50,
};
const calls = [];
const gate = createNominatimGate({
settings: () => active,
fetchImpl: async (url) => {
calls.push(new URL(url).host);
return calls.length === 1
? new Response('', {
status: 429,
headers: { 'Retry-After': '60' },
})
: json([]);
},
...quick,
});
const request = () => gate.requestJson((endpoint) => `${endpoint}?q=x`);
await assert.rejects(request(), { code: 'NOMINATIM_REFUSED' });
active = { ...active, endpoint: 'https://b.example.test/search' };
assert.deepEqual(await request(), []);
active = { ...active, endpoint: 'https://a.example.test/search' };
await assert.rejects(request(), { code: 'NOMINATIM_PAUSED' });
assert.deepEqual(calls, ['a.example.test', 'b.example.test']);
});
test('a stalled body times out instead of holding the shared queue', async () => {
let cancelled = 0;
const stalled = (signal) => {
@@ -217,7 +372,7 @@ test('a stalled body times out instead of holding the shared queue', async () =>
test('cache keys are request-equivalent: a miss for "Paris???" never answers "Paris"', async () => {
const calls = [];
const gate = createNominatimGate({
settings: PUBLIC,
settings: LOCAL,
fetchImpl: async (url) => {
calls.push(new URL(url).searchParams.get('q'));
return json(calls.length === 1 ? [] : [{ lat: '48.85', lon: '2.35', display_name: 'Paris, France', name: 'Paris', addresstype: 'city' }]);
@@ -236,7 +391,7 @@ test('a malformed answer is a failure, not a cached "no result"', async () => {
let shape = { error: 'Unexpected' };
let calls = 0;
const gate = createNominatimGate({
settings: PUBLIC,
settings: LOCAL,
fetchImpl: async () => {
calls += 1;
return json(shape);
@@ -301,9 +456,29 @@ test('administrative boundaries must be at the asked level', () => {
const state = boundary({ name: 'Texas', addresstype: 'state', place_rank: 8 });
assert.equal(selectOutlineResult([state], { kind: 'city', query: 'Texas' }).outline, null);
assert.equal(selectOutlineResult([city], { kind: 'admin', query: 'Austin' }).outline, null);
assert.equal(
selectOutlineResult(
[boundary({ name: 'Travis', addresstype: 'county', place_rank: 12 })],
{ kind: 'city', query: 'Travis' },
).outline,
null,
);
assert.equal(
selectOutlineResult(
[boundary({ name: 'Austin', addresstype: 'city', place_rank: 12 })],
{ kind: 'admin', query: 'Austin' },
).outline,
null,
);
assert.equal(selectOutlineResult([state], { kind: 'neighborhood', query: 'Texas' }).outline, null);
assert.ok(selectOutlineResult([county], { kind: 'admin', query: 'Travis County' }).outline);
assert.ok(selectOutlineResult([boundary({ name: 'Somewhere', place_rank: 16 })], { kind: 'city', query: 'Somewhere' }).outline);
assert.ok(
selectOutlineResult(
[boundary({ name: 'Somewhere', addresstype: 'locality', place_rank: 16 })],
{ kind: 'city', query: 'Somewhere' },
).outline,
);
assert.equal(
selectOutlineResult([boundary({ name: 'Somewhere' })], { kind: 'city', query: 'Somewhere' }).outline,
null,
@@ -317,6 +492,103 @@ test('administrative boundaries must be at the asked level', () => {
assert.equal(picked.outline.osm, 'relation/9');
});
test('admin identity comes only from unambiguous evidence on the selected row', () => {
for (const [name, addresstype, adminLevel] of [
['Nepal', 'country', 'country'],
['Bagmati Province', 'province', 'admin1'],
['Province of Bagmati', 'province', 'admin1'],
['Texas', 'state', 'admin1'],
['Travis County', 'county', 'admin2'],
['County of Travis', 'county', 'admin2'],
['Kathmandu District', 'state_district', 'admin2'],
]) {
const { outline } = selectOutlineResult(
[boundary({ name, addresstype })],
{ kind: 'admin', query: name },
);
assert.equal(outline.adminArea, name);
assert.equal(outline.adminLevel, adminLevel);
}
for (const row of [
boundary({ name: 'Northern Region', addresstype: 'region', place_rank: 8 }),
boundary({ name: 'Central District', addresstype: 'district', place_rank: 10 }),
boundary({ name: 'Mystery Province', place_rank: 8 }),
boundary({ name: 'Example State', addresstype: 'county' }),
boundary({ name: 'County of Example', addresstype: 'state' }),
boundary({ name: 'Province of Example', addresstype: 'county' }),
boundary({ name: 'Example County', addresstype: 'state' }),
boundary({ name: 'Example Province', addresstype: 'county' }),
boundary({ name: 'County of Example Province', addresstype: 'county' }),
]) {
const { outline } = selectOutlineResult([row], {
kind: 'admin',
query: row.name,
});
assert.ok(outline, row.name);
assert.equal(outline.adminArea, undefined, row.name);
assert.equal(outline.adminLevel, undefined, row.name);
}
const retired = selectOutlineResult(
[boundary({ name: 'Bagmati Zone', addresstype: 'state' })],
{ kind: 'admin', query: 'Bagmati Zone' },
).outline;
assert.equal(retired.adminArea, 'Bagmati Zone');
assert.equal(retired.adminLevel, undefined);
const retiredPrefix = selectOutlineResult(
[boundary({ name: 'Zone of Bagmati', addresstype: 'state' })],
{ kind: 'admin', query: 'Zone of Bagmati' },
).outline;
assert.equal(retiredPrefix.adminArea, 'Zone of Bagmati');
assert.equal(retiredPrefix.adminLevel, undefined);
});
test('admin identity validates the full name before display truncation', () => {
const longValidName = `Province of ${'Example '.repeat(24)}`.trim();
const valid = selectOutlineResult(
[boundary({ name: longValidName, addresstype: 'province' })],
{ kind: 'admin', query: longValidName },
).outline;
assert.equal(valid.adminArea, longValidName.slice(0, 160));
assert.equal(valid.adminLevel, 'admin1');
const longContradictoryName = `${'Very Long Administrative Name '.repeat(7)}County`;
assert.ok(longContradictoryName.length > 160);
const contradictory = selectOutlineResult(
[boundary({ name: longContradictoryName, addresstype: 'state' })],
{ kind: 'admin', query: longContradictoryName },
).outline;
assert.ok(contradictory);
assert.equal(contradictory.adminArea, undefined);
assert.equal(contradictory.adminLevel, undefined);
});
test('non-administrative outline kinds never manufacture admin identity', () => {
const city = selectOutlineResult(
[boundary({ name: 'County of Austin', addresstype: 'city', place_rank: 16 })],
{ kind: 'city', query: 'County of Austin' },
).outline;
const park = selectOutlineResult(
[
{
category: 'leisure',
type: 'park',
name: 'Zilker Park',
lat: '1',
lon: '1',
geojson: { type: 'Polygon', coordinates: square(1, 1, 0.01) },
},
],
{ kind: 'landmark', query: 'Zilker Park' },
).outline;
for (const outline of [city, park]) {
assert.equal(outline.adminArea, undefined);
assert.equal(outline.adminLevel, undefined);
}
});
test('building asks need building evidence', () => {
const park = { category: 'tourism', type: 'theme_park', name: 'Big Top', lat: '1', lon: '1', geojson: { type: 'Polygon', coordinates: square(1, 1, 0.0001) } };
const camp = { ...park, type: 'camp_site' };