mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 08:12:24 +00:00
fix(outlines): harden shared cache and provider lifecycle
This commit is contained in:
1 parent
9559e7a282
commit
177b5951a0
27 files changed
+3363
-248
No files matched your search
@@ -25,6 +25,7 @@ import {
|
||||
} from '../../server/providers/regional/place.js';
|
||||
|
||||
const PUBLIC = { endpoint: PUBLIC_NOMINATIM_SEARCH, isPublic: true, dailyCap: 50 };
|
||||
const LOCAL = { ...PUBLIC, isPublic: false };
|
||||
const tempDir = (t) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-nominatim-'));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
@@ -117,21 +118,103 @@ test('a sweep removes expired and corrupt entries, never ones it could not read'
|
||||
assert.deepEqual([1, 2, 3].map((n) => fs.existsSync(file(n))), [true, false, false]);
|
||||
});
|
||||
|
||||
test('processes sharing the state file add to one count', async (t) => {
|
||||
const file = path.join(tempDir(t), 'state.json');
|
||||
test('processes sharing state reserve public spacing and cap atomically', async (t) => {
|
||||
const dir = tempDir(t);
|
||||
const module = new URL('../../server/providers/regional/nominatimGate.js', import.meta.url).href;
|
||||
const script = `
|
||||
const run = (file, dailyCap) => {
|
||||
const script = `
|
||||
const { createGateStateStore } = await import(${JSON.stringify(module)});
|
||||
const store = createGateStateStore({ file: ${JSON.stringify(file)} });
|
||||
for (let i = 0; i < 25; i++) store.increment('2026-09-29');
|
||||
console.log(JSON.stringify(store.reserve('2026-09-29', {
|
||||
now: 1000,
|
||||
dailyCap: ${dailyCap},
|
||||
minSpacingMs: 1100,
|
||||
latestStartAt: 10000,
|
||||
})));
|
||||
`;
|
||||
const run = () =>
|
||||
new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: 'inherit' });
|
||||
child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`exit ${code}`))));
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ['--input-type=module', '-e', script]);
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', (chunk) => (stdout += chunk));
|
||||
child.stderr.on('data', (chunk) => (stderr += chunk));
|
||||
child.on('exit', (code) =>
|
||||
code === 0
|
||||
? resolve(JSON.parse(stdout))
|
||||
: reject(new Error(`exit ${code}: ${stderr}`)),
|
||||
);
|
||||
});
|
||||
await Promise.all([run(), run(), run()]);
|
||||
assert.equal(createGateStateStore({ file }).count('2026-09-29'), 75);
|
||||
};
|
||||
|
||||
const spacedFile = path.join(dir, 'spaced.json');
|
||||
const reservations = await Promise.all([
|
||||
run(spacedFile, 3),
|
||||
run(spacedFile, 3),
|
||||
run(spacedFile, 3),
|
||||
]);
|
||||
assert.deepEqual(
|
||||
reservations.map((result) => result.status),
|
||||
['reserved', 'reserved', 'reserved'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
reservations.map((result) => result.startAt).sort((a, b) => a - b),
|
||||
[1000, 2100, 3200],
|
||||
);
|
||||
assert.equal(createGateStateStore({ file: spacedFile }).count('2026-09-29'), 3);
|
||||
|
||||
const cappedFile = path.join(dir, 'capped.json');
|
||||
const capped = await Promise.all([run(cappedFile, 1), run(cappedFile, 1)]);
|
||||
assert.deepEqual(
|
||||
capped.map((result) => result.status).sort(),
|
||||
['cap', 'reserved'],
|
||||
);
|
||||
assert.equal(createGateStateStore({ file: cappedFile }).count('2026-09-29'), 1);
|
||||
});
|
||||
|
||||
test('independent public gates sharing state enforce spacing and cap', async (t) => {
|
||||
const dir = tempDir(t);
|
||||
const starts = [];
|
||||
const fetchImpl = async () => {
|
||||
starts.push(Date.now());
|
||||
return json([]);
|
||||
};
|
||||
const file = path.join(dir, 'spacing.json');
|
||||
const make = (options = {}) =>
|
||||
createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl,
|
||||
usage: createGateStateStore({ file: options.file || file }),
|
||||
minSpacingMs: options.minSpacingMs || 40,
|
||||
});
|
||||
await Promise.all([
|
||||
createNominatimSearchProvider({ gate: make() })('one', null),
|
||||
createNominatimSearchProvider({ gate: make() })('two', null),
|
||||
]);
|
||||
starts.sort((a, b) => a - b);
|
||||
assert.ok(starts[1] - starts[0] >= 35, `${starts[1] - starts[0]}ms apart`);
|
||||
|
||||
const capFile = path.join(dir, 'cap.json');
|
||||
let sent = 0;
|
||||
const capGate = () =>
|
||||
createNominatimGate({
|
||||
settings: { ...PUBLIC, dailyCap: 1 },
|
||||
fetchImpl: async () => {
|
||||
sent += 1;
|
||||
return json([]);
|
||||
},
|
||||
usage: createGateStateStore({ file: capFile }),
|
||||
minSpacingMs: 1,
|
||||
});
|
||||
const results = await Promise.allSettled([
|
||||
createNominatimSearchProvider({ gate: capGate() })('three', null),
|
||||
createNominatimSearchProvider({ gate: capGate() })('four', null),
|
||||
]);
|
||||
assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1);
|
||||
assert.equal(
|
||||
results.find((result) => result.status === 'rejected').reason.code,
|
||||
'NOMINATIM_DAILY_CAP',
|
||||
);
|
||||
assert.equal(sent, 1);
|
||||
});
|
||||
|
||||
test('a persistence failure is reported and never lowers the count', (t) => {
|
||||
@@ -151,6 +234,49 @@ test('a persistence failure is reported and never lowers the count', (t) => {
|
||||
assert.equal(errors.length, 1, 'reported once, not per request');
|
||||
});
|
||||
|
||||
test('parseable but invalid public usage state fails closed', async (t) => {
|
||||
const dir = tempDir(t);
|
||||
const file = path.join(dir, 'state.json');
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
JSON.stringify({
|
||||
day: '2026-09-30',
|
||||
count: '50',
|
||||
pausedUntil: '9999999999999',
|
||||
}),
|
||||
);
|
||||
let sent = 0;
|
||||
const gate = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
fetchImpl: async () => {
|
||||
sent += 1;
|
||||
return json([]);
|
||||
},
|
||||
usage: createGateStateStore({ file, onError: () => {} }),
|
||||
...quick,
|
||||
});
|
||||
await assert.rejects(
|
||||
gate.requestJson((endpoint) => `${endpoint}?q=x`),
|
||||
{ code: 'NOMINATIM_STATE_UNAVAILABLE' },
|
||||
);
|
||||
assert.equal(sent, 0);
|
||||
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
JSON.stringify({ day: '2026-09-30', count: 1, pausedUntil: 0 }),
|
||||
);
|
||||
const migrated = createGateStateStore({ file });
|
||||
assert.equal(
|
||||
migrated.reserve('2026-09-30', {
|
||||
now: 1000,
|
||||
dailyCap: 50,
|
||||
minSpacingMs: 1100,
|
||||
}).status,
|
||||
'reserved',
|
||||
'the original state format without nextStartAt remains supported',
|
||||
);
|
||||
});
|
||||
|
||||
test('a long Retry-After is honoured in full and survives a restart', async (t) => {
|
||||
const file = path.join(tempDir(t), 'state.json');
|
||||
let clock = Date.UTC(2026, 8, 29, 12);
|
||||
@@ -189,6 +315,35 @@ test('a long Retry-After is honoured in full and survives a restart', async (t)
|
||||
assert.equal(calls.length, 1);
|
||||
});
|
||||
|
||||
test('an operator refusal pauses only the endpoint that refused', async () => {
|
||||
let active = {
|
||||
endpoint: 'https://a.example.test/search',
|
||||
isPublic: false,
|
||||
dailyCap: 50,
|
||||
};
|
||||
const calls = [];
|
||||
const gate = createNominatimGate({
|
||||
settings: () => active,
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(new URL(url).host);
|
||||
return calls.length === 1
|
||||
? new Response('', {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '60' },
|
||||
})
|
||||
: json([]);
|
||||
},
|
||||
...quick,
|
||||
});
|
||||
const request = () => gate.requestJson((endpoint) => `${endpoint}?q=x`);
|
||||
await assert.rejects(request(), { code: 'NOMINATIM_REFUSED' });
|
||||
active = { ...active, endpoint: 'https://b.example.test/search' };
|
||||
assert.deepEqual(await request(), []);
|
||||
active = { ...active, endpoint: 'https://a.example.test/search' };
|
||||
await assert.rejects(request(), { code: 'NOMINATIM_PAUSED' });
|
||||
assert.deepEqual(calls, ['a.example.test', 'b.example.test']);
|
||||
});
|
||||
|
||||
test('a stalled body times out instead of holding the shared queue', async () => {
|
||||
let cancelled = 0;
|
||||
const stalled = (signal) => {
|
||||
@@ -217,7 +372,7 @@ test('a stalled body times out instead of holding the shared queue', async () =>
|
||||
test('cache keys are request-equivalent: a miss for "Paris???" never answers "Paris"', async () => {
|
||||
const calls = [];
|
||||
const gate = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
settings: LOCAL,
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(new URL(url).searchParams.get('q'));
|
||||
return json(calls.length === 1 ? [] : [{ lat: '48.85', lon: '2.35', display_name: 'Paris, France', name: 'Paris', addresstype: 'city' }]);
|
||||
@@ -236,7 +391,7 @@ test('a malformed answer is a failure, not a cached "no result"', async () => {
|
||||
let shape = { error: 'Unexpected' };
|
||||
let calls = 0;
|
||||
const gate = createNominatimGate({
|
||||
settings: PUBLIC,
|
||||
settings: LOCAL,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return json(shape);
|
||||
@@ -301,9 +456,29 @@ test('administrative boundaries must be at the asked level', () => {
|
||||
const state = boundary({ name: 'Texas', addresstype: 'state', place_rank: 8 });
|
||||
assert.equal(selectOutlineResult([state], { kind: 'city', query: 'Texas' }).outline, null);
|
||||
assert.equal(selectOutlineResult([city], { kind: 'admin', query: 'Austin' }).outline, null);
|
||||
assert.equal(
|
||||
selectOutlineResult(
|
||||
[boundary({ name: 'Travis', addresstype: 'county', place_rank: 12 })],
|
||||
{ kind: 'city', query: 'Travis' },
|
||||
).outline,
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
selectOutlineResult(
|
||||
[boundary({ name: 'Austin', addresstype: 'city', place_rank: 12 })],
|
||||
{ kind: 'admin', query: 'Austin' },
|
||||
).outline,
|
||||
null,
|
||||
);
|
||||
assert.equal(selectOutlineResult([state], { kind: 'neighborhood', query: 'Texas' }).outline, null);
|
||||
assert.ok(selectOutlineResult([county], { kind: 'admin', query: 'Travis County' }).outline);
|
||||
assert.ok(selectOutlineResult([boundary({ name: 'Somewhere', place_rank: 16 })], { kind: 'city', query: 'Somewhere' }).outline);
|
||||
assert.ok(
|
||||
selectOutlineResult(
|
||||
[boundary({ name: 'Somewhere', addresstype: 'locality', place_rank: 16 })],
|
||||
{ kind: 'city', query: 'Somewhere' },
|
||||
).outline,
|
||||
);
|
||||
assert.equal(
|
||||
selectOutlineResult([boundary({ name: 'Somewhere' })], { kind: 'city', query: 'Somewhere' }).outline,
|
||||
null,
|
||||
@@ -317,6 +492,103 @@ test('administrative boundaries must be at the asked level', () => {
|
||||
assert.equal(picked.outline.osm, 'relation/9');
|
||||
});
|
||||
|
||||
test('admin identity comes only from unambiguous evidence on the selected row', () => {
|
||||
for (const [name, addresstype, adminLevel] of [
|
||||
['Nepal', 'country', 'country'],
|
||||
['Bagmati Province', 'province', 'admin1'],
|
||||
['Province of Bagmati', 'province', 'admin1'],
|
||||
['Texas', 'state', 'admin1'],
|
||||
['Travis County', 'county', 'admin2'],
|
||||
['County of Travis', 'county', 'admin2'],
|
||||
['Kathmandu District', 'state_district', 'admin2'],
|
||||
]) {
|
||||
const { outline } = selectOutlineResult(
|
||||
[boundary({ name, addresstype })],
|
||||
{ kind: 'admin', query: name },
|
||||
);
|
||||
assert.equal(outline.adminArea, name);
|
||||
assert.equal(outline.adminLevel, adminLevel);
|
||||
}
|
||||
|
||||
for (const row of [
|
||||
boundary({ name: 'Northern Region', addresstype: 'region', place_rank: 8 }),
|
||||
boundary({ name: 'Central District', addresstype: 'district', place_rank: 10 }),
|
||||
boundary({ name: 'Mystery Province', place_rank: 8 }),
|
||||
boundary({ name: 'Example State', addresstype: 'county' }),
|
||||
boundary({ name: 'County of Example', addresstype: 'state' }),
|
||||
boundary({ name: 'Province of Example', addresstype: 'county' }),
|
||||
boundary({ name: 'Example County', addresstype: 'state' }),
|
||||
boundary({ name: 'Example Province', addresstype: 'county' }),
|
||||
boundary({ name: 'County of Example Province', addresstype: 'county' }),
|
||||
]) {
|
||||
const { outline } = selectOutlineResult([row], {
|
||||
kind: 'admin',
|
||||
query: row.name,
|
||||
});
|
||||
assert.ok(outline, row.name);
|
||||
assert.equal(outline.adminArea, undefined, row.name);
|
||||
assert.equal(outline.adminLevel, undefined, row.name);
|
||||
}
|
||||
|
||||
const retired = selectOutlineResult(
|
||||
[boundary({ name: 'Bagmati Zone', addresstype: 'state' })],
|
||||
{ kind: 'admin', query: 'Bagmati Zone' },
|
||||
).outline;
|
||||
assert.equal(retired.adminArea, 'Bagmati Zone');
|
||||
assert.equal(retired.adminLevel, undefined);
|
||||
|
||||
const retiredPrefix = selectOutlineResult(
|
||||
[boundary({ name: 'Zone of Bagmati', addresstype: 'state' })],
|
||||
{ kind: 'admin', query: 'Zone of Bagmati' },
|
||||
).outline;
|
||||
assert.equal(retiredPrefix.adminArea, 'Zone of Bagmati');
|
||||
assert.equal(retiredPrefix.adminLevel, undefined);
|
||||
});
|
||||
|
||||
test('admin identity validates the full name before display truncation', () => {
|
||||
const longValidName = `Province of ${'Example '.repeat(24)}`.trim();
|
||||
const valid = selectOutlineResult(
|
||||
[boundary({ name: longValidName, addresstype: 'province' })],
|
||||
{ kind: 'admin', query: longValidName },
|
||||
).outline;
|
||||
assert.equal(valid.adminArea, longValidName.slice(0, 160));
|
||||
assert.equal(valid.adminLevel, 'admin1');
|
||||
|
||||
const longContradictoryName = `${'Very Long Administrative Name '.repeat(7)}County`;
|
||||
assert.ok(longContradictoryName.length > 160);
|
||||
const contradictory = selectOutlineResult(
|
||||
[boundary({ name: longContradictoryName, addresstype: 'state' })],
|
||||
{ kind: 'admin', query: longContradictoryName },
|
||||
).outline;
|
||||
assert.ok(contradictory);
|
||||
assert.equal(contradictory.adminArea, undefined);
|
||||
assert.equal(contradictory.adminLevel, undefined);
|
||||
});
|
||||
|
||||
test('non-administrative outline kinds never manufacture admin identity', () => {
|
||||
const city = selectOutlineResult(
|
||||
[boundary({ name: 'County of Austin', addresstype: 'city', place_rank: 16 })],
|
||||
{ kind: 'city', query: 'County of Austin' },
|
||||
).outline;
|
||||
const park = selectOutlineResult(
|
||||
[
|
||||
{
|
||||
category: 'leisure',
|
||||
type: 'park',
|
||||
name: 'Zilker Park',
|
||||
lat: '1',
|
||||
lon: '1',
|
||||
geojson: { type: 'Polygon', coordinates: square(1, 1, 0.01) },
|
||||
},
|
||||
],
|
||||
{ kind: 'landmark', query: 'Zilker Park' },
|
||||
).outline;
|
||||
for (const outline of [city, park]) {
|
||||
assert.equal(outline.adminArea, undefined);
|
||||
assert.equal(outline.adminLevel, undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test('building asks need building evidence', () => {
|
||||
const park = { category: 'tourism', type: 'theme_park', name: 'Big Top', lat: '1', lon: '1', geojson: { type: 'Polygon', coordinates: square(1, 1, 0.0001) } };
|
||||
const camp = { ...park, type: 'camp_site' };
|
||||
|
||||
Reference in new issue
Block a user