diff --git a/changelog.d/1838-nearby-places-namesake.md b/changelog.d/1838-nearby-places-namesake.md new file mode 100644 index 0000000000..ffda7e347e --- /dev/null +++ b/changelog.d/1838-nearby-places-namesake.md @@ -0,0 +1,4 @@ +- "Villages near" a name picks the place of that name in view over a far + bigger namesake, a place lookup reads the detail tiles within 500 km at + most, and rebuilding the places pack never leaves it half removed + (manjunath22466, samehkhamis, #1838). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 6cd948e474..809ceb47e0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -966,7 +966,8 @@ the fleet set loaded at startup, so opening a card fetches no model. country and Wikidata id. `src/data/wikidataCities.js` reads it on the first lookup and answers `nearestCity` (the largest city within 12 km, else the nearest within 60 km), `citiesAround`, `findPlace` ("Paris, France", or the -nearest namesake within 150 km of the view, else the largest) and +nearest namesake within 150 km of the view, else the largest; asked to, it +counts the smaller places of the detail tiles near the view too) and `countryName`. Names match without case or accents ("lodz" finds Łódź). The app never calls Wikidata; `scripts/build-wikidata-cities.mjs` rebuilds the pack and fails if well-known cities are missing or a known grouping such as @@ -8257,7 +8258,12 @@ clears them but the browser's own site data. Setting `PHOTON_URL` empty turns the suggestions off along with it. Small places come from the bundled Wikidata packs and are never looked up online; a small place with the name of a city within 3 km in the same country is that city, and only -the city is listed. At most 64 detail tiles are kept in memory, the least +the city is listed. A place named in "villages near …" is the nearest of +that name within 150 km of the view, small places included, so a village in +view comes before a bigger namesake far away; with none near, the largest +city of the name. Small places are looked for within 500 km at most, so a +lookup asked for a wider radius reads only the detail tiles around its +centre (the search box asks for 150 km at most). At most 64 detail tiles are kept in memory, the least recently used dropped first. ### Place-search providers diff --git a/scripts/build-wikidata-cities.mjs b/scripts/build-wikidata-cities.mjs index a276062831..d2016fa3d7 100644 --- a/scripts/build-wikidata-cities.mjs +++ b/scripts/build-wikidata-cities.mjs @@ -39,6 +39,8 @@ import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { + keepBuiltIfUnchanged, + keepPackBuiltIfUnchanged, placeTileFiles, withoutCities, withoutNearTwins, @@ -68,6 +70,8 @@ const outDir = path.join( : 'src/data/local_data/wikidata_cities', ); +/** A file's text, or undefined when there is none yet. */ +const readIfThere = (file) => readFile(file, 'utf8').catch(() => undefined); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1); @@ -499,17 +503,30 @@ if (tilesMode) { count: tileRows.length, maxPopulation, }); - // Tiles from an earlier build that this one has no rows for go. + // The new files are all written before tiles from an earlier build that + // this one has no rows for go, so a build that stops part way never leaves + // an index naming a removed tile. + const earlier = new Map(); + for (const name of await readdir(outDir)) + if (/^-?\d+_-?\d+\.json$|^index\.json$|^files\.js$/.test(name)) + earlier.set(name, await readFile(path.join(outDir, name), 'utf8')); + keepPackBuiltIfUnchanged(files, earlier); + for (const [name, text] of files) + await writeFile(path.join(outDir, name), text); for (const name of await readdir(outDir)) if (/^-?\d+_-?\d+\.json$/.test(name) && !files.has(name)) await rm(path.join(outDir, name)); - for (const [name, text] of files) - await writeFile(path.join(outDir, name), text); console.log( `Wrote ${tileRows.length} places in ${files.size - 2} tiles to ${path.relative(root, outDir)}`, ); process.exit(0); } const file = path.join(outDir, 'cities.json'); -await writeFile(file, `${JSON.stringify({ meta, rows })}\n`); +await writeFile( + file, + keepBuiltIfUnchanged( + `${JSON.stringify({ meta, rows })}\n`, + await readIfThere(file), + ), +); console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`); diff --git a/scripts/wikidataPlaceTiles.mjs b/scripts/wikidataPlaceTiles.mjs index 15f0c89e3a..d3396937f6 100644 --- a/scripts/wikidataPlaceTiles.mjs +++ b/scripts/wikidataPlaceTiles.mjs @@ -89,6 +89,51 @@ export function placeTileFiles(rows, meta) { return files; } +/** + * A pack file's new text, keeping the earlier build's `meta.built` when + * nothing else in the file changed, so rebuilding unchanged data leaves the + * file as it was. A file that does not hold all of its pack's data (the + * places index) is passed through this only when the rest of the pack is + * unchanged too. + * @param {string} text The new file, JSON with a `meta.built` stamp. + * @param {string | undefined} earlier The file as it is now, if there is one. + */ +export function keepBuiltIfUnchanged(text, earlier) { + if (!earlier) return text; + let before; + try { + before = JSON.parse(earlier); + } catch { + return text; + } + const after = JSON.parse(text); + if (typeof before?.meta?.built !== 'string' || !after.meta) return text; + const unstamped = (value) => + JSON.stringify({ ...value, meta: { ...value.meta, built: null } }); + if (unstamped(before) !== unstamped(after)) return text; + return `${JSON.stringify({ ...after, meta: { ...after.meta, built: before.meta.built } })}\n`; +} + +/** + * Keeps the earlier build's `meta.built` in the places pack's index when the + * rebuild changed nothing: every tile and `files.js` the same text, no tile + * of the earlier build left out, and nothing else in the index changed. The + * index holds only row counts, so it alone cannot tell. + * @param {Map} files The new pack, as placeTileFiles gives it; its index is replaced in place. + * @param {Map} earlier The pack's files on disk now, by name. + */ +export function keepPackBuiltIfUnchanged(files, earlier) { + for (const name of earlier.keys()) + if (name !== 'index.json' && !files.has(name)) return files; + for (const [name, text] of files) + if (name !== 'index.json' && earlier.get(name) !== text) return files; + files.set( + 'index.json', + keepBuiltIfUnchanged(files.get('index.json'), earlier.get('index.json')), + ); + return files; +} + const kmBetween = (a, b) => greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] }); diff --git a/src/data/local_data/wikidata_cities/README.md b/src/data/local_data/wikidata_cities/README.md index 8c2abf3818..026f412988 100644 --- a/src/data/local_data/wikidata_cities/README.md +++ b/src/data/local_data/wikidata_cities/README.md @@ -36,7 +36,13 @@ places without an English label. The build runs one query at a time with pauses and honours the service's Retry-After; it takes about ten minutes. It fails if a short list of well-known cities is missing from the result, or a known grouping gets in. +`meta.built` keeps its earlier date when the rows are unchanged. **Limits:** populations are whatever Wikidata states and differ in date and definition between places (city proper, municipality, urban district). Some districts and suburbs appear as their own rows. + +**Finding a place by name:** `findPlace` takes the nearest place of the name +within 150 km of the view, else the largest city of the name. When asked, +the smaller places of `../wikidata_places` near the view count too, so a +village in view comes before a bigger namesake far away. diff --git a/src/data/local_data/wikidata_places/README.md b/src/data/local_data/wikidata_places/README.md index 2831f89a56..efad9ea1b3 100644 --- a/src/data/local_data/wikidata_places/README.md +++ b/src/data/local_data/wikidata_places/README.md @@ -1,8 +1,11 @@ # Wikidata places detail tiles Smaller towns and villages, below the cut-off of the cities pack -(`../wikidata_cities`), for "places nearby" around one picked or named spot. Read by `placesNear` in `src/data/wikidataCities.js`. The app reads -these files; it does not call Wikidata or any place service for them. +(`../wikidata_cities`), for "places nearby" around one picked or named spot. Read by `placesNear` in `src/data/wikidataCities.js`, and by `findPlace` +when asked to look for a small place by name near the view: a place of that +name in view, or close to it, comes before a bigger city of the name far +away. The app reads these files; it does not call Wikidata or any place +service for them. | File | Contents | |------|----------| @@ -17,7 +20,12 @@ entry in the index. **Loading:** nothing here loads with the app. A lookup reads `files.js` and the index once, then only the tiles its search box touches (one for a short radius well inside a tile, more near tile edges and for wider searches), -once each; at most 64 tiles are kept in memory, the least recently used +once each. A lookup reaches 500 km at most (`PLACE_TILE_KM`; the search box +asks for 150 km at most), so one asked for a wider radius reads only the +tiles around its centre: at most one +tile past the centre's own on each side, a few more columns toward the +poles, and never all 732. Cities still count to the full radius. At most +64 tiles are kept in memory, the least recently used dropped first. Rows whose name has no letter (one row, `380` in Nicaragua) are dropped when a tile is read. @@ -39,7 +47,10 @@ names from Wikidata (CC0)" as a courtesy once a name from the pack is shown. `node scripts/build-wikidata-cities.mjs --tiles --min-population 1000 --max-population 15000`. It writes only to this folder (tiles, `index.json`, `files.js`, through `scripts/wikidataPlaceTiles.mjs`) and leaves `../wikidata_cities` alone. It -runs one query at a time with pauses and takes a long while. +runs one query at a time with pauses and takes a long while. The new files +are all written before tiles that the new build has no rows for are +removed, and `meta.built` keeps its earlier date when no tile and +nothing else in `index.json` changed. **Reproducibility:** the step from rows to files is checked offline: `src/tooling/wikidataPlaceTiles.test.mjs` sorts the shipped rows as the diff --git a/src/data/wikidataCities.js b/src/data/wikidataCities.js index 70dcb3e67a..e106d2bc4c 100644 --- a/src/data/wikidataCities.js +++ b/src/data/wikidataCities.js @@ -217,6 +217,15 @@ const TILE_DEG = 5; * far more than one lookup touches, and bounds what a long session keeps. */ const MAX_TILES = 64; +/** + * How far out small places are looked for, in km. 500 km is under the 556 km + * that 5 degrees of latitude span, so a lookup's tile box reaches at most one + * tile past the point's own on each side: 3 rows, and 3 columns at the + * equator (more toward the poles, where tiles narrow). A lookup asked for a + * wider radius, up to the whole globe, still reads only those tiles; cities + * count to its full radius. + */ +export const PLACE_TILE_KM = 500; const NAMED = /\p{L}/u; const tiles = new Map(); let loadTileIndex; @@ -248,9 +257,11 @@ async function readPlaceTiles(latitude, longitude, radiusKm, fetchImpl) { readPlacePack(INDEX_URL(), fetchImpl), ); const index = await loadTileIndex(); - const keys = tileKeysAround(latitude, longitude, radiusKm).filter( - (key) => index.tiles[key], - ); + const keys = tileKeysAround( + latitude, + longitude, + Math.min(radiusKm, PLACE_TILE_KM), + ).filter((key) => index.tiles[key]); // A tile that cannot be read is left out: the places of the others, // and the cities, are still the answer. const settled = await Promise.allSettled( @@ -281,7 +292,10 @@ async function readPlacePack(url, fetchImpl) { return response.json(); } -/** Named cities and smaller places around one point, nearest first. */ +/** + * Named cities and smaller places around one point, nearest first. Smaller + * places are looked for within PLACE_TILE_KM only. + */ export async function placesNear( { latitude, longitude, radiusKm = 10, limit = 6 }, fetchImpl = (...args) => globalThis.fetch(...args), @@ -325,7 +339,14 @@ export async function placesNear( const found = [ ...cities.filter((city) => city.distanceKm <= radius), ...detail - .flatMap((source) => citiesWithin(source, latitude, longitude, radius)) + .flatMap((source) => + citiesWithin( + source, + latitude, + longitude, + Math.min(radius, PLACE_TILE_KM), + ), + ) .filter((place) => !twinOfCity(place)), ]; const seen = new Set(); @@ -339,12 +360,13 @@ export async function placesNear( const TWIN_KM = 3; /** - * The city with this name, or null. A qualifier after a comma that names a - * country ("Paris, France", "Paris, US") keeps only cities there, and finds + * The place with this name, or null. A qualifier after a comma that names a + * country ("Paris, France", "Paris, US") keeps only places there, and finds * nothing when none is; one that names no country ("Paris, Texas") is - * ignored. A city of that name within `radiusKm` of `near` (the view) wins, - * the nearest first, else the largest city of that name anywhere. Omnibar - * can also ask for a smaller place nearby when the cities pack has no match. + * ignored. A place of that name within `radiusKm` of `near` (the view) wins, + * the nearest first, else the largest city of that name anywhere. With + * `includeSmallPlaces`, the smaller places of the detail tiles near the view + * count too: a village in view comes before a far bigger city of its name. */ export async function findPlace( name, @@ -360,20 +382,6 @@ export async function findPlace( const cities = (names.get(wanted) || []).filter( (row) => !country || row[4] === country, ); - if (!cities.length) { - if (!includeSmallPlaces || !near) return null; - const nearby = await placesNear( - { ...near, radiusKm, limit: Infinity }, - fetchImpl, - ); - return ( - nearby.find( - (place) => - foldName(place.name) === wanted && - (!country || place.country === country), - ) ?? null - ); - } if (Number.isFinite(near?.latitude) && Number.isFinite(near?.longitude)) { let best = null; for (const row of cities) { @@ -381,7 +389,24 @@ export async function findPlace( if (km <= radiusKm && (!best || km < best.distanceKm)) best = toCity(row, km); } + if (includeSmallPlaces) { + const nearby = await placesNear( + { + latitude: near.latitude, + longitude: near.longitude, + radiusKm, + limit: Infinity, + }, + fetchImpl, + ); + const small = nearby.find( + (place) => + foldName(place.name) === wanted && + (!country || place.country === country), + ); + if (small && (!best || small.distanceKm < best.distanceKm)) best = small; + } if (best) return best; } - return toCity(cities[0], 0); + return cities.length ? toCity(cities[0], 0) : null; } diff --git a/src/data/wikidataCities.test.mjs b/src/data/wikidataCities.test.mjs index ae96f1e462..4c62b1b65b 100644 --- a/src/data/wikidataCities.test.mjs +++ b/src/data/wikidataCities.test.mjs @@ -96,6 +96,66 @@ test('findPlace takes a country qualifier and prefers a namesake in view', async ); }); +test('a small place in view comes before a far bigger city of its name', async () => { + const module = await import(`./wikidataCities.js?small-${Date.now()}`); + // A village called Paris near Austin, in the 30_-100 detail tile. + const village = ['Paris', 30.4, -97.6, 1200, 'US', 'Q-village']; + const fetchImpl = async (url) => { + const file = url.pathname.split('/').at(-1); + const body = + file === 'cities.json' + ? { meta: {}, rows: PARIS } + : file === 'index.json' + ? { tiles: { '30_-100': 1 } } + : { rows: file === '30_-100.json' ? [village] : [] }; + return { ok: true, json: async () => body }; + }; + const find = (name, near) => + module.findPlace(name, { near, includeSmallPlaces: true }, fetchImpl); + const austin = { latitude: 30.27, longitude: -97.74 }; + assert.equal((await find('Paris', austin)).wikidata, 'Q-village'); + assert.ok((await find('Paris', austin)).distanceKm < 20); + // A bigger namesake inside a wider search: the nearer village still wins. + assert.equal( + ( + await module.findPlace( + 'Paris', + { near: austin, radiusKm: 600, includeSmallPlaces: true }, + fetchImpl, + ) + ).wikidata, + 'Q-village', + ); + // Only when small places are asked for. + assert.equal( + (await module.findPlace('Paris', { near: austin }, fetchImpl)).wikidata, + 'Q90', + ); + // A qualifier still decides. + assert.equal((await find('Paris, France', austin)).wikidata, 'Q90'); + // Nowhere near the village: the largest city of the name. + assert.equal( + (await find('Paris', { latitude: 0, longitude: 0 })).wikidata, + 'Q90', + ); + assert.equal( + (await module.findPlace('Paris', {}, fetchImpl)).wikidata, + 'Q90', + ); + // Paris, Texas is nearer a view over it than the village is. + const texas = { latitude: 33.6, longitude: -95.6 }; + assert.equal( + ( + await module.findPlace( + 'Paris', + { near: texas, radiusKm: 600, includeSmallPlaces: true }, + fetchImpl, + ) + ).wikidata, + 'Q830149', + ); +}); + test('a circle that reaches a pole spans every longitude', () => { assert.equal(lonHalfSpan(89, 150), 180); assert.ok(lonHalfSpan(60, 150) < 3); @@ -232,3 +292,50 @@ test('a tile that cannot be read leaves the other places and the cities', async assert.ok(near.some((place) => place.name === 'Austin')); assert.ok(near.some((place) => place.longitude > -95)); }); + +test('a lookup asked for the whole globe reads only the tiles near its centre', async () => { + const { placesNear, PLACE_TILE_KM } = + await import('./wikidataCities.js?globe'); + const fetched = []; + const fetchPack = async (url) => { + fetched.push(url.pathname.split('/').at(-1)); + return new Response(await readFile(url)); + }; + const near = await placesNear( + { latitude: 30.27, longitude: -97.74, radiusKm: 20000, limit: 50 }, + fetchPack, + ); + const tileReads = fetched + .filter((name) => /^-?\d+_-?\d+\.json$/.test(name)) + .sort(); + // 500 km around Austin: latitude 25.8 to 34.8, longitude -103 to -92.5. + assert.equal(PLACE_TILE_KM, 500); + assert.deepEqual(tileReads, [ + '25_-100.json', + '25_-105.json', + '25_-95.json', + '30_-100.json', + '30_-105.json', + '30_-95.json', + ]); + assert.equal(near.length, 50); + assert.ok(near.every((place) => place.distanceKm <= PLACE_TILE_KM)); + // Cities still count to the full radius. + const far = await placesNear( + { latitude: 30.27, longitude: -97.74, radiusKm: 20000, limit: Infinity }, + fetchPack, + ); + assert.ok(far.some((place) => place.wikidata === 'Q90')); + // Places from the tiles it read stop at 500 km too: Tahoka, Texas + // (Q982084), a town of about 2,500, is 500.6 km away. + assert.ok(!far.some((place) => place.wikidata === 'Q982084')); + assert.ok( + far + .filter((place) => place.distanceKm > PLACE_TILE_KM) + .every((place) => place.population >= 15000), + ); + assert.equal( + fetched.filter((name) => /^-?\d+_-?\d+\.json$/.test(name)).length, + tileReads.length, + ); +}); diff --git a/src/tooling/wikidataPlaceTiles.test.mjs b/src/tooling/wikidataPlaceTiles.test.mjs index 9e9cf54179..9f807c20e5 100644 --- a/src/tooling/wikidataPlaceTiles.test.mjs +++ b/src/tooling/wikidataPlaceTiles.test.mjs @@ -8,6 +8,8 @@ import { createHash } from 'node:crypto'; import { readdir, readFile } from 'node:fs/promises'; import test from 'node:test'; import { + keepBuiltIfUnchanged, + keepPackBuiltIfUnchanged, placeTileFiles, tileKey, withoutCities, @@ -103,3 +105,41 @@ test('twins are found where longitude cells narrow, and across 180°', () => { ['Q1', 'Q3'], ); }); + +test('a rebuild of unchanged rows keeps the earlier build stamp', () => { + const earlier = `${JSON.stringify({ meta: { built: '2026-01-01T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`; + const same = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`; + const changed = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 3 }, rows: [1, 2, 3] })}\n`; + assert.equal(keepBuiltIfUnchanged(same, earlier), earlier); + assert.equal(keepBuiltIfUnchanged(changed, earlier), changed); + assert.equal(keepBuiltIfUnchanged(same, undefined), same); + assert.equal(keepBuiltIfUnchanged(same, 'not json'), same); +}); + +test('the places index keeps its build stamp only when no tile changed', () => { + const rows = [ + ['Aa', 31, -98, 2000, 'US', 'Q1'], + ['Bb', 41, -88, 1500, 'US', 'Q2'], + ]; + const meta = (built) => ({ + source: 's', + license: 'l', + built, + minPopulation: 1000, + maxPopulation: 15000, + columns: [], + }); + const earlier = placeTileFiles(rows, meta('2026-01-01T00:00:00.000Z')); + const built = (files) => JSON.parse(files.get('index.json')).meta.built; + const rebuild = (newRows) => + keepPackBuiltIfUnchanged( + placeTileFiles(newRows, meta('2026-11-18T00:00:00.000Z')), + earlier, + ); + assert.equal(built(rebuild(rows)), '2026-01-01T00:00:00.000Z'); + // Same row count in every tile, new population: a new stamp. + const changed = [['Aa', 31, -98, 2100, 'US', 'Q1'], rows[1]]; + assert.equal(built(rebuild(changed)), '2026-11-18T00:00:00.000Z'); + // A tile of the earlier build left out: a new stamp. + assert.equal(built(rebuild([rows[0]])), '2026-11-18T00:00:00.000Z'); +});