From f414df97ebe9e438e3a0a8f8377fa33943b55b6b Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 18 Nov 2026 22:23:25 -0800 Subject: [PATCH 1/7] fix(places): a small place in view comes before a far bigger namesake findPlace looked at the detail tiles only when no city had the typed name, so a village called Paris in view lost to Paris, France. With includeSmallPlaces it now reads the places near the view as well and takes the nearest place of the name there, city or not. With none near, the largest city of the name still wins. Co-authored-by: Manjunath Paparaju --- docs/CURRENT-STATE.md | 8 ++- src/data/local_data/wikidata_cities/README.md | 5 ++ src/data/local_data/wikidata_places/README.md | 7 ++- src/data/wikidataCities.js | 44 +++++++++-------- src/data/wikidataCities.test.mjs | 49 +++++++++++++++++++ 5 files changed, 89 insertions(+), 24 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 6b0cc95cdb..724fa05523 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -966,7 +966,8 @@ the fleet set loaded at startup, so opening a card fetches no model. country and Wikidata id. `src/data/wikidataCities.js` reads it on the first lookup and answers `nearestCity` (the largest city within 12 km, else the nearest within 60 km), `citiesAround`, `findPlace` ("Paris, France", or the -nearest namesake within 150 km of the view, else the largest) and +nearest namesake within 150 km of the view, else the largest; asked to, it +counts the smaller places of the detail tiles near the view too) and `countryName`. Names match without case or accents ("lodz" finds Łódź). The app never calls Wikidata; `scripts/build-wikidata-cities.mjs` rebuilds the pack and fails if well-known cities are missing or a known grouping such as @@ -8249,7 +8250,10 @@ clears them but the browser's own site data. Setting `PHOTON_URL` empty turns the suggestions off along with it. Small places come from the bundled Wikidata packs and are never looked up online; a small place with the name of a city within 3 km in the same country is that city, and only -the city is listed. At most 64 detail tiles are kept in memory, the least +the city is listed. A place named in "villages near …" is the nearest of +that name within 150 km of the view, small places included, so a village in +view comes before a bigger namesake far away; with none near, the largest +city of the name. At most 64 detail tiles are kept in memory, the least recently used dropped first. ### Place-search providers diff --git a/src/data/local_data/wikidata_cities/README.md b/src/data/local_data/wikidata_cities/README.md index 8c2abf3818..ff80a6dc56 100644 --- a/src/data/local_data/wikidata_cities/README.md +++ b/src/data/local_data/wikidata_cities/README.md @@ -40,3 +40,8 @@ well-known cities is missing from the result, or a known grouping gets in. **Limits:** populations are whatever Wikidata states and differ in date and definition between places (city proper, municipality, urban district). Some districts and suburbs appear as their own rows. + +**Finding a place by name:** `findPlace` takes the nearest place of the name +within 150 km of the view, else the largest city of the name. When asked, +the smaller places of `../wikidata_places` near the view count too, so a +village in view comes before a bigger namesake far away. diff --git a/src/data/local_data/wikidata_places/README.md b/src/data/local_data/wikidata_places/README.md index 2831f89a56..0694fe5c5c 100644 --- a/src/data/local_data/wikidata_places/README.md +++ b/src/data/local_data/wikidata_places/README.md @@ -1,8 +1,11 @@ # Wikidata places detail tiles Smaller towns and villages, below the cut-off of the cities pack -(`../wikidata_cities`), for "places nearby" around one picked or named spot. Read by `placesNear` in `src/data/wikidataCities.js`. The app reads -these files; it does not call Wikidata or any place service for them. +(`../wikidata_cities`), for "places nearby" around one picked or named spot. Read by `placesNear` in `src/data/wikidataCities.js`, and by `findPlace` +when asked to look for a small place by name near the view: a place of that +name in view, or close to it, comes before a bigger city of the name far +away. The app reads these files; it does not call Wikidata or any place +service for them. | File | Contents | |------|----------| diff --git a/src/data/wikidataCities.js b/src/data/wikidataCities.js index 70dcb3e67a..1cc7eae468 100644 --- a/src/data/wikidataCities.js +++ b/src/data/wikidataCities.js @@ -339,12 +339,13 @@ export async function placesNear( const TWIN_KM = 3; /** - * The city with this name, or null. A qualifier after a comma that names a - * country ("Paris, France", "Paris, US") keeps only cities there, and finds + * The place with this name, or null. A qualifier after a comma that names a + * country ("Paris, France", "Paris, US") keeps only places there, and finds * nothing when none is; one that names no country ("Paris, Texas") is - * ignored. A city of that name within `radiusKm` of `near` (the view) wins, - * the nearest first, else the largest city of that name anywhere. Omnibar - * can also ask for a smaller place nearby when the cities pack has no match. + * ignored. A place of that name within `radiusKm` of `near` (the view) wins, + * the nearest first, else the largest city of that name anywhere. With + * `includeSmallPlaces`, the smaller places of the detail tiles near the view + * count too: a village in view comes before a far bigger city of its name. */ export async function findPlace( name, @@ -360,20 +361,6 @@ export async function findPlace( const cities = (names.get(wanted) || []).filter( (row) => !country || row[4] === country, ); - if (!cities.length) { - if (!includeSmallPlaces || !near) return null; - const nearby = await placesNear( - { ...near, radiusKm, limit: Infinity }, - fetchImpl, - ); - return ( - nearby.find( - (place) => - foldName(place.name) === wanted && - (!country || place.country === country), - ) ?? null - ); - } if (Number.isFinite(near?.latitude) && Number.isFinite(near?.longitude)) { let best = null; for (const row of cities) { @@ -381,7 +368,24 @@ export async function findPlace( if (km <= radiusKm && (!best || km < best.distanceKm)) best = toCity(row, km); } + if (includeSmallPlaces) { + const nearby = await placesNear( + { + latitude: near.latitude, + longitude: near.longitude, + radiusKm, + limit: Infinity, + }, + fetchImpl, + ); + const small = nearby.find( + (place) => + foldName(place.name) === wanted && + (!country || place.country === country), + ); + if (small && (!best || small.distanceKm < best.distanceKm)) best = small; + } if (best) return best; } - return toCity(cities[0], 0); + return cities.length ? toCity(cities[0], 0) : null; } diff --git a/src/data/wikidataCities.test.mjs b/src/data/wikidataCities.test.mjs index ae96f1e462..dd3259efa1 100644 --- a/src/data/wikidataCities.test.mjs +++ b/src/data/wikidataCities.test.mjs @@ -96,6 +96,55 @@ test('findPlace takes a country qualifier and prefers a namesake in view', async ); }); +test('a small place in view comes before a far bigger city of its name', async () => { + const module = await import(`./wikidataCities.js?small-${Date.now()}`); + // A village called Paris near Austin, in the 30_-100 detail tile. + const village = ['Paris', 30.4, -97.6, 1200, 'US', 'Q-village']; + const fetchImpl = async (url) => { + const file = url.pathname.split('/').at(-1); + const body = + file === 'cities.json' + ? { meta: {}, rows: PARIS } + : file === 'index.json' + ? { tiles: { '30_-100': 1 } } + : { rows: file === '30_-100.json' ? [village] : [] }; + return { ok: true, json: async () => body }; + }; + const find = (name, near) => + module.findPlace(name, { near, includeSmallPlaces: true }, fetchImpl); + const austin = { latitude: 30.27, longitude: -97.74 }; + assert.equal((await find('Paris', austin)).wikidata, 'Q-village'); + assert.ok((await find('Paris', austin)).distanceKm < 20); + // Only when small places are asked for. + assert.equal( + (await module.findPlace('Paris', { near: austin }, fetchImpl)).wikidata, + 'Q90', + ); + // A qualifier still decides. + assert.equal((await find('Paris, France', austin)).wikidata, 'Q90'); + // Nowhere near the village: the largest city of the name. + assert.equal( + (await find('Paris', { latitude: 0, longitude: 0 })).wikidata, + 'Q90', + ); + assert.equal( + (await module.findPlace('Paris', {}, fetchImpl)).wikidata, + 'Q90', + ); + // Paris, Texas is nearer a view over it than the village is. + const texas = { latitude: 33.6, longitude: -95.6 }; + assert.equal( + ( + await module.findPlace( + 'Paris', + { near: texas, radiusKm: 600, includeSmallPlaces: true }, + fetchImpl, + ) + ).wikidata, + 'Q830149', + ); +}); + test('a circle that reaches a pole spans every longitude', () => { assert.equal(lonHalfSpan(89, 150), 180); assert.ok(lonHalfSpan(60, 150) < 3); From 30ec362c13131113e23058cac39e3e2063803e64 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 18 Nov 2026 22:34:54 -0800 Subject: [PATCH 2/7] fix(places): look for small places within 500 km, so a whole-globe view reads only the tiles near its centre Co-authored-by: Manjunath Paparaju --- docs/CURRENT-STATE.md | 4 +- src/data/local_data/wikidata_places/README.md | 11 +++++- src/data/wikidataCities.js | 21 ++++++++-- src/data/wikidataCities.test.mjs | 39 +++++++++++++++++++ 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 724fa05523..f51d96efcb 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -8253,7 +8253,9 @@ the name of a city within 3 km in the same country is that city, and only the city is listed. A place named in "villages near …" is the nearest of that name within 150 km of the view, small places included, so a village in view comes before a bigger namesake far away; with none near, the largest -city of the name. At most 64 detail tiles are kept in memory, the least +city of the name. Small places are looked for within 500 km at most, so a +view zoomed out to the whole globe reads only the detail tiles around its +centre. At most 64 detail tiles are kept in memory, the least recently used dropped first. ### Place-search providers diff --git a/src/data/local_data/wikidata_places/README.md b/src/data/local_data/wikidata_places/README.md index 0694fe5c5c..144f907209 100644 --- a/src/data/local_data/wikidata_places/README.md +++ b/src/data/local_data/wikidata_places/README.md @@ -20,7 +20,11 @@ entry in the index. **Loading:** nothing here loads with the app. A lookup reads `files.js` and the index once, then only the tiles its search box touches (one for a short radius well inside a tile, more near tile edges and for wider searches), -once each; at most 64 tiles are kept in memory, the least recently used +once each. The search box reaches 500 km at most (`PLACE_TILE_KM`), so a +view of the whole globe reads only the tiles around its centre: at most one +tile past the centre's own on each side, a few more columns toward the +poles, and never all 732. Cities still count to the full radius. At most +64 tiles are kept in memory, the least recently used dropped first. Rows whose name has no letter (one row, `380` in Nicaragua) are dropped when a tile is read. @@ -42,7 +46,10 @@ names from Wikidata (CC0)" as a courtesy once a name from the pack is shown. `node scripts/build-wikidata-cities.mjs --tiles --min-population 1000 --max-population 15000`. It writes only to this folder (tiles, `index.json`, `files.js`, through `scripts/wikidataPlaceTiles.mjs`) and leaves `../wikidata_cities` alone. It -runs one query at a time with pauses and takes a long while. +runs one query at a time with pauses and takes a long while. The new files +are all written before tiles that the new build has no rows for are +removed, and `meta.built` keeps its earlier date when nothing else in +`index.json` changed. **Reproducibility:** the step from rows to files is checked offline: `src/tooling/wikidataPlaceTiles.test.mjs` sorts the shipped rows as the diff --git a/src/data/wikidataCities.js b/src/data/wikidataCities.js index 1cc7eae468..2b62ac62b0 100644 --- a/src/data/wikidataCities.js +++ b/src/data/wikidataCities.js @@ -217,6 +217,14 @@ const TILE_DEG = 5; * far more than one lookup touches, and bounds what a long session keeps. */ const MAX_TILES = 64; +/** + * How far out small places are looked for, in km. 500 km is under the 556 km + * that 5 degrees of latitude span, so the search box reaches at most one tile + * past the point's own on each side: 3 rows, and 3 columns at the equator + * (more toward the poles, where tiles narrow). A wider view, out to the whole + * globe, still reads only those tiles; cities count to its full radius. + */ +export const PLACE_TILE_KM = 500; const NAMED = /\p{L}/u; const tiles = new Map(); let loadTileIndex; @@ -248,9 +256,11 @@ async function readPlaceTiles(latitude, longitude, radiusKm, fetchImpl) { readPlacePack(INDEX_URL(), fetchImpl), ); const index = await loadTileIndex(); - const keys = tileKeysAround(latitude, longitude, radiusKm).filter( - (key) => index.tiles[key], - ); + const keys = tileKeysAround( + latitude, + longitude, + Math.min(radiusKm, PLACE_TILE_KM), + ).filter((key) => index.tiles[key]); // A tile that cannot be read is left out: the places of the others, // and the cities, are still the answer. const settled = await Promise.allSettled( @@ -281,7 +291,10 @@ async function readPlacePack(url, fetchImpl) { return response.json(); } -/** Named cities and smaller places around one point, nearest first. */ +/** + * Named cities and smaller places around one point, nearest first. Smaller + * places are looked for within PLACE_TILE_KM only. + */ export async function placesNear( { latitude, longitude, radiusKm = 10, limit = 6 }, fetchImpl = (...args) => globalThis.fetch(...args), diff --git a/src/data/wikidataCities.test.mjs b/src/data/wikidataCities.test.mjs index dd3259efa1..10cec8c834 100644 --- a/src/data/wikidataCities.test.mjs +++ b/src/data/wikidataCities.test.mjs @@ -281,3 +281,42 @@ test('a tile that cannot be read leaves the other places and the cities', async assert.ok(near.some((place) => place.name === 'Austin')); assert.ok(near.some((place) => place.longitude > -95)); }); + +test('a view of the whole globe reads only the tiles near its centre', async () => { + const { placesNear, PLACE_TILE_KM } = + await import('./wikidataCities.js?globe'); + const fetched = []; + const fetchPack = async (url) => { + fetched.push(url.pathname.split('/').at(-1)); + return new Response(await readFile(url)); + }; + const near = await placesNear( + { latitude: 30.27, longitude: -97.74, radiusKm: 20000, limit: 50 }, + fetchPack, + ); + const tileReads = fetched + .filter((name) => /^-?\d+_-?\d+\.json$/.test(name)) + .sort(); + // 500 km around Austin: latitude 25.8 to 34.8, longitude -103 to -92.5. + assert.equal(PLACE_TILE_KM, 500); + assert.deepEqual(tileReads, [ + '25_-100.json', + '25_-105.json', + '25_-95.json', + '30_-100.json', + '30_-105.json', + '30_-95.json', + ]); + assert.equal(near.length, 50); + assert.ok(near.every((place) => place.distanceKm <= PLACE_TILE_KM)); + // Cities still count to the full radius. + const far = await placesNear( + { latitude: 30.27, longitude: -97.74, radiusKm: 20000, limit: Infinity }, + fetchPack, + ); + assert.ok(far.some((place) => place.wikidata === 'Q90')); + assert.equal( + fetched.filter((name) => /^-?\d+_-?\d+\.json$/.test(name)).length, + tileReads.length, + ); +}); From a2c86c0fed3d253342799cc200f57fbfef7f99bd Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 18 Nov 2026 22:34:54 -0800 Subject: [PATCH 3/7] fix(places): a rebuild writes its files before removing stale tiles, and keeps the build date when nothing changed Co-authored-by: Manjunath Paparaju --- scripts/build-wikidata-cities.mjs | 24 +++++++++++++++---- scripts/wikidataPlaceTiles.mjs | 23 ++++++++++++++++++ src/data/local_data/wikidata_cities/README.md | 1 + src/tooling/wikidataPlaceTiles.test.mjs | 11 +++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/scripts/build-wikidata-cities.mjs b/scripts/build-wikidata-cities.mjs index a276062831..00dc36f0c0 100644 --- a/scripts/build-wikidata-cities.mjs +++ b/scripts/build-wikidata-cities.mjs @@ -39,6 +39,7 @@ import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { + keepBuiltIfUnchanged, placeTileFiles, withoutCities, withoutNearTwins, @@ -68,6 +69,8 @@ const outDir = path.join( : 'src/data/local_data/wikidata_cities', ); +/** A file's text, or undefined when there is none yet. */ +const readIfThere = (file) => readFile(file, 'utf8').catch(() => undefined); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1); @@ -499,17 +502,30 @@ if (tilesMode) { count: tileRows.length, maxPopulation, }); - // Tiles from an earlier build that this one has no rows for go. + // The new files are all written before tiles from an earlier build that + // this one has no rows for go, so a build that stops part way never leaves + // an index naming a removed tile. + const indexFile = path.join(outDir, 'index.json'); + files.set( + 'index.json', + keepBuiltIfUnchanged(files.get('index.json'), await readIfThere(indexFile)), + ); + for (const [name, text] of files) + await writeFile(path.join(outDir, name), text); for (const name of await readdir(outDir)) if (/^-?\d+_-?\d+\.json$/.test(name) && !files.has(name)) await rm(path.join(outDir, name)); - for (const [name, text] of files) - await writeFile(path.join(outDir, name), text); console.log( `Wrote ${tileRows.length} places in ${files.size - 2} tiles to ${path.relative(root, outDir)}`, ); process.exit(0); } const file = path.join(outDir, 'cities.json'); -await writeFile(file, `${JSON.stringify({ meta, rows })}\n`); +await writeFile( + file, + keepBuiltIfUnchanged( + `${JSON.stringify({ meta, rows })}\n`, + await readIfThere(file), + ), +); console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`); diff --git a/scripts/wikidataPlaceTiles.mjs b/scripts/wikidataPlaceTiles.mjs index 15f0c89e3a..bae1774bb7 100644 --- a/scripts/wikidataPlaceTiles.mjs +++ b/scripts/wikidataPlaceTiles.mjs @@ -89,6 +89,29 @@ export function placeTileFiles(rows, meta) { return files; } +/** + * A pack file's new text, keeping the earlier build's `meta.built` when + * nothing else in the file changed, so rebuilding unchanged data leaves the + * file as it was. + * @param {string} text The new file, JSON with a `meta.built` stamp. + * @param {string | undefined} earlier The file as it is now, if there is one. + */ +export function keepBuiltIfUnchanged(text, earlier) { + if (!earlier) return text; + let before; + try { + before = JSON.parse(earlier); + } catch { + return text; + } + const after = JSON.parse(text); + if (typeof before?.meta?.built !== 'string' || !after.meta) return text; + const unstamped = (value) => + JSON.stringify({ ...value, meta: { ...value.meta, built: null } }); + if (unstamped(before) !== unstamped(after)) return text; + return `${JSON.stringify({ ...after, meta: { ...after.meta, built: before.meta.built } })}\n`; +} + const kmBetween = (a, b) => greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] }); diff --git a/src/data/local_data/wikidata_cities/README.md b/src/data/local_data/wikidata_cities/README.md index ff80a6dc56..026f412988 100644 --- a/src/data/local_data/wikidata_cities/README.md +++ b/src/data/local_data/wikidata_cities/README.md @@ -36,6 +36,7 @@ places without an English label. The build runs one query at a time with pauses and honours the service's Retry-After; it takes about ten minutes. It fails if a short list of well-known cities is missing from the result, or a known grouping gets in. +`meta.built` keeps its earlier date when the rows are unchanged. **Limits:** populations are whatever Wikidata states and differ in date and definition between places (city proper, municipality, urban district). Some diff --git a/src/tooling/wikidataPlaceTiles.test.mjs b/src/tooling/wikidataPlaceTiles.test.mjs index 9e9cf54179..10e77b8dd3 100644 --- a/src/tooling/wikidataPlaceTiles.test.mjs +++ b/src/tooling/wikidataPlaceTiles.test.mjs @@ -8,6 +8,7 @@ import { createHash } from 'node:crypto'; import { readdir, readFile } from 'node:fs/promises'; import test from 'node:test'; import { + keepBuiltIfUnchanged, placeTileFiles, tileKey, withoutCities, @@ -103,3 +104,13 @@ test('twins are found where longitude cells narrow, and across 180°', () => { ['Q1', 'Q3'], ); }); + +test('a rebuild of unchanged rows keeps the earlier build stamp', () => { + const earlier = `${JSON.stringify({ meta: { built: '2026-01-01T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`; + const same = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`; + const changed = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 3 }, rows: [1, 2, 3] })}\n`; + assert.equal(keepBuiltIfUnchanged(same, earlier), earlier); + assert.equal(keepBuiltIfUnchanged(changed, earlier), changed); + assert.equal(keepBuiltIfUnchanged(same, undefined), same); + assert.equal(keepBuiltIfUnchanged(same, 'not json'), same); +}); From ecb7e7cd68df4b50e86933882bb6955147543e1c Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 18 Nov 2026 22:41:36 -0800 Subject: [PATCH 4/7] Add the changelog entry for #1838 Co-authored-by: Manjunath Paparaju --- changelog.d/1838-nearby-places-namesake.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/1838-nearby-places-namesake.md diff --git a/changelog.d/1838-nearby-places-namesake.md b/changelog.d/1838-nearby-places-namesake.md new file mode 100644 index 0000000000..a9efb54a0c --- /dev/null +++ b/changelog.d/1838-nearby-places-namesake.md @@ -0,0 +1,4 @@ +- "Villages near" a name picks the place of that name in view over a far + bigger namesake, a view of the whole globe reads only the place tiles + around its centre, and rebuilding the places pack never leaves it half + removed (manjunath22466, samehkhamis, #1838). From d2797dbad29aeb12460a99489fb9c02808f47ee8 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 18 Nov 2026 22:44:12 -0800 Subject: [PATCH 5/7] fix(places): keep the places index's build date only when every tile is unchanged; say the 500 km cap is for wide lookups Co-authored-by: Manjunath Paparaju --- changelog.d/1838-nearby-places-namesake.md | 6 ++--- docs/CURRENT-STATE.md | 4 ++-- scripts/build-wikidata-cities.mjs | 22 +++++++++++++++---- scripts/wikidataPlaceTiles.mjs | 4 +++- src/data/local_data/wikidata_places/README.md | 9 ++++---- src/data/wikidataCities.js | 9 ++++---- src/data/wikidataCities.test.mjs | 2 +- 7 files changed, 37 insertions(+), 19 deletions(-) diff --git a/changelog.d/1838-nearby-places-namesake.md b/changelog.d/1838-nearby-places-namesake.md index a9efb54a0c..ffda7e347e 100644 --- a/changelog.d/1838-nearby-places-namesake.md +++ b/changelog.d/1838-nearby-places-namesake.md @@ -1,4 +1,4 @@ - "Villages near" a name picks the place of that name in view over a far - bigger namesake, a view of the whole globe reads only the place tiles - around its centre, and rebuilding the places pack never leaves it half - removed (manjunath22466, samehkhamis, #1838). + bigger namesake, a place lookup reads the detail tiles within 500 km at + most, and rebuilding the places pack never leaves it half removed + (manjunath22466, samehkhamis, #1838). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index f51d96efcb..ff51fca9fe 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -8254,8 +8254,8 @@ the city is listed. A place named in "villages near …" is the nearest of that name within 150 km of the view, small places included, so a village in view comes before a bigger namesake far away; with none near, the largest city of the name. Small places are looked for within 500 km at most, so a -view zoomed out to the whole globe reads only the detail tiles around its -centre. At most 64 detail tiles are kept in memory, the least +lookup asked for a wider radius reads only the detail tiles around its +centre (the search box asks for 150 km at most). At most 64 detail tiles are kept in memory, the least recently used dropped first. ### Place-search providers diff --git a/scripts/build-wikidata-cities.mjs b/scripts/build-wikidata-cities.mjs index 00dc36f0c0..1c66eea87b 100644 --- a/scripts/build-wikidata-cities.mjs +++ b/scripts/build-wikidata-cities.mjs @@ -505,11 +505,25 @@ if (tilesMode) { // The new files are all written before tiles from an earlier build that // this one has no rows for go, so a build that stops part way never leaves // an index naming a removed tile. - const indexFile = path.join(outDir, 'index.json'); - files.set( - 'index.json', - keepBuiltIfUnchanged(files.get('index.json'), await readIfThere(indexFile)), + // The index holds only row counts, so its build date is kept only when + // every tile is unchanged too and no earlier tile is left over. + const earlierTiles = (await readdir(outDir)).filter((name) => + /^-?\d+_-?\d+\.json$/.test(name), ); + let tilesUnchanged = earlierTiles.every((name) => files.has(name)); + for (const [name, text] of files) { + if (!tilesUnchanged) break; + if (name === 'index.json') continue; + tilesUnchanged = (await readIfThere(path.join(outDir, name))) === text; + } + if (tilesUnchanged) + files.set( + 'index.json', + keepBuiltIfUnchanged( + files.get('index.json'), + await readIfThere(path.join(outDir, 'index.json')), + ), + ); for (const [name, text] of files) await writeFile(path.join(outDir, name), text); for (const name of await readdir(outDir)) diff --git a/scripts/wikidataPlaceTiles.mjs b/scripts/wikidataPlaceTiles.mjs index bae1774bb7..98c2b10651 100644 --- a/scripts/wikidataPlaceTiles.mjs +++ b/scripts/wikidataPlaceTiles.mjs @@ -92,7 +92,9 @@ export function placeTileFiles(rows, meta) { /** * A pack file's new text, keeping the earlier build's `meta.built` when * nothing else in the file changed, so rebuilding unchanged data leaves the - * file as it was. + * file as it was. A file that does not hold all of its pack's data (the + * places index) is passed through this only when the rest of the pack is + * unchanged too. * @param {string} text The new file, JSON with a `meta.built` stamp. * @param {string | undefined} earlier The file as it is now, if there is one. */ diff --git a/src/data/local_data/wikidata_places/README.md b/src/data/local_data/wikidata_places/README.md index 144f907209..efad9ea1b3 100644 --- a/src/data/local_data/wikidata_places/README.md +++ b/src/data/local_data/wikidata_places/README.md @@ -20,8 +20,9 @@ entry in the index. **Loading:** nothing here loads with the app. A lookup reads `files.js` and the index once, then only the tiles its search box touches (one for a short radius well inside a tile, more near tile edges and for wider searches), -once each. The search box reaches 500 km at most (`PLACE_TILE_KM`), so a -view of the whole globe reads only the tiles around its centre: at most one +once each. A lookup reaches 500 km at most (`PLACE_TILE_KM`; the search box +asks for 150 km at most), so one asked for a wider radius reads only the +tiles around its centre: at most one tile past the centre's own on each side, a few more columns toward the poles, and never all 732. Cities still count to the full radius. At most 64 tiles are kept in memory, the least recently used @@ -48,8 +49,8 @@ It writes only to this folder (tiles, `index.json`, `files.js`, through `scripts/wikidataPlaceTiles.mjs`) and leaves `../wikidata_cities` alone. It runs one query at a time with pauses and takes a long while. The new files are all written before tiles that the new build has no rows for are -removed, and `meta.built` keeps its earlier date when nothing else in -`index.json` changed. +removed, and `meta.built` keeps its earlier date when no tile and +nothing else in `index.json` changed. **Reproducibility:** the step from rows to files is checked offline: `src/tooling/wikidataPlaceTiles.test.mjs` sorts the shipped rows as the diff --git a/src/data/wikidataCities.js b/src/data/wikidataCities.js index 2b62ac62b0..c36dd89c3a 100644 --- a/src/data/wikidataCities.js +++ b/src/data/wikidataCities.js @@ -219,10 +219,11 @@ const TILE_DEG = 5; const MAX_TILES = 64; /** * How far out small places are looked for, in km. 500 km is under the 556 km - * that 5 degrees of latitude span, so the search box reaches at most one tile - * past the point's own on each side: 3 rows, and 3 columns at the equator - * (more toward the poles, where tiles narrow). A wider view, out to the whole - * globe, still reads only those tiles; cities count to its full radius. + * that 5 degrees of latitude span, so a lookup's tile box reaches at most one + * tile past the point's own on each side: 3 rows, and 3 columns at the + * equator (more toward the poles, where tiles narrow). A lookup asked for a + * wider radius, up to the whole globe, still reads only those tiles; cities + * count to its full radius. */ export const PLACE_TILE_KM = 500; const NAMED = /\p{L}/u; diff --git a/src/data/wikidataCities.test.mjs b/src/data/wikidataCities.test.mjs index 10cec8c834..6e6f742007 100644 --- a/src/data/wikidataCities.test.mjs +++ b/src/data/wikidataCities.test.mjs @@ -282,7 +282,7 @@ test('a tile that cannot be read leaves the other places and the cities', async assert.ok(near.some((place) => place.longitude > -95)); }); -test('a view of the whole globe reads only the tiles near its centre', async () => { +test('a lookup asked for the whole globe reads only the tiles near its centre', async () => { const { placesNear, PLACE_TILE_KM } = await import('./wikidataCities.js?globe'); const fetched = []; From 75e345aa07a2a919815613c81ddb23210936497f Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 18 Nov 2026 22:44:36 -0800 Subject: [PATCH 6/7] fix(places): small places stop at 500 km even in tiles that reach past it Co-authored-by: Manjunath Paparaju --- src/data/wikidataCities.js | 9 ++++++++- src/data/wikidataCities.test.mjs | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/data/wikidataCities.js b/src/data/wikidataCities.js index c36dd89c3a..e106d2bc4c 100644 --- a/src/data/wikidataCities.js +++ b/src/data/wikidataCities.js @@ -339,7 +339,14 @@ export async function placesNear( const found = [ ...cities.filter((city) => city.distanceKm <= radius), ...detail - .flatMap((source) => citiesWithin(source, latitude, longitude, radius)) + .flatMap((source) => + citiesWithin( + source, + latitude, + longitude, + Math.min(radius, PLACE_TILE_KM), + ), + ) .filter((place) => !twinOfCity(place)), ]; const seen = new Set(); diff --git a/src/data/wikidataCities.test.mjs b/src/data/wikidataCities.test.mjs index 6e6f742007..84c5a8c112 100644 --- a/src/data/wikidataCities.test.mjs +++ b/src/data/wikidataCities.test.mjs @@ -315,6 +315,14 @@ test('a lookup asked for the whole globe reads only the tiles near its centre', fetchPack, ); assert.ok(far.some((place) => place.wikidata === 'Q90')); + // Places from the tiles it read stop at 500 km too: Tahoka, Texas + // (Q982084), a town of about 2,500, is 500.6 km away. + assert.ok(!far.some((place) => place.wikidata === 'Q982084')); + assert.ok( + far + .filter((place) => place.distanceKm > PLACE_TILE_KM) + .every((place) => place.population >= 15000), + ); assert.equal( fetched.filter((name) => /^-?\d+_-?\d+\.json$/.test(name)).length, tileReads.length, From 50824c9a65bfe5d5be95e04467f2d152be405eef Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 18 Nov 2026 22:54:53 -0800 Subject: [PATCH 7/7] test(places): pin the pack build-date rule and a nearer village beating a namesake inside a wide search Co-authored-by: Manjunath Paparaju --- scripts/build-wikidata-cities.mjs | 25 +++++---------------- scripts/wikidataPlaceTiles.mjs | 20 +++++++++++++++++ src/data/wikidataCities.test.mjs | 11 ++++++++++ src/tooling/wikidataPlaceTiles.test.mjs | 29 +++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 19 deletions(-) diff --git a/scripts/build-wikidata-cities.mjs b/scripts/build-wikidata-cities.mjs index 1c66eea87b..d2016fa3d7 100644 --- a/scripts/build-wikidata-cities.mjs +++ b/scripts/build-wikidata-cities.mjs @@ -40,6 +40,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { keepBuiltIfUnchanged, + keepPackBuiltIfUnchanged, placeTileFiles, withoutCities, withoutNearTwins, @@ -505,25 +506,11 @@ if (tilesMode) { // The new files are all written before tiles from an earlier build that // this one has no rows for go, so a build that stops part way never leaves // an index naming a removed tile. - // The index holds only row counts, so its build date is kept only when - // every tile is unchanged too and no earlier tile is left over. - const earlierTiles = (await readdir(outDir)).filter((name) => - /^-?\d+_-?\d+\.json$/.test(name), - ); - let tilesUnchanged = earlierTiles.every((name) => files.has(name)); - for (const [name, text] of files) { - if (!tilesUnchanged) break; - if (name === 'index.json') continue; - tilesUnchanged = (await readIfThere(path.join(outDir, name))) === text; - } - if (tilesUnchanged) - files.set( - 'index.json', - keepBuiltIfUnchanged( - files.get('index.json'), - await readIfThere(path.join(outDir, 'index.json')), - ), - ); + const earlier = new Map(); + for (const name of await readdir(outDir)) + if (/^-?\d+_-?\d+\.json$|^index\.json$|^files\.js$/.test(name)) + earlier.set(name, await readFile(path.join(outDir, name), 'utf8')); + keepPackBuiltIfUnchanged(files, earlier); for (const [name, text] of files) await writeFile(path.join(outDir, name), text); for (const name of await readdir(outDir)) diff --git a/scripts/wikidataPlaceTiles.mjs b/scripts/wikidataPlaceTiles.mjs index 98c2b10651..d3396937f6 100644 --- a/scripts/wikidataPlaceTiles.mjs +++ b/scripts/wikidataPlaceTiles.mjs @@ -114,6 +114,26 @@ export function keepBuiltIfUnchanged(text, earlier) { return `${JSON.stringify({ ...after, meta: { ...after.meta, built: before.meta.built } })}\n`; } +/** + * Keeps the earlier build's `meta.built` in the places pack's index when the + * rebuild changed nothing: every tile and `files.js` the same text, no tile + * of the earlier build left out, and nothing else in the index changed. The + * index holds only row counts, so it alone cannot tell. + * @param {Map} files The new pack, as placeTileFiles gives it; its index is replaced in place. + * @param {Map} earlier The pack's files on disk now, by name. + */ +export function keepPackBuiltIfUnchanged(files, earlier) { + for (const name of earlier.keys()) + if (name !== 'index.json' && !files.has(name)) return files; + for (const [name, text] of files) + if (name !== 'index.json' && earlier.get(name) !== text) return files; + files.set( + 'index.json', + keepBuiltIfUnchanged(files.get('index.json'), earlier.get('index.json')), + ); + return files; +} + const kmBetween = (a, b) => greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] }); diff --git a/src/data/wikidataCities.test.mjs b/src/data/wikidataCities.test.mjs index 84c5a8c112..4c62b1b65b 100644 --- a/src/data/wikidataCities.test.mjs +++ b/src/data/wikidataCities.test.mjs @@ -115,6 +115,17 @@ test('a small place in view comes before a far bigger city of its name', async ( const austin = { latitude: 30.27, longitude: -97.74 }; assert.equal((await find('Paris', austin)).wikidata, 'Q-village'); assert.ok((await find('Paris', austin)).distanceKm < 20); + // A bigger namesake inside a wider search: the nearer village still wins. + assert.equal( + ( + await module.findPlace( + 'Paris', + { near: austin, radiusKm: 600, includeSmallPlaces: true }, + fetchImpl, + ) + ).wikidata, + 'Q-village', + ); // Only when small places are asked for. assert.equal( (await module.findPlace('Paris', { near: austin }, fetchImpl)).wikidata, diff --git a/src/tooling/wikidataPlaceTiles.test.mjs b/src/tooling/wikidataPlaceTiles.test.mjs index 10e77b8dd3..9f807c20e5 100644 --- a/src/tooling/wikidataPlaceTiles.test.mjs +++ b/src/tooling/wikidataPlaceTiles.test.mjs @@ -9,6 +9,7 @@ import { readdir, readFile } from 'node:fs/promises'; import test from 'node:test'; import { keepBuiltIfUnchanged, + keepPackBuiltIfUnchanged, placeTileFiles, tileKey, withoutCities, @@ -114,3 +115,31 @@ test('a rebuild of unchanged rows keeps the earlier build stamp', () => { assert.equal(keepBuiltIfUnchanged(same, undefined), same); assert.equal(keepBuiltIfUnchanged(same, 'not json'), same); }); + +test('the places index keeps its build stamp only when no tile changed', () => { + const rows = [ + ['Aa', 31, -98, 2000, 'US', 'Q1'], + ['Bb', 41, -88, 1500, 'US', 'Q2'], + ]; + const meta = (built) => ({ + source: 's', + license: 'l', + built, + minPopulation: 1000, + maxPopulation: 15000, + columns: [], + }); + const earlier = placeTileFiles(rows, meta('2026-01-01T00:00:00.000Z')); + const built = (files) => JSON.parse(files.get('index.json')).meta.built; + const rebuild = (newRows) => + keepPackBuiltIfUnchanged( + placeTileFiles(newRows, meta('2026-11-18T00:00:00.000Z')), + earlier, + ); + assert.equal(built(rebuild(rows)), '2026-01-01T00:00:00.000Z'); + // Same row count in every tile, new population: a new stamp. + const changed = [['Aa', 31, -98, 2100, 'US', 'Q1'], rows[1]]; + assert.equal(built(rebuild(changed)), '2026-11-18T00:00:00.000Z'); + // A tile of the earlier build left out: a new stamp. + assert.equal(built(rebuild([rows[0]])), '2026-11-18T00:00:00.000Z'); +});