Bundle smaller Wikidata settlements for nearby-place lookups

137,662 places of 1,000 to 14,999 people that Wikidata gives a population
and coordinates for, none already in the cities pack, in 732 tiles of
5 x 5 degrees with an index of row counts (CC0). Nothing loads with the
app: a lookup reads the index once and then only the tiles it touches.
placesNear lists the places around a point, and findPlace finds a small
place by name near the view when no city of that name is near. The cities
build script writes the tiles with --tiles; the step from rows to files is
checked offline.

Co-authored-by: Sameh Khamis <[email protected]>
This commit is contained in:
Manjunath PaparajuandSameh Khamis authored and Sameh Khamis committed 2026-11-18 12:36:16 -08:00
1 parent 328b8f5d1b
commit 78305411eb
743 files changed
+2087 -14

No files matched your search

+66 -5
View File
@@ -20,13 +20,29 @@
* request runs at a time, with a pause between requests, and a refusal's
* Retry-After is honoured.
*
* With `--tiles`, the same query between `--min-population` and
* `--max-population` (1,000 and 15,000 by default) builds the detail tiles
* instead, and writes only to:
*
* src/data/local_data/wikidata_places/
*
* replacing that folder's tiles, `index.json` and `files.js` (see
* scripts/wikidataPlaceTiles.mjs). The checks that well-known cities are
* present apply to the cities pack only.
*
* Usage:
* node scripts/build-wikidata-cities.mjs [--min-population 15000]
* node scripts/build-wikidata-cities.mjs --tiles [--min-population 1000] [--max-population 15000]
*/
import { mkdir, writeFile } from 'node:fs/promises';
import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
placeTileFiles,
withoutCities,
withoutNearTwins,
} from './wikidataPlaceTiles.mjs';
const ENDPOINT = 'https://query.wikidata.org/sparql';
const USER_AGENT =
@@ -38,9 +54,20 @@ const LABEL_BATCH = 400;
const args = process.argv.slice(2);
const flag = (name) =>
args.includes(name) ? Number(args[args.indexOf(name) + 1]) : NaN;
const minPopulation = flag('--min-population') || 15000;
const tilesMode = args.includes('--tiles');
const minPopulation = flag('--min-population') || (tilesMode ? 1000 : 15000);
const maxPopulation = tilesMode
? flag('--max-population') || 15000
: flag('--max-population') || Infinity;
if (!(maxPopulation > minPopulation))
throw new Error('--max-population must be above --min-population');
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const outDir = path.join(root, 'src/data/local_data/wikidata_cities');
const outDir = path.join(
root,
tilesMode
? 'src/data/local_data/wikidata_places'
: 'src/data/local_data/wikidata_cities',
);
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1);
@@ -222,7 +249,7 @@ const grouped = new Set();
const settled = new Set();
const places = new Map();
for (const [low, high] of bands(minPopulation, Infinity)) {
for (const [low, high] of bands(minPopulation, maxPopulation)) {
console.log(`Population ${low}–${Number.isFinite(high) ? high : '∞'}…`);
const rows = await bandRows(low, high);
let kept = 0;
@@ -434,7 +461,7 @@ if (intruders.length) {
`Groupings got in: ${intruders.map(([id, name]) => `${name} (${id})`).join(', ')}`,
);
}
if (missing.length) {
if (missing.length && !tilesMode) {
throw new Error(
`Expected cities are missing: ${missing.map(([id, name]) => `${name} (${id})`).join(', ')}`,
);
@@ -450,6 +477,40 @@ const meta = {
};
await mkdir(outDir, { recursive: true });
if (tilesMode) {
// Places the cities pack already has stay out of the tiles: an item with
// one stated population under the cut-off and another over it is in both
// query ranges. Build the cities pack first.
const citiesFile = path.join(
root,
'src/data/local_data/wikidata_cities/cities.json',
);
const cityIds = JSON.parse(await readFile(citiesFile, 'utf8')).rows.map(
(row) => row[5],
);
const outsideCities = withoutCities(rows, cityIds);
console.log(
`${rows.length - outsideCities.length} places left to the cities pack`,
);
// One place listed twice nearby under one name keeps its larger listing.
const tileRows = withoutNearTwins(outsideCities);
console.log(`${outsideCities.length - tileRows.length} near twins left out`);
const files = placeTileFiles(tileRows, {
...meta,
count: tileRows.length,
maxPopulation,
});
// Tiles from an earlier build that this one has no rows for go.
for (const name of await readdir(outDir))
if (/^-?\d+_-?\d+\.json$/.test(name) && !files.has(name))
await rm(path.join(outDir, name));
for (const [name, text] of files)
await writeFile(path.join(outDir, name), text);
console.log(
`Wrote ${tileRows.length} places in ${files.size - 2} tiles to ${path.relative(root, outDir)}`,
);
process.exit(0);
}
const file = path.join(outDir, 'cities.json');
await writeFile(file, `${JSON.stringify({ meta, rows })}\n`);
console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`);