Bundle smaller Wikidata settlements for nearby-place lookups

137,662 places of 1,000 to 14,999 people that Wikidata gives a population
and coordinates for, none already in the cities pack, in 732 tiles of
5 x 5 degrees with an index of row counts (CC0). Nothing loads with the
app: a lookup reads the index once and then only the tiles it touches.
placesNear lists the places around a point, and findPlace finds a small
place by name near the view when no city of that name is near. The cities
build script writes the tiles with --tiles; the step from rows to files is
checked offline.

Co-authored-by: Sameh Khamis <[email protected]>
This commit is contained in:
Manjunath PaparajuandSameh Khamis authored and Sameh Khamis committed 2026-11-18 12:36:16 -08:00
1 parent 328b8f5d1b
commit 78305411eb
743 files changed
+2087 -14

No files matched your search

+88
View File
@@ -0,0 +1,88 @@
// The Wikidata places pack is written by scripts/wikidataPlaceTiles.mjs. Its
// rows come from a Wikidata query that cannot be repeated offline, so this
// checks the steps after it: the shipped rows, sorted and filtered as the
// build does, give back every shipped file byte for byte, and hash to the
// value the pack's README records.
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { readdir, readFile } from 'node:fs/promises';
import test from 'node:test';
import {
placeTileFiles,
tileKey,
withoutCities,
withoutNearTwins,
} from '../../scripts/wikidataPlaceTiles.mjs';
const PACK = new URL('../data/local_data/wikidata_places/', import.meta.url);
const CITIES = new URL(
'../data/local_data/wikidata_cities/cities.json',
import.meta.url,
);
test('a tile is named by its south-west corner', () => {
assert.equal(tileKey(30.343, -97.557), '30_-100');
assert.equal(tileKey(-0.5, 0), '-5_0');
assert.equal(tileKey(10.682, -71.036), '10_-75');
});
test('re-tiles the shipped rows deterministically', async () => {
const index = JSON.parse(await readFile(new URL('index.json', PACK), 'utf8'));
const rows = [];
for (const key of Object.keys(index.tiles)) {
const tile = JSON.parse(
await readFile(new URL(`${key}.json`, PACK), 'utf8'),
);
rows.push(...tile.rows);
}
assert.equal(rows.length, 137662);
// As scripts/build-wikidata-cities.mjs orders and filters its rows.
rows.sort((a, b) => b[3] - a[3] || a[5].localeCompare(b[5]));
const cityIds = JSON.parse(await readFile(CITIES, 'utf8')).rows.map(
(row) => row[5],
);
const kept = withoutNearTwins(withoutCities(rows, cityIds));
assert.equal(
kept.length,
rows.length,
'the shipped rows are already filtered',
);
const readme = await readFile(new URL('README.md', PACK), 'utf8');
const digest = createHash('sha256')
.update(JSON.stringify(kept))
.digest('hex');
assert.ok(readme.includes(`\`${digest}\``), 'README rows hash');
const files = placeTileFiles(kept, index.meta);
const shipped = (await readdir(PACK)).filter((name) => name !== 'README.md');
assert.deepEqual([...files.keys()].sort(), shipped.sort());
for (const [name, text] of files)
assert.equal(text, await readFile(new URL(name, PACK), 'utf8'), name);
});
test('a place the cities pack holds is left out of the tiles', () => {
const rows = [
['Palmares', 8.1, -82.1, 11000, 'CR', 'Q1542741'],
['Smallville', 8.2, -82.2, 2000, 'CR', 'Q1'],
];
assert.deepEqual(
withoutCities(rows, ['Q1542741']).map((row) => row[5]),
['Q1'],
);
});
test('one place listed twice nearby keeps its larger listing', () => {
const rows = [
['Orange', 41.278, -73.026, 14280, 'US', 'Q1'],
['Orange', 41.28, -73.02, 13950, 'US', 'Q2'],
// Across a border: a namesake, kept.
['Orange', 41.281, -73.03, 3000, 'CA', 'Q3'],
// Far away: another place.
['Orange', 33.79, -117.85, 2000, 'US', 'Q4'],
['Zábřeh', 49.88, 16.87, 13320, 'CZ', 'Q5'],
['Zabreh', 49.881, 16.871, 1200, 'CZ', 'Q6'],
];
assert.deepEqual(
withoutNearTwins(rows).map((row) => row[5]),
['Q1', 'Q3', 'Q4', 'Q5'],
);
});