mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-19 09:42:21 +00:00
Bundle smaller Wikidata settlements for nearby-place lookups
137,662 places of 1,000 to 14,999 people that Wikidata gives a population and coordinates for, none already in the cities pack, in 732 tiles of 5 x 5 degrees with an index of row counts (CC0). Nothing loads with the app: a lookup reads the index once and then only the tiles it touches. placesNear lists the places around a point, and findPlace finds a small place by name near the view when no city of that name is near. The cities build script writes the tiles with --tiles; the step from rows to files is checked offline. Co-authored-by: Sameh Khamis <[email protected]>
This commit is contained in:
1 parent
328b8f5d1b
commit
78305411eb
743 files changed
+2087
-14
No files matched your search
@@ -0,0 +1,88 @@
|
||||
// The Wikidata places pack is written by scripts/wikidataPlaceTiles.mjs. Its
|
||||
// rows come from a Wikidata query that cannot be repeated offline, so this
|
||||
// checks the steps after it: the shipped rows, sorted and filtered as the
|
||||
// build does, give back every shipped file byte for byte, and hash to the
|
||||
// value the pack's README records.
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readdir, readFile } from 'node:fs/promises';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
placeTileFiles,
|
||||
tileKey,
|
||||
withoutCities,
|
||||
withoutNearTwins,
|
||||
} from '../../scripts/wikidataPlaceTiles.mjs';
|
||||
|
||||
const PACK = new URL('../data/local_data/wikidata_places/', import.meta.url);
|
||||
const CITIES = new URL(
|
||||
'../data/local_data/wikidata_cities/cities.json',
|
||||
import.meta.url,
|
||||
);
|
||||
|
||||
test('a tile is named by its south-west corner', () => {
|
||||
assert.equal(tileKey(30.343, -97.557), '30_-100');
|
||||
assert.equal(tileKey(-0.5, 0), '-5_0');
|
||||
assert.equal(tileKey(10.682, -71.036), '10_-75');
|
||||
});
|
||||
|
||||
test('re-tiles the shipped rows deterministically', async () => {
|
||||
const index = JSON.parse(await readFile(new URL('index.json', PACK), 'utf8'));
|
||||
const rows = [];
|
||||
for (const key of Object.keys(index.tiles)) {
|
||||
const tile = JSON.parse(
|
||||
await readFile(new URL(`${key}.json`, PACK), 'utf8'),
|
||||
);
|
||||
rows.push(...tile.rows);
|
||||
}
|
||||
assert.equal(rows.length, 137662);
|
||||
// As scripts/build-wikidata-cities.mjs orders and filters its rows.
|
||||
rows.sort((a, b) => b[3] - a[3] || a[5].localeCompare(b[5]));
|
||||
const cityIds = JSON.parse(await readFile(CITIES, 'utf8')).rows.map(
|
||||
(row) => row[5],
|
||||
);
|
||||
const kept = withoutNearTwins(withoutCities(rows, cityIds));
|
||||
assert.equal(
|
||||
kept.length,
|
||||
rows.length,
|
||||
'the shipped rows are already filtered',
|
||||
);
|
||||
const readme = await readFile(new URL('README.md', PACK), 'utf8');
|
||||
const digest = createHash('sha256')
|
||||
.update(JSON.stringify(kept))
|
||||
.digest('hex');
|
||||
assert.ok(readme.includes(`\`${digest}\``), 'README rows hash');
|
||||
const files = placeTileFiles(kept, index.meta);
|
||||
const shipped = (await readdir(PACK)).filter((name) => name !== 'README.md');
|
||||
assert.deepEqual([...files.keys()].sort(), shipped.sort());
|
||||
for (const [name, text] of files)
|
||||
assert.equal(text, await readFile(new URL(name, PACK), 'utf8'), name);
|
||||
});
|
||||
|
||||
test('a place the cities pack holds is left out of the tiles', () => {
|
||||
const rows = [
|
||||
['Palmares', 8.1, -82.1, 11000, 'CR', 'Q1542741'],
|
||||
['Smallville', 8.2, -82.2, 2000, 'CR', 'Q1'],
|
||||
];
|
||||
assert.deepEqual(
|
||||
withoutCities(rows, ['Q1542741']).map((row) => row[5]),
|
||||
['Q1'],
|
||||
);
|
||||
});
|
||||
|
||||
test('one place listed twice nearby keeps its larger listing', () => {
|
||||
const rows = [
|
||||
['Orange', 41.278, -73.026, 14280, 'US', 'Q1'],
|
||||
['Orange', 41.28, -73.02, 13950, 'US', 'Q2'],
|
||||
// Across a border: a namesake, kept.
|
||||
['Orange', 41.281, -73.03, 3000, 'CA', 'Q3'],
|
||||
// Far away: another place.
|
||||
['Orange', 33.79, -117.85, 2000, 'US', 'Q4'],
|
||||
['Zábřeh', 49.88, 16.87, 13320, 'CZ', 'Q5'],
|
||||
['Zabreh', 49.881, 16.871, 1200, 'CZ', 'Q6'],
|
||||
];
|
||||
assert.deepEqual(
|
||||
withoutNearTwins(rows).map((row) => row[5]),
|
||||
['Q1', 'Q3', 'Q4', 'Q5'],
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user