Files
gods-eye-view/src/tooling/wikidataPlaceTiles.test.mjs
T

146 lines
5.4 KiB
JavaScript

// The Wikidata places pack is written by scripts/wikidataPlaceTiles.mjs. Its
// rows come from a Wikidata query that cannot be repeated offline, so this
// checks the steps after it: the shipped rows, sorted and filtered as the
// build does, give back every shipped file byte for byte, and hash to the
// value the pack's README records.
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { readdir, readFile } from 'node:fs/promises';
import test from 'node:test';
import {
keepBuiltIfUnchanged,
keepPackBuiltIfUnchanged,
placeTileFiles,
tileKey,
withoutCities,
withoutNearTwins,
} from '../../scripts/wikidataPlaceTiles.mjs';
const PACK = new URL('../data/local_data/wikidata_places/', import.meta.url);
const CITIES = new URL(
'../data/local_data/wikidata_cities/cities.json',
import.meta.url,
);
test('a tile is named by its south-west corner', () => {
assert.equal(tileKey(30.343, -97.557), '30_-100');
assert.equal(tileKey(-0.5, 0), '-5_0');
assert.equal(tileKey(10.682, -71.036), '10_-75');
// In the tiles the lookups read: the top row, and -180 for 180.
assert.equal(tileKey(90, 180), '85_-180');
});
test('re-tiles the shipped rows deterministically', async () => {
const index = JSON.parse(await readFile(new URL('index.json', PACK), 'utf8'));
const rows = [];
for (const key of Object.keys(index.tiles)) {
const tile = JSON.parse(
await readFile(new URL(`${key}.json`, PACK), 'utf8'),
);
rows.push(...tile.rows);
}
assert.equal(rows.length, 137662);
// As scripts/build-wikidata-cities.mjs orders and filters its rows.
rows.sort((a, b) => b[3] - a[3] || a[5].localeCompare(b[5]));
const cityIds = JSON.parse(await readFile(CITIES, 'utf8')).rows.map(
(row) => row[5],
);
const kept = withoutNearTwins(withoutCities(rows, cityIds));
assert.equal(
kept.length,
rows.length,
'the shipped rows are already filtered',
);
const readme = await readFile(new URL('README.md', PACK), 'utf8');
const digest = createHash('sha256')
.update(JSON.stringify(kept))
.digest('hex');
assert.ok(readme.includes(`\`${digest}\``), 'README rows hash');
const files = placeTileFiles(kept, index.meta);
const shipped = (await readdir(PACK)).filter((name) => name !== 'README.md');
assert.deepEqual([...files.keys()].sort(), shipped.sort());
for (const [name, text] of files)
assert.equal(text, await readFile(new URL(name, PACK), 'utf8'), name);
});
test('a place the cities pack holds is left out of the tiles', () => {
const rows = [
['Palmares', 8.1, -82.1, 11000, 'CR', 'Q1542741'],
['Smallville', 8.2, -82.2, 2000, 'CR', 'Q1'],
];
assert.deepEqual(
withoutCities(rows, ['Q1542741']).map((row) => row[5]),
['Q1'],
);
});
test('one place listed twice nearby keeps its larger listing', () => {
const rows = [
['Orange', 41.278, -73.026, 14280, 'US', 'Q1'],
['Orange', 41.28, -73.02, 13950, 'US', 'Q2'],
// Across a border: a namesake, kept.
['Orange', 41.281, -73.03, 3000, 'CA', 'Q3'],
// Far away: another place.
['Orange', 33.79, -117.85, 2000, 'US', 'Q4'],
['Zábřeh', 49.88, 16.87, 13320, 'CZ', 'Q5'],
['Zabreh', 49.881, 16.871, 1200, 'CZ', 'Q6'],
];
assert.deepEqual(
withoutNearTwins(rows).map((row) => row[5]),
['Q1', 'Q3', 'Q4', 'Q5'],
);
});
test('twins are found where longitude cells narrow, and across 180°', () => {
const rows = [
// 1.98 km apart at 70° N, two longitude cells apart.
['Nord', 70, 10.049, 9000, 'NO', 'Q1'],
['Nord', 70, 10.101, 2000, 'NO', 'Q2'],
// Either side of the antimeridian, under 2 km apart.
['Kai', -16.5, 179.995, 5000, 'FJ', 'Q3'],
['Kai', -16.5, -179.995, 1000, 'FJ', 'Q4'],
];
assert.deepEqual(
withoutNearTwins(rows).map((row) => row[5]),
['Q1', 'Q3'],
);
});
test('a rebuild of unchanged rows keeps the earlier build stamp', () => {
const earlier = `${JSON.stringify({ meta: { built: '2026-01-01T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`;
const same = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`;
const changed = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 3 }, rows: [1, 2, 3] })}\n`;
assert.equal(keepBuiltIfUnchanged(same, earlier), earlier);
assert.equal(keepBuiltIfUnchanged(changed, earlier), changed);
assert.equal(keepBuiltIfUnchanged(same, undefined), same);
assert.equal(keepBuiltIfUnchanged(same, 'not json'), same);
});
test('the places index keeps its build stamp only when no tile changed', () => {
const rows = [
['Aa', 31, -98, 2000, 'US', 'Q1'],
['Bb', 41, -88, 1500, 'US', 'Q2'],
];
const meta = (built) => ({
source: 's',
license: 'l',
built,
minPopulation: 1000,
maxPopulation: 15000,
columns: [],
});
const earlier = placeTileFiles(rows, meta('2026-01-01T00:00:00.000Z'));
const built = (files) => JSON.parse(files.get('index.json')).meta.built;
const rebuild = (newRows) =>
keepPackBuiltIfUnchanged(
placeTileFiles(newRows, meta('2026-11-18T00:00:00.000Z')),
earlier,
);
assert.equal(built(rebuild(rows)), '2026-01-01T00:00:00.000Z');
// Same row count in every tile, new population: a new stamp.
const changed = [['Aa', 31, -98, 2100, 'US', 'Q1'], rows[1]];
assert.equal(built(rebuild(changed)), '2026-11-18T00:00:00.000Z');
// A tile of the earlier build left out: a new stamp.
assert.equal(built(rebuild([rows[0]])), '2026-11-18T00:00:00.000Z');
});