Files
gods-eye-view/scripts/wikidataPlaceTiles.mjs
T
Sameh KhamisandManjunath Paparaju 88919f760f Keep a lookup's places when one tile fails, and ship every tile as its own file
- A tile that cannot be read is left out of placesNear's answer instead of
  failing it; without the pack's index the cities are still the answer.
- The build no longer inlines the small tiles into one chunk that loaded
  with the first lookup: all 732 are their own files.
- A row at latitude 90 or longitude 180 is tiled where the lookups read it.
- --min-population 0 is taken as 0, not as the default.

Co-authored-by: Manjunath Paparaju <[email protected]>
2026-11-18 12:36:16 -08:00

124 lines
4.5 KiB
JavaScript

/**
* The files of the Wikidata places detail pack
* (`src/data/local_data/wikidata_places/`), from its rows. Used by
* `scripts/build-wikidata-cities.mjs --tiles`; kept apart from the query so
* the tiling can be checked offline against the shipped pack.
*
* Rows are `[name, lat, lon, population, country, wikidata]`. Each goes into
* the 5 x 5 degree tile named by its south-west corner (`30_-100` covers 30
* to 35 N, 100 to 95 W). Rows keep the order they are given in, largest
* first; tiles without a row get no file.
*/
import { foldName } from '../src/data/wikidataCities.js';
import { greatCircleKm } from '../src/geoDistance.js';
export const TILE_DEGREES = 5;
/** How near two rows of one name in one country are taken as one, in km. */
export const TWIN_KM = 3;
/**
* The rows the cities pack does not already hold. An item with several
* stated populations can fall in both packs (one figure under the cut-off,
* another over it); the cities pack keeps it, so the detail tiles never name
* a place twice.
* @param {Array<Array>} rows Detail rows, `[name, lat, lon, population, country, wikidata]`.
* @param {Iterable<string>} cityIds Wikidata ids in the cities pack.
*/
export function withoutCities(rows, cityIds) {
const cities = new Set(cityIds);
return rows.filter((row) => !cities.has(row[5]));
}
/** The tile a point falls in, named by its south-west corner. */
export function tileKey(lat, lon) {
const corner = (value) => Math.floor(value / TILE_DEGREES) * TILE_DEGREES;
// As the lookups read them: latitude 90 is in the top row of tiles, and
// longitude 180 is longitude -180.
const top = 90 - TILE_DEGREES;
return `${Math.min(top, corner(lat))}_${corner((((lon % 360) + 540) % 360) - 180)}`;
}
/**
* Every file of the pack, by file name.
* @param {Array<Array>} rows Largest first.
* @param {object} meta The build's `source`, `license`, `built`,
* `minPopulation`, `maxPopulation` and `columns`.
* @returns {Map<string, string>}
*/
export function placeTileFiles(rows, meta) {
const tiles = new Map();
for (const row of rows) {
const key = tileKey(row[1], row[2]);
if (!tiles.has(key)) tiles.set(key, []);
tiles.get(key).push(row);
}
// In locale order, as the shipped pack was written.
const keys = [...tiles.keys()].sort((a, b) => a.localeCompare(b, 'en'));
const files = new Map();
for (const key of keys)
files.set(`${key}.json`, `${JSON.stringify({ rows: tiles.get(key) })}\n`);
const index = {
meta: {
source: meta.source,
license: meta.license,
built: meta.built,
minPopulation: meta.minPopulation,
maxPopulation: meta.maxPopulation,
count: rows.length,
columns: meta.columns,
tileDegrees: TILE_DEGREES,
tiles: keys.length,
},
tiles: Object.fromEntries(keys.map((key) => [key, tiles.get(key).length])),
};
files.set('index.json', `${JSON.stringify(index)}\n`);
files.set(
'files.js',
[
'// Generated by scripts/build-wikidata-cities.mjs --tiles; do not edit.',
"export const INDEX_URL = () => new URL('./index.json', import.meta.url);",
'export const TILE_URLS = {',
...keys.map(
(key) => ` '${key}': () => new URL('./${key}.json', import.meta.url),`,
),
'};',
'',
].join('\n'),
);
return files;
}
const kmBetween = (a, b) =>
greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] });
/**
* The rows without a second listing of one place: a row whose name (folded,
* as lookups compare names) and country match a row already kept within
* TWIN_KM is left out, so the larger of the two, given first, stays.
* Namesakes across a border both stay.
* @param {Array<Array>} rows Largest first.
*/
export function withoutNearTwins(rows) {
// Cells of 0.05 degrees (about 5 km), searched with their neighbours.
const cell = (lat, lon) => `${Math.floor(lat * 20)}_${Math.floor(lon * 20)}`;
const kept = new Map();
return rows.filter((row) => {
const name = foldName(row[0]);
const [la, lo] = cell(row[1], row[2]).split('_').map(Number);
for (let i = -1; i <= 1; i++)
for (let j = -1; j <= 1; j++)
for (const other of kept.get(`${la + i}_${lo + j}`) || [])
if (
other.name === name &&
other.row[4] === row[4] &&
kmBetween(other.row, row) <= TWIN_KM
)
return false;
const key = `${la}_${lo}`;
if (!kept.has(key)) kept.set(key, []);
kept.get(key).push({ name, row });
return true;
});
}