/** * The files of the Wikidata places detail pack * (`src/data/local_data/wikidata_places/`), from its rows. Used by * `scripts/build-wikidata-cities.mjs --tiles`; kept apart from the query so * the tiling can be checked offline against the shipped pack. * * Rows are `[name, lat, lon, population, country, wikidata]`. Each goes into * the 5 x 5 degree tile named by its south-west corner (`30_-100` covers 30 * to 35 N, 100 to 95 W). Rows keep the order they are given in, largest * first; tiles without a row get no file. */ import { foldName } from '../src/data/wikidataCities.js'; import { greatCircleKm } from '../src/geoDistance.js'; export const TILE_DEGREES = 5; /** How near two rows of one name in one country are taken as one, in km. */ export const TWIN_KM = 3; /** * The rows the cities pack does not already hold. An item with several * stated populations can fall in both packs (one figure under the cut-off, * another over it); the cities pack keeps it, so the detail tiles never name * a place twice. * @param {Array} rows Detail rows, `[name, lat, lon, population, country, wikidata]`. * @param {Iterable} cityIds Wikidata ids in the cities pack. */ export function withoutCities(rows, cityIds) { const cities = new Set(cityIds); return rows.filter((row) => !cities.has(row[5])); } /** The tile a point falls in, named by its south-west corner. */ export function tileKey(lat, lon) { const corner = (value) => Math.floor(value / TILE_DEGREES) * TILE_DEGREES; // As the lookups read them: latitude 90 is in the top row of tiles, and // longitude 180 is longitude -180. const top = 90 - TILE_DEGREES; return `${Math.min(top, corner(lat))}_${corner((((lon % 360) + 540) % 360) - 180)}`; } /** * Every file of the pack, by file name. * @param {Array} rows Largest first. * @param {object} meta The build's `source`, `license`, `built`, * `minPopulation`, `maxPopulation` and `columns`. * @returns {Map} */ export function placeTileFiles(rows, meta) { const tiles = new Map(); for (const row of rows) { const key = tileKey(row[1], row[2]); if (!tiles.has(key)) tiles.set(key, []); tiles.get(key).push(row); } // In locale order, as the shipped pack was written. const keys = [...tiles.keys()].sort((a, b) => a.localeCompare(b, 'en')); const files = new Map(); for (const key of keys) files.set(`${key}.json`, `${JSON.stringify({ rows: tiles.get(key) })}\n`); const index = { meta: { source: meta.source, license: meta.license, built: meta.built, minPopulation: meta.minPopulation, maxPopulation: meta.maxPopulation, count: rows.length, columns: meta.columns, tileDegrees: TILE_DEGREES, tiles: keys.length, }, tiles: Object.fromEntries(keys.map((key) => [key, tiles.get(key).length])), }; files.set('index.json', `${JSON.stringify(index)}\n`); files.set( 'files.js', [ '// Generated by scripts/build-wikidata-cities.mjs --tiles; do not edit.', "export const INDEX_URL = () => new URL('./index.json', import.meta.url);", 'export const TILE_URLS = {', ...keys.map( (key) => ` '${key}': () => new URL('./${key}.json', import.meta.url),`, ), '};', '', ].join('\n'), ); return files; } /** * A pack file's new text, keeping the earlier build's `meta.built` when * nothing else in the file changed, so rebuilding unchanged data leaves the * file as it was. A file that does not hold all of its pack's data (the * places index) is passed through this only when the rest of the pack is * unchanged too. * @param {string} text The new file, JSON with a `meta.built` stamp. * @param {string | undefined} earlier The file as it is now, if there is one. */ export function keepBuiltIfUnchanged(text, earlier) { if (!earlier) return text; let before; try { before = JSON.parse(earlier); } catch { return text; } const after = JSON.parse(text); if (typeof before?.meta?.built !== 'string' || !after.meta) return text; const unstamped = (value) => JSON.stringify({ ...value, meta: { ...value.meta, built: null } }); if (unstamped(before) !== unstamped(after)) return text; return `${JSON.stringify({ ...after, meta: { ...after.meta, built: before.meta.built } })}\n`; } const kmBetween = (a, b) => greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] }); /** * The rows without a second listing of one place: a row whose name (folded, * as lookups compare names) and country match a row already kept within * TWIN_KM is left out, so the larger of the two, given first, stays. * Namesakes across a border both stay. * @param {Array} rows Largest first. */ export function withoutNearTwins(rows) { // Cells of 0.05 degrees (about 5 km of latitude), searched with their // neighbours. A cell narrows toward the poles, so as many longitude cells // are searched as TWIN_KM spans there, wrapping at the antimeridian. const CELLS = 20; const AROUND = 360 * CELLS; const kept = new Map(); return rows.filter((row) => { const name = foldName(row[0]); const la = Math.floor(row[1] * CELLS); const lo = Math.floor(row[2] * CELLS); const cos = Math.cos( (Math.min(89.9, Math.abs(row[1]) + 0.1) * Math.PI) / 180, ); const span = Math.min( AROUND / 2, Math.ceil((TWIN_KM / (111.32 * cos)) * CELLS) + 1, ); for (let i = -1; i <= 1; i++) for (let j = -span; j <= span; j++) for (const other of kept.get( `${la + i}_${(((lo + j) % AROUND) + AROUND) % AROUND}`, ) || []) if ( other.name === name && other.row[4] === row[4] && kmBetween(other.row, row) <= TWIN_KM ) return false; const key = `${la}_${((lo % AROUND) + AROUND) % AROUND}`; if (!kept.has(key)) kept.set(key, []); kept.get(key).push({ name, row }); return true; }); }