mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-19 01:32:19 +00:00
Bundle smaller Wikidata settlements for nearby-place lookups
137,662 places of 1,000 to 14,999 people that Wikidata gives a population and coordinates for, none already in the cities pack, in 732 tiles of 5 x 5 degrees with an index of row counts (CC0). Nothing loads with the app: a lookup reads the index once and then only the tiles it touches. placesNear lists the places around a point, and findPlace finds a small place by name near the view when no city of that name is near. The cities build script writes the tiles with --tiles; the step from rows to files is checked offline. Co-authored-by: Sameh Khamis <[email protected]>
This commit is contained in:
1 parent
328b8f5d1b
commit
78305411eb
743 files changed
+2087
-14
No files matched your search
@@ -20,13 +20,29 @@
|
||||
* request runs at a time, with a pause between requests, and a refusal's
|
||||
* Retry-After is honoured.
|
||||
*
|
||||
* With `--tiles`, the same query between `--min-population` and
|
||||
* `--max-population` (1,000 and 15,000 by default) builds the detail tiles
|
||||
* instead, and writes only to:
|
||||
*
|
||||
* src/data/local_data/wikidata_places/
|
||||
*
|
||||
* replacing that folder's tiles, `index.json` and `files.js` (see
|
||||
* scripts/wikidataPlaceTiles.mjs). The checks that well-known cities are
|
||||
* present apply to the cities pack only.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/build-wikidata-cities.mjs [--min-population 15000]
|
||||
* node scripts/build-wikidata-cities.mjs --tiles [--min-population 1000] [--max-population 15000]
|
||||
*/
|
||||
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
placeTileFiles,
|
||||
withoutCities,
|
||||
withoutNearTwins,
|
||||
} from './wikidataPlaceTiles.mjs';
|
||||
|
||||
const ENDPOINT = 'https://query.wikidata.org/sparql';
|
||||
const USER_AGENT =
|
||||
@@ -38,9 +54,20 @@ const LABEL_BATCH = 400;
|
||||
const args = process.argv.slice(2);
|
||||
const flag = (name) =>
|
||||
args.includes(name) ? Number(args[args.indexOf(name) + 1]) : NaN;
|
||||
const minPopulation = flag('--min-population') || 15000;
|
||||
const tilesMode = args.includes('--tiles');
|
||||
const minPopulation = flag('--min-population') || (tilesMode ? 1000 : 15000);
|
||||
const maxPopulation = tilesMode
|
||||
? flag('--max-population') || 15000
|
||||
: flag('--max-population') || Infinity;
|
||||
if (!(maxPopulation > minPopulation))
|
||||
throw new Error('--max-population must be above --min-population');
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const outDir = path.join(root, 'src/data/local_data/wikidata_cities');
|
||||
const outDir = path.join(
|
||||
root,
|
||||
tilesMode
|
||||
? 'src/data/local_data/wikidata_places'
|
||||
: 'src/data/local_data/wikidata_cities',
|
||||
);
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1);
|
||||
@@ -222,7 +249,7 @@ const grouped = new Set();
|
||||
const settled = new Set();
|
||||
|
||||
const places = new Map();
|
||||
for (const [low, high] of bands(minPopulation, Infinity)) {
|
||||
for (const [low, high] of bands(minPopulation, maxPopulation)) {
|
||||
console.log(`Population ${low}–${Number.isFinite(high) ? high : '∞'}…`);
|
||||
const rows = await bandRows(low, high);
|
||||
let kept = 0;
|
||||
@@ -434,7 +461,7 @@ if (intruders.length) {
|
||||
`Groupings got in: ${intruders.map(([id, name]) => `${name} (${id})`).join(', ')}`,
|
||||
);
|
||||
}
|
||||
if (missing.length) {
|
||||
if (missing.length && !tilesMode) {
|
||||
throw new Error(
|
||||
`Expected cities are missing: ${missing.map(([id, name]) => `${name} (${id})`).join(', ')}`,
|
||||
);
|
||||
@@ -450,6 +477,40 @@ const meta = {
|
||||
};
|
||||
|
||||
await mkdir(outDir, { recursive: true });
|
||||
if (tilesMode) {
|
||||
// Places the cities pack already has stay out of the tiles: an item with
|
||||
// one stated population under the cut-off and another over it is in both
|
||||
// query ranges. Build the cities pack first.
|
||||
const citiesFile = path.join(
|
||||
root,
|
||||
'src/data/local_data/wikidata_cities/cities.json',
|
||||
);
|
||||
const cityIds = JSON.parse(await readFile(citiesFile, 'utf8')).rows.map(
|
||||
(row) => row[5],
|
||||
);
|
||||
const outsideCities = withoutCities(rows, cityIds);
|
||||
console.log(
|
||||
`${rows.length - outsideCities.length} places left to the cities pack`,
|
||||
);
|
||||
// One place listed twice nearby under one name keeps its larger listing.
|
||||
const tileRows = withoutNearTwins(outsideCities);
|
||||
console.log(`${outsideCities.length - tileRows.length} near twins left out`);
|
||||
const files = placeTileFiles(tileRows, {
|
||||
...meta,
|
||||
count: tileRows.length,
|
||||
maxPopulation,
|
||||
});
|
||||
// Tiles from an earlier build that this one has no rows for go.
|
||||
for (const name of await readdir(outDir))
|
||||
if (/^-?\d+_-?\d+\.json$/.test(name) && !files.has(name))
|
||||
await rm(path.join(outDir, name));
|
||||
for (const [name, text] of files)
|
||||
await writeFile(path.join(outDir, name), text);
|
||||
console.log(
|
||||
`Wrote ${tileRows.length} places in ${files.size - 2} tiles to ${path.relative(root, outDir)}`,
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
const file = path.join(outDir, 'cities.json');
|
||||
await writeFile(file, `${JSON.stringify({ meta, rows })}\n`);
|
||||
console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`);
|
||||
@@ -1584,5 +1584,7 @@
|
||||
"src/data/wikidataCities.js",
|
||||
"src/data/wikidataCities.test.mjs",
|
||||
"scripts/build-wikidata-cities.mjs",
|
||||
"src/search/placeSuggestions.test.mjs"
|
||||
"src/search/placeSuggestions.test.mjs",
|
||||
"scripts/wikidataPlaceTiles.mjs",
|
||||
"src/tooling/wikidataPlaceTiles.test.mjs"
|
||||
]
|
||||
@@ -287,7 +287,9 @@
|
||||
"src/data/vesselTypes.js",
|
||||
"src/data/fleetModels.js",
|
||||
"src/render/modelParts.js",
|
||||
"src/ui/views/panels/AircraftModelHero.vue"
|
||||
"src/ui/views/panels/AircraftModelHero.vue",
|
||||
"src/data/retryableLoad.js",
|
||||
"src/data/local_data/wikidata_places/files.js"
|
||||
],
|
||||
"external": ["vue", "pinia", "vue-i18n"]
|
||||
},
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* The files of the Wikidata places detail pack
|
||||
* (`src/data/local_data/wikidata_places/`), from its rows. Used by
|
||||
* `scripts/build-wikidata-cities.mjs --tiles`; kept apart from the query so
|
||||
* the tiling can be checked offline against the shipped pack.
|
||||
*
|
||||
* Rows are `[name, lat, lon, population, country, wikidata]`. Each goes into
|
||||
* the 5 x 5 degree tile named by its south-west corner (`30_-100` covers 30
|
||||
* to 35 N, 100 to 95 W). Rows keep the order they are given in, largest
|
||||
* first; tiles without a row get no file.
|
||||
*/
|
||||
|
||||
import { foldName } from '../src/data/wikidataCities.js';
|
||||
import { greatCircleKm } from '../src/geoDistance.js';
|
||||
|
||||
export const TILE_DEGREES = 5;
|
||||
/** How near two rows of one name in one country are taken as one, in km. */
|
||||
export const TWIN_KM = 3;
|
||||
|
||||
/**
|
||||
* The rows the cities pack does not already hold. An item with several
|
||||
* stated populations can fall in both packs (one figure under the cut-off,
|
||||
* another over it); the cities pack keeps it, so the detail tiles never name
|
||||
* a place twice.
|
||||
* @param {Array<Array>} rows Detail rows, `[name, lat, lon, population, country, wikidata]`.
|
||||
* @param {Iterable<string>} cityIds Wikidata ids in the cities pack.
|
||||
*/
|
||||
export function withoutCities(rows, cityIds) {
|
||||
const cities = new Set(cityIds);
|
||||
return rows.filter((row) => !cities.has(row[5]));
|
||||
}
|
||||
|
||||
/** The tile a point falls in, named by its south-west corner. */
|
||||
export function tileKey(lat, lon) {
|
||||
const corner = (value) => Math.floor(value / TILE_DEGREES) * TILE_DEGREES;
|
||||
return `${corner(lat)}_${corner(lon)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every file of the pack, by file name.
|
||||
* @param {Array<Array>} rows Largest first.
|
||||
* @param {object} meta The build's `source`, `license`, `built`,
|
||||
* `minPopulation`, `maxPopulation` and `columns`.
|
||||
* @returns {Map<string, string>}
|
||||
*/
|
||||
export function placeTileFiles(rows, meta) {
|
||||
const tiles = new Map();
|
||||
for (const row of rows) {
|
||||
const key = tileKey(row[1], row[2]);
|
||||
if (!tiles.has(key)) tiles.set(key, []);
|
||||
tiles.get(key).push(row);
|
||||
}
|
||||
// In locale order, as the shipped pack was written.
|
||||
const keys = [...tiles.keys()].sort((a, b) => a.localeCompare(b, 'en'));
|
||||
const files = new Map();
|
||||
for (const key of keys)
|
||||
files.set(`${key}.json`, `${JSON.stringify({ rows: tiles.get(key) })}\n`);
|
||||
const index = {
|
||||
meta: {
|
||||
source: meta.source,
|
||||
license: meta.license,
|
||||
built: meta.built,
|
||||
minPopulation: meta.minPopulation,
|
||||
maxPopulation: meta.maxPopulation,
|
||||
count: rows.length,
|
||||
columns: meta.columns,
|
||||
tileDegrees: TILE_DEGREES,
|
||||
tiles: keys.length,
|
||||
},
|
||||
tiles: Object.fromEntries(keys.map((key) => [key, tiles.get(key).length])),
|
||||
};
|
||||
files.set('index.json', `${JSON.stringify(index)}\n`);
|
||||
files.set(
|
||||
'files.js',
|
||||
[
|
||||
'// Generated by scripts/build-wikidata-cities.mjs --tiles; do not edit.',
|
||||
"export const INDEX_URL = () => new URL('./index.json', import.meta.url);",
|
||||
'export const TILE_URLS = {',
|
||||
...keys.map(
|
||||
(key) => ` '${key}': () => new URL('./${key}.json', import.meta.url),`,
|
||||
),
|
||||
'};',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
return files;
|
||||
}
|
||||
|
||||
const kmBetween = (a, b) =>
|
||||
greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] });
|
||||
|
||||
/**
|
||||
* The rows without a second listing of one place: a row whose name (folded,
|
||||
* as lookups compare names) and country match a row already kept within
|
||||
* TWIN_KM is left out, so the larger of the two, given first, stays.
|
||||
* Namesakes across a border both stay.
|
||||
* @param {Array<Array>} rows Largest first.
|
||||
*/
|
||||
export function withoutNearTwins(rows) {
|
||||
// Cells of 0.05 degrees (about 5 km), searched with their neighbours.
|
||||
const cell = (lat, lon) => `${Math.floor(lat * 20)}_${Math.floor(lon * 20)}`;
|
||||
const kept = new Map();
|
||||
return rows.filter((row) => {
|
||||
const name = foldName(row[0]);
|
||||
const [la, lo] = cell(row[1], row[2]).split('_').map(Number);
|
||||
for (let i = -1; i <= 1; i++)
|
||||
for (let j = -1; j <= 1; j++)
|
||||
for (const other of kept.get(`${la + i}_${lo + j}`) || [])
|
||||
if (
|
||||
other.name === name &&
|
||||
other.row[4] === row[4] &&
|
||||
kmBetween(other.row, row) <= TWIN_KM
|
||||
)
|
||||
return false;
|
||||
const key = `${la}_${lo}`;
|
||||
if (!kept.has(key)) kept.set(key, []);
|
||||
kept.get(key).push({ name, row });
|
||||
return true;
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user