mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-15 07:42:23 +00:00
Bundle Wikidata cities for offline place lookups
A pack of the world's settlements of 15,000 people or more (31,827 places,
1.6 MB, CC0) built from the Wikidata Query Service by
scripts/build-wikidata-cities.mjs. src/data/wikidataCities.js reads it on the
first lookup and answers the nearest city to a point, the cities around it,
a city by name ("Paris, France", or the nearest namesake in view) and a
country's name in the page language. Names match without accents or case.
The build leaves out groupings of settlements (metropolitan areas,
agglomerations) and places that no longer stand, keeps a city that is also a
narrower grouping kind (Istanbul) or a unit dissolved where nothing else
names the spot (the Vietnamese provincial cities of 2025), fills a country
from the area a place lies in, splits a population band the service cannot
answer in time, and fails if well-known cities are missing or a known
grouping gets in. The app never calls Wikidata itself.
Co-authored-by: Sameh Khamis <[email protected]>
This commit is contained in:
Manjunath PaparajuandSameh Khamis authored and Sameh Khamis committed
1 parent
c2e42b5dfd
commit
1dc5a67518
8 files changed
+891
-1
No files matched your search
@@ -0,0 +1,455 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the bundled world cities pack read by `src/data/wikidataCities.js`:
|
||||
*
|
||||
* src/data/local_data/wikidata_cities/cities.json
|
||||
*
|
||||
* Source: Wikidata (CC0), read once from the public query service at build
|
||||
* time. The app itself never calls Wikidata: it reads this file.
|
||||
*
|
||||
* What goes in: items that are a human settlement (an instance of any subclass
|
||||
* of Q486972), have coordinates (P625) and a population (P1082) at or above
|
||||
* the cut-off. Each row keeps the English label, coordinates to three
|
||||
* decimals, the largest stated population, the ISO 3166-1 alpha-2 country
|
||||
* code and the Wikidata id. Groupings of settlements (metropolitan areas,
|
||||
* agglomerations, conurbations) and places that no longer stand are left out.
|
||||
*
|
||||
* The query service limits each query to 60 s, so the population range is
|
||||
* read in bands without labels, the settlement test runs here against the
|
||||
* class list, and labels and countries are read afterwards in batches. One
|
||||
* request runs at a time, with a pause between requests, and a refusal's
|
||||
* Retry-After is honoured.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/build-wikidata-cities.mjs [--min-population 15000]
|
||||
*/
|
||||
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ENDPOINT = 'https://query.wikidata.org/sparql';
|
||||
const USER_AGENT =
|
||||
'gods-eye-view-places-builder/0.2 (offline data pack; one-off build; https://github.com/bilawalsidhu/gods-eye-view)';
|
||||
const SETTLEMENT = 'Q486972';
|
||||
const PAUSE_MS = 1500;
|
||||
const LABEL_BATCH = 400;
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const flag = (name) =>
|
||||
args.includes(name) ? Number(args[args.indexOf(name) + 1]) : NaN;
|
||||
const minPopulation = flag('--min-population') || 15000;
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const outDir = path.join(root, 'src/data/local_data/wikidata_cities');
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1);
|
||||
|
||||
/** The service ran out of time on a query: a smaller one may still answer. */
|
||||
class QueryTimeout extends Error {}
|
||||
|
||||
async function sparql(query, attempt = 1) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'User-Agent': USER_AGENT,
|
||||
Accept: 'application/sparql-results+json',
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
},
|
||||
body: `query=${encodeURIComponent(query)}`,
|
||||
// The service stops a query at 60 s; past that the answer is lost.
|
||||
signal: AbortSignal.timeout(90000),
|
||||
});
|
||||
} catch (error) {
|
||||
// Our own time limit: the query is too big, as with a 504.
|
||||
if (error.name === 'TimeoutError' && attempt >= 2)
|
||||
throw new QueryTimeout('Wikidata did not answer in time');
|
||||
if (attempt >= 4) throw error;
|
||||
console.warn(` ${error.message}; waiting ${20 * attempt} s`);
|
||||
await sleep(20000 * attempt);
|
||||
return sparql(query, attempt + 1);
|
||||
}
|
||||
if (response.status === 504 && attempt >= 2)
|
||||
throw new QueryTimeout('Wikidata answered 504');
|
||||
// A malformed query fails the same way every time.
|
||||
if (
|
||||
response.status >= 400 &&
|
||||
response.status < 500 &&
|
||||
response.status !== 429
|
||||
)
|
||||
throw new Error(
|
||||
`Wikidata answered ${response.status}: ${(await response.text()).slice(0, 300)}`,
|
||||
);
|
||||
if (!response.ok) {
|
||||
if (attempt >= 4) throw new Error(`Wikidata answered ${response.status}`);
|
||||
const wait =
|
||||
Number(response.headers.get('retry-after')) * 1000 || 20000 * attempt;
|
||||
console.warn(` ${response.status}; waiting ${Math.round(wait / 1000)} s`);
|
||||
await sleep(wait);
|
||||
return sparql(query, attempt + 1);
|
||||
}
|
||||
// The service can stop a long answer partway; that arrives as broken JSON.
|
||||
let rows;
|
||||
try {
|
||||
rows = JSON.parse(await response.text()).results.bindings;
|
||||
} catch (error) {
|
||||
// The service stops a query that runs too long partway through its answer.
|
||||
if (attempt >= 2)
|
||||
throw new QueryTimeout(`Wikidata answer was cut short: ${error.message}`);
|
||||
console.warn(` answer cut short; waiting ${20 * attempt} s`);
|
||||
await sleep(20000 * attempt);
|
||||
return sparql(query, attempt + 1);
|
||||
}
|
||||
await sleep(PAUSE_MS);
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Population bands, narrow where places are many, so each answer stays well
|
||||
* under the service's 60 s limit (about 35,000 places a band).
|
||||
*/
|
||||
function bands(min, max) {
|
||||
const stepAt = (pop) =>
|
||||
pop < 2000
|
||||
? 200
|
||||
: pop < 5000
|
||||
? 500
|
||||
: pop < 10000
|
||||
? 1000
|
||||
: pop < 20000
|
||||
? 2500
|
||||
: pop < 30000
|
||||
? 10000
|
||||
: pop < 50000
|
||||
? 20000
|
||||
: pop < 100000
|
||||
? 50000
|
||||
: Infinity;
|
||||
const list = [];
|
||||
for (let low = min; low < max;) {
|
||||
const high = Math.min(max, low + stepAt(low));
|
||||
list.push([low, high]);
|
||||
low = high;
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
console.log('Settlement classes…');
|
||||
const classes = new Set(
|
||||
(await sparql(`SELECT ?c WHERE { ?c wdt:P279* wd:${SETTLEMENT} }`)).map(
|
||||
(row) => qid(row.c.value),
|
||||
),
|
||||
);
|
||||
console.log(` ${classes.size} classes`);
|
||||
|
||||
// Groupings of settlements (metropolitan areas, agglomerations, conurbations,
|
||||
// capital regions)
|
||||
// are settlements in Wikidata too, and are left out so a view is named after
|
||||
// the city rather than the area around it. An item stated as one of these
|
||||
// classes goes, whatever else it is ("New York metropolitan area" is also a
|
||||
// megacity). An item of a narrower kind only goes when it is nothing else:
|
||||
// Istanbul is a "metropolitan municipality in Turkey", a kind of
|
||||
// metropolitan area, and a megacity, so it stays.
|
||||
const GROUPING_TREES = [
|
||||
'Q1907114', // metropolitan area
|
||||
'Q159313', // urban agglomeration
|
||||
'Q245260', // conurbation
|
||||
'Q1062177', // capital region
|
||||
'Q1768043', // metropolitan statistical area
|
||||
'Q3175539', // functional urban area
|
||||
];
|
||||
// Left out as stated only: "city in the United States" is a subclass of
|
||||
// these, so their subclasses must stay.
|
||||
const GROUPING_CLASSES = [
|
||||
'Q702492', // urban area
|
||||
'Q2342385', // urban area (statistical)
|
||||
'Q5951278', // urban area of the United States
|
||||
];
|
||||
// Places that no longer stand as settlements (former entities, ghost towns,
|
||||
// abandoned villages, archaeological sites) are settlements in Wikidata too.
|
||||
// An item typed only as one of these, with every subclass, is left out; one
|
||||
// that is also a living settlement (a city that is also a former capital)
|
||||
// stays.
|
||||
const HISTORICAL_TREES = [
|
||||
'Q15893266', // former entity
|
||||
'Q19953632', // former administrative territorial entity
|
||||
'Q74047', // ghost town
|
||||
'Q350895', // abandoned village
|
||||
'Q839954', // archaeological site
|
||||
];
|
||||
const former = new Set(
|
||||
(
|
||||
await sparql(
|
||||
`SELECT ?c WHERE { VALUES ?top { ${HISTORICAL_TREES.map((id) => `wd:${id}`).join(' ')} } ?c wdt:P279* ?top }`,
|
||||
)
|
||||
).map((row) => qid(row.c.value)),
|
||||
);
|
||||
console.log(` ${former.size} former-place classes`);
|
||||
const groupings = new Set([
|
||||
...(
|
||||
await sparql(
|
||||
`SELECT ?c WHERE { VALUES ?top { ${GROUPING_TREES.map((id) => `wd:${id}`).join(' ')} } ?c wdt:P279* ?top }`,
|
||||
)
|
||||
).map((row) => qid(row.c.value)),
|
||||
...GROUPING_CLASSES,
|
||||
]);
|
||||
console.log(` ${groupings.size} grouping classes left out`);
|
||||
// A band's rows; one the service cannot answer in time is asked again in two
|
||||
// halves.
|
||||
async function bandRows(low, high) {
|
||||
const range = Number.isFinite(high)
|
||||
? `?pop >= ${low} && ?pop < ${high}`
|
||||
: `?pop >= ${low}`;
|
||||
try {
|
||||
return await sparql(
|
||||
`SELECT ?item ?pop ?coord ?type WHERE { ?item wdt:P1082 ?pop . FILTER(${range}) ?item wdt:P625 ?coord ; wdt:P31 ?type . }`,
|
||||
);
|
||||
} catch (error) {
|
||||
const mid = Number.isFinite(high) ? Math.round((low + high) / 2) : low * 4;
|
||||
if (!(error instanceof QueryTimeout) || mid <= low || mid === high)
|
||||
throw error;
|
||||
console.warn(` splitting ${low}–${high} at ${mid}`);
|
||||
return [...(await bandRows(low, mid)), ...(await bandRows(mid, high))];
|
||||
}
|
||||
}
|
||||
|
||||
const GROUPING_STATED = new Set([...GROUPING_TREES, ...GROUPING_CLASSES]);
|
||||
const living = new Set();
|
||||
const stated = new Set();
|
||||
const grouped = new Set();
|
||||
const settled = new Set();
|
||||
|
||||
const places = new Map();
|
||||
for (const [low, high] of bands(minPopulation, Infinity)) {
|
||||
console.log(`Population ${low}–${Number.isFinite(high) ? high : '∞'}…`);
|
||||
const rows = await bandRows(low, high);
|
||||
let kept = 0;
|
||||
for (const row of rows) {
|
||||
const type = qid(row.type.value);
|
||||
if (GROUPING_STATED.has(type)) stated.add(qid(row.item.value));
|
||||
if (groupings.has(type)) grouped.add(qid(row.item.value));
|
||||
if (!classes.has(type)) continue;
|
||||
if (!groupings.has(type) && !former.has(type))
|
||||
settled.add(qid(row.item.value));
|
||||
if (!former.has(type)) living.add(qid(row.item.value));
|
||||
const match = /^Point\(([-\d.eE]+) ([-\d.eE]+)\)$/.exec(row.coord.value);
|
||||
if (!match) continue;
|
||||
const id = qid(row.item.value);
|
||||
const pop = Math.round(Number(row.pop.value));
|
||||
const known = places.get(id);
|
||||
if (known) {
|
||||
known.pop = Math.max(known.pop, pop);
|
||||
// Of several stated coordinates, the same one on every build.
|
||||
const lon = Number(match[1]);
|
||||
const lat = Number(match[2]);
|
||||
if (lat < known.lat || (lat === known.lat && lon < known.lon)) {
|
||||
known.lat = lat;
|
||||
known.lon = lon;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
places.set(id, { id, pop, lon: Number(match[1]), lat: Number(match[2]) });
|
||||
kept++;
|
||||
}
|
||||
console.log(` ${rows.length} rows, ${kept} new settlements`);
|
||||
}
|
||||
|
||||
let groupingsLeftOut = 0;
|
||||
for (const id of grouped) {
|
||||
if ((settled.has(id) && !stated.has(id)) || !places.delete(id)) continue;
|
||||
groupingsLeftOut++;
|
||||
}
|
||||
console.log(`${groupingsLeftOut} groupings left out`);
|
||||
// A place typed only as a former kind is held until its end date is known: a
|
||||
// unit dissolved in a recent reform still stands (below).
|
||||
for (const [id, place] of places) if (!living.has(id)) place.formerOnly = true;
|
||||
const ids = [...places.keys()];
|
||||
console.log(`Labels and countries for ${ids.length} places…`);
|
||||
for (let start = 0; start < ids.length; start += LABEL_BATCH) {
|
||||
const values = ids
|
||||
.slice(start, start + LABEL_BATCH)
|
||||
.map((id) => `wd:${id}`)
|
||||
.join(' ');
|
||||
const rows = await sparql(
|
||||
`SELECT ?item ?label ?iso ?ended WHERE { VALUES ?item { ${values} } OPTIONAL { ?item rdfs:label ?label . FILTER(LANG(?label) = "en") } OPTIONAL { ?item wdt:P17 ?country . ?country wdt:P297 ?iso } OPTIONAL { ?item wdt:P576 ?ended } }`,
|
||||
);
|
||||
for (const row of rows) {
|
||||
const place = places.get(qid(row.item.value));
|
||||
if (!place) continue;
|
||||
if (row.label && !place.name) place.name = row.label.value;
|
||||
// Of several stated countries, the same one on every build.
|
||||
if (row.iso && (!place.cc || row.iso.value < place.cc))
|
||||
place.cc = row.iso.value;
|
||||
// Dissolved, abolished or merged as a unit (P576), and when.
|
||||
if (row.ended) {
|
||||
place.ended = true;
|
||||
const year = Number(/^-?\d+/.exec(row.ended.value)?.[0]);
|
||||
if (Number.isFinite(year))
|
||||
place.endedYear = Math.max(place.endedYear ?? -Infinity, year);
|
||||
}
|
||||
}
|
||||
if ((start / LABEL_BATCH) % 10 === 0)
|
||||
console.log(
|
||||
` ${Math.min(start + LABEL_BATCH, ids.length)} / ${ids.length}`,
|
||||
);
|
||||
}
|
||||
|
||||
// A dissolved place goes only where another place still names the spot (a
|
||||
// city merged into its neighbour, a former district inside its city). Where
|
||||
// nothing else does, the town is still there under its name and stays: many
|
||||
// of Vietnam's provincial cities ended as units in 2025, and with them their
|
||||
// class, so they are typed only as a former kind. A place typed only as a
|
||||
// former kind stays only that way: dissolved within the last ten years with
|
||||
// nothing else near it. Ghost towns and ancient cities go.
|
||||
const SUCCESSOR_KM = 15;
|
||||
const RECENT_YEARS = 10;
|
||||
const recent = new Date().getUTCFullYear() - RECENT_YEARS;
|
||||
const standing = [...places.values()].filter(
|
||||
(place) => place.name && !place.ended && !place.formerOnly,
|
||||
);
|
||||
const near = (a, b) => {
|
||||
const rad = Math.PI / 180;
|
||||
const h =
|
||||
Math.sin(((b.lat - a.lat) * rad) / 2) ** 2 +
|
||||
Math.cos(a.lat * rad) *
|
||||
Math.cos(b.lat * rad) *
|
||||
Math.sin(((b.lon - a.lon) * rad) / 2) ** 2;
|
||||
return 2 * 6371 * Math.asin(Math.min(1, Math.sqrt(h))) <= SUCCESSOR_KM;
|
||||
};
|
||||
const byCell = new Map();
|
||||
const cellOf = (lat, lon) => `${Math.floor(lat)},${Math.floor(lon)}`;
|
||||
for (const place of standing) {
|
||||
const key = cellOf(place.lat, place.lon);
|
||||
if (!byCell.has(key)) byCell.set(key, []);
|
||||
byCell.get(key).push(place);
|
||||
}
|
||||
let dissolved = 0;
|
||||
let formerOnly = 0;
|
||||
for (const [id, place] of places) {
|
||||
if (place.formerOnly && !(place.ended && place.endedYear >= recent)) {
|
||||
places.delete(id);
|
||||
formerOnly++;
|
||||
continue;
|
||||
}
|
||||
if (!place.ended) continue;
|
||||
let succeeded = false;
|
||||
for (let dLat = -1; dLat <= 1 && !succeeded; dLat++)
|
||||
for (let dLon = -1; dLon <= 1 && !succeeded; dLon++)
|
||||
succeeded = (
|
||||
byCell.get(cellOf(place.lat + dLat, place.lon + dLon)) || []
|
||||
).some((other) => near(place, other));
|
||||
if (succeeded) dissolved++;
|
||||
else place.ended = false;
|
||||
}
|
||||
console.log(`${formerOnly} former places left out`);
|
||||
console.log(`${dissolved} dissolved places left out`);
|
||||
|
||||
// A place whose stated country is a former one (no ISO code) takes the
|
||||
// country of the area it lies in, one area level up at a time (a path of
|
||||
// several levels in one query runs past the service's time limit).
|
||||
const COUNTRY_BATCH = 50;
|
||||
const countryless = () =>
|
||||
[...places.values()]
|
||||
.filter((place) => place.name && !place.cc && !place.ended)
|
||||
.map((place) => place.id);
|
||||
for (const hops of [1, 2, 3]) {
|
||||
const ids = countryless();
|
||||
if (!ids.length) break;
|
||||
console.log(`Countries ${hops} area level(s) up for ${ids.length} places…`);
|
||||
const pathToArea = Array.from({ length: hops }, () => 'wdt:P131').join('/');
|
||||
for (let start = 0; start < ids.length; start += COUNTRY_BATCH) {
|
||||
const values = ids
|
||||
.slice(start, start + COUNTRY_BATCH)
|
||||
.map((id) => `wd:${id}`)
|
||||
.join(' ');
|
||||
const rows = await sparql(
|
||||
`SELECT ?item ?iso WHERE { VALUES ?item { ${values} } ?item ${pathToArea} ?area . ?area wdt:P17 ?country . ?country wdt:P297 ?iso }`,
|
||||
);
|
||||
for (const row of rows) {
|
||||
const place = places.get(qid(row.item.value));
|
||||
if (!place || (place.cc && place.hops !== hops)) continue;
|
||||
if (!place.cc || row.iso.value < place.cc) {
|
||||
place.cc = row.iso.value;
|
||||
place.hops = hops;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(` ${countryless().length} places still without a country`);
|
||||
|
||||
const rows = [...places.values()]
|
||||
.filter(
|
||||
(place) =>
|
||||
place.name &&
|
||||
!place.ended &&
|
||||
Math.abs(place.lat) <= 90 &&
|
||||
Math.abs(place.lon) <= 180,
|
||||
)
|
||||
.sort((a, b) => b.pop - a.pop || a.id.localeCompare(b.id))
|
||||
.map((place) => [
|
||||
place.name,
|
||||
Number(place.lat.toFixed(3)),
|
||||
Number(place.lon.toFixed(3)),
|
||||
place.pop,
|
||||
place.cc || '',
|
||||
place.id,
|
||||
]);
|
||||
|
||||
// A filter that is too wide drops real cities without any error; these must
|
||||
// survive, and no settlement grouping may.
|
||||
const MUST_HAVE = {
|
||||
Q1490: 'Tokyo',
|
||||
Q84: 'London',
|
||||
Q16559: 'Austin',
|
||||
Q16555: 'Houston',
|
||||
Q1355: 'Bengaluru',
|
||||
Q3630: 'Jakarta',
|
||||
Q174: 'São Paulo',
|
||||
Q34600: 'Kyoto',
|
||||
Q656: 'Saint Petersburg',
|
||||
Q406: 'Istanbul',
|
||||
Q8678: 'Rio de Janeiro',
|
||||
Q1345: 'Philadelphia',
|
||||
Q586: 'Bonn',
|
||||
Q1449: 'Genoa',
|
||||
Q1563: 'Havana',
|
||||
// Provincial cities dissolved as units in 2025 that still stand.
|
||||
Q19491: 'Nha Trang',
|
||||
Q36081: 'Quy Nhon',
|
||||
};
|
||||
// Groupings that are also typed as cities; none may get in.
|
||||
const MUST_NOT = {
|
||||
Q683705: 'New York metropolitan area',
|
||||
Q665894: 'Greater Mexico City',
|
||||
Q13580: 'Metro Manila',
|
||||
Q3114019: 'Katowice urban area',
|
||||
};
|
||||
const present = new Set(rows.map((row) => row[5]));
|
||||
const missing = Object.entries(MUST_HAVE).filter(([id]) => !present.has(id));
|
||||
const intruders = Object.entries(MUST_NOT).filter(([id]) => present.has(id));
|
||||
if (intruders.length) {
|
||||
throw new Error(
|
||||
`Groupings got in: ${intruders.map(([id, name]) => `${name} (${id})`).join(', ')}`,
|
||||
);
|
||||
}
|
||||
if (missing.length) {
|
||||
throw new Error(
|
||||
`Expected cities are missing: ${missing.map(([id, name]) => `${name} (${id})`).join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
const meta = {
|
||||
source: 'Wikidata (https://www.wikidata.org), via the Wikidata Query Service',
|
||||
license: 'CC0 1.0',
|
||||
built: new Date().toISOString(),
|
||||
minPopulation,
|
||||
count: rows.length,
|
||||
columns: ['name', 'lat', 'lon', 'population', 'country', 'wikidata'],
|
||||
};
|
||||
|
||||
await mkdir(outDir, { recursive: true });
|
||||
const file = path.join(outDir, 'cities.json');
|
||||
await writeFile(file, `${JSON.stringify({ meta, rows })}\n`);
|
||||
console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`);
|
||||
Reference in new issue
Block a user