Bundle smaller Wikidata settlements for nearby-place lookups

137,662 places of 1,000 to 14,999 people that Wikidata gives a population
and coordinates for, none already in the cities pack, in 732 tiles of
5 x 5 degrees with an index of row counts (CC0). Nothing loads with the
app: a lookup reads the index once and then only the tiles it touches.
placesNear lists the places around a point, and findPlace finds a small
place by name near the view when no city of that name is near. The cities
build script writes the tiles with --tiles; the step from rows to files is
checked offline.

Co-authored-by: Sameh Khamis <[email protected]>
This commit is contained in:
Manjunath PaparajuandSameh Khamis authored and Sameh Khamis committed 2026-11-18 12:36:16 -08:00
1 parent 328b8f5d1b
commit 78305411eb
743 files changed
+2087 -14

No files matched your search

+142 -5
View File
@@ -1,12 +1,14 @@
/**
* The bundled world cities pack (Wikidata, CC0), read from
* `local_data/wikidata_cities/cities.json` (built by
* scripts/build-wikidata-cities.mjs). The file loads once, on first use, and
* every answer comes from it: no place service is called.
* scripts/build-wikidata-cities.mjs). The cities pack and optional detail
* tiles load on first use. All answers are offline.
*
* Rows are `[name, lat, lon, population, country, wikidata]`, largest first.
*/
import { createRetryableLoader } from './retryableLoad.js';
const CELL_DEG = 1;
// Letters that are not a base letter plus an accent, so NFD leaves them.
@@ -209,16 +211,138 @@ export function countryName(
}
}
const TILE_DEG = 5;
/**
* Detail tiles kept in memory, least recently used first out: 64 tiles is
* far more than one lookup touches, and bounds what a long session keeps.
*/
const MAX_TILES = 64;
const NAMED = /\p{L}/u;
const tiles = new Map();
let loadTileIndex;
/** The detail tiles touched by the circle's enclosing latitude/longitude box. */
function tileKeysAround(latitude, longitude, radiusKm) {
const latSpan = latHalfSpan(radiusKm);
const lonSpan = lonHalfSpan(latitude, radiusKm);
const latOrigin = (lat) =>
Math.min(85, Math.floor(lat / TILE_DEG) * TILE_DEG);
const south = latOrigin(Math.max(-90, latitude - latSpan));
const north = latOrigin(Math.min(90, latitude + latSpan));
const lon = (((longitude % 360) + 540) % 360) - 180;
const west = Math.floor((lon - lonSpan) / TILE_DEG) * TILE_DEG;
const east = Math.floor((lon + lonSpan) / TILE_DEG) * TILE_DEG;
const keys = [];
for (let lat = south; lat <= north; lat += TILE_DEG) {
for (let lon = west; lon <= east && lon < west + 360; lon += TILE_DEG) {
keys.push(`${lat}_${((lon + 540) % 360) - 180}`);
}
}
return keys;
}
async function readPlaceTiles(latitude, longitude, radiusKm, fetchImpl) {
const { INDEX_URL, TILE_URLS } =
await import('./local_data/wikidata_places/files.js');
loadTileIndex ??= createRetryableLoader(() =>
readPlacePack(INDEX_URL(), fetchImpl),
);
const index = await loadTileIndex();
const keys = tileKeysAround(latitude, longitude, radiusKm).filter(
(key) => index.tiles[key],
);
return Promise.all(
keys.map((key) => {
let load = tiles.get(key);
if (load) tiles.delete(key);
else
load = createRetryableLoader(async () => {
const pack = await readPlacePack(TILE_URLS[key](), fetchImpl);
// A name without a letter ("380") is no name to show.
return indexCities(pack.rows.filter((row) => NAMED.test(row[0])));
});
// Most recently used last; the least recently used goes past the cap.
tiles.set(key, load);
while (tiles.size > MAX_TILES) tiles.delete(tiles.keys().next().value);
return load();
}),
);
}
async function readPlacePack(url, fetchImpl) {
const response = await fetchImpl(url);
if (!response.ok)
throw new Error(`places pack: HTTP ${response.status} for ${url.pathname}`);
return response.json();
}
/** Named cities and smaller places around one point, nearest first. */
export async function placesNear(
{ latitude, longitude, radiusKm = 10, limit = 6 },
fetchImpl = (...args) => globalThis.fetch(...args),
) {
if (
!Number.isFinite(latitude) ||
Math.abs(latitude) > 90 ||
!Number.isFinite(longitude) ||
!Number.isFinite(radiusKm) ||
radiusKm < 0
)
return [];
const radius = Math.min(radiusKm, Math.PI * EARTH_KM);
const [{ cells }, detail] = await Promise.all([
loadCities(fetchImpl),
readPlaceTiles(latitude, longitude, radius, fetchImpl),
]);
// A smaller place that has the name of a city within TWIN_KM in the same
// country is the same place listed twice (a town and its municipality):
// the city's row wins. Cities just past the radius count, so a twin
// inside it is still caught; namesakes across a border both stay.
const cities = citiesWithin(cells, latitude, longitude, radius + TWIN_KM);
const byName = new Map();
for (const city of cities) {
const key = foldName(city.name);
if (!byName.has(key)) byName.set(key, []);
byName.get(key).push(city);
}
const twinOfCity = (place) =>
(byName.get(foldName(place.name)) || []).some(
(city) =>
(city.country || '') === (place.country || '') &&
distanceKm(
city.latitude,
city.longitude,
place.latitude,
place.longitude,
) <= TWIN_KM,
);
const found = [
...cities.filter((city) => city.distanceKm <= radius),
...detail
.flatMap((source) => citiesWithin(source, latitude, longitude, radius))
.filter((place) => !twinOfCity(place)),
];
const seen = new Set();
return found
.sort((a, b) => a.distanceKm - b.distanceKm)
.filter((place) => !seen.has(place.wikidata) && seen.add(place.wikidata))
.slice(0, limit);
}
/** How near two places of one name in one country are taken as one, in km. */
const TWIN_KM = 3;
/**
* The city with this name, or null. A qualifier after a comma that names a
* country ("Paris, France", "Paris, US") keeps only cities there, and finds
* nothing when none is; one that names no country ("Paris, Texas") is
* ignored. A city of that name within `radiusKm` of `near` (the view) wins,
* the nearest first, else the largest city of that name anywhere.
* the nearest first, else the largest city of that name anywhere. Omnibar
* can also ask for a smaller place nearby when the cities pack has no match.
*/
export async function findPlace(
name,
{ near = null, radiusKm = 150 } = {},
{ near = null, radiusKm = 150, includeSmallPlaces = false } = {},
fetchImpl = (...args) => globalThis.fetch(...args),
) {
const [head, ...rest] = String(name || '').split(',');
@@ -230,7 +354,20 @@ export async function findPlace(
const cities = (names.get(wanted) || []).filter(
(row) => !country || row[4] === country,
);
if (!cities.length) return null;
if (!cities.length) {
if (!includeSmallPlaces || !near) return null;
const nearby = await placesNear(
{ ...near, radiusKm, limit: Infinity },
fetchImpl,
);
return (
nearby.find(
(place) =>
foldName(place.name) === wanted &&
(!country || place.country === country),
) ?? null
);
}
if (Number.isFinite(near?.latitude) && Number.isFinite(near?.longitude)) {
let best = null;
for (const row of cities) {