Merge branch 'main' into samehkhamis/rail-and-hud-cleanup

This commit is contained in:
Sameh Khamis authored and GitHub committed 2026-11-18 23:33:51 -08:00
commit 6502028fbe
9 files changed
+296 -35

No files matched your search

@@ -36,7 +36,13 @@ places without an English label.
The build runs one query at a time with pauses and honours the service's
Retry-After; it takes about ten minutes. It fails if a short list of
well-known cities is missing from the result, or a known grouping gets in.
`meta.built` keeps its earlier date when the rows are unchanged.
**Limits:** populations are whatever Wikidata states and differ in date and
definition between places (city proper, municipality, urban district). Some
districts and suburbs appear as their own rows.
**Finding a place by name:** `findPlace` takes the nearest place of the name
within 150 km of the view, else the largest city of the name. When asked,
the smaller places of `../wikidata_places` near the view count too, so a
village in view comes before a bigger namesake far away.
+15 -4
View File
@@ -1,8 +1,11 @@
# Wikidata places detail tiles
Smaller towns and villages, below the cut-off of the cities pack
(`../wikidata_cities`), for "places nearby" around one picked or named spot. Read by `placesNear` in `src/data/wikidataCities.js`. The app reads
these files; it does not call Wikidata or any place service for them.
(`../wikidata_cities`), for "places nearby" around one picked or named spot. Read by `placesNear` in `src/data/wikidataCities.js`, and by `findPlace`
when asked to look for a small place by name near the view: a place of that
name in view, or close to it, comes before a bigger city of the name far
away. The app reads these files; it does not call Wikidata or any place
service for them.
| File | Contents |
|------|----------|
@@ -17,7 +20,12 @@ entry in the index.
**Loading:** nothing here loads with the app. A lookup reads `files.js` and
the index once, then only the tiles its search box touches (one for a short
radius well inside a tile, more near tile edges and for wider searches),
once each; at most 64 tiles are kept in memory, the least recently used
once each. A lookup reaches 500 km at most (`PLACE_TILE_KM`; the search box
asks for 150 km at most), so one asked for a wider radius reads only the
tiles around its centre: at most one
tile past the centre's own on each side, a few more columns toward the
poles, and never all 732. Cities still count to the full radius. At most
64 tiles are kept in memory, the least recently used
dropped first. Rows whose name has no letter (one row, `380` in Nicaragua) are
dropped when a tile is read.
@@ -39,7 +47,10 @@ names from Wikidata (CC0)" as a courtesy once a name from the pack is shown.
`node scripts/build-wikidata-cities.mjs --tiles --min-population 1000 --max-population 15000`.
It writes only to this folder (tiles, `index.json`, `files.js`, through
`scripts/wikidataPlaceTiles.mjs`) and leaves `../wikidata_cities` alone. It
runs one query at a time with pauses and takes a long while.
runs one query at a time with pauses and takes a long while. The new files
are all written before tiles that the new build has no rows for are
removed, and `meta.built` keeps its earlier date when no tile and
nothing else in `index.json` changed.
**Reproducibility:** the step from rows to files is checked offline:
`src/tooling/wikidataPlaceTiles.test.mjs` sorts the shipped rows as the
+50 -25
View File
@@ -217,6 +217,15 @@ const TILE_DEG = 5;
* far more than one lookup touches, and bounds what a long session keeps.
*/
const MAX_TILES = 64;
/**
* How far out small places are looked for, in km. 500 km is under the 556 km
* that 5 degrees of latitude span, so a lookup's tile box reaches at most one
* tile past the point's own on each side: 3 rows, and 3 columns at the
* equator (more toward the poles, where tiles narrow). A lookup asked for a
* wider radius, up to the whole globe, still reads only those tiles; cities
* count to its full radius.
*/
export const PLACE_TILE_KM = 500;
const NAMED = /\p{L}/u;
const tiles = new Map();
let loadTileIndex;
@@ -248,9 +257,11 @@ async function readPlaceTiles(latitude, longitude, radiusKm, fetchImpl) {
readPlacePack(INDEX_URL(), fetchImpl),
);
const index = await loadTileIndex();
const keys = tileKeysAround(latitude, longitude, radiusKm).filter(
(key) => index.tiles[key],
);
const keys = tileKeysAround(
latitude,
longitude,
Math.min(radiusKm, PLACE_TILE_KM),
).filter((key) => index.tiles[key]);
// A tile that cannot be read is left out: the places of the others,
// and the cities, are still the answer.
const settled = await Promise.allSettled(
@@ -281,7 +292,10 @@ async function readPlacePack(url, fetchImpl) {
return response.json();
}
/** Named cities and smaller places around one point, nearest first. */
/**
* Named cities and smaller places around one point, nearest first. Smaller
* places are looked for within PLACE_TILE_KM only.
*/
export async function placesNear(
{ latitude, longitude, radiusKm = 10, limit = 6 },
fetchImpl = (...args) => globalThis.fetch(...args),
@@ -325,7 +339,14 @@ export async function placesNear(
const found = [
...cities.filter((city) => city.distanceKm <= radius),
...detail
.flatMap((source) => citiesWithin(source, latitude, longitude, radius))
.flatMap((source) =>
citiesWithin(
source,
latitude,
longitude,
Math.min(radius, PLACE_TILE_KM),
),
)
.filter((place) => !twinOfCity(place)),
];
const seen = new Set();
@@ -339,12 +360,13 @@ export async function placesNear(
const TWIN_KM = 3;
/**
* The city with this name, or null. A qualifier after a comma that names a
* country ("Paris, France", "Paris, US") keeps only cities there, and finds
* The place with this name, or null. A qualifier after a comma that names a
* country ("Paris, France", "Paris, US") keeps only places there, and finds
* nothing when none is; one that names no country ("Paris, Texas") is
* ignored. A city of that name within `radiusKm` of `near` (the view) wins,
* the nearest first, else the largest city of that name anywhere. Omnibar
* can also ask for a smaller place nearby when the cities pack has no match.
* ignored. A place of that name within `radiusKm` of `near` (the view) wins,
* the nearest first, else the largest city of that name anywhere. With
* `includeSmallPlaces`, the smaller places of the detail tiles near the view
* count too: a village in view comes before a far bigger city of its name.
*/
export async function findPlace(
name,
@@ -360,20 +382,6 @@ export async function findPlace(
const cities = (names.get(wanted) || []).filter(
(row) => !country || row[4] === country,
);
if (!cities.length) {
if (!includeSmallPlaces || !near) return null;
const nearby = await placesNear(
{ ...near, radiusKm, limit: Infinity },
fetchImpl,
);
return (
nearby.find(
(place) =>
foldName(place.name) === wanted &&
(!country || place.country === country),
) ?? null
);
}
if (Number.isFinite(near?.latitude) && Number.isFinite(near?.longitude)) {
let best = null;
for (const row of cities) {
@@ -381,7 +389,24 @@ export async function findPlace(
if (km <= radiusKm && (!best || km < best.distanceKm))
best = toCity(row, km);
}
if (includeSmallPlaces) {
const nearby = await placesNear(
{
latitude: near.latitude,
longitude: near.longitude,
radiusKm,
limit: Infinity,
},
fetchImpl,
);
const small = nearby.find(
(place) =>
foldName(place.name) === wanted &&
(!country || place.country === country),
);
if (small && (!best || small.distanceKm < best.distanceKm)) best = small;
}
if (best) return best;
}
return toCity(cities[0], 0);
return cities.length ? toCity(cities[0], 0) : null;
}
+107
View File
@@ -96,6 +96,66 @@ test('findPlace takes a country qualifier and prefers a namesake in view', async
);
});
test('a small place in view comes before a far bigger city of its name', async () => {
const module = await import(`./wikidataCities.js?small-${Date.now()}`);
// A village called Paris near Austin, in the 30_-100 detail tile.
const village = ['Paris', 30.4, -97.6, 1200, 'US', 'Q-village'];
const fetchImpl = async (url) => {
const file = url.pathname.split('/').at(-1);
const body =
file === 'cities.json'
? { meta: {}, rows: PARIS }
: file === 'index.json'
? { tiles: { '30_-100': 1 } }
: { rows: file === '30_-100.json' ? [village] : [] };
return { ok: true, json: async () => body };
};
const find = (name, near) =>
module.findPlace(name, { near, includeSmallPlaces: true }, fetchImpl);
const austin = { latitude: 30.27, longitude: -97.74 };
assert.equal((await find('Paris', austin)).wikidata, 'Q-village');
assert.ok((await find('Paris', austin)).distanceKm < 20);
// A bigger namesake inside a wider search: the nearer village still wins.
assert.equal(
(
await module.findPlace(
'Paris',
{ near: austin, radiusKm: 600, includeSmallPlaces: true },
fetchImpl,
)
).wikidata,
'Q-village',
);
// Only when small places are asked for.
assert.equal(
(await module.findPlace('Paris', { near: austin }, fetchImpl)).wikidata,
'Q90',
);
// A qualifier still decides.
assert.equal((await find('Paris, France', austin)).wikidata, 'Q90');
// Nowhere near the village: the largest city of the name.
assert.equal(
(await find('Paris', { latitude: 0, longitude: 0 })).wikidata,
'Q90',
);
assert.equal(
(await module.findPlace('Paris', {}, fetchImpl)).wikidata,
'Q90',
);
// Paris, Texas is nearer a view over it than the village is.
const texas = { latitude: 33.6, longitude: -95.6 };
assert.equal(
(
await module.findPlace(
'Paris',
{ near: texas, radiusKm: 600, includeSmallPlaces: true },
fetchImpl,
)
).wikidata,
'Q830149',
);
});
test('a circle that reaches a pole spans every longitude', () => {
assert.equal(lonHalfSpan(89, 150), 180);
assert.ok(lonHalfSpan(60, 150) < 3);
@@ -232,3 +292,50 @@ test('a tile that cannot be read leaves the other places and the cities', async
assert.ok(near.some((place) => place.name === 'Austin'));
assert.ok(near.some((place) => place.longitude > -95));
});
test('a lookup asked for the whole globe reads only the tiles near its centre', async () => {
const { placesNear, PLACE_TILE_KM } =
await import('./wikidataCities.js?globe');
const fetched = [];
const fetchPack = async (url) => {
fetched.push(url.pathname.split('/').at(-1));
return new Response(await readFile(url));
};
const near = await placesNear(
{ latitude: 30.27, longitude: -97.74, radiusKm: 20000, limit: 50 },
fetchPack,
);
const tileReads = fetched
.filter((name) => /^-?\d+_-?\d+\.json$/.test(name))
.sort();
// 500 km around Austin: latitude 25.8 to 34.8, longitude -103 to -92.5.
assert.equal(PLACE_TILE_KM, 500);
assert.deepEqual(tileReads, [
'25_-100.json',
'25_-105.json',
'25_-95.json',
'30_-100.json',
'30_-105.json',
'30_-95.json',
]);
assert.equal(near.length, 50);
assert.ok(near.every((place) => place.distanceKm <= PLACE_TILE_KM));
// Cities still count to the full radius.
const far = await placesNear(
{ latitude: 30.27, longitude: -97.74, radiusKm: 20000, limit: Infinity },
fetchPack,
);
assert.ok(far.some((place) => place.wikidata === 'Q90'));
// Places from the tiles it read stop at 500 km too: Tahoka, Texas
// (Q982084), a town of about 2,500, is 500.6 km away.
assert.ok(!far.some((place) => place.wikidata === 'Q982084'));
assert.ok(
far
.filter((place) => place.distanceKm > PLACE_TILE_KM)
.every((place) => place.population >= 15000),
);
assert.equal(
fetched.filter((name) => /^-?\d+_-?\d+\.json$/.test(name)).length,
tileReads.length,
);
});
+40
View File
@@ -8,6 +8,8 @@ import { createHash } from 'node:crypto';
import { readdir, readFile } from 'node:fs/promises';
import test from 'node:test';
import {
keepBuiltIfUnchanged,
keepPackBuiltIfUnchanged,
placeTileFiles,
tileKey,
withoutCities,
@@ -103,3 +105,41 @@ test('twins are found where longitude cells narrow, and across 180°', () => {
['Q1', 'Q3'],
);
});
test('a rebuild of unchanged rows keeps the earlier build stamp', () => {
const earlier = `${JSON.stringify({ meta: { built: '2026-01-01T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`;
const same = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`;
const changed = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 3 }, rows: [1, 2, 3] })}\n`;
assert.equal(keepBuiltIfUnchanged(same, earlier), earlier);
assert.equal(keepBuiltIfUnchanged(changed, earlier), changed);
assert.equal(keepBuiltIfUnchanged(same, undefined), same);
assert.equal(keepBuiltIfUnchanged(same, 'not json'), same);
});
test('the places index keeps its build stamp only when no tile changed', () => {
const rows = [
['Aa', 31, -98, 2000, 'US', 'Q1'],
['Bb', 41, -88, 1500, 'US', 'Q2'],
];
const meta = (built) => ({
source: 's',
license: 'l',
built,
minPopulation: 1000,
maxPopulation: 15000,
columns: [],
});
const earlier = placeTileFiles(rows, meta('2026-01-01T00:00:00.000Z'));
const built = (files) => JSON.parse(files.get('index.json')).meta.built;
const rebuild = (newRows) =>
keepPackBuiltIfUnchanged(
placeTileFiles(newRows, meta('2026-11-18T00:00:00.000Z')),
earlier,
);
assert.equal(built(rebuild(rows)), '2026-01-01T00:00:00.000Z');
// Same row count in every tile, new population: a new stamp.
const changed = [['Aa', 31, -98, 2100, 'US', 'Q1'], rows[1]];
assert.equal(built(rebuild(changed)), '2026-11-18T00:00:00.000Z');
// A tile of the earlier build left out: a new stamp.
assert.equal(built(rebuild([rows[0]])), '2026-11-18T00:00:00.000Z');
});