Merge branch 'main' into samehkhamis/rail-and-hud-cleanup

This commit is contained in:
Sameh Khamis authored and GitHub committed 2026-11-18 23:33:51 -08:00
commit 6502028fbe
9 files changed
+296 -35

No files matched your search

@@ -0,0 +1,4 @@
- "Villages near" a name picks the place of that name in view over a far
bigger namesake, a place lookup reads the detail tiles within 500 km at
most, and rebuilding the places pack never leaves it half removed
(manjunath22466, samehkhamis, #1838).
+8 -2
View File
@@ -966,7 +966,8 @@ the fleet set loaded at startup, so opening a card fetches no model.
country and Wikidata id. `src/data/wikidataCities.js` reads it on the first
lookup and answers `nearestCity` (the largest city within 12 km, else the
nearest within 60 km), `citiesAround`, `findPlace` ("Paris, France", or the
nearest namesake within 150 km of the view, else the largest) and
nearest namesake within 150 km of the view, else the largest; asked to, it
counts the smaller places of the detail tiles near the view too) and
`countryName`. Names match without case or accents ("lodz" finds Łódź). The
app never calls Wikidata; `scripts/build-wikidata-cities.mjs` rebuilds the
pack and fails if well-known cities are missing or a known grouping such as
@@ -8257,7 +8258,12 @@ clears them but the browser's own site data. Setting `PHOTON_URL`
empty turns the suggestions off along with it. Small places come from the
bundled Wikidata packs and are never looked up online; a small place with
the name of a city within 3 km in the same country is that city, and only
the city is listed. At most 64 detail tiles are kept in memory, the least
the city is listed. A place named in "villages near …" is the nearest of
that name within 150 km of the view, small places included, so a village in
view comes before a bigger namesake far away; with none near, the largest
city of the name. Small places are looked for within 500 km at most, so a
lookup asked for a wider radius reads only the detail tiles around its
centre (the search box asks for 150 km at most). At most 64 detail tiles are kept in memory, the least
recently used dropped first.
### Place-search providers
+21 -4
View File
@@ -39,6 +39,8 @@ import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
keepBuiltIfUnchanged,
keepPackBuiltIfUnchanged,
placeTileFiles,
withoutCities,
withoutNearTwins,
@@ -68,6 +70,8 @@ const outDir = path.join(
: 'src/data/local_data/wikidata_cities',
);
/** A file's text, or undefined when there is none yet. */
const readIfThere = (file) => readFile(file, 'utf8').catch(() => undefined);
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1);
@@ -499,17 +503,30 @@ if (tilesMode) {
count: tileRows.length,
maxPopulation,
});
// Tiles from an earlier build that this one has no rows for go.
// The new files are all written before tiles from an earlier build that
// this one has no rows for go, so a build that stops part way never leaves
// an index naming a removed tile.
const earlier = new Map();
for (const name of await readdir(outDir))
if (/^-?\d+_-?\d+\.json$|^index\.json$|^files\.js$/.test(name))
earlier.set(name, await readFile(path.join(outDir, name), 'utf8'));
keepPackBuiltIfUnchanged(files, earlier);
for (const [name, text] of files)
await writeFile(path.join(outDir, name), text);
for (const name of await readdir(outDir))
if (/^-?\d+_-?\d+\.json$/.test(name) && !files.has(name))
await rm(path.join(outDir, name));
for (const [name, text] of files)
await writeFile(path.join(outDir, name), text);
console.log(
`Wrote ${tileRows.length} places in ${files.size - 2} tiles to ${path.relative(root, outDir)}`,
);
process.exit(0);
}
const file = path.join(outDir, 'cities.json');
await writeFile(file, `${JSON.stringify({ meta, rows })}\n`);
await writeFile(
file,
keepBuiltIfUnchanged(
`${JSON.stringify({ meta, rows })}\n`,
await readIfThere(file),
),
);
console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`);
+45
View File
@@ -89,6 +89,51 @@ export function placeTileFiles(rows, meta) {
return files;
}
/**
* A pack file's new text, keeping the earlier build's `meta.built` when
* nothing else in the file changed, so rebuilding unchanged data leaves the
* file as it was. A file that does not hold all of its pack's data (the
* places index) is passed through this only when the rest of the pack is
* unchanged too.
* @param {string} text The new file, JSON with a `meta.built` stamp.
* @param {string | undefined} earlier The file as it is now, if there is one.
*/
export function keepBuiltIfUnchanged(text, earlier) {
if (!earlier) return text;
let before;
try {
before = JSON.parse(earlier);
} catch {
return text;
}
const after = JSON.parse(text);
if (typeof before?.meta?.built !== 'string' || !after.meta) return text;
const unstamped = (value) =>
JSON.stringify({ ...value, meta: { ...value.meta, built: null } });
if (unstamped(before) !== unstamped(after)) return text;
return `${JSON.stringify({ ...after, meta: { ...after.meta, built: before.meta.built } })}\n`;
}
/**
* Keeps the earlier build's `meta.built` in the places pack's index when the
* rebuild changed nothing: every tile and `files.js` the same text, no tile
* of the earlier build left out, and nothing else in the index changed. The
* index holds only row counts, so it alone cannot tell.
* @param {Map<string, string>} files The new pack, as placeTileFiles gives it; its index is replaced in place.
* @param {Map<string, string>} earlier The pack's files on disk now, by name.
*/
export function keepPackBuiltIfUnchanged(files, earlier) {
for (const name of earlier.keys())
if (name !== 'index.json' && !files.has(name)) return files;
for (const [name, text] of files)
if (name !== 'index.json' && earlier.get(name) !== text) return files;
files.set(
'index.json',
keepBuiltIfUnchanged(files.get('index.json'), earlier.get('index.json')),
);
return files;
}
const kmBetween = (a, b) =>
greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] });
@@ -36,7 +36,13 @@ places without an English label.
The build runs one query at a time with pauses and honours the service's
Retry-After; it takes about ten minutes. It fails if a short list of
well-known cities is missing from the result, or a known grouping gets in.
`meta.built` keeps its earlier date when the rows are unchanged.
**Limits:** populations are whatever Wikidata states and differ in date and
definition between places (city proper, municipality, urban district). Some
districts and suburbs appear as their own rows.
**Finding a place by name:** `findPlace` takes the nearest place of the name
within 150 km of the view, else the largest city of the name. When asked,
the smaller places of `../wikidata_places` near the view count too, so a
village in view comes before a bigger namesake far away.
+15 -4
View File
@@ -1,8 +1,11 @@
# Wikidata places detail tiles
Smaller towns and villages, below the cut-off of the cities pack
(`../wikidata_cities`), for "places nearby" around one picked or named spot. Read by `placesNear` in `src/data/wikidataCities.js`. The app reads
these files; it does not call Wikidata or any place service for them.
(`../wikidata_cities`), for "places nearby" around one picked or named spot. Read by `placesNear` in `src/data/wikidataCities.js`, and by `findPlace`
when asked to look for a small place by name near the view: a place of that
name in view, or close to it, comes before a bigger city of the name far
away. The app reads these files; it does not call Wikidata or any place
service for them.
| File | Contents |
|------|----------|
@@ -17,7 +20,12 @@ entry in the index.
**Loading:** nothing here loads with the app. A lookup reads `files.js` and
the index once, then only the tiles its search box touches (one for a short
radius well inside a tile, more near tile edges and for wider searches),
once each; at most 64 tiles are kept in memory, the least recently used
once each. A lookup reaches 500 km at most (`PLACE_TILE_KM`; the search box
asks for 150 km at most), so one asked for a wider radius reads only the
tiles around its centre: at most one
tile past the centre's own on each side, a few more columns toward the
poles, and never all 732. Cities still count to the full radius. At most
64 tiles are kept in memory, the least recently used
dropped first. Rows whose name has no letter (one row, `380` in Nicaragua) are
dropped when a tile is read.
@@ -39,7 +47,10 @@ names from Wikidata (CC0)" as a courtesy once a name from the pack is shown.
`node scripts/build-wikidata-cities.mjs --tiles --min-population 1000 --max-population 15000`.
It writes only to this folder (tiles, `index.json`, `files.js`, through
`scripts/wikidataPlaceTiles.mjs`) and leaves `../wikidata_cities` alone. It
runs one query at a time with pauses and takes a long while.
runs one query at a time with pauses and takes a long while. The new files
are all written before tiles that the new build has no rows for are
removed, and `meta.built` keeps its earlier date when no tile and
nothing else in `index.json` changed.
**Reproducibility:** the step from rows to files is checked offline:
`src/tooling/wikidataPlaceTiles.test.mjs` sorts the shipped rows as the
+50 -25
View File
@@ -217,6 +217,15 @@ const TILE_DEG = 5;
* far more than one lookup touches, and bounds what a long session keeps.
*/
const MAX_TILES = 64;
/**
* How far out small places are looked for, in km. 500 km is under the 556 km
* that 5 degrees of latitude span, so a lookup's tile box reaches at most one
* tile past the point's own on each side: 3 rows, and 3 columns at the
* equator (more toward the poles, where tiles narrow). A lookup asked for a
* wider radius, up to the whole globe, still reads only those tiles; cities
* count to its full radius.
*/
export const PLACE_TILE_KM = 500;
const NAMED = /\p{L}/u;
const tiles = new Map();
let loadTileIndex;
@@ -248,9 +257,11 @@ async function readPlaceTiles(latitude, longitude, radiusKm, fetchImpl) {
readPlacePack(INDEX_URL(), fetchImpl),
);
const index = await loadTileIndex();
const keys = tileKeysAround(latitude, longitude, radiusKm).filter(
(key) => index.tiles[key],
);
const keys = tileKeysAround(
latitude,
longitude,
Math.min(radiusKm, PLACE_TILE_KM),
).filter((key) => index.tiles[key]);
// A tile that cannot be read is left out: the places of the others,
// and the cities, are still the answer.
const settled = await Promise.allSettled(
@@ -281,7 +292,10 @@ async function readPlacePack(url, fetchImpl) {
return response.json();
}
/** Named cities and smaller places around one point, nearest first. */
/**
* Named cities and smaller places around one point, nearest first. Smaller
* places are looked for within PLACE_TILE_KM only.
*/
export async function placesNear(
{ latitude, longitude, radiusKm = 10, limit = 6 },
fetchImpl = (...args) => globalThis.fetch(...args),
@@ -325,7 +339,14 @@ export async function placesNear(
const found = [
...cities.filter((city) => city.distanceKm <= radius),
...detail
.flatMap((source) => citiesWithin(source, latitude, longitude, radius))
.flatMap((source) =>
citiesWithin(
source,
latitude,
longitude,
Math.min(radius, PLACE_TILE_KM),
),
)
.filter((place) => !twinOfCity(place)),
];
const seen = new Set();
@@ -339,12 +360,13 @@ export async function placesNear(
const TWIN_KM = 3;
/**
* The city with this name, or null. A qualifier after a comma that names a
* country ("Paris, France", "Paris, US") keeps only cities there, and finds
* The place with this name, or null. A qualifier after a comma that names a
* country ("Paris, France", "Paris, US") keeps only places there, and finds
* nothing when none is; one that names no country ("Paris, Texas") is
* ignored. A city of that name within `radiusKm` of `near` (the view) wins,
* the nearest first, else the largest city of that name anywhere. Omnibar
* can also ask for a smaller place nearby when the cities pack has no match.
* ignored. A place of that name within `radiusKm` of `near` (the view) wins,
* the nearest first, else the largest city of that name anywhere. With
* `includeSmallPlaces`, the smaller places of the detail tiles near the view
* count too: a village in view comes before a far bigger city of its name.
*/
export async function findPlace(
name,
@@ -360,20 +382,6 @@ export async function findPlace(
const cities = (names.get(wanted) || []).filter(
(row) => !country || row[4] === country,
);
if (!cities.length) {
if (!includeSmallPlaces || !near) return null;
const nearby = await placesNear(
{ ...near, radiusKm, limit: Infinity },
fetchImpl,
);
return (
nearby.find(
(place) =>
foldName(place.name) === wanted &&
(!country || place.country === country),
) ?? null
);
}
if (Number.isFinite(near?.latitude) && Number.isFinite(near?.longitude)) {
let best = null;
for (const row of cities) {
@@ -381,7 +389,24 @@ export async function findPlace(
if (km <= radiusKm && (!best || km < best.distanceKm))
best = toCity(row, km);
}
if (includeSmallPlaces) {
const nearby = await placesNear(
{
latitude: near.latitude,
longitude: near.longitude,
radiusKm,
limit: Infinity,
},
fetchImpl,
);
const small = nearby.find(
(place) =>
foldName(place.name) === wanted &&
(!country || place.country === country),
);
if (small && (!best || small.distanceKm < best.distanceKm)) best = small;
}
if (best) return best;
}
return toCity(cities[0], 0);
return cities.length ? toCity(cities[0], 0) : null;
}
+107
View File
@@ -96,6 +96,66 @@ test('findPlace takes a country qualifier and prefers a namesake in view', async
);
});
test('a small place in view comes before a far bigger city of its name', async () => {
const module = await import(`./wikidataCities.js?small-${Date.now()}`);
// A village called Paris near Austin, in the 30_-100 detail tile.
const village = ['Paris', 30.4, -97.6, 1200, 'US', 'Q-village'];
const fetchImpl = async (url) => {
const file = url.pathname.split('/').at(-1);
const body =
file === 'cities.json'
? { meta: {}, rows: PARIS }
: file === 'index.json'
? { tiles: { '30_-100': 1 } }
: { rows: file === '30_-100.json' ? [village] : [] };
return { ok: true, json: async () => body };
};
const find = (name, near) =>
module.findPlace(name, { near, includeSmallPlaces: true }, fetchImpl);
const austin = { latitude: 30.27, longitude: -97.74 };
assert.equal((await find('Paris', austin)).wikidata, 'Q-village');
assert.ok((await find('Paris', austin)).distanceKm < 20);
// A bigger namesake inside a wider search: the nearer village still wins.
assert.equal(
(
await module.findPlace(
'Paris',
{ near: austin, radiusKm: 600, includeSmallPlaces: true },
fetchImpl,
)
).wikidata,
'Q-village',
);
// Only when small places are asked for.
assert.equal(
(await module.findPlace('Paris', { near: austin }, fetchImpl)).wikidata,
'Q90',
);
// A qualifier still decides.
assert.equal((await find('Paris, France', austin)).wikidata, 'Q90');
// Nowhere near the village: the largest city of the name.
assert.equal(
(await find('Paris', { latitude: 0, longitude: 0 })).wikidata,
'Q90',
);
assert.equal(
(await module.findPlace('Paris', {}, fetchImpl)).wikidata,
'Q90',
);
// Paris, Texas is nearer a view over it than the village is.
const texas = { latitude: 33.6, longitude: -95.6 };
assert.equal(
(
await module.findPlace(
'Paris',
{ near: texas, radiusKm: 600, includeSmallPlaces: true },
fetchImpl,
)
).wikidata,
'Q830149',
);
});
test('a circle that reaches a pole spans every longitude', () => {
assert.equal(lonHalfSpan(89, 150), 180);
assert.ok(lonHalfSpan(60, 150) < 3);
@@ -232,3 +292,50 @@ test('a tile that cannot be read leaves the other places and the cities', async
assert.ok(near.some((place) => place.name === 'Austin'));
assert.ok(near.some((place) => place.longitude > -95));
});
test('a lookup asked for the whole globe reads only the tiles near its centre', async () => {
const { placesNear, PLACE_TILE_KM } =
await import('./wikidataCities.js?globe');
const fetched = [];
const fetchPack = async (url) => {
fetched.push(url.pathname.split('/').at(-1));
return new Response(await readFile(url));
};
const near = await placesNear(
{ latitude: 30.27, longitude: -97.74, radiusKm: 20000, limit: 50 },
fetchPack,
);
const tileReads = fetched
.filter((name) => /^-?\d+_-?\d+\.json$/.test(name))
.sort();
// 500 km around Austin: latitude 25.8 to 34.8, longitude -103 to -92.5.
assert.equal(PLACE_TILE_KM, 500);
assert.deepEqual(tileReads, [
'25_-100.json',
'25_-105.json',
'25_-95.json',
'30_-100.json',
'30_-105.json',
'30_-95.json',
]);
assert.equal(near.length, 50);
assert.ok(near.every((place) => place.distanceKm <= PLACE_TILE_KM));
// Cities still count to the full radius.
const far = await placesNear(
{ latitude: 30.27, longitude: -97.74, radiusKm: 20000, limit: Infinity },
fetchPack,
);
assert.ok(far.some((place) => place.wikidata === 'Q90'));
// Places from the tiles it read stop at 500 km too: Tahoka, Texas
// (Q982084), a town of about 2,500, is 500.6 km away.
assert.ok(!far.some((place) => place.wikidata === 'Q982084'));
assert.ok(
far
.filter((place) => place.distanceKm > PLACE_TILE_KM)
.every((place) => place.population >= 15000),
);
assert.equal(
fetched.filter((name) => /^-?\d+_-?\d+\.json$/.test(name)).length,
tileReads.length,
);
});
+40
View File
@@ -8,6 +8,8 @@ import { createHash } from 'node:crypto';
import { readdir, readFile } from 'node:fs/promises';
import test from 'node:test';
import {
keepBuiltIfUnchanged,
keepPackBuiltIfUnchanged,
placeTileFiles,
tileKey,
withoutCities,
@@ -103,3 +105,41 @@ test('twins are found where longitude cells narrow, and across 180°', () => {
['Q1', 'Q3'],
);
});
test('a rebuild of unchanged rows keeps the earlier build stamp', () => {
const earlier = `${JSON.stringify({ meta: { built: '2026-01-01T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`;
const same = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`;
const changed = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 3 }, rows: [1, 2, 3] })}\n`;
assert.equal(keepBuiltIfUnchanged(same, earlier), earlier);
assert.equal(keepBuiltIfUnchanged(changed, earlier), changed);
assert.equal(keepBuiltIfUnchanged(same, undefined), same);
assert.equal(keepBuiltIfUnchanged(same, 'not json'), same);
});
test('the places index keeps its build stamp only when no tile changed', () => {
const rows = [
['Aa', 31, -98, 2000, 'US', 'Q1'],
['Bb', 41, -88, 1500, 'US', 'Q2'],
];
const meta = (built) => ({
source: 's',
license: 'l',
built,
minPopulation: 1000,
maxPopulation: 15000,
columns: [],
});
const earlier = placeTileFiles(rows, meta('2026-01-01T00:00:00.000Z'));
const built = (files) => JSON.parse(files.get('index.json')).meta.built;
const rebuild = (newRows) =>
keepPackBuiltIfUnchanged(
placeTileFiles(newRows, meta('2026-11-18T00:00:00.000Z')),
earlier,
);
assert.equal(built(rebuild(rows)), '2026-01-01T00:00:00.000Z');
// Same row count in every tile, new population: a new stamp.
const changed = [['Aa', 31, -98, 2100, 'US', 'Q1'], rows[1]];
assert.equal(built(rebuild(changed)), '2026-11-18T00:00:00.000Z');
// A tile of the earlier build left out: a new stamp.
assert.equal(built(rebuild([rows[0]])), '2026-11-18T00:00:00.000Z');
});