diff --git a/scripts/build-wikidata-cities.mjs b/scripts/build-wikidata-cities.mjs index a276062831..00dc36f0c0 100644 --- a/scripts/build-wikidata-cities.mjs +++ b/scripts/build-wikidata-cities.mjs @@ -39,6 +39,7 @@ import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { + keepBuiltIfUnchanged, placeTileFiles, withoutCities, withoutNearTwins, @@ -68,6 +69,8 @@ const outDir = path.join( : 'src/data/local_data/wikidata_cities', ); +/** A file's text, or undefined when there is none yet. */ +const readIfThere = (file) => readFile(file, 'utf8').catch(() => undefined); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1); @@ -499,17 +502,30 @@ if (tilesMode) { count: tileRows.length, maxPopulation, }); - // Tiles from an earlier build that this one has no rows for go. + // The new files are all written before tiles from an earlier build that + // this one has no rows for go, so a build that stops part way never leaves + // an index naming a removed tile. + const indexFile = path.join(outDir, 'index.json'); + files.set( + 'index.json', + keepBuiltIfUnchanged(files.get('index.json'), await readIfThere(indexFile)), + ); + for (const [name, text] of files) + await writeFile(path.join(outDir, name), text); for (const name of await readdir(outDir)) if (/^-?\d+_-?\d+\.json$/.test(name) && !files.has(name)) await rm(path.join(outDir, name)); - for (const [name, text] of files) - await writeFile(path.join(outDir, name), text); console.log( `Wrote ${tileRows.length} places in ${files.size - 2} tiles to ${path.relative(root, outDir)}`, ); process.exit(0); } const file = path.join(outDir, 'cities.json'); -await writeFile(file, `${JSON.stringify({ meta, rows })}\n`); +await writeFile( + file, + keepBuiltIfUnchanged( + `${JSON.stringify({ meta, rows })}\n`, + await readIfThere(file), + ), +); console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`); diff --git a/scripts/wikidataPlaceTiles.mjs b/scripts/wikidataPlaceTiles.mjs index 15f0c89e3a..bae1774bb7 100644 --- a/scripts/wikidataPlaceTiles.mjs +++ b/scripts/wikidataPlaceTiles.mjs @@ -89,6 +89,29 @@ export function placeTileFiles(rows, meta) { return files; } +/** + * A pack file's new text, keeping the earlier build's `meta.built` when + * nothing else in the file changed, so rebuilding unchanged data leaves the + * file as it was. + * @param {string} text The new file, JSON with a `meta.built` stamp. + * @param {string | undefined} earlier The file as it is now, if there is one. + */ +export function keepBuiltIfUnchanged(text, earlier) { + if (!earlier) return text; + let before; + try { + before = JSON.parse(earlier); + } catch { + return text; + } + const after = JSON.parse(text); + if (typeof before?.meta?.built !== 'string' || !after.meta) return text; + const unstamped = (value) => + JSON.stringify({ ...value, meta: { ...value.meta, built: null } }); + if (unstamped(before) !== unstamped(after)) return text; + return `${JSON.stringify({ ...after, meta: { ...after.meta, built: before.meta.built } })}\n`; +} + const kmBetween = (a, b) => greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] }); diff --git a/src/data/local_data/wikidata_cities/README.md b/src/data/local_data/wikidata_cities/README.md index ff80a6dc56..026f412988 100644 --- a/src/data/local_data/wikidata_cities/README.md +++ b/src/data/local_data/wikidata_cities/README.md @@ -36,6 +36,7 @@ places without an English label. The build runs one query at a time with pauses and honours the service's Retry-After; it takes about ten minutes. It fails if a short list of well-known cities is missing from the result, or a known grouping gets in. +`meta.built` keeps its earlier date when the rows are unchanged. **Limits:** populations are whatever Wikidata states and differ in date and definition between places (city proper, municipality, urban district). Some diff --git a/src/tooling/wikidataPlaceTiles.test.mjs b/src/tooling/wikidataPlaceTiles.test.mjs index 9e9cf54179..10e77b8dd3 100644 --- a/src/tooling/wikidataPlaceTiles.test.mjs +++ b/src/tooling/wikidataPlaceTiles.test.mjs @@ -8,6 +8,7 @@ import { createHash } from 'node:crypto'; import { readdir, readFile } from 'node:fs/promises'; import test from 'node:test'; import { + keepBuiltIfUnchanged, placeTileFiles, tileKey, withoutCities, @@ -103,3 +104,13 @@ test('twins are found where longitude cells narrow, and across 180°', () => { ['Q1', 'Q3'], ); }); + +test('a rebuild of unchanged rows keeps the earlier build stamp', () => { + const earlier = `${JSON.stringify({ meta: { built: '2026-01-01T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`; + const same = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`; + const changed = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 3 }, rows: [1, 2, 3] })}\n`; + assert.equal(keepBuiltIfUnchanged(same, earlier), earlier); + assert.equal(keepBuiltIfUnchanged(changed, earlier), changed); + assert.equal(keepBuiltIfUnchanged(same, undefined), same); + assert.equal(keepBuiltIfUnchanged(same, 'not json'), same); +});