mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-19 09:42:21 +00:00
fix(places): a rebuild writes its files before removing stale tiles, and keeps the build date when nothing changed
Co-authored-by: Manjunath Paparaju <[email protected]>
This commit is contained in:
1 parent
30ec362c13
commit
a2c86c0fed
4 files changed
+55
-4
No files matched your search
@@ -39,6 +39,7 @@ import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import {
|
import {
|
||||||
|
keepBuiltIfUnchanged,
|
||||||
placeTileFiles,
|
placeTileFiles,
|
||||||
withoutCities,
|
withoutCities,
|
||||||
withoutNearTwins,
|
withoutNearTwins,
|
||||||
@@ -68,6 +69,8 @@ const outDir = path.join(
|
|||||||
: 'src/data/local_data/wikidata_cities',
|
: 'src/data/local_data/wikidata_cities',
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/** A file's text, or undefined when there is none yet. */
|
||||||
|
const readIfThere = (file) => readFile(file, 'utf8').catch(() => undefined);
|
||||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1);
|
const qid = (uri) => uri.slice(uri.lastIndexOf('/') + 1);
|
||||||
|
|
||||||
@@ -499,17 +502,30 @@ if (tilesMode) {
|
|||||||
count: tileRows.length,
|
count: tileRows.length,
|
||||||
maxPopulation,
|
maxPopulation,
|
||||||
});
|
});
|
||||||
// Tiles from an earlier build that this one has no rows for go.
|
// The new files are all written before tiles from an earlier build that
|
||||||
|
// this one has no rows for go, so a build that stops part way never leaves
|
||||||
|
// an index naming a removed tile.
|
||||||
|
const indexFile = path.join(outDir, 'index.json');
|
||||||
|
files.set(
|
||||||
|
'index.json',
|
||||||
|
keepBuiltIfUnchanged(files.get('index.json'), await readIfThere(indexFile)),
|
||||||
|
);
|
||||||
|
for (const [name, text] of files)
|
||||||
|
await writeFile(path.join(outDir, name), text);
|
||||||
for (const name of await readdir(outDir))
|
for (const name of await readdir(outDir))
|
||||||
if (/^-?\d+_-?\d+\.json$/.test(name) && !files.has(name))
|
if (/^-?\d+_-?\d+\.json$/.test(name) && !files.has(name))
|
||||||
await rm(path.join(outDir, name));
|
await rm(path.join(outDir, name));
|
||||||
for (const [name, text] of files)
|
|
||||||
await writeFile(path.join(outDir, name), text);
|
|
||||||
console.log(
|
console.log(
|
||||||
`Wrote ${tileRows.length} places in ${files.size - 2} tiles to ${path.relative(root, outDir)}`,
|
`Wrote ${tileRows.length} places in ${files.size - 2} tiles to ${path.relative(root, outDir)}`,
|
||||||
);
|
);
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
const file = path.join(outDir, 'cities.json');
|
const file = path.join(outDir, 'cities.json');
|
||||||
await writeFile(file, `${JSON.stringify({ meta, rows })}\n`);
|
await writeFile(
|
||||||
|
file,
|
||||||
|
keepBuiltIfUnchanged(
|
||||||
|
`${JSON.stringify({ meta, rows })}\n`,
|
||||||
|
await readIfThere(file),
|
||||||
|
),
|
||||||
|
);
|
||||||
console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`);
|
console.log(`Wrote ${rows.length} places to ${path.relative(root, file)}`);
|
||||||
@@ -89,6 +89,29 @@ export function placeTileFiles(rows, meta) {
|
|||||||
return files;
|
return files;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A pack file's new text, keeping the earlier build's `meta.built` when
|
||||||
|
* nothing else in the file changed, so rebuilding unchanged data leaves the
|
||||||
|
* file as it was.
|
||||||
|
* @param {string} text The new file, JSON with a `meta.built` stamp.
|
||||||
|
* @param {string | undefined} earlier The file as it is now, if there is one.
|
||||||
|
*/
|
||||||
|
export function keepBuiltIfUnchanged(text, earlier) {
|
||||||
|
if (!earlier) return text;
|
||||||
|
let before;
|
||||||
|
try {
|
||||||
|
before = JSON.parse(earlier);
|
||||||
|
} catch {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
const after = JSON.parse(text);
|
||||||
|
if (typeof before?.meta?.built !== 'string' || !after.meta) return text;
|
||||||
|
const unstamped = (value) =>
|
||||||
|
JSON.stringify({ ...value, meta: { ...value.meta, built: null } });
|
||||||
|
if (unstamped(before) !== unstamped(after)) return text;
|
||||||
|
return `${JSON.stringify({ ...after, meta: { ...after.meta, built: before.meta.built } })}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
const kmBetween = (a, b) =>
|
const kmBetween = (a, b) =>
|
||||||
greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] });
|
greatCircleKm({ lat: a[1], lon: a[2] }, { lat: b[1], lon: b[2] });
|
||||||
|
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ places without an English label.
|
|||||||
The build runs one query at a time with pauses and honours the service's
|
The build runs one query at a time with pauses and honours the service's
|
||||||
Retry-After; it takes about ten minutes. It fails if a short list of
|
Retry-After; it takes about ten minutes. It fails if a short list of
|
||||||
well-known cities is missing from the result, or a known grouping gets in.
|
well-known cities is missing from the result, or a known grouping gets in.
|
||||||
|
`meta.built` keeps its earlier date when the rows are unchanged.
|
||||||
|
|
||||||
**Limits:** populations are whatever Wikidata states and differ in date and
|
**Limits:** populations are whatever Wikidata states and differ in date and
|
||||||
definition between places (city proper, municipality, urban district). Some
|
definition between places (city proper, municipality, urban district). Some
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import { createHash } from 'node:crypto';
|
|||||||
import { readdir, readFile } from 'node:fs/promises';
|
import { readdir, readFile } from 'node:fs/promises';
|
||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import {
|
import {
|
||||||
|
keepBuiltIfUnchanged,
|
||||||
placeTileFiles,
|
placeTileFiles,
|
||||||
tileKey,
|
tileKey,
|
||||||
withoutCities,
|
withoutCities,
|
||||||
@@ -103,3 +104,13 @@ test('twins are found where longitude cells narrow, and across 180°', () => {
|
|||||||
['Q1', 'Q3'],
|
['Q1', 'Q3'],
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('a rebuild of unchanged rows keeps the earlier build stamp', () => {
|
||||||
|
const earlier = `${JSON.stringify({ meta: { built: '2026-01-01T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`;
|
||||||
|
const same = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 2 }, rows: [1, 2] })}\n`;
|
||||||
|
const changed = `${JSON.stringify({ meta: { built: '2026-11-18T00:00:00.000Z', count: 3 }, rows: [1, 2, 3] })}\n`;
|
||||||
|
assert.equal(keepBuiltIfUnchanged(same, earlier), earlier);
|
||||||
|
assert.equal(keepBuiltIfUnchanged(changed, earlier), changed);
|
||||||
|
assert.equal(keepBuiltIfUnchanged(same, undefined), same);
|
||||||
|
assert.equal(keepBuiltIfUnchanged(same, 'not json'), same);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user