mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-17 08:42:21 +00:00
322 lines
12 KiB
JavaScript
322 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Data Library maintenance. See docs/DATA-LIBRARY.md.
|
|
*
|
|
* npm run library:check [-- --base-ref origin/main]
|
|
* Validate every manifest, the share-token ledger and the generated
|
|
* DATA_SOURCES.md section. With --base-ref, also prove the ledger only
|
|
* appended rows since that ref (published tokens never change).
|
|
* npm run library:next -- <entry-id>
|
|
* Print the next free share token for a new switchable entry.
|
|
* npm run library:docs
|
|
* Rewrite the generated Data Library section of DATA_SOURCES.md.
|
|
* npm run library:probe [-- --json] [-- --strict]
|
|
* Hit every dataset and tile upstream once (bounded) and report
|
|
* status, latency and feature count. --strict exits 1 on a failure.
|
|
*/
|
|
import { execFileSync } from 'node:child_process';
|
|
import { readFileSync, writeFileSync } from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { LIBRARY_ENTRIES, LIBRARY_LEDGER } from '../src/library/catalog.js';
|
|
import { libraryTileUrl } from '../src/library/manifest.js';
|
|
import {
|
|
arcgisPageUrl,
|
|
datasetRequestUrl,
|
|
parseLibraryBody,
|
|
} from '../src/library/formats.js';
|
|
import {
|
|
nextLibraryToken,
|
|
readLibraryLedger,
|
|
} from '../src/library/selection.js';
|
|
|
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
const DATA_SOURCES = path.join(root, 'DATA_SOURCES.md');
|
|
const START = '<!-- data-library:start (generated by npm run library:docs) -->';
|
|
const END = '<!-- data-library:end -->';
|
|
const LEDGER_PATH = 'src/library/ledger.json';
|
|
|
|
const cell = (text) => String(text).replace(/\|/g, '\\|').replace(/\s+/g, ' ');
|
|
|
|
/** The generated DATA_SOURCES.md section, one row per entry. */
|
|
export function libraryDocsSection(entries = LIBRARY_ENTRIES) {
|
|
const rows = [...entries]
|
|
.sort((a, b) => a.id.localeCompare(b.id))
|
|
.map((entry) => {
|
|
const host = (() => {
|
|
try {
|
|
return new URL(entry.source.url.replace(/\{[xyz]\}/g, '0')).host;
|
|
} catch {
|
|
return '';
|
|
}
|
|
})();
|
|
const how =
|
|
entry.kind === 'pack'
|
|
? `CCTV pack; operator sets \`${entry.pack.env}=1\``
|
|
: `${entry.kind}, ${entry.source.format}, ${entry.source.fetch}` +
|
|
(entry.licence.operatorOptIn
|
|
? `; operator adds \`${entry.id}\` to \`GEV_LIBRARY_ENABLE\``
|
|
: '') +
|
|
(entry.source.key
|
|
? `; needs the \`${entry.source.key.id}\` key`
|
|
: '');
|
|
return `| **${cell(entry.title)}** (\`${entry.id}\`, \`${host}\`) | ${cell(how)} | [${cell(entry.licence.name)}](${entry.licence.url}) | "${cell(entry.attribution.text)}" |`;
|
|
});
|
|
return [
|
|
START,
|
|
'',
|
|
'## Data Library entries (fetched at runtime — not stored in this repo)',
|
|
'',
|
|
'Each Data Library entry is a manifest in `src/library/entries/` pointing at a live upstream its provider hosts; see [docs/DATA-LIBRARY.md](docs/DATA-LIBRARY.md). An entry appears on the map only after a viewer adds it from **DATA LAYERS ▸ Browse data library**, and its attribution is registered in the Data attribution popover while it is in use. This table is generated from the manifests: edit the manifest, then run `npm run library:docs`.',
|
|
'',
|
|
'| Entry | How it is served | Licence / terms | Attribution |',
|
|
'| --- | --- | --- | --- |',
|
|
...rows,
|
|
'',
|
|
END,
|
|
].join('\n');
|
|
}
|
|
|
|
function replaceSection(text, section) {
|
|
const start = text.indexOf(START);
|
|
const end = text.indexOf(END);
|
|
if (start < 0 || end < start) {
|
|
const anchor = text.indexOf('\n## Bundled snapshots');
|
|
if (anchor < 0)
|
|
throw new Error('DATA_SOURCES.md has no Bundled snapshots heading');
|
|
return `${text.slice(0, anchor)}\n${section}\n${text.slice(anchor)}`;
|
|
}
|
|
return text.slice(0, start) + section + text.slice(end + END.length);
|
|
}
|
|
|
|
function check(baseRef) {
|
|
const errors = [];
|
|
const ledger = readLibraryLedger(LIBRARY_LEDGER);
|
|
errors.push(...ledger.errors.map((error) => `ledger ${error}`));
|
|
for (const entry of LIBRARY_ENTRIES) {
|
|
const switchable = entry.kind !== 'pack';
|
|
if (switchable && !ledger.tokenById.has(entry.id))
|
|
errors.push(
|
|
`${entry.id}: no share token (npm run library:next -- ${entry.id})`,
|
|
);
|
|
if (!switchable && ledger.tokenById.has(entry.id))
|
|
errors.push(
|
|
`${entry.id}: packs are switched by their host and take no token`,
|
|
);
|
|
}
|
|
if (baseRef) {
|
|
let published = [];
|
|
try {
|
|
published = JSON.parse(
|
|
execFileSync('git', ['show', `${baseRef}:${LEDGER_PATH}`], {
|
|
cwd: root,
|
|
encoding: 'utf8',
|
|
stdio: ['ignore', 'pipe', 'ignore'],
|
|
}),
|
|
);
|
|
} catch {
|
|
published = [];
|
|
}
|
|
published.forEach(([id, token], index) => {
|
|
const row = LIBRARY_LEDGER[index];
|
|
if (!row || row[0] !== id || row[1] !== token)
|
|
errors.push(
|
|
`ledger row ${index} (${id} → ${token}) was published on ${baseRef} and must not change`,
|
|
);
|
|
});
|
|
LIBRARY_LEDGER.slice(published.length).forEach(([id, token], offset) => {
|
|
const expected = nextLibraryToken(
|
|
LIBRARY_LEDGER.slice(0, published.length + offset),
|
|
);
|
|
if (token !== expected)
|
|
errors.push(
|
|
`ledger row for ${id} should take ${expected}, not ${token}`,
|
|
);
|
|
});
|
|
}
|
|
const text = readFileSync(DATA_SOURCES, 'utf8');
|
|
if (!text.includes(libraryDocsSection()))
|
|
errors.push(
|
|
'DATA_SOURCES.md Data Library section is out of date (npm run library:docs)',
|
|
);
|
|
if (errors.length) {
|
|
for (const error of errors) console.error(`✗ ${error}`);
|
|
process.exit(1);
|
|
}
|
|
console.log(
|
|
`Data Library valid: ${LIBRARY_ENTRIES.length} entries, ${LIBRARY_LEDGER.length} share tokens${baseRef ? `, ledger append-only against ${baseRef}` : ''}.`,
|
|
);
|
|
}
|
|
|
|
/** Points inside a coverage bbox to try, centre first, as [fx, fy] shares. */
|
|
const PROBE_POINTS = [
|
|
[0.5, 0.5],
|
|
[0.4, 0.45],
|
|
[0.6, 0.55],
|
|
[0.25, 0.25],
|
|
[0.75, 0.75],
|
|
];
|
|
|
|
/**
|
|
* A Web Mercator tile to probe. Regional entries try points inside their
|
|
* coverage bbox (a regional server answers 404 or a blank fill outside its
|
|
* photos, and a bbox centre can be sea); global entries probe a mid-world
|
|
* tile.
|
|
*/
|
|
function probeTile(entry, z, [fx, fy] = PROBE_POINTS[0]) {
|
|
const n = 2 ** z;
|
|
const bbox = entry.coverage?.bbox;
|
|
if (!bbox) return { x: Math.floor(n / 2), y: Math.floor(n / 3) };
|
|
const lon = bbox[0] + (bbox[2] - bbox[0]) * fx;
|
|
const lat = Math.max(-85, Math.min(85, bbox[1] + (bbox[3] - bbox[1]) * fy));
|
|
const rad = (lat * Math.PI) / 180;
|
|
const clamp = (value) => Math.min(n - 1, Math.max(0, Math.floor(value)));
|
|
return {
|
|
x: clamp(((lon + 180) / 360) * n),
|
|
y: clamp(
|
|
((1 - Math.log(Math.tan(rad) + 1 / Math.cos(rad)) / Math.PI) / 2) * n,
|
|
),
|
|
};
|
|
}
|
|
|
|
async function probeEntry(entry) {
|
|
const started = Date.now();
|
|
const signal = AbortSignal.timeout(30_000);
|
|
try {
|
|
if (entry.kind === 'tiles') {
|
|
// Regional imagery is probed at z13, the Open Imagery Register's
|
|
// validation zoom (some caches start there); global layers at their
|
|
// coarsest level.
|
|
const minZoom = entry.source.minZoom ?? 0;
|
|
const z = entry.coverage.bbox
|
|
? Math.min(entry.source.maxZoom ?? 18, Math.max(minZoom, 13))
|
|
: minZoom;
|
|
const points = entry.coverage.bbox ? PROBE_POINTS : [PROBE_POINTS[0]];
|
|
let status = null;
|
|
let lastError = null;
|
|
for (const point of points) {
|
|
const { x, y } = probeTile(entry, z, point);
|
|
const url = libraryTileUrl(entry.source.url, z, x, y);
|
|
try {
|
|
const response = await fetch(url, {
|
|
signal: AbortSignal.timeout(20_000),
|
|
redirect: 'error',
|
|
});
|
|
await response.arrayBuffer();
|
|
status = response.status;
|
|
if (
|
|
response.ok &&
|
|
/^image\//.test(response.headers.get('content-type') || '')
|
|
)
|
|
return { id: entry.id, ok: true, status, ms: Date.now() - started };
|
|
} catch (error) {
|
|
lastError = error;
|
|
}
|
|
}
|
|
if (status === null && lastError) throw lastError;
|
|
return { id: entry.id, ok: false, status, ms: Date.now() - started };
|
|
}
|
|
if (entry.source.format === 'geoparquet') {
|
|
// List the partitions and read one footer by range request: proof the
|
|
// bucket answers and the files are GeoParquet, without reading data.
|
|
const { listPartitions, rangeBuffer, geoParquetInfo } =
|
|
await import('../src/library/parquet.js');
|
|
const { parquetMetadataAsync } = await import('hyparquet');
|
|
const fetchImpl = (target, options = {}) =>
|
|
fetch(target, { ...options, signal });
|
|
const objects = entry.source.partitions
|
|
? await listPartitions(entry, { fetchImpl, signal })
|
|
: [{ key: entry.source.url, size: null }];
|
|
const first = objects[0];
|
|
const url = entry.source.partitions
|
|
? new URL(first.key, entry.source.partitions.keyBase).href
|
|
: entry.source.url;
|
|
const size =
|
|
first.size ??
|
|
Number(
|
|
(await fetch(url, { method: 'HEAD', signal })).headers.get(
|
|
'content-length',
|
|
),
|
|
);
|
|
const metadata = await parquetMetadataAsync(
|
|
rangeBuffer(url, size, { fetchImpl, signal }),
|
|
{ initialFetchSize: 16 * 1024 },
|
|
);
|
|
return {
|
|
id: entry.id,
|
|
ok: Boolean(geoParquetInfo(metadata).column),
|
|
status: 200,
|
|
ms: Date.now() - started,
|
|
features: Number(metadata.num_rows),
|
|
partitions: objects.length,
|
|
};
|
|
}
|
|
const url =
|
|
entry.source.format === 'arcgis'
|
|
? arcgisPageUrl(
|
|
entry.source.url,
|
|
entry.source.query,
|
|
0,
|
|
// Some huge view services refuse a page smaller than their own.
|
|
entry.source.view ? 2000 : 100,
|
|
// A view-bounded layer is probed over its sample area.
|
|
entry.source.view?.sample ?? null,
|
|
)
|
|
: datasetRequestUrl(entry.source);
|
|
const response = await fetch(url, { signal, redirect: 'error' });
|
|
const text = await response.text();
|
|
const parsed = response.ok
|
|
? parseLibraryBody(entry.source.format, text, entry.source)
|
|
: null;
|
|
return {
|
|
id: entry.id,
|
|
ok: Boolean(parsed?.candidates.length),
|
|
status: response.status,
|
|
ms: Date.now() - started,
|
|
features: parsed?.candidates.length ?? 0,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
id: entry.id,
|
|
ok: false,
|
|
status: null,
|
|
ms: Date.now() - started,
|
|
error: error.message,
|
|
};
|
|
}
|
|
}
|
|
|
|
const [command = 'check', ...rest] = process.argv.slice(2);
|
|
const flag = (name) => rest.includes(name);
|
|
const option = (name) => {
|
|
const index = rest.indexOf(name);
|
|
return index >= 0 ? rest[index + 1] : null;
|
|
};
|
|
|
|
if (command === 'check') check(option('--base-ref'));
|
|
else if (command === 'next') {
|
|
const id = rest.find((arg) => !arg.startsWith('--'));
|
|
if (!id) throw new Error('usage: npm run library:next -- <entry-id>');
|
|
if (LIBRARY_LEDGER.some(([existing]) => existing === id))
|
|
throw new Error(`${id} already owns a token`);
|
|
console.log(nextLibraryToken(LIBRARY_LEDGER));
|
|
} else if (command === 'docs') {
|
|
const text = readFileSync(DATA_SOURCES, 'utf8');
|
|
writeFileSync(DATA_SOURCES, replaceSection(text, libraryDocsSection()));
|
|
console.log('DATA_SOURCES.md Data Library section updated.');
|
|
} else if (command === 'probe') {
|
|
const results = [];
|
|
for (const entry of LIBRARY_ENTRIES.filter(({ kind }) => kind !== 'pack'))
|
|
results.push(await probeEntry(entry));
|
|
if (flag('--json')) console.log(JSON.stringify(results, null, 2));
|
|
else
|
|
for (const result of results)
|
|
console.log(
|
|
`${result.ok ? 'ok ' : 'FAIL'} ${result.id} · ${result.status ?? '—'} · ${result.ms} ms${result.features !== undefined ? ` · ${result.features} features` : ''}${result.partitions !== undefined ? ` in the first of ${result.partitions} partitions` : ''}${result.error ? ` · ${result.error}` : ''}`,
|
|
);
|
|
if (flag('--strict') && results.some(({ ok }) => !ok)) process.exit(1);
|
|
} else {
|
|
console.error(`unknown command: ${command}`);
|
|
process.exit(2);
|
|
}
|