mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-17 08:42:21 +00:00
Add Data Library: live-fetched dataset, tile and pack layers
This commit is contained in:
1 parent
5a1598fd9e
commit
976fa6d6ce
169 files changed
+14637
-31
No files matched your search
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Data Library maintenance. See docs/DATA-LIBRARY.md.
|
||||
*
|
||||
* npm run library:check [-- --base-ref origin/main]
|
||||
* Validate every manifest, the share-token ledger and the generated
|
||||
* DATA_SOURCES.md section. With --base-ref, also prove the ledger only
|
||||
* appended rows since that ref (published tokens never change).
|
||||
* npm run library:next -- <entry-id>
|
||||
* Print the next free share token for a new switchable entry.
|
||||
* npm run library:docs
|
||||
* Rewrite the generated Data Library section of DATA_SOURCES.md.
|
||||
* npm run library:probe [-- --json] [-- --strict]
|
||||
* Hit every dataset and tile upstream once (bounded) and report
|
||||
* status, latency and feature count. --strict exits 1 on a failure.
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { LIBRARY_ENTRIES, LIBRARY_LEDGER } from '../src/library/catalog.js';
|
||||
import { libraryTileUrl } from '../src/library/manifest.js';
|
||||
import {
|
||||
arcgisPageUrl,
|
||||
datasetRequestUrl,
|
||||
parseLibraryBody,
|
||||
} from '../src/library/formats.js';
|
||||
import {
|
||||
nextLibraryToken,
|
||||
readLibraryLedger,
|
||||
} from '../src/library/selection.js';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const DATA_SOURCES = path.join(root, 'DATA_SOURCES.md');
|
||||
const START = '<!-- data-library:start (generated by npm run library:docs) -->';
|
||||
const END = '<!-- data-library:end -->';
|
||||
const LEDGER_PATH = 'src/library/ledger.json';
|
||||
|
||||
const cell = (text) => String(text).replace(/\|/g, '\\|').replace(/\s+/g, ' ');
|
||||
|
||||
/** The generated DATA_SOURCES.md section, one row per entry. */
|
||||
export function libraryDocsSection(entries = LIBRARY_ENTRIES) {
|
||||
const rows = [...entries]
|
||||
.sort((a, b) => a.id.localeCompare(b.id))
|
||||
.map((entry) => {
|
||||
const host = (() => {
|
||||
try {
|
||||
return new URL(entry.source.url.replace(/\{[xyz]\}/g, '0')).host;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
})();
|
||||
const how =
|
||||
entry.kind === 'pack'
|
||||
? `CCTV pack; operator sets \`${entry.pack.env}=1\``
|
||||
: `${entry.kind}, ${entry.source.format}, ${entry.source.fetch}` +
|
||||
(entry.licence.operatorOptIn
|
||||
? `; operator adds \`${entry.id}\` to \`GEV_LIBRARY_ENABLE\``
|
||||
: '') +
|
||||
(entry.source.key
|
||||
? `; needs the \`${entry.source.key.id}\` key`
|
||||
: '');
|
||||
return `| **${cell(entry.title)}** (\`${entry.id}\`, \`${host}\`) | ${cell(how)} | [${cell(entry.licence.name)}](${entry.licence.url}) | "${cell(entry.attribution.text)}" |`;
|
||||
});
|
||||
return [
|
||||
START,
|
||||
'',
|
||||
'## Data Library entries (fetched at runtime — not stored in this repo)',
|
||||
'',
|
||||
'Each Data Library entry is a manifest in `src/library/entries/` pointing at a live upstream its provider hosts; see [docs/DATA-LIBRARY.md](docs/DATA-LIBRARY.md). An entry appears on the map only after a viewer adds it from **DATA LAYERS ▸ Browse data library**, and its attribution is registered in the Data attribution popover while it is in use. This table is generated from the manifests: edit the manifest, then run `npm run library:docs`.',
|
||||
'',
|
||||
'| Entry | How it is served | Licence / terms | Attribution |',
|
||||
'| --- | --- | --- | --- |',
|
||||
...rows,
|
||||
'',
|
||||
END,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function replaceSection(text, section) {
|
||||
const start = text.indexOf(START);
|
||||
const end = text.indexOf(END);
|
||||
if (start < 0 || end < start) {
|
||||
const anchor = text.indexOf('\n## Bundled snapshots');
|
||||
if (anchor < 0)
|
||||
throw new Error('DATA_SOURCES.md has no Bundled snapshots heading');
|
||||
return `${text.slice(0, anchor)}\n${section}\n${text.slice(anchor)}`;
|
||||
}
|
||||
return text.slice(0, start) + section + text.slice(end + END.length);
|
||||
}
|
||||
|
||||
function check(baseRef) {
|
||||
const errors = [];
|
||||
const ledger = readLibraryLedger(LIBRARY_LEDGER);
|
||||
errors.push(...ledger.errors.map((error) => `ledger ${error}`));
|
||||
for (const entry of LIBRARY_ENTRIES) {
|
||||
const switchable = entry.kind !== 'pack';
|
||||
if (switchable && !ledger.tokenById.has(entry.id))
|
||||
errors.push(
|
||||
`${entry.id}: no share token (npm run library:next -- ${entry.id})`,
|
||||
);
|
||||
if (!switchable && ledger.tokenById.has(entry.id))
|
||||
errors.push(
|
||||
`${entry.id}: packs are switched by their host and take no token`,
|
||||
);
|
||||
}
|
||||
if (baseRef) {
|
||||
let published = [];
|
||||
try {
|
||||
published = JSON.parse(
|
||||
execFileSync('git', ['show', `${baseRef}:${LEDGER_PATH}`], {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
published = [];
|
||||
}
|
||||
published.forEach(([id, token], index) => {
|
||||
const row = LIBRARY_LEDGER[index];
|
||||
if (!row || row[0] !== id || row[1] !== token)
|
||||
errors.push(
|
||||
`ledger row ${index} (${id} → ${token}) was published on ${baseRef} and must not change`,
|
||||
);
|
||||
});
|
||||
LIBRARY_LEDGER.slice(published.length).forEach(([id, token], offset) => {
|
||||
const expected = nextLibraryToken(
|
||||
LIBRARY_LEDGER.slice(0, published.length + offset),
|
||||
);
|
||||
if (token !== expected)
|
||||
errors.push(
|
||||
`ledger row for ${id} should take ${expected}, not ${token}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
const text = readFileSync(DATA_SOURCES, 'utf8');
|
||||
if (!text.includes(libraryDocsSection()))
|
||||
errors.push(
|
||||
'DATA_SOURCES.md Data Library section is out of date (npm run library:docs)',
|
||||
);
|
||||
if (errors.length) {
|
||||
for (const error of errors) console.error(`✗ ${error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(
|
||||
`Data Library valid: ${LIBRARY_ENTRIES.length} entries, ${LIBRARY_LEDGER.length} share tokens${baseRef ? `, ledger append-only against ${baseRef}` : ''}.`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Points inside a coverage bbox to try, centre first, as [fx, fy] shares. */
|
||||
const PROBE_POINTS = [
|
||||
[0.5, 0.5],
|
||||
[0.4, 0.45],
|
||||
[0.6, 0.55],
|
||||
[0.25, 0.25],
|
||||
[0.75, 0.75],
|
||||
];
|
||||
|
||||
/**
|
||||
* A Web Mercator tile to probe. Regional entries try points inside their
|
||||
* coverage bbox (a regional server answers 404 or a blank fill outside its
|
||||
* photos, and a bbox centre can be sea); global entries probe a mid-world
|
||||
* tile.
|
||||
*/
|
||||
function probeTile(entry, z, [fx, fy] = PROBE_POINTS[0]) {
|
||||
const n = 2 ** z;
|
||||
const bbox = entry.coverage?.bbox;
|
||||
if (!bbox) return { x: Math.floor(n / 2), y: Math.floor(n / 3) };
|
||||
const lon = bbox[0] + (bbox[2] - bbox[0]) * fx;
|
||||
const lat = Math.max(-85, Math.min(85, bbox[1] + (bbox[3] - bbox[1]) * fy));
|
||||
const rad = (lat * Math.PI) / 180;
|
||||
const clamp = (value) => Math.min(n - 1, Math.max(0, Math.floor(value)));
|
||||
return {
|
||||
x: clamp(((lon + 180) / 360) * n),
|
||||
y: clamp(
|
||||
((1 - Math.log(Math.tan(rad) + 1 / Math.cos(rad)) / Math.PI) / 2) * n,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function probeEntry(entry) {
|
||||
const started = Date.now();
|
||||
const signal = AbortSignal.timeout(30_000);
|
||||
try {
|
||||
if (entry.kind === 'tiles') {
|
||||
// Regional imagery is probed at z13, the Open Imagery Register's
|
||||
// validation zoom (some caches start there); global layers at their
|
||||
// coarsest level.
|
||||
const minZoom = entry.source.minZoom ?? 0;
|
||||
const z = entry.coverage.bbox
|
||||
? Math.min(entry.source.maxZoom ?? 18, Math.max(minZoom, 13))
|
||||
: minZoom;
|
||||
const points = entry.coverage.bbox ? PROBE_POINTS : [PROBE_POINTS[0]];
|
||||
let status = null;
|
||||
let lastError = null;
|
||||
for (const point of points) {
|
||||
const { x, y } = probeTile(entry, z, point);
|
||||
const url = libraryTileUrl(entry.source.url, z, x, y);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(20_000),
|
||||
redirect: 'error',
|
||||
});
|
||||
await response.arrayBuffer();
|
||||
status = response.status;
|
||||
if (
|
||||
response.ok &&
|
||||
/^image\//.test(response.headers.get('content-type') || '')
|
||||
)
|
||||
return { id: entry.id, ok: true, status, ms: Date.now() - started };
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
if (status === null && lastError) throw lastError;
|
||||
return { id: entry.id, ok: false, status, ms: Date.now() - started };
|
||||
}
|
||||
if (entry.source.format === 'geoparquet') {
|
||||
// List the partitions and read one footer by range request: proof the
|
||||
// bucket answers and the files are GeoParquet, without reading data.
|
||||
const { listPartitions, rangeBuffer, geoParquetInfo } =
|
||||
await import('../src/library/parquet.js');
|
||||
const { parquetMetadataAsync } = await import('hyparquet');
|
||||
const fetchImpl = (target, options = {}) =>
|
||||
fetch(target, { ...options, signal });
|
||||
const objects = entry.source.partitions
|
||||
? await listPartitions(entry, { fetchImpl, signal })
|
||||
: [{ key: entry.source.url, size: null }];
|
||||
const first = objects[0];
|
||||
const url = entry.source.partitions
|
||||
? new URL(first.key, entry.source.partitions.keyBase).href
|
||||
: entry.source.url;
|
||||
const size =
|
||||
first.size ??
|
||||
Number(
|
||||
(await fetch(url, { method: 'HEAD', signal })).headers.get(
|
||||
'content-length',
|
||||
),
|
||||
);
|
||||
const metadata = await parquetMetadataAsync(
|
||||
rangeBuffer(url, size, { fetchImpl, signal }),
|
||||
{ initialFetchSize: 16 * 1024 },
|
||||
);
|
||||
return {
|
||||
id: entry.id,
|
||||
ok: Boolean(geoParquetInfo(metadata).column),
|
||||
status: 200,
|
||||
ms: Date.now() - started,
|
||||
features: Number(metadata.num_rows),
|
||||
partitions: objects.length,
|
||||
};
|
||||
}
|
||||
const url =
|
||||
entry.source.format === 'arcgis'
|
||||
? arcgisPageUrl(
|
||||
entry.source.url,
|
||||
entry.source.query,
|
||||
0,
|
||||
// Some huge view services refuse a page smaller than their own.
|
||||
entry.source.view ? 2000 : 100,
|
||||
// A view-bounded layer is probed over its sample area.
|
||||
entry.source.view?.sample ?? null,
|
||||
)
|
||||
: datasetRequestUrl(entry.source);
|
||||
const response = await fetch(url, { signal, redirect: 'error' });
|
||||
const text = await response.text();
|
||||
const parsed = response.ok
|
||||
? parseLibraryBody(entry.source.format, text, entry.source)
|
||||
: null;
|
||||
return {
|
||||
id: entry.id,
|
||||
ok: Boolean(parsed?.candidates.length),
|
||||
status: response.status,
|
||||
ms: Date.now() - started,
|
||||
features: parsed?.candidates.length ?? 0,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: entry.id,
|
||||
ok: false,
|
||||
status: null,
|
||||
ms: Date.now() - started,
|
||||
error: error.message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const [command = 'check', ...rest] = process.argv.slice(2);
|
||||
const flag = (name) => rest.includes(name);
|
||||
const option = (name) => {
|
||||
const index = rest.indexOf(name);
|
||||
return index >= 0 ? rest[index + 1] : null;
|
||||
};
|
||||
|
||||
if (command === 'check') check(option('--base-ref'));
|
||||
else if (command === 'next') {
|
||||
const id = rest.find((arg) => !arg.startsWith('--'));
|
||||
if (!id) throw new Error('usage: npm run library:next -- <entry-id>');
|
||||
if (LIBRARY_LEDGER.some(([existing]) => existing === id))
|
||||
throw new Error(`${id} already owns a token`);
|
||||
console.log(nextLibraryToken(LIBRARY_LEDGER));
|
||||
} else if (command === 'docs') {
|
||||
const text = readFileSync(DATA_SOURCES, 'utf8');
|
||||
writeFileSync(DATA_SOURCES, replaceSection(text, libraryDocsSection()));
|
||||
console.log('DATA_SOURCES.md Data Library section updated.');
|
||||
} else if (command === 'probe') {
|
||||
const results = [];
|
||||
for (const entry of LIBRARY_ENTRIES.filter(({ kind }) => kind !== 'pack'))
|
||||
results.push(await probeEntry(entry));
|
||||
if (flag('--json')) console.log(JSON.stringify(results, null, 2));
|
||||
else
|
||||
for (const result of results)
|
||||
console.log(
|
||||
`${result.ok ? 'ok ' : 'FAIL'} ${result.id} · ${result.status ?? '—'} · ${result.ms} ms${result.features !== undefined ? ` · ${result.features} features` : ''}${result.partitions !== undefined ? ` in the first of ${result.partitions} partitions` : ''}${result.error ? ` · ${result.error}` : ''}`,
|
||||
);
|
||||
if (flag('--strict') && results.some(({ ok }) => !ok)) process.exit(1);
|
||||
} else {
|
||||
console.error(`unknown command: ${command}`);
|
||||
process.exit(2);
|
||||
}
|
||||
Reference in new issue
Block a user