mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-13 06:42:23 +00:00
Add the Data Library: live regional datasets, tiles and packs
DATA LAYERS keeps its curated core layers; regional and niche sources live in the Data Library, opened from Browse data library at the foot of the panel. Each entry is a JSON manifest in src/library/entries/ pointing at a live upstream (query API, provider-hosted file or tile endpoint), so nothing is bundled and a self-hosted instance hosts no data. Sources without CORS go through one allowlisted /api/library route. Kinds: dataset (GeoJSON, CSV, JSON records, ArcGIS, partitioned GeoParquet), tiles (XYZ, WMS/ArcGIS bbox export, terrarium relief) and pack. Selections persist and travel in share links as lib=, separate from the core layer tokens. Library rows get Zoom and Remove icon chips that work while the layer is off, rendered through RailChips. Strings are translated in every locale.
This commit is contained in:
Said Turksever committed
1 parent
10a788db78
commit
db80d82b8d
182 files changed
+15805
-44
No files matched your search
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Data Library maintenance. See docs/DATA-LIBRARY.md.
|
||||
*
|
||||
* npm run library:check [-- --base-ref origin/main]
|
||||
* Validate every manifest, the share-token ledger and the generated
|
||||
* DATA_SOURCES.md section. With --base-ref, also prove the ledger only
|
||||
* appended rows since that ref (published tokens never change).
|
||||
* npm run library:next -- <entry-id>
|
||||
* Print the next free share token for a new switchable entry.
|
||||
* npm run library:docs
|
||||
* Rewrite the generated Data Library section of DATA_SOURCES.md.
|
||||
* npm run library:probe [-- --json] [-- --strict]
|
||||
* Hit every dataset and tile upstream once (bounded) and report
|
||||
* status, latency and feature count. --strict exits 1 on a failure.
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { LIBRARY_ENTRIES, LIBRARY_LEDGER } from '../src/library/catalog.js';
|
||||
import { libraryTileUrl } from '../src/library/manifest.js';
|
||||
import {
|
||||
arcgisPageUrl,
|
||||
datasetRequestUrl,
|
||||
parseLibraryBody,
|
||||
} from '../src/library/formats.js';
|
||||
import {
|
||||
nextLibraryToken,
|
||||
readLibraryLedger,
|
||||
} from '../src/library/selection.js';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const DATA_SOURCES = path.join(root, 'DATA_SOURCES.md');
|
||||
const START = '<!-- data-library:start (generated by npm run library:docs) -->';
|
||||
const END = '<!-- data-library:end -->';
|
||||
const LEDGER_PATH = 'src/library/ledger.json';
|
||||
|
||||
const cell = (text) => String(text).replace(/\|/g, '\\|').replace(/\s+/g, ' ');
|
||||
|
||||
/** The generated DATA_SOURCES.md section, one row per entry. */
|
||||
export function libraryDocsSection(entries = LIBRARY_ENTRIES) {
|
||||
const rows = [...entries]
|
||||
.sort((a, b) => a.id.localeCompare(b.id))
|
||||
.map((entry) => {
|
||||
const host = (() => {
|
||||
try {
|
||||
return new URL(entry.source.url.replace(/\{[xyz]\}/g, '0')).host;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
})();
|
||||
const how =
|
||||
entry.kind === 'pack'
|
||||
? `CCTV pack; operator sets \`${entry.pack.env}=1\``
|
||||
: `${entry.kind}, ${entry.source.format}, ${entry.source.fetch}` +
|
||||
(entry.licence.operatorOptIn
|
||||
? `; operator adds \`${entry.id}\` to \`GEV_LIBRARY_ENABLE\``
|
||||
: '') +
|
||||
(entry.source.key
|
||||
? `; needs the \`${entry.source.key.id}\` key`
|
||||
: '');
|
||||
return `| **${cell(entry.title)}** (\`${entry.id}\`, \`${host}\`) | ${cell(how)} | [${cell(entry.licence.name)}](${entry.licence.url}) | "${cell(entry.attribution.text)}" |`;
|
||||
});
|
||||
return [
|
||||
START,
|
||||
'',
|
||||
'## Data Library entries (fetched at runtime — not stored in this repo)',
|
||||
'',
|
||||
'Each Data Library entry is a manifest in `src/library/entries/` pointing at a live upstream its provider hosts; see [docs/DATA-LIBRARY.md](docs/DATA-LIBRARY.md). An entry appears on the map only after a viewer adds it from **DATA LAYERS ▸ Browse data library**, and its attribution is registered in the Data attribution popover while it is in use. This table is generated from the manifests: edit the manifest, then run `npm run library:docs`.',
|
||||
'',
|
||||
'| Entry | How it is served | Licence / terms | Attribution |',
|
||||
'| --- | --- | --- | --- |',
|
||||
...rows,
|
||||
'',
|
||||
END,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function replaceSection(text, section) {
|
||||
const start = text.indexOf(START);
|
||||
const end = text.indexOf(END);
|
||||
if (start < 0 || end < start) {
|
||||
const anchor = text.indexOf('\n## Bundled snapshots');
|
||||
if (anchor < 0)
|
||||
throw new Error('DATA_SOURCES.md has no Bundled snapshots heading');
|
||||
return `${text.slice(0, anchor)}\n${section}\n${text.slice(anchor)}`;
|
||||
}
|
||||
return text.slice(0, start) + section + text.slice(end + END.length);
|
||||
}
|
||||
|
||||
function check(baseRef) {
|
||||
const errors = [];
|
||||
const ledger = readLibraryLedger(LIBRARY_LEDGER);
|
||||
errors.push(...ledger.errors.map((error) => `ledger ${error}`));
|
||||
for (const entry of LIBRARY_ENTRIES) {
|
||||
const switchable = entry.kind !== 'pack';
|
||||
if (switchable && !ledger.tokenById.has(entry.id))
|
||||
errors.push(
|
||||
`${entry.id}: no share token (npm run library:next -- ${entry.id})`,
|
||||
);
|
||||
if (!switchable && ledger.tokenById.has(entry.id))
|
||||
errors.push(
|
||||
`${entry.id}: packs are switched by their host and take no token`,
|
||||
);
|
||||
}
|
||||
if (baseRef) {
|
||||
let published = [];
|
||||
try {
|
||||
published = JSON.parse(
|
||||
execFileSync('git', ['show', `${baseRef}:${LEDGER_PATH}`], {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
published = [];
|
||||
}
|
||||
published.forEach(([id, token], index) => {
|
||||
const row = LIBRARY_LEDGER[index];
|
||||
if (!row || row[0] !== id || row[1] !== token)
|
||||
errors.push(
|
||||
`ledger row ${index} (${id} → ${token}) was published on ${baseRef} and must not change`,
|
||||
);
|
||||
});
|
||||
LIBRARY_LEDGER.slice(published.length).forEach(([id, token], offset) => {
|
||||
const expected = nextLibraryToken(
|
||||
LIBRARY_LEDGER.slice(0, published.length + offset),
|
||||
);
|
||||
if (token !== expected)
|
||||
errors.push(
|
||||
`ledger row for ${id} should take ${expected}, not ${token}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
const text = readFileSync(DATA_SOURCES, 'utf8');
|
||||
if (!text.includes(libraryDocsSection()))
|
||||
errors.push(
|
||||
'DATA_SOURCES.md Data Library section is out of date (npm run library:docs)',
|
||||
);
|
||||
if (errors.length) {
|
||||
for (const error of errors) console.error(`✗ ${error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(
|
||||
`Data Library valid: ${LIBRARY_ENTRIES.length} entries, ${LIBRARY_LEDGER.length} share tokens${baseRef ? `, ledger append-only against ${baseRef}` : ''}.`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Points inside a coverage bbox to try, centre first, as [fx, fy] shares. */
|
||||
const PROBE_POINTS = [
|
||||
[0.5, 0.5],
|
||||
[0.4, 0.45],
|
||||
[0.6, 0.55],
|
||||
[0.25, 0.25],
|
||||
[0.75, 0.75],
|
||||
];
|
||||
|
||||
/**
|
||||
* A Web Mercator tile to probe. Regional entries try points inside their
|
||||
* coverage bbox (a regional server answers 404 or a blank fill outside its
|
||||
* photos, and a bbox centre can be sea); global entries probe a mid-world
|
||||
* tile.
|
||||
*/
|
||||
function probeTile(entry, z, [fx, fy] = PROBE_POINTS[0]) {
|
||||
const n = 2 ** z;
|
||||
const bbox = entry.coverage?.bbox;
|
||||
if (!bbox) return { x: Math.floor(n / 2), y: Math.floor(n / 3) };
|
||||
const lon = bbox[0] + (bbox[2] - bbox[0]) * fx;
|
||||
const lat = Math.max(-85, Math.min(85, bbox[1] + (bbox[3] - bbox[1]) * fy));
|
||||
const rad = (lat * Math.PI) / 180;
|
||||
const clamp = (value) => Math.min(n - 1, Math.max(0, Math.floor(value)));
|
||||
return {
|
||||
x: clamp(((lon + 180) / 360) * n),
|
||||
y: clamp(
|
||||
((1 - Math.log(Math.tan(rad) + 1 / Math.cos(rad)) / Math.PI) / 2) * n,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function probeEntry(entry) {
|
||||
const started = Date.now();
|
||||
const signal = AbortSignal.timeout(30_000);
|
||||
try {
|
||||
if (entry.kind === 'tiles') {
|
||||
// Regional imagery is probed at z13, the Open Imagery Register's
|
||||
// validation zoom (some caches start there); global layers at their
|
||||
// coarsest level.
|
||||
const minZoom = entry.source.minZoom ?? 0;
|
||||
const z = entry.coverage.bbox
|
||||
? Math.min(entry.source.maxZoom ?? 18, Math.max(minZoom, 13))
|
||||
: minZoom;
|
||||
const points = entry.coverage.bbox ? PROBE_POINTS : [PROBE_POINTS[0]];
|
||||
let status = null;
|
||||
let lastError = null;
|
||||
for (const point of points) {
|
||||
const { x, y } = probeTile(entry, z, point);
|
||||
const url = libraryTileUrl(entry.source.url, z, x, y);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(20_000),
|
||||
redirect: 'error',
|
||||
});
|
||||
await response.arrayBuffer();
|
||||
status = response.status;
|
||||
if (
|
||||
response.ok &&
|
||||
/^image\//.test(response.headers.get('content-type') || '')
|
||||
)
|
||||
return { id: entry.id, ok: true, status, ms: Date.now() - started };
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
if (status === null && lastError) throw lastError;
|
||||
return { id: entry.id, ok: false, status, ms: Date.now() - started };
|
||||
}
|
||||
if (entry.source.format === 'geoparquet') {
|
||||
// List the partitions and read one footer by range request: proof the
|
||||
// bucket answers and the files are GeoParquet, without reading data.
|
||||
const { listPartitions, rangeBuffer, geoParquetInfo } =
|
||||
await import('../src/library/parquet.js');
|
||||
const { parquetMetadataAsync } = await import('hyparquet');
|
||||
const fetchImpl = (target, options = {}) =>
|
||||
fetch(target, { ...options, signal });
|
||||
const objects = entry.source.partitions
|
||||
? await listPartitions(entry, { fetchImpl, signal })
|
||||
: [{ key: entry.source.url, size: null }];
|
||||
const first = objects[0];
|
||||
const url = entry.source.partitions
|
||||
? new URL(first.key, entry.source.partitions.keyBase).href
|
||||
: entry.source.url;
|
||||
const size =
|
||||
first.size ??
|
||||
Number(
|
||||
(await fetch(url, { method: 'HEAD', signal })).headers.get(
|
||||
'content-length',
|
||||
),
|
||||
);
|
||||
const metadata = await parquetMetadataAsync(
|
||||
rangeBuffer(url, size, { fetchImpl, signal }),
|
||||
{ initialFetchSize: 16 * 1024 },
|
||||
);
|
||||
return {
|
||||
id: entry.id,
|
||||
ok: Boolean(geoParquetInfo(metadata).column),
|
||||
status: 200,
|
||||
ms: Date.now() - started,
|
||||
features: Number(metadata.num_rows),
|
||||
partitions: objects.length,
|
||||
};
|
||||
}
|
||||
const url =
|
||||
entry.source.format === 'arcgis'
|
||||
? arcgisPageUrl(
|
||||
entry.source.url,
|
||||
entry.source.query,
|
||||
0,
|
||||
// Some huge view services refuse a page smaller than their own.
|
||||
entry.source.view ? 2000 : 100,
|
||||
// A view-bounded layer is probed over its sample area.
|
||||
entry.source.view?.sample ?? null,
|
||||
)
|
||||
: datasetRequestUrl(entry.source);
|
||||
const response = await fetch(url, { signal, redirect: 'error' });
|
||||
const text = await response.text();
|
||||
const parsed = response.ok
|
||||
? parseLibraryBody(entry.source.format, text, entry.source)
|
||||
: null;
|
||||
return {
|
||||
id: entry.id,
|
||||
ok: Boolean(parsed?.candidates.length),
|
||||
status: response.status,
|
||||
ms: Date.now() - started,
|
||||
features: parsed?.candidates.length ?? 0,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: entry.id,
|
||||
ok: false,
|
||||
status: null,
|
||||
ms: Date.now() - started,
|
||||
error: error.message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const [command = 'check', ...rest] = process.argv.slice(2);
|
||||
const flag = (name) => rest.includes(name);
|
||||
const option = (name) => {
|
||||
const index = rest.indexOf(name);
|
||||
return index >= 0 ? rest[index + 1] : null;
|
||||
};
|
||||
|
||||
if (command === 'check') check(option('--base-ref'));
|
||||
else if (command === 'next') {
|
||||
const id = rest.find((arg) => !arg.startsWith('--'));
|
||||
if (!id) throw new Error('usage: npm run library:next -- <entry-id>');
|
||||
if (LIBRARY_LEDGER.some(([existing]) => existing === id))
|
||||
throw new Error(`${id} already owns a token`);
|
||||
console.log(nextLibraryToken(LIBRARY_LEDGER));
|
||||
} else if (command === 'docs') {
|
||||
const text = readFileSync(DATA_SOURCES, 'utf8');
|
||||
writeFileSync(DATA_SOURCES, replaceSection(text, libraryDocsSection()));
|
||||
console.log('DATA_SOURCES.md Data Library section updated.');
|
||||
} else if (command === 'probe') {
|
||||
const results = [];
|
||||
for (const entry of LIBRARY_ENTRIES.filter(({ kind }) => kind !== 'pack'))
|
||||
results.push(await probeEntry(entry));
|
||||
if (flag('--json')) console.log(JSON.stringify(results, null, 2));
|
||||
else
|
||||
for (const result of results)
|
||||
console.log(
|
||||
`${result.ok ? 'ok ' : 'FAIL'} ${result.id} · ${result.status ?? '—'} · ${result.ms} ms${result.features !== undefined ? ` · ${result.features} features` : ''}${result.partitions !== undefined ? ` in the first of ${result.partitions} partitions` : ''}${result.error ? ` · ${result.error}` : ''}`,
|
||||
);
|
||||
if (flag('--strict') && results.some(({ ok }) => !ok)) process.exit(1);
|
||||
} else {
|
||||
console.error(`unknown command: ${command}`);
|
||||
process.exit(2);
|
||||
}
|
||||
Reference in new issue
Block a user