Add the Data Library: live regional datasets, tiles and packs

DATA LAYERS keeps its curated core layers; regional and niche sources
live in the Data Library, opened from Browse data library at the foot
of the panel. Each entry is a JSON manifest in src/library/entries/
pointing at a live upstream (query API, provider-hosted file or tile
endpoint), so nothing is bundled and a self-hosted instance hosts no
data. Sources without CORS go through one allowlisted /api/library
route. Kinds: dataset (GeoJSON, CSV, JSON records, ArcGIS, partitioned
GeoParquet), tiles (XYZ, WMS/ArcGIS bbox export, terrarium relief) and
pack. Selections persist and travel in share links as lib=, separate
from the core layer tokens. Library rows get Zoom and Remove icon
chips that work while the layer is off, rendered through RailChips.
Strings are translated in every locale.
This commit is contained in:
Said Turksever committed 2026-11-12 12:26:45 +00:00
1 parent 10a788db78
commit db80d82b8d
182 files changed
+15805 -44

No files matched your search

+363
View File
@@ -0,0 +1,363 @@
import { LIBRARY_ENTRIES } from '../../src/library/catalog.js';
import { loadLibraryDataset } from '../../src/library/formats.js';
import { buildPartitionIndex } from '../../src/library/parquet.js';
import {
libraryUpstreamUrl,
parseLibraryIdList,
libraryTileUrl,
} from '../../src/library/manifest.js';
import { KEY_SETUP_KEYS } from '../../src/keySetupCore.mjs';
import {
readResponseBytesCapped,
readResponseTextCapped,
coalesceProxyRequest,
} from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSite, admitSameSiteKeyless } from './common/same-site.js';
const DATASET_TIMEOUT_MS = 30_000;
const TILE_TIMEOUT_MS = 10_000;
/** Each listing page or footer read while building a partition index. */
const INDEX_TIMEOUT_MS = 20_000;
const TILE_MAX_BYTES = 2 * 1024 * 1024;
const TILE_CACHE_LIMIT = 512;
const IN_FLIGHT_LIMIT = 64;
/**
* Availability of every entry on this instance, decided by the operator's
* environment alone: `GEV_LIBRARY_DISABLE` removes entries, opt-in entries
* need `GEV_LIBRARY_ENABLE` (or, for a pack, its host's own switch), and a
* keyed entry needs its key. The answer names a reason, never a secret.
* @param {object[]} entries
* @param {Record<string, string|undefined>} env
* @returns {Map<string, { available: boolean, reason: string|null }>}
*/
export function libraryAvailability(entries, env) {
const enabled = parseLibraryIdList(env.GEV_LIBRARY_ENABLE);
const disabled = parseLibraryIdList(env.GEV_LIBRARY_DISABLE);
const result = new Map();
for (const entry of entries) {
let reason = null;
if (disabled.has(entry.id)) reason = 'operator-disabled';
else if (entry.kind === 'pack') {
if (String(env[entry.pack.env] || '').trim() !== '1')
reason = 'operator-opt-in';
} else if (entry.licence.operatorOptIn && !enabled.has(entry.id))
reason = 'operator-opt-in';
else if (entry.source.key && !libraryKeyValue(entry.source.key, env))
reason = 'key-missing';
result.set(entry.id, { available: reason === null, reason });
}
return result;
}
/** The configured value for a manifest key, through the key registry. */
export function libraryKeyValue(key, env) {
const registered = KEY_SETUP_KEYS.find(({ id }) => id === key?.id);
for (const name of registered?.envVars || []) {
const value = String(env[name] || '').trim();
if (value) return value;
}
return null;
}
function withKey(url, key, env, headers) {
if (!key) return url;
const value = libraryKeyValue(key, env);
if (!value) throw Object.assign(new Error('key_missing'), { status: 503 });
if (key.in === 'header') {
headers[key.name] = value;
return url;
}
const keyed = new URL(url);
keyed.searchParams.set(key.name, value);
return keyed.href;
}
/**
* Parse `/<id>/<z>/<x>/<y>` against a tiles entry's zoom range.
* @returns {{ z: number, x: number, y: number } | null}
*/
export function parseLibraryTilePath(parts, source) {
if (parts.length !== 3 || !parts.every((part) => /^\d{1,8}$/.test(part)))
return null;
const [z, x, y] = parts.map(Number);
const minZoom = source.minZoom ?? 0;
const maxZoom = source.maxZoom ?? 22;
if (z < minZoom || z > maxZoom) return null;
const span = 2 ** z;
if (x >= span || y >= span) return null;
return { z, x, y };
}
/**
* The Data Library's one server route. Proxied datasets and tiles reach their
* manifest's fixed upstream only: the browser names an entry id, never a URL,
* so the allowlist is exactly the catalog. Redirects are refused, bodies are
* capped while streaming, keys come from this server's environment and
* upstream error text is never returned.
*/
export function libraryProxy({
entries = LIBRARY_ENTRIES,
env = process.env,
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
} = {}) {
const byId = new Map(entries.map((entry) => [entry.id, entry]));
const datasets = new Map();
const tiles = new Map();
const indexes = new Map();
const inFlight = new Map();
const allowData = makeRateLimiter({
windowMs: 60_000,
max: 30,
globalMax: 600,
});
const allowTiles = makeRateLimiter({
windowMs: 60_000,
max: 600,
globalMax: 6000,
});
async function upstream(url, entry, headers, timeoutMs) {
// Defense in depth: every derived request stays on the manifest's host.
const target = libraryUpstreamUrl(url);
const declared = libraryUpstreamUrl(entry.source.url);
if (!target || !declared || target.host !== declared.host)
throw Object.assign(new Error('off_manifest_host'), { status: 502 });
const signal = AbortSignal.timeout(timeoutMs);
const response = await fetchImpl(url, {
signal,
redirect: 'error',
headers: { Accept: '*/*', ...headers },
});
if (!response.ok) {
await response.body?.cancel?.().catch?.(() => {});
throw Object.assign(new Error('upstream_unavailable'), { status: 502 });
}
return { response, signal };
}
async function fetchDataset(entry) {
const headers = {};
const collection = await loadLibraryDataset(entry, async (url, budget) => {
const keyed = withKey(url, entry.source.key, env, headers);
const { response, signal } = await upstream(
keyed,
entry,
headers,
DATASET_TIMEOUT_MS,
);
const text = await readResponseTextCapped(
response,
budget.remainingBytes,
signal,
);
budget.remainingBytes -= new TextEncoder().encode(text).byteLength;
return text;
});
return { ...collection, fetchedAt: now() };
}
async function cached(store, key, ttlMs, load, limit = Infinity) {
const previous = store.get(key);
if (previous && now() - previous.savedAt < ttlMs)
return { value: previous.value, stale: false };
if (!inFlight.has(key) && inFlight.size >= IN_FLIGHT_LIMIT)
throw Object.assign(new Error('busy'), { status: 429 });
try {
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
const value = await load();
store.delete(key);
store.set(key, { value, savedAt: now() });
while (store.size > limit) store.delete(store.keys().next().value);
return value;
});
return { value: await promise, stale: false };
} catch (error) {
if (previous) return { value: previous.value, stale: true };
throw error;
}
}
function json(res, status, value, extra = {}) {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...(status === 429 ? { 'Retry-After': '60' } : {}),
...extra,
});
res.end(JSON.stringify(value));
}
function gate(req, res, entry) {
// A keyed entry spends the operator's credential: refuse forwarded
// requests like the other cost-bearing routes. Keyless entries keep
// working behind a reverse proxy, like terrain heights.
return entry?.source?.key
? admitSameSite(req, res)
: admitSameSiteKeyless(req, res);
}
async function handler(req, res) {
if (req.method !== 'GET')
return json(res, 405, { error: 'method_not_allowed' });
const path = (req.url || '/').split('?')[0];
const parts = path.split('/').filter(Boolean);
const availability = libraryAvailability(entries, env);
if (parts.length === 1 && parts[0] === 'catalog') {
if (admitSameSiteKeyless(req, res)) return undefined;
return json(res, 200, {
entries: entries.map(({ id }) => ({ id, ...availability.get(id) })),
});
}
const [route, id, ...rest] = parts;
const entry = byId.get(id);
if (!entry || !['data', 'tile', 'index'].includes(route))
return json(res, 404, { error: 'unknown_entry' });
// A partitioned GeoParquet entry's index: which file covers where, read
// from each file's footer. Built once per cacheSeconds and shared, so
// visitors do not each list the bucket and read every footer.
if (route === 'index') {
if (rest.length || !entry.source?.partitions)
return json(res, 404, { error: 'not_partitioned' });
if (admitSameSiteKeyless(req, res)) return undefined;
const { available, reason } = availability.get(id);
if (!available) return json(res, 403, { error: reason });
if (!allowData(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
try {
const { value } = await cached(
indexes,
`index:${id}`,
entry.source.cacheSeconds * 1000,
async () => ({
partitions: await buildPartitionIndex(entry, {
fetchImpl: (url, options = {}) =>
fetchImpl(url, {
...options,
signal:
options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS),
}),
}),
builtAt: now(),
}),
);
return json(res, 200, value);
} catch (error) {
const status = error?.status === 429 ? 429 : 502;
return json(res, status, { error: 'library_upstream_unavailable' });
}
}
const isDataset = route === 'data' && rest.length === 0;
const isTile = route === 'tile';
if (
entry.source.fetch !== 'proxied' ||
(isDataset && entry.kind !== 'dataset') ||
(isTile && entry.kind !== 'tiles') ||
(!isDataset && !isTile)
)
return json(res, 404, { error: 'not_proxied' });
if (gate(req, res, entry)) return undefined;
const { available, reason } = availability.get(id);
if (!available) return json(res, 403, { error: reason });
const ttlMs = entry.source.cacheSeconds * 1000;
if (isDataset) {
if (!allowData(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
try {
const { value, stale } = await cached(
datasets,
`data:${id}`,
ttlMs,
() => fetchDataset(entry),
);
return json(
res,
200,
stale ? { ...value, stale: true } : value,
stale ? { 'X-Data-Stale': 'true' } : {},
);
} catch (error) {
const status = [429, 503].includes(error?.status) ? error.status : 502;
return json(res, status, {
error:
status === 503 ? 'key_missing' : 'library_upstream_unavailable',
});
}
}
const tile = parseLibraryTilePath(rest, entry.source);
if (!tile) return json(res, 400, { error: 'invalid_tile' });
if (!allowTiles(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
try {
const key = `tile:${id}:${tile.z}/${tile.x}/${tile.y}`;
const { value } = await cached(
tiles,
key,
ttlMs,
async () => {
const headers = {};
const url = withKey(
libraryTileUrl(entry.source.url, tile.z, tile.x, tile.y),
entry.source.key,
env,
headers,
);
const { response, signal } = await upstream(
url,
entry,
headers,
TILE_TIMEOUT_MS,
);
const type = String(response.headers.get('content-type') || '')
.split(';')[0]
.trim()
.toLowerCase();
if (!/^image\/(png|jpeg|webp)$/.test(type)) {
await response.body?.cancel?.().catch?.(() => {});
throw Object.assign(new Error('not_an_image'), { status: 502 });
}
const bytes = await readResponseBytesCapped(
response,
TILE_MAX_BYTES,
signal,
);
return { type, bytes };
},
TILE_CACHE_LIMIT,
);
if (res.destroyed) return undefined;
res.writeHead(200, {
'Content-Type': value.type,
'Cache-Control': `private, max-age=${entry.source.cacheSeconds}`,
'X-Content-Type-Options': 'nosniff',
});
res.end(Buffer.from(value.bytes));
return undefined;
} catch (error) {
const status = [429, 503].includes(error?.status) ? error.status : 502;
return json(res, status, {
error: status === 503 ? 'key_missing' : 'library_upstream_unavailable',
});
}
}
return {
name: 'data-library',
handler,
configureServer({ middlewares }) {
middlewares.use('/api/library', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/library', handler);
},
};
}
+332
View File
@@ -0,0 +1,332 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
libraryAvailability,
libraryProxy,
parseLibraryTilePath,
} from './library.js';
import { validateLibraryCatalog } from '../../src/library/manifest.js';
const base = {
family: 'events',
provider: 'Fixture',
description: 'Fixture entry.',
maintainer: 'test',
coverage: { global: true },
licence: { name: 'Public domain', url: 'https://example.org/licence' },
attribution: { text: 'Fixture' },
status: 'community',
};
const { entries, errors } = validateLibraryCatalog([
{
...base,
id: 'proxied-points',
kind: 'dataset',
title: 'Proxied points',
source: {
format: 'geojson',
url: 'https://data.example.org/points.json',
fetch: 'proxied',
maxBytes: 4096,
},
display: { label: 'title' },
},
{
...base,
id: 'direct-points',
kind: 'dataset',
title: 'Direct points',
source: {
format: 'geojson',
url: 'https://data.example.org/direct.json',
fetch: 'browser-direct',
},
},
{
...base,
id: 'keyed-points',
kind: 'dataset',
title: 'Keyed points',
source: {
format: 'geojson',
url: 'https://keyed.example.org/points.json',
fetch: 'proxied',
key: { id: 'firms', in: 'query', name: 'MAP_KEY' },
},
},
{
...base,
id: 'opt-in-points',
kind: 'dataset',
title: 'Opt-in points',
source: {
format: 'geojson',
url: 'https://data.example.org/opt.json',
fetch: 'proxied',
},
licence: { ...base.licence, operatorOptIn: true },
},
{
...base,
id: 'proxied-tiles',
kind: 'tiles',
title: 'Proxied tiles',
source: {
format: 'xyz',
url: 'https://tiles.example.org/{z}/{x}/{y}.png',
fetch: 'proxied',
minZoom: 2,
maxZoom: 5,
},
},
{
...base,
id: 'fixture-pack',
kind: 'pack',
family: 'cameras',
title: 'Fixture cameras',
source: { format: 'host', url: 'https://cams.example.org/' },
pack: { host: 'cctv', env: 'CCTV_FIXTURE_ENABLED' },
licence: { ...base.licence, operatorOptIn: true },
},
]);
assert.deepEqual(errors, []);
const collection = (features) =>
JSON.stringify({ type: 'FeatureCollection', features });
const point = (title) => ({
type: 'Feature',
geometry: { type: 'Point', coordinates: [1, 2] },
properties: { title },
});
function install({ env = {}, fetchImpl, now } = {}) {
const plugin = libraryProxy({ entries, env, fetchImpl, now });
let handler;
plugin.configurePreviewServer({
middlewares: {
use(path, callback) {
assert.equal(path, '/api/library');
handler = callback;
},
},
});
return async (url, headers = {}) => {
const res = {
headers: {},
writeHead(status, values) {
this.status = status;
this.headers = values;
},
setHeader(name, value) {
this.headers[name] = value;
},
end(body) {
this.raw = body;
try {
this.body = JSON.parse(body);
} catch {
this.body = null;
}
},
set statusCode(value) {
this.status = value;
},
};
await handler(
{ url, method: 'GET', headers, socket: { remoteAddress: '127.0.0.1' } },
res,
);
return res;
};
}
test('availability is decided by the operator environment and names no secret', () => {
const none = libraryAvailability(entries, {});
assert.deepEqual(none.get('proxied-points'), {
available: true,
reason: null,
});
assert.equal(none.get('keyed-points').reason, 'key-missing');
assert.equal(none.get('opt-in-points').reason, 'operator-opt-in');
assert.equal(none.get('fixture-pack').reason, 'operator-opt-in');
const configured = libraryAvailability(entries, {
FIRMS_MAP_KEY: 'secret',
GEV_LIBRARY_ENABLE: 'opt-in-points',
GEV_LIBRARY_DISABLE: 'proxied-points',
CCTV_FIXTURE_ENABLED: '1',
});
assert.equal(configured.get('keyed-points').available, true);
assert.equal(configured.get('opt-in-points').available, true);
assert.equal(configured.get('proxied-points').reason, 'operator-disabled');
assert.equal(configured.get('fixture-pack').available, true);
assert.doesNotMatch(JSON.stringify([...configured]), /secret/);
});
test('the catalog route lists every entry with its availability', async () => {
const request = install();
const res = await request('/catalog');
assert.equal(res.status, 200);
assert.equal(res.body.entries.length, entries.length);
assert.deepEqual(
res.body.entries.find(({ id }) => id === 'opt-in-points'),
{ id: 'opt-in-points', available: false, reason: 'operator-opt-in' },
);
});
test('a proxied dataset reaches only its manifest URL, refuses redirects and is normalized', async () => {
const calls = [];
const request = install({
now: () => 42,
fetchImpl: async (url, options) => {
calls.push({ url, options });
return new Response(collection([point('Storm'), { geometry: null }]));
},
});
const res = await request('/data/proxied-points?url=https://evil.example');
assert.equal(res.status, 200);
assert.equal(calls.length, 1);
assert.equal(calls[0].url, 'https://data.example.org/points.json');
assert.equal(calls[0].options.redirect, 'error');
assert.ok(calls[0].options.signal instanceof AbortSignal);
assert.equal(res.body.features.length, 1);
assert.equal(res.body.features[0].properties.name, 'Storm');
assert.equal(res.body.dropped, 1);
assert.equal(res.body.fetchedAt, 42);
// A fresh cache answers without another upstream request.
await request('/data/proxied-points');
assert.equal(calls.length, 1);
});
test('browser-direct, unknown, opt-in and wrong-kind entries are not proxied', async () => {
let calls = 0;
const request = install({
fetchImpl: async () => {
calls++;
return new Response(collection([]));
},
});
assert.equal((await request('/data/direct-points')).status, 404);
assert.equal((await request('/data/nope-nope')).status, 404);
assert.equal((await request('/data/proxied-tiles')).status, 404);
assert.equal((await request('/tile/proxied-points/2/0/0')).status, 404);
const optIn = await request('/data/opt-in-points');
assert.equal(optIn.status, 403);
assert.equal(optIn.body.error, 'operator-opt-in');
assert.equal(calls, 0);
});
test('a keyed entry gets the operator key server-side and refuses forwarded requests', async () => {
const calls = [];
const request = install({
env: { FIRMS_MAP_KEY: 'k3y' },
fetchImpl: async (url) => {
calls.push(new URL(url));
return new Response(collection([point('Fire')]));
},
});
const forwarded = await request('/data/keyed-points', {
'x-forwarded-for': '203.0.113.9',
});
assert.equal(forwarded.status, 403);
assert.equal(calls.length, 0);
const res = await request('/data/keyed-points');
assert.equal(res.status, 200);
assert.equal(calls[0].searchParams.get('MAP_KEY'), 'k3y');
assert.doesNotMatch(res.raw, /k3y/);
});
test('an oversized or failing upstream answers 502 without its text, then serves stale', async () => {
let clock = 0;
let mode = 'ok';
const request = install({
now: () => clock,
fetchImpl: async () => {
if (mode === 'big') return new Response('x'.repeat(8192));
if (mode === 'fail')
return new Response('secret upstream text', { status: 500 });
return new Response(collection([point('A')]));
},
});
mode = 'big';
const big = await request('/data/proxied-points');
assert.equal(big.status, 502);
assert.doesNotMatch(big.raw, /x{20}/);
mode = 'ok';
assert.equal((await request('/data/proxied-points')).status, 200);
clock = 10 * 3600 * 1000;
mode = 'fail';
const stale = await request('/data/proxied-points');
assert.equal(stale.status, 200);
assert.equal(stale.body.stale, true);
assert.equal(stale.headers['X-Data-Stale'], 'true');
assert.doesNotMatch(stale.raw, /secret upstream text/);
});
test('cross-site browser requests are refused before any upstream request', async () => {
let calls = 0;
const request = install({
fetchImpl: async () => {
calls++;
return new Response(collection([]));
},
});
const res = await request('/data/proxied-points', {
host: 'localhost:5173',
origin: 'https://evil.example',
'sec-fetch-site': 'cross-site',
});
assert.equal(res.status, 403);
assert.equal(calls, 0);
});
test('tiles: bounded z/x/y, image types only, cached', async () => {
const calls = [];
const request = install({
fetchImpl: async (url) => {
calls.push(url);
if (url.includes('/3/'))
return new Response('<html>', {
headers: { 'content-type': 'text/html' },
});
return new Response(new Uint8Array([137, 80, 78, 71]), {
headers: { 'content-type': 'image/png' },
});
},
});
assert.equal((await request('/tile/proxied-tiles/1/0/0')).status, 400);
assert.equal((await request('/tile/proxied-tiles/2/4/0')).status, 400);
assert.equal((await request('/tile/proxied-tiles/2/a/0')).status, 400);
const ok = await request('/tile/proxied-tiles/2/1/3');
assert.equal(ok.status, 200);
assert.equal(ok.headers['Content-Type'], 'image/png');
assert.equal(calls[0], 'https://tiles.example.org/2/1/3.png');
await request('/tile/proxied-tiles/2/1/3');
assert.equal(calls.length, 1);
assert.equal((await request('/tile/proxied-tiles/3/1/1')).status, 502);
});
test('tile paths are parsed against the zoom range', () => {
const source = { minZoom: 0, maxZoom: 3 };
assert.deepEqual(parseLibraryTilePath(['3', '7', '7'], source), {
z: 3,
x: 7,
y: 7,
});
assert.equal(parseLibraryTilePath(['3', '8', '0'], source), null);
assert.equal(parseLibraryTilePath(['4', '0', '0'], source), null);
assert.equal(parseLibraryTilePath(['1', '0'], source), null);
});
test('only GET is served', async () => {
const plugin = libraryProxy({ entries, env: {} });
const res = {
writeHead(status, headers) {
this.status = status;
this.headers = headers;
},
end() {},
};
await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res);
assert.equal(res.status, 405);
});
+2
View File
@@ -36,6 +36,7 @@ import { inaturalistProxy } from './inaturalist/observations.js';
import { acartiaProxy } from './acartia.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
import { libraryProxy } from './library.js';
/**
* Construct the local provider plugins in their established order.
@@ -85,6 +86,7 @@ function localProviderPlugins({
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
libraryProxy(),
ghostMapsProxy(),
shipwrecksProxy(),
droneAirspaceProxy(),