Add Data Library: live-fetched dataset, tile and pack layers

This commit is contained in:
Said Turksever committed 2026-11-06 12:12:15 +00:00
1 parent 5a1598fd9e
commit 976fa6d6ce
169 files changed
+14637 -31

No files matched your search

+363
View File
@@ -0,0 +1,363 @@
import { LIBRARY_ENTRIES } from '../../src/library/catalog.js';
import { loadLibraryDataset } from '../../src/library/formats.js';
import { buildPartitionIndex } from '../../src/library/parquet.js';
import {
libraryUpstreamUrl,
parseLibraryIdList,
libraryTileUrl,
} from '../../src/library/manifest.js';
import { KEY_SETUP_KEYS } from '../../src/keySetupCore.mjs';
import {
readResponseBytesCapped,
readResponseTextCapped,
coalesceProxyRequest,
} from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSite, admitSameSiteKeyless } from './common/same-site.js';
const DATASET_TIMEOUT_MS = 30_000;
const TILE_TIMEOUT_MS = 10_000;
/** Each listing page or footer read while building a partition index. */
const INDEX_TIMEOUT_MS = 20_000;
const TILE_MAX_BYTES = 2 * 1024 * 1024;
const TILE_CACHE_LIMIT = 512;
const IN_FLIGHT_LIMIT = 64;
/**
* Availability of every entry on this instance, decided by the operator's
* environment alone: `GEV_LIBRARY_DISABLE` removes entries, opt-in entries
* need `GEV_LIBRARY_ENABLE` (or, for a pack, its host's own switch), and a
* keyed entry needs its key. The answer names a reason, never a secret.
* @param {object[]} entries
* @param {Record<string, string|undefined>} env
* @returns {Map<string, { available: boolean, reason: string|null }>}
*/
export function libraryAvailability(entries, env) {
const enabled = parseLibraryIdList(env.GEV_LIBRARY_ENABLE);
const disabled = parseLibraryIdList(env.GEV_LIBRARY_DISABLE);
const result = new Map();
for (const entry of entries) {
let reason = null;
if (disabled.has(entry.id)) reason = 'operator-disabled';
else if (entry.kind === 'pack') {
if (String(env[entry.pack.env] || '').trim() !== '1')
reason = 'operator-opt-in';
} else if (entry.licence.operatorOptIn && !enabled.has(entry.id))
reason = 'operator-opt-in';
else if (entry.source.key && !libraryKeyValue(entry.source.key, env))
reason = 'key-missing';
result.set(entry.id, { available: reason === null, reason });
}
return result;
}
/** The configured value for a manifest key, through the key registry. */
export function libraryKeyValue(key, env) {
const registered = KEY_SETUP_KEYS.find(({ id }) => id === key?.id);
for (const name of registered?.envVars || []) {
const value = String(env[name] || '').trim();
if (value) return value;
}
return null;
}
function withKey(url, key, env, headers) {
if (!key) return url;
const value = libraryKeyValue(key, env);
if (!value) throw Object.assign(new Error('key_missing'), { status: 503 });
if (key.in === 'header') {
headers[key.name] = value;
return url;
}
const keyed = new URL(url);
keyed.searchParams.set(key.name, value);
return keyed.href;
}
/**
* Parse `/<id>/<z>/<x>/<y>` against a tiles entry's zoom range.
* @returns {{ z: number, x: number, y: number } | null}
*/
export function parseLibraryTilePath(parts, source) {
if (parts.length !== 3 || !parts.every((part) => /^\d{1,8}$/.test(part)))
return null;
const [z, x, y] = parts.map(Number);
const minZoom = source.minZoom ?? 0;
const maxZoom = source.maxZoom ?? 22;
if (z < minZoom || z > maxZoom) return null;
const span = 2 ** z;
if (x >= span || y >= span) return null;
return { z, x, y };
}
/**
* The Data Library's one server route. Proxied datasets and tiles reach their
* manifest's fixed upstream only: the browser names an entry id, never a URL,
* so the allowlist is exactly the catalog. Redirects are refused, bodies are
* capped while streaming, keys come from this server's environment and
* upstream error text is never returned.
*/
export function libraryProxy({
entries = LIBRARY_ENTRIES,
env = process.env,
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
} = {}) {
const byId = new Map(entries.map((entry) => [entry.id, entry]));
const datasets = new Map();
const tiles = new Map();
const indexes = new Map();
const inFlight = new Map();
const allowData = makeRateLimiter({
windowMs: 60_000,
max: 30,
globalMax: 600,
});
const allowTiles = makeRateLimiter({
windowMs: 60_000,
max: 600,
globalMax: 6000,
});
async function upstream(url, entry, headers, timeoutMs) {
// Defense in depth: every derived request stays on the manifest's host.
const target = libraryUpstreamUrl(url);
const declared = libraryUpstreamUrl(entry.source.url);
if (!target || !declared || target.host !== declared.host)
throw Object.assign(new Error('off_manifest_host'), { status: 502 });
const signal = AbortSignal.timeout(timeoutMs);
const response = await fetchImpl(url, {
signal,
redirect: 'error',
headers: { Accept: '*/*', ...headers },
});
if (!response.ok) {
await response.body?.cancel?.().catch?.(() => {});
throw Object.assign(new Error('upstream_unavailable'), { status: 502 });
}
return { response, signal };
}
async function fetchDataset(entry) {
const headers = {};
const collection = await loadLibraryDataset(entry, async (url, budget) => {
const keyed = withKey(url, entry.source.key, env, headers);
const { response, signal } = await upstream(
keyed,
entry,
headers,
DATASET_TIMEOUT_MS,
);
const text = await readResponseTextCapped(
response,
budget.remainingBytes,
signal,
);
budget.remainingBytes -= new TextEncoder().encode(text).byteLength;
return text;
});
return { ...collection, fetchedAt: now() };
}
async function cached(store, key, ttlMs, load, limit = Infinity) {
const previous = store.get(key);
if (previous && now() - previous.savedAt < ttlMs)
return { value: previous.value, stale: false };
if (!inFlight.has(key) && inFlight.size >= IN_FLIGHT_LIMIT)
throw Object.assign(new Error('busy'), { status: 429 });
try {
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
const value = await load();
store.delete(key);
store.set(key, { value, savedAt: now() });
while (store.size > limit) store.delete(store.keys().next().value);
return value;
});
return { value: await promise, stale: false };
} catch (error) {
if (previous) return { value: previous.value, stale: true };
throw error;
}
}
function json(res, status, value, extra = {}) {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...(status === 429 ? { 'Retry-After': '60' } : {}),
...extra,
});
res.end(JSON.stringify(value));
}
function gate(req, res, entry) {
// A keyed entry spends the operator's credential: refuse forwarded
// requests like the other cost-bearing routes. Keyless entries keep
// working behind a reverse proxy, like terrain heights.
return entry?.source?.key
? admitSameSite(req, res)
: admitSameSiteKeyless(req, res);
}
async function handler(req, res) {
if (req.method !== 'GET')
return json(res, 405, { error: 'method_not_allowed' });
const path = (req.url || '/').split('?')[0];
const parts = path.split('/').filter(Boolean);
const availability = libraryAvailability(entries, env);
if (parts.length === 1 && parts[0] === 'catalog') {
if (admitSameSiteKeyless(req, res)) return undefined;
return json(res, 200, {
entries: entries.map(({ id }) => ({ id, ...availability.get(id) })),
});
}
const [route, id, ...rest] = parts;
const entry = byId.get(id);
if (!entry || !['data', 'tile', 'index'].includes(route))
return json(res, 404, { error: 'unknown_entry' });
// A partitioned GeoParquet entry's index: which file covers where, read
// from each file's footer. Built once per cacheSeconds and shared, so
// visitors do not each list the bucket and read every footer.
if (route === 'index') {
if (rest.length || !entry.source?.partitions)
return json(res, 404, { error: 'not_partitioned' });
if (admitSameSiteKeyless(req, res)) return undefined;
const { available, reason } = availability.get(id);
if (!available) return json(res, 403, { error: reason });
if (!allowData(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
try {
const { value } = await cached(
indexes,
`index:${id}`,
entry.source.cacheSeconds * 1000,
async () => ({
partitions: await buildPartitionIndex(entry, {
fetchImpl: (url, options = {}) =>
fetchImpl(url, {
...options,
signal:
options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS),
}),
}),
builtAt: now(),
}),
);
return json(res, 200, value);
} catch (error) {
const status = error?.status === 429 ? 429 : 502;
return json(res, status, { error: 'library_upstream_unavailable' });
}
}
const isDataset = route === 'data' && rest.length === 0;
const isTile = route === 'tile';
if (
entry.source.fetch !== 'proxied' ||
(isDataset && entry.kind !== 'dataset') ||
(isTile && entry.kind !== 'tiles') ||
(!isDataset && !isTile)
)
return json(res, 404, { error: 'not_proxied' });
if (gate(req, res, entry)) return undefined;
const { available, reason } = availability.get(id);
if (!available) return json(res, 403, { error: reason });
const ttlMs = entry.source.cacheSeconds * 1000;
if (isDataset) {
if (!allowData(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
try {
const { value, stale } = await cached(
datasets,
`data:${id}`,
ttlMs,
() => fetchDataset(entry),
);
return json(
res,
200,
stale ? { ...value, stale: true } : value,
stale ? { 'X-Data-Stale': 'true' } : {},
);
} catch (error) {
const status = [429, 503].includes(error?.status) ? error.status : 502;
return json(res, status, {
error:
status === 503 ? 'key_missing' : 'library_upstream_unavailable',
});
}
}
const tile = parseLibraryTilePath(rest, entry.source);
if (!tile) return json(res, 400, { error: 'invalid_tile' });
if (!allowTiles(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
try {
const key = `tile:${id}:${tile.z}/${tile.x}/${tile.y}`;
const { value } = await cached(
tiles,
key,
ttlMs,
async () => {
const headers = {};
const url = withKey(
libraryTileUrl(entry.source.url, tile.z, tile.x, tile.y),
entry.source.key,
env,
headers,
);
const { response, signal } = await upstream(
url,
entry,
headers,
TILE_TIMEOUT_MS,
);
const type = String(response.headers.get('content-type') || '')
.split(';')[0]
.trim()
.toLowerCase();
if (!/^image\/(png|jpeg|webp)$/.test(type)) {
await response.body?.cancel?.().catch?.(() => {});
throw Object.assign(new Error('not_an_image'), { status: 502 });
}
const bytes = await readResponseBytesCapped(
response,
TILE_MAX_BYTES,
signal,
);
return { type, bytes };
},
TILE_CACHE_LIMIT,
);
if (res.destroyed) return undefined;
res.writeHead(200, {
'Content-Type': value.type,
'Cache-Control': `private, max-age=${entry.source.cacheSeconds}`,
'X-Content-Type-Options': 'nosniff',
});
res.end(Buffer.from(value.bytes));
return undefined;
} catch (error) {
const status = [429, 503].includes(error?.status) ? error.status : 502;
return json(res, status, {
error: status === 503 ? 'key_missing' : 'library_upstream_unavailable',
});
}
}
return {
name: 'data-library',
handler,
configureServer({ middlewares }) {
middlewares.use('/api/library', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/library', handler);
},
};
}
+332
View File
@@ -0,0 +1,332 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
libraryAvailability,
libraryProxy,
parseLibraryTilePath,
} from './library.js';
import { validateLibraryCatalog } from '../../src/library/manifest.js';
const base = {
family: 'events',
provider: 'Fixture',
description: 'Fixture entry.',
maintainer: 'test',
coverage: { global: true },
licence: { name: 'Public domain', url: 'https://example.org/licence' },
attribution: { text: 'Fixture' },
status: 'community',
};
const { entries, errors } = validateLibraryCatalog([
{
...base,
id: 'proxied-points',
kind: 'dataset',
title: 'Proxied points',
source: {
format: 'geojson',
url: 'https://data.example.org/points.json',
fetch: 'proxied',
maxBytes: 4096,
},
display: { label: 'title' },
},
{
...base,
id: 'direct-points',
kind: 'dataset',
title: 'Direct points',
source: {
format: 'geojson',
url: 'https://data.example.org/direct.json',
fetch: 'browser-direct',
},
},
{
...base,
id: 'keyed-points',
kind: 'dataset',
title: 'Keyed points',
source: {
format: 'geojson',
url: 'https://keyed.example.org/points.json',
fetch: 'proxied',
key: { id: 'firms', in: 'query', name: 'MAP_KEY' },
},
},
{
...base,
id: 'opt-in-points',
kind: 'dataset',
title: 'Opt-in points',
source: {
format: 'geojson',
url: 'https://data.example.org/opt.json',
fetch: 'proxied',
},
licence: { ...base.licence, operatorOptIn: true },
},
{
...base,
id: 'proxied-tiles',
kind: 'tiles',
title: 'Proxied tiles',
source: {
format: 'xyz',
url: 'https://tiles.example.org/{z}/{x}/{y}.png',
fetch: 'proxied',
minZoom: 2,
maxZoom: 5,
},
},
{
...base,
id: 'fixture-pack',
kind: 'pack',
family: 'cameras',
title: 'Fixture cameras',
source: { format: 'host', url: 'https://cams.example.org/' },
pack: { host: 'cctv', env: 'CCTV_FIXTURE_ENABLED' },
licence: { ...base.licence, operatorOptIn: true },
},
]);
assert.deepEqual(errors, []);
const collection = (features) =>
JSON.stringify({ type: 'FeatureCollection', features });
const point = (title) => ({
type: 'Feature',
geometry: { type: 'Point', coordinates: [1, 2] },
properties: { title },
});
function install({ env = {}, fetchImpl, now } = {}) {
const plugin = libraryProxy({ entries, env, fetchImpl, now });
let handler;
plugin.configurePreviewServer({
middlewares: {
use(path, callback) {
assert.equal(path, '/api/library');
handler = callback;
},
},
});
return async (url, headers = {}) => {
const res = {
headers: {},
writeHead(status, values) {
this.status = status;
this.headers = values;
},
setHeader(name, value) {
this.headers[name] = value;
},
end(body) {
this.raw = body;
try {
this.body = JSON.parse(body);
} catch {
this.body = null;
}
},
set statusCode(value) {
this.status = value;
},
};
await handler(
{ url, method: 'GET', headers, socket: { remoteAddress: '127.0.0.1' } },
res,
);
return res;
};
}
test('availability is decided by the operator environment and names no secret', () => {
const none = libraryAvailability(entries, {});
assert.deepEqual(none.get('proxied-points'), {
available: true,
reason: null,
});
assert.equal(none.get('keyed-points').reason, 'key-missing');
assert.equal(none.get('opt-in-points').reason, 'operator-opt-in');
assert.equal(none.get('fixture-pack').reason, 'operator-opt-in');
const configured = libraryAvailability(entries, {
FIRMS_MAP_KEY: 'secret',
GEV_LIBRARY_ENABLE: 'opt-in-points',
GEV_LIBRARY_DISABLE: 'proxied-points',
CCTV_FIXTURE_ENABLED: '1',
});
assert.equal(configured.get('keyed-points').available, true);
assert.equal(configured.get('opt-in-points').available, true);
assert.equal(configured.get('proxied-points').reason, 'operator-disabled');
assert.equal(configured.get('fixture-pack').available, true);
assert.doesNotMatch(JSON.stringify([...configured]), /secret/);
});
test('the catalog route lists every entry with its availability', async () => {
const request = install();
const res = await request('/catalog');
assert.equal(res.status, 200);
assert.equal(res.body.entries.length, entries.length);
assert.deepEqual(
res.body.entries.find(({ id }) => id === 'opt-in-points'),
{ id: 'opt-in-points', available: false, reason: 'operator-opt-in' },
);
});
test('a proxied dataset reaches only its manifest URL, refuses redirects and is normalized', async () => {
const calls = [];
const request = install({
now: () => 42,
fetchImpl: async (url, options) => {
calls.push({ url, options });
return new Response(collection([point('Storm'), { geometry: null }]));
},
});
const res = await request('/data/proxied-points?url=https://evil.example');
assert.equal(res.status, 200);
assert.equal(calls.length, 1);
assert.equal(calls[0].url, 'https://data.example.org/points.json');
assert.equal(calls[0].options.redirect, 'error');
assert.ok(calls[0].options.signal instanceof AbortSignal);
assert.equal(res.body.features.length, 1);
assert.equal(res.body.features[0].properties.name, 'Storm');
assert.equal(res.body.dropped, 1);
assert.equal(res.body.fetchedAt, 42);
// A fresh cache answers without another upstream request.
await request('/data/proxied-points');
assert.equal(calls.length, 1);
});
test('browser-direct, unknown, opt-in and wrong-kind entries are not proxied', async () => {
let calls = 0;
const request = install({
fetchImpl: async () => {
calls++;
return new Response(collection([]));
},
});
assert.equal((await request('/data/direct-points')).status, 404);
assert.equal((await request('/data/nope-nope')).status, 404);
assert.equal((await request('/data/proxied-tiles')).status, 404);
assert.equal((await request('/tile/proxied-points/2/0/0')).status, 404);
const optIn = await request('/data/opt-in-points');
assert.equal(optIn.status, 403);
assert.equal(optIn.body.error, 'operator-opt-in');
assert.equal(calls, 0);
});
test('a keyed entry gets the operator key server-side and refuses forwarded requests', async () => {
const calls = [];
const request = install({
env: { FIRMS_MAP_KEY: 'k3y' },
fetchImpl: async (url) => {
calls.push(new URL(url));
return new Response(collection([point('Fire')]));
},
});
const forwarded = await request('/data/keyed-points', {
'x-forwarded-for': '203.0.113.9',
});
assert.equal(forwarded.status, 403);
assert.equal(calls.length, 0);
const res = await request('/data/keyed-points');
assert.equal(res.status, 200);
assert.equal(calls[0].searchParams.get('MAP_KEY'), 'k3y');
assert.doesNotMatch(res.raw, /k3y/);
});
test('an oversized or failing upstream answers 502 without its text, then serves stale', async () => {
let clock = 0;
let mode = 'ok';
const request = install({
now: () => clock,
fetchImpl: async () => {
if (mode === 'big') return new Response('x'.repeat(8192));
if (mode === 'fail')
return new Response('secret upstream text', { status: 500 });
return new Response(collection([point('A')]));
},
});
mode = 'big';
const big = await request('/data/proxied-points');
assert.equal(big.status, 502);
assert.doesNotMatch(big.raw, /x{20}/);
mode = 'ok';
assert.equal((await request('/data/proxied-points')).status, 200);
clock = 10 * 3600 * 1000;
mode = 'fail';
const stale = await request('/data/proxied-points');
assert.equal(stale.status, 200);
assert.equal(stale.body.stale, true);
assert.equal(stale.headers['X-Data-Stale'], 'true');
assert.doesNotMatch(stale.raw, /secret upstream text/);
});
test('cross-site browser requests are refused before any upstream request', async () => {
let calls = 0;
const request = install({
fetchImpl: async () => {
calls++;
return new Response(collection([]));
},
});
const res = await request('/data/proxied-points', {
host: 'localhost:5173',
origin: 'https://evil.example',
'sec-fetch-site': 'cross-site',
});
assert.equal(res.status, 403);
assert.equal(calls, 0);
});
test('tiles: bounded z/x/y, image types only, cached', async () => {
const calls = [];
const request = install({
fetchImpl: async (url) => {
calls.push(url);
if (url.includes('/3/'))
return new Response('<html>', {
headers: { 'content-type': 'text/html' },
});
return new Response(new Uint8Array([137, 80, 78, 71]), {
headers: { 'content-type': 'image/png' },
});
},
});
assert.equal((await request('/tile/proxied-tiles/1/0/0')).status, 400);
assert.equal((await request('/tile/proxied-tiles/2/4/0')).status, 400);
assert.equal((await request('/tile/proxied-tiles/2/a/0')).status, 400);
const ok = await request('/tile/proxied-tiles/2/1/3');
assert.equal(ok.status, 200);
assert.equal(ok.headers['Content-Type'], 'image/png');
assert.equal(calls[0], 'https://tiles.example.org/2/1/3.png');
await request('/tile/proxied-tiles/2/1/3');
assert.equal(calls.length, 1);
assert.equal((await request('/tile/proxied-tiles/3/1/1')).status, 502);
});
test('tile paths are parsed against the zoom range', () => {
const source = { minZoom: 0, maxZoom: 3 };
assert.deepEqual(parseLibraryTilePath(['3', '7', '7'], source), {
z: 3,
x: 7,
y: 7,
});
assert.equal(parseLibraryTilePath(['3', '8', '0'], source), null);
assert.equal(parseLibraryTilePath(['4', '0', '0'], source), null);
assert.equal(parseLibraryTilePath(['1', '0'], source), null);
});
test('only GET is served', async () => {
const plugin = libraryProxy({ entries, env: {} });
const res = {
writeHead(status, headers) {
this.status = status;
this.headers = headers;
},
end() {},
};
await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res);
assert.equal(res.status, 405);
});
+2
View File
@@ -28,6 +28,7 @@ import { firePerimetersProxy } from './firePerimeters.js';
import { gnssIntegrityProxy } from './gnssIntegrity.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
import { libraryProxy } from './library.js';
/**
* Construct the local provider plugins in their established order.
@@ -64,6 +65,7 @@ function localProviderPlugins({ realtime, gemini } = {}) {
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
libraryProxy(),
keySetupEndpoint(),
];
}