Serve the Data Library from any server, and let a server shape it (#1807)

A server can answer the library with createLibraryHandler, a fetch handler, and shape what it offers: hide entries (listed: false) or route browser-direct ones through itself (fetch: proxied). View-bounded entries ask for a grid-snapped box, and GeoParquet is read on the server within it. Applications can pass their own tile and data fetchers (createApplicationTools({library: {fetchTile, fetchData}})), and with library.requireCatalog a catalog that does not answer offers nothing and is asked again later. Tile paths have one spelling: integer z/x/y without leading zeros, within the grid and the entry's zooms. A server's index and partition reads are bounded and counted against its subrequest limit.
This commit is contained in:
Sameh Khamis authored and GitHub committed 2026-11-17 23:34:00 -08:00
1 parent 7b4c84a717
commit 9554d6ea71
15 files changed
+1510 -395

No files matched your search

+10
View File
@@ -48,6 +48,10 @@ import { overlayHost } from './layers/overlayHost.js';
* Attach scene tools, rendering listeners and the application debug handle.
* `scene` is what `createApplicationScene` returns; its `trimStaleCommands`
* runs after each layer disable (a scene without one keeps that memory).
* `library.fetchTile` and `library.fetchData` fetch the Data Library's tiles
* and proxied datasets in place of the defaults (`createLibrarySource`).
* `library.requireCatalog` offers only what the server's library catalog
* names, and nothing while it does not answer.
*/
export function createApplicationTools({
scene,
@@ -60,6 +64,7 @@ export function createApplicationTools({
onSceneDirector,
sceneDataPacks,
drawMarkActions = [],
library = {},
signal,
defer,
}) {
@@ -158,6 +163,11 @@ export function createApplicationTools({
mapStackController,
shareLink: styleManager.shareLinkManager,
requestRender: governorRequestRender,
// An application's own tile and dataset fetches, when it keeps them in
// its own cache.
fetchTile: library.fetchTile ?? null,
fetchData: library.fetchData ?? null,
requireCatalog: library.requireCatalog === true,
// Zoom to layer: an immediate navigation, so Cockpit can refuse it and
// the current follow owner is released first (as for imported files).
zoomTo: (extent) =>
+66 -2
View File
@@ -25,6 +25,10 @@ import {
} from './layers.js';
import { isExplicitLayerStateOrigin } from '../data/layerState.js';
/** A required catalog that did not answer is asked again after this, doubling. */
const CATALOG_RETRY_MS = 30_000;
const CATALOG_RETRY_MAX_MS = 10 * 60_000;
function safeLocalStorage() {
try {
return globalThis.localStorage ?? null;
@@ -44,6 +48,12 @@ function safeLocalStorage() {
* @param {(reason: string) => void} [options.requestRender]
* @param {(extent: object) => boolean} [options.zoomTo] Fly to an extent
* through the shell's navigation authority; false when refused.
* @param {Function} [options.fetchTile] A tiles entry's tile fetch in place
* of the renderer's (`createLibrarySource`).
* @param {Function} [options.fetchData] A proxied dataset's fetch
* (`createLibrarySource`).
* @param {boolean} [options.requireCatalog] Offer only what the server's
* catalog names, nothing while it does not answer; it is asked again later.
*/
export function createDataLibrary({
entries = LIBRARY_ENTRIES,
@@ -57,7 +67,19 @@ export function createDataLibrary({
requestRender = () => {},
zoomTo = null,
storage = safeLocalStorage(),
source = createLibrarySource({ entries }),
fetchTile = null,
fetchData = null,
requireCatalog = false,
source = createLibrarySource({
entries,
storage,
fetchTile,
fetchData,
requireCatalog,
}),
retryMs = CATALOG_RETRY_MS,
setTimer = (run, ms) => setTimeout(run, ms),
clearTimer = (timer) => clearTimeout(timer),
}) {
if (!dataManager?.registerLibraryLayer)
throw new TypeError('A sealed layer manager is required');
@@ -80,6 +102,8 @@ export function createDataLibrary({
let destroyed = false;
const listeners = new Set();
const pending = new Map();
let retryTimer = null;
let retryDelay = retryMs;
const notify = () => {
for (const listener of [...listeners]) {
@@ -315,6 +339,7 @@ export function createDataLibrary({
if (destroyed) return;
availability = answer.availability;
serverPresent = answer.server;
if (answer.retry) scheduleRetry(signal);
const stored = readStoredLibrarySelection(storage, switchable);
savedOn = stored.on;
// A share opened in this tab while availability loaded wins over the
@@ -342,6 +367,37 @@ export function createDataLibrary({
applySelection(next, linked ? 'share-restore' : 'local-restore');
}
/**
* Ask a required catalog again until it answers, then restore what the
* library holds as start-up would have.
*/
function scheduleRetry(signal) {
if (destroyed || retryTimer !== null || signal?.aborted) return;
retryTimer = setTimer(async () => {
retryTimer = null;
let answer;
try {
answer = await source.getAvailability({ signal });
} catch {
return;
}
if (destroyed) return;
availability = answer.availability;
serverPresent = answer.server;
if (answer.retry) {
retryDelay = Math.min(retryDelay * 2, CATALOG_RETRY_MAX_MS);
scheduleRetry(signal);
notify();
return;
}
for (const id of selection.added) {
const entry = byId.get(id);
if (entry && addable(id)) register(entry);
}
applySelection(selection, 'local-restore');
}, retryDelay);
}
/**
* Add an entry to the panel and switch it on.
* @param {string} id
@@ -460,7 +516,13 @@ export function createDataLibrary({
* @returns {Array<{ entry: object, available: boolean, reason: string|null, added: boolean, on: boolean, busy: boolean }>}
*/
function list() {
return entries.map((entry) => {
// A required catalog shows nothing until it answers, and an entry this
// instance's server does not list is not shown at all.
if (source.requireCatalog && !loaded) return [];
const shown = entries.filter(
(entry) => availability.get(entry.id)?.listed !== false,
);
return shown.map((entry) => {
const state = availability.get(entry.id) || {
available: false,
reason: started ? 'server-unavailable' : null,
@@ -538,6 +600,8 @@ export function createDataLibrary({
destroy() {
if (destroyed) return;
destroyed = true;
if (retryTimer !== null) clearTimer(retryTimer);
retryTimer = null;
unsubscribeManager?.();
shareLink?.setLibraryParamsEncoder?.(null);
shareLink?.setLibraryShareListener?.(null);
+95 -5
View File
@@ -5,6 +5,8 @@ import { createDataLibrary } from './controller.js';
import { LIBRARY_ENTRIES, LIBRARY_LEDGER } from './catalog.js';
import { validateLibraryCatalog } from './manifest.js';
import { LIBRARY_STORAGE_KEY } from './selection.js';
import { createLibrarySource } from './source.js';
import { libraryLayerId } from '../data/libraryLayerIds.js';
const base = {
family: 'imagery',
@@ -502,7 +504,10 @@ test('entries this instance cannot serve still show as added and can be removed'
assert.equal(row.added, true);
assert.equal(library.summary().added, 1);
assert.equal(await library.remove('night-lights'), true);
assert.deepEqual(JSON.parse(storage.store.get(LIBRARY_STORAGE_KEY)).added, []);
assert.deepEqual(
JSON.parse(storage.store.get(LIBRARY_STORAGE_KEY)).added,
[],
);
library.destroy();
});
@@ -530,10 +535,9 @@ test('a layer that fails to tear down stays in the library', async (t) => {
assert.equal(await library.remove('night-lights'), false);
assert.equal(await library.reset(), 0);
assert.equal(manager.layers.has('lib-night-lights'), true);
assert.deepEqual(
JSON.parse(storage.store.get(LIBRARY_STORAGE_KEY)).added,
['night-lights'],
);
assert.deepEqual(JSON.parse(storage.store.get(LIBRARY_STORAGE_KEY)).added, [
'night-lights',
]);
assert.equal(
library.list().find((row) => row.entry.id === 'night-lights').added,
true,
@@ -772,3 +776,89 @@ test('every shipped entry kind registers, view-bounded datasets included', async
assert.equal(module.getDiagnostics().tooHigh, true);
assert.equal(await library.add('nasa-black-marble'), true);
});
test('an entry the server does not list is neither shown nor added', async (t) => {
const manager = sealedManager(t);
const library = createDataLibrary({
entries,
ledger,
dataManager: manager,
services: {},
storage: memoryStorage(),
initialHash: '',
source: source({
'sea-marks': {
available: false,
reason: 'operator-disabled',
listed: false,
},
}),
});
await library.start();
assert.deepEqual(
library.list().map(({ entry }) => entry.id),
['night-lights', 'fixture-pack'],
);
assert.equal(library.summary().entries, 2);
assert.equal(await library.add('sea-marks'), false);
});
test('a required catalog that does not answer offers nothing and is asked again later', async (t) => {
const manager = sealedManager(t);
const calls = [];
let catalog = new Response('<html>busy</html>', { status: 503 });
const timers = [];
const library = createDataLibrary({
entries,
ledger,
dataManager: manager,
services: {},
storage: memoryStorage(
JSON.stringify({ added: ['night-lights'], on: ['night-lights'] }),
),
initialHash: '',
requireCatalog: true,
source: createLibrarySource({
entries,
storage: null,
requireCatalog: true,
fetchImpl: async (url) => {
calls.push(String(url));
return String(url).endsWith('/catalog')
? catalog
: new Response('', { status: 500 });
},
}),
setTimer: (run, ms) => timers.push({ run, ms }),
clearTimer: () => {},
});
await library.start();
assert.deepEqual(library.list(), []);
assert.equal(library.summary().entries, 0);
assert.equal(await library.add('night-lights'), false);
assert.deepEqual(calls, ['/api/library/catalog']);
assert.equal(timers.length, 1);
assert.equal(timers[0].ms, 30_000);
// Still failing: asked again, later still.
catalog = new Response('', { status: 429 });
await timers[0].run();
assert.equal(timers.length, 2);
assert.equal(timers[1].ms, 60_000);
assert.deepEqual(library.list(), []);
// Answered: what it names is offered, and the stored selection returns.
catalog = Response.json({
entries: [{ id: 'night-lights', available: true, reason: null }],
});
await timers[1].run();
assert.equal(timers.length, 2);
assert.deepEqual(
library.list().map(({ entry }) => entry.id),
['night-lights'],
);
assert.ok(manager.layers.has(libraryLayerId('night-lights')));
// No publisher was ever asked.
assert.ok(calls.every((url) => url === '/api/library/catalog'));
library.destroy();
});
+9 -2
View File
@@ -410,9 +410,11 @@ export const LIBRARY_AERIAL_Z = 10;
* The globe port imagery spec for a tiles entry.
* @param {object} entry Validated tiles entry.
* @param {string} url Tile URL template.
* @param {import('../render/contract/globe').ImagerySpec['fetchTile']|null} [fetchTile]
* Fetches each tile in place of the renderer (`source.tileFetch`).
* @returns {import('../render/contract/globe').ImagerySpec}
*/
export function libraryImagerySpec(entry, url) {
export function libraryImagerySpec(entry, url, fetchTile = null) {
const bbox = entry.source.bounded ? entry.coverage.bbox : null;
const minZoom = entry.source.minZoom ?? 0;
// A dense entry (`minimumTerrainLevelFor`) is hidden where the view is
@@ -442,6 +444,7 @@ export function libraryImagerySpec(entry, url) {
: {}),
...(entry.display.aerial ? { zOrder: LIBRARY_AERIAL_Z } : {}),
...(tileTransform ? { tileTransform } : {}),
...(fetchTile ? { fetchTile } : {}),
};
}
@@ -563,7 +566,11 @@ export function createLibraryTileLayer(
if (host.kind === 'tileset') markBaseTileset(tileset);
try {
imagery = globe.imagery.add(
libraryImagerySpec(entry, source.tileTemplate(entry)),
libraryImagerySpec(
entry,
source.tileTemplate(entry),
source.tileFetch?.(entry) ?? null,
),
);
} catch (error) {
// A renderer without tile hooks cannot draw relief or clear the
+51 -1
View File
@@ -67,6 +67,55 @@ export function libraryTileUrl(template, z, x, y) {
.replaceAll('{y}', String(y));
}
/** The finest and coarsest grid a proxied view box snaps to, in degrees. */
const VIEW_GRID_MIN_DEG = 2 ** -10;
const VIEW_GRID_MAX_DEG = 1;
/**
* The box a view-bounded entry asks a proxy for: the camera's box snapped
* out to a grid of two cells a side, the cell the smallest power of two
* (degrees) at least as large as the box. Nearby views name the same box,
* so a proxy and the caches in front of it see few distinct requests, and
* each box always holds the box it came from.
* @param {number[]} box `[west, south, east, north]` in degrees.
* @returns {number[]|null} The snapped box, or null for one too large.
*/
export function libraryViewBox([west, south, east, north]) {
const span = Math.max(east - west, north - south);
if (!(span > 0)) return null;
let cell = VIEW_GRID_MIN_DEG;
while (cell < span) cell *= 2;
if (cell > VIEW_GRID_MAX_DEG) return null;
const left = Math.floor(west / cell) * cell;
const bottom = Math.floor(south / cell) * cell;
return [left, bottom, left + 2 * cell, bottom + 2 * cell];
}
/**
* Parse a proxied view box (`west,south,east,north`) as
* {@link libraryViewBox} writes it: two grid cells a side, on the grid, and
* written exactly as `String` writes its numbers, so every box has one
* spelling.
* @param {string|null} text
* @returns {number[]|null}
*/
export function parseLibraryViewBox(text) {
if (typeof text !== 'string' || text.length > 128) return null;
const box = text.split(',').map(Number);
if (box.length !== 4 || !box.every(Number.isFinite)) return null;
if (box.join(',') !== text) return null;
const [west, south, east, north] = box;
const cell = (east - west) / 2;
if (north - south !== 2 * cell) return null;
if (!(cell >= VIEW_GRID_MIN_DEG && cell <= VIEW_GRID_MAX_DEG)) return null;
if (2 ** Math.round(Math.log2(cell)) !== cell) return null;
if (!Number.isInteger(west / cell) || !Number.isInteger(south / cell))
return null;
if (west < -180 - 2 * cell || east > 180 + 2 * cell) return null;
if (south < -90 - 2 * cell || north > 90 + 2 * cell) return null;
return box;
}
/** How the browser reaches the upstream. */
export const FETCH_MODES = Object.freeze(['browser-direct', 'proxied']);
@@ -338,7 +387,8 @@ function checkSource(entry, errors) {
`source.view needs maxAltitudeM (${MIN_VIEW_ALTITUDE_M}–${MAX_VIEW_ALTITUDE_M}) and an optional sample bbox`,
);
// View-bounded loading queries the camera's box; this build does that
// for ArcGIS layers and partitioned GeoParquet, fetched by the browser.
// for ArcGIS layers and partitioned GeoParquet, fetched by the browser
// (or, when a server routes the entry, by its proxy).
if (
!['arcgis', 'geoparquet'].includes(source.format) ||
source.fetch !== 'browser-direct'
+533
View File
@@ -0,0 +1,533 @@
/**
* The Data Library's server side as a fetch handler: a `Request` in, a
* `Response` out, so any server (Node, a Worker, a test) can answer the
* library routes the browser asks for.
*
* GET <base>/catalog what this server offers, per entry
* GET <base>/data/<id>[?bbox=w,s,e,n] a dataset's features
* GET <base>/tile/<id>/<z>/<x>/<y> one tile's image
* GET <base>/index/<id> a partitioned GeoParquet's index
*
* The browser names an entry id, never a URL: every upstream request is built
* from that entry's manifest and stays on its host, so the allowlist is
* exactly the catalog. Redirects are refused, bodies are capped while
* streaming, keys come from the server's environment and upstream error text
* is never returned.
*
* A server may hide entries (`listed`: never offered, never served) and route
* browser-direct entries through itself (`proxied`); the catalog route tells
* the browser both. Same-site gates and rate limits belong to the server
* that mounts the handler.
*/
import { LIBRARY_ENTRIES } from './catalog.js';
import { loadLibraryDataset } from './formats.js';
import { buildPartitionIndex } from './parquet.js';
import {
libraryTileUrl,
libraryUpstreamUrl,
parseLibraryIdList,
parseLibraryViewBox,
} from './manifest.js';
import { createLibrarySource } from './source.js';
import { KEY_SETUP_KEYS } from '../keySetupCore.mjs';
import {
coalesceProxyRequest,
readResponseBytesCapped,
readResponseTextCapped,
} from '../sources/httpBody.js';
/** The catalog and its view grid, for a server that decides what it serves. */
export { LIBRARY_ENTRIES } from './catalog.js';
export { libraryViewBox, parseLibraryViewBox } from './manifest.js';
const DATASET_TIMEOUT_MS = 30_000;
const TILE_TIMEOUT_MS = 10_000;
/** Each listing page or footer read while building a partition index. */
const INDEX_TIMEOUT_MS = 20_000;
/** How long an incomplete partition index is served before a rebuild. */
const PARTIAL_INDEX_MS = 5 * 60_000;
const TILE_MAX_BYTES = 2 * 1024 * 1024;
const TILE_CACHE_LIMIT = 512;
const VIEW_CACHE_LIMIT = 64;
const IN_FLIGHT_LIMIT = 64;
const TILE_TYPES = /^image\/(png|jpeg|webp)$/;
/** The configured value for a manifest key, through the key registry. */
export function libraryKeyValue(key, env) {
const registered = KEY_SETUP_KEYS.find(({ id }) => id === key?.id);
for (const name of registered?.envVars || []) {
const value = String(env[name] || '').trim();
if (value) return value;
}
return null;
}
/**
* Availability of every entry on this instance, decided by the operator's
* environment alone: `GEV_LIBRARY_DISABLE` removes entries, opt-in entries
* need `GEV_LIBRARY_ENABLE` (or, for a pack, its host's own switch), and a
* keyed entry needs its key. The answer names a reason, never a secret.
* @param {object[]} entries
* @param {Record<string, string|undefined>} env
* @returns {Map<string, { available: boolean, reason: string|null }>}
*/
export function libraryAvailability(entries, env) {
const enabled = parseLibraryIdList(env.GEV_LIBRARY_ENABLE);
const disabled = parseLibraryIdList(env.GEV_LIBRARY_DISABLE);
const result = new Map();
for (const entry of entries) {
let reason = null;
if (disabled.has(entry.id)) reason = 'operator-disabled';
else if (entry.kind === 'pack') {
if (String(env[entry.pack.env] || '').trim() !== '1')
reason = 'operator-opt-in';
} else if (entry.licence.operatorOptIn && !enabled.has(entry.id))
reason = 'operator-opt-in';
else if (entry.source.key && !libraryKeyValue(entry.source.key, env))
reason = 'key-missing';
result.set(entry.id, { available: reason === null, reason });
}
return result;
}
/**
* Parse `/<id>/<z>/<x>/<y>` against a tiles entry's zoom range.
* @returns {{ z: number, x: number, y: number } | null}
*/
export function parseLibraryTilePath(parts, source) {
if (
parts.length !== 3 ||
!parts.every((part) => /^(0|[1-9]\d{0,7})$/.test(part))
)
return null;
const [z, x, y] = parts.map(Number);
const minZoom = source.minZoom ?? 0;
const maxZoom = source.maxZoom ?? 22;
if (z < minZoom || z > maxZoom) return null;
const span = 2 ** z;
if (x >= span || y >= span) return null;
return { z, x, y };
}
function withKey(url, key, env, headers) {
if (!key) return url;
const value = libraryKeyValue(key, env);
if (!value) throw Object.assign(new Error('key_missing'), { status: 503 });
if (key.in === 'header') {
headers[key.name] = value;
return url;
}
const keyed = new URL(url);
keyed.searchParams.set(key.name, value);
return keyed.href;
}
/** A timeout that also ends with the caller's signal, when there is one. */
function timeoutSignal(timeoutMs, signal) {
const timeout = AbortSignal.timeout(timeoutMs);
return signal && typeof AbortSignal.any === 'function'
? AbortSignal.any([timeout, signal])
: timeout;
}
function json(status, value, headers = {}) {
return new Response(JSON.stringify(value), {
status,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...(status === 429 ? { 'Retry-After': '60' } : {}),
...headers,
},
});
}
/**
* How long a browser or shared cache may keep an entry's answer: its
* `cacheSeconds`, shared unless a key fetched it (a keyed answer is the
* operator's, kept by the browser alone).
*/
function cacheControl(entry) {
const scope = entry.source.key ? 'private' : 'public';
return `${scope}, max-age=${entry.source.cacheSeconds}`;
}
/**
* @param {object} [options]
* @param {object[]} [options.entries] The catalog.
* @param {Record<string, string|undefined>} [options.env] Keys and the
* `GEV_LIBRARY_*` switches (`libraryAvailability`).
* @param {typeof fetch} [options.fetchImpl]
* @param {() => number} [options.now]
* @param {string} [options.base] Path the routes are mounted under.
* @param {(entry: object) => boolean} [options.listed] False hides an entry:
* the catalog leaves it out of the browser's list and no route serves it.
* @param {(entry: object) => boolean} [options.proxied] True routes a
* browser-direct entry through this handler.
* @param {boolean} [options.keep] Keep answers in this process's memory for
* `cacheSeconds` (and serve a stale dataset when a refresh fails). A server
* with a shared cache in front turns it off.
* @param {Map<string, object[]>} [options.partitionCache] Decoded GeoParquet
* partitions, shared by every request this map is given to; at most four
* are kept (pass one map per process to cap them there).
* @param {(entry: object, init: { signal?: AbortSignal }) => Promise<Response>} [options.fetchIndex]
* Fetches a partitioned entry's index, e.g. this handler's own `/index/<id>`
* through a shared cache. By default the index route is answered in
* process. A dataset request never builds an index itself.
* @param {number} [options.subrequestLimit] Most upstream requests one
* dataset or tile request may make (a host's per-request subrequest
* limit); a request that would need more is refused with 503
* `subrequest_limit` before passing it.
* @returns {(request: Request) => Promise<Response>}
*/
export function createLibraryHandler({
entries = LIBRARY_ENTRIES,
env = {},
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
base = '/api/library',
listed = () => true,
proxied = () => false,
keep = true,
partitionCache = new Map(),
fetchIndex = null,
subrequestLimit = Infinity,
} = {}) {
const shown = entries.filter((entry) => listed(entry));
const byId = new Map(shown.map((entry) => [entry.id, entry]));
const datasets = new Map();
const views = new Map();
const tiles = new Map();
const indexes = new Map();
const partialIndexes = new Map();
const inFlight = new Map();
const routed = (entry) =>
entry.kind !== 'pack' &&
(entry.source.fetch === 'proxied' || proxied(entry) === true);
/** The upstream request for `url`, on the manifest's host only. */
async function upstream(entry, url, { timeoutMs, signal, headers = {} }) {
// Defense in depth: every derived request stays on the manifest's host.
const target = libraryUpstreamUrl(url);
const declared = libraryUpstreamUrl(entry.source.url);
if (!target || !declared || target.host !== declared.host)
throw Object.assign(new Error('off_manifest_host'), { status: 502 });
const keyHeaders = {};
const keyed = withKey(url, entry.source.key, env, keyHeaders);
const timed = timeoutSignal(timeoutMs, signal);
const response = await fetchImpl(keyed, {
signal: timed,
redirect: 'error',
headers: { Accept: '*/*', ...headers, ...keyHeaders },
});
return { response, signal: timed };
}
async function okUpstream(entry, url, options) {
const result = await upstream(entry, url, options);
if (!result.response.ok) {
await result.response.body?.cancel?.().catch?.(() => {});
throw Object.assign(new Error('upstream_unavailable'), { status: 502 });
}
return result;
}
/** Counts one request's upstream fetches against `subrequestLimit`. */
function subrequests() {
let used = 0;
return () => {
used += 1;
if (used > subrequestLimit)
throw Object.assign(new Error('subrequest_limit'), { status: 503 });
};
}
/**
* A GeoParquet entry's reader for one request: the browser's own, fetching
* upstream from here, with its index fetched (never built) and decoded
* partitions shared through `partitionCache`.
*/
function readerFor(entry, count) {
const direct = {
...entry,
source: { ...entry.source, fetch: 'browser-direct' },
};
const source = createLibrarySource({
entries: [direct],
base: null,
storage: null,
now,
partitionCache,
buildIndex: false,
fetchIndex: async (_entry, init) => {
count();
return fetchIndex ? fetchIndex(entry, init) : index(entry);
},
fetchImpl: async (url, init = {}) => {
count();
const range = new Headers(init.headers).get('range');
const { response } = await upstream(entry, String(url), {
timeoutMs: DATASET_TIMEOUT_MS,
signal: init.signal,
headers: range ? { Range: range } : {},
});
return response;
},
});
return { entry: direct, source };
}
async function fetchDataset(entry, bbox) {
const count = subrequests();
if (entry.source.format === 'geoparquet') {
const reader = readerFor(entry, count);
const result = await reader.source.loadDataset(reader.entry, { bbox });
return { ...result, fetchedAt: now() };
}
const collection = await loadLibraryDataset(
entry,
async (url, budget) => {
count();
const { response, signal } = await okUpstream(entry, url, {
timeoutMs: DATASET_TIMEOUT_MS,
});
const text = await readResponseTextCapped(
response,
budget.remainingBytes,
signal,
);
budget.remainingBytes -= new TextEncoder().encode(text).byteLength;
return text;
},
{ bbox },
);
return { ...collection, fetchedAt: now() };
}
async function cached(store, key, ttlMs, load, limit = Infinity) {
const previous = keep ? store.get(key) : undefined;
if (previous && now() - previous.savedAt < ttlMs)
return { value: previous.value, stale: false };
if (!inFlight.has(key) && inFlight.size >= IN_FLIGHT_LIMIT)
throw Object.assign(new Error('busy'), { status: 429 });
try {
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
const value = await load();
if (keep) {
store.delete(key);
store.set(key, { value, savedAt: now() });
while (store.size > limit) store.delete(store.keys().next().value);
}
return value;
});
return { value: await promise, stale: false };
} catch (error) {
if (previous) return { value: previous.value, stale: true };
throw error;
}
}
function failure(error) {
if (error?.status === 429) return json(429, { error: 'rate_limited' });
if (error?.status === 503)
return json(503, {
error: ['subrequest_limit', 'index_unavailable'].includes(error.message)
? error.message
: 'key_missing',
});
return json(502, { error: 'library_upstream_unavailable' });
}
async function index(entry) {
// Only a complete index is kept for cacheSeconds. An incomplete one
// (footers that kept failing) is served as `complete: false` and kept
// for PARTIAL_INDEX_MS only, so the next request soon rebuilds it; a
// complete index from before keeps being served while rebuilds fail.
const key = `index:${entry.id}`;
const partial = partialIndexes.get(key);
if (partial && now() - partial.savedAt < PARTIAL_INDEX_MS) {
const previous = indexes.get(key);
return json(200, previous?.value ?? partial.value);
}
try {
const { value } = await cached(
indexes,
key,
entry.source.cacheSeconds * 1000,
async () => {
const count = subrequests();
const built = await buildPartitionIndex(entry, {
fetchImpl: async (url, options = {}) => {
count();
const range = new Headers(options.headers).get('range');
const { response } = await upstream(entry, String(url), {
timeoutMs: INDEX_TIMEOUT_MS,
signal: options.signal,
headers: range ? { Range: range } : {},
});
return response;
},
});
const value = {
partitions: built.partitions,
complete: built.complete,
builtAt: now(),
};
if (!built.complete) {
// Recorded here too: with an earlier complete index, cached()
// serves that one and swallows this error, and the rebuild
// still waits PARTIAL_INDEX_MS.
partialIndexes.set(key, { value, savedAt: now() });
throw Object.assign(new Error('index_incomplete'), {
partialIndex: value,
});
}
partialIndexes.delete(key);
return value;
},
);
return json(200, value, { 'Cache-Control': cacheControl(entry) });
} catch (error) {
if (error?.partialIndex) return json(200, error.partialIndex);
return error?.status === 429
? json(429, { error: 'rate_limited' })
: json(502, { error: 'library_upstream_unavailable' });
}
}
async function data(entry, params) {
const view = Boolean(entry.source.view);
const bboxText = params.get('bbox');
const bbox = view ? parseLibraryViewBox(bboxText) : null;
if (view && !bbox) return json(400, { error: 'invalid_bbox' });
if (!view && bboxText !== null) return json(400, { error: 'invalid_bbox' });
try {
const { value, stale } = await cached(
view ? views : datasets,
view ? `view:${entry.id}:${bboxText}` : `data:${entry.id}`,
entry.source.cacheSeconds * 1000,
() => fetchDataset(entry, bbox),
view ? VIEW_CACHE_LIMIT : Infinity,
);
return stale
? json(200, { ...value, stale: true }, { 'X-Data-Stale': 'true' })
: json(200, value, { 'Cache-Control': cacheControl(entry) });
} catch (error) {
return failure(error);
}
}
async function tile(entry, parts) {
const at = parseLibraryTilePath(parts, entry.source);
if (!at) return json(400, { error: 'invalid_tile' });
try {
const { value } = await cached(
tiles,
`tile:${entry.id}:${at.z}/${at.x}/${at.y}`,
entry.source.cacheSeconds * 1000,
async () => {
const { response, signal } = await okUpstream(
entry,
libraryTileUrl(entry.source.url, at.z, at.x, at.y),
{ timeoutMs: TILE_TIMEOUT_MS },
);
const type = String(response.headers.get('content-type') || '')
.split(';')[0]
.trim()
.toLowerCase();
if (!TILE_TYPES.test(type)) {
await response.body?.cancel?.().catch?.(() => {});
throw Object.assign(new Error('not_an_image'), { status: 502 });
}
const bytes = await readResponseBytesCapped(
response,
TILE_MAX_BYTES,
signal,
);
return { type, bytes };
},
TILE_CACHE_LIMIT,
);
return new Response(value.bytes, {
status: 200,
headers: {
'Content-Type': value.type,
'Cache-Control': cacheControl(entry),
'X-Content-Type-Options': 'nosniff',
},
});
} catch (error) {
return failure(error);
}
}
return async function handle(request) {
if (request.method !== 'GET')
return json(405, { error: 'method_not_allowed' });
const url = new URL(request.url);
const path = url.pathname.startsWith(base)
? url.pathname.slice(base.length)
: url.pathname;
// One spelling per resource, so a cache keyed on the path holds each
// tile or answer once: no empty segments, no trailing slash.
const parts = path.replace(/^\//, '').split('/');
if (parts.some((part) => part === ''))
return json(404, { error: 'unknown_entry' });
const availability = libraryAvailability(shown, env);
if (parts.length === 1 && parts[0] === 'catalog') {
return json(200, {
entries: [
...shown.map(({ id }) => {
const entry = byId.get(id);
return {
id,
...availability.get(id),
...(entry.kind !== 'pack' &&
entry.source.fetch !== 'proxied' &&
routed(entry)
? { fetch: 'proxied' }
: {}),
};
}),
...entries
.filter((entry) => !byId.has(entry.id))
.map(({ id }) => ({
id,
available: false,
reason: 'operator-disabled',
listed: false,
})),
],
});
}
const [route, id, ...rest] = parts;
const entry = byId.get(id);
if (!entry || !['data', 'tile', 'index'].includes(route))
return json(404, { error: 'unknown_entry' });
if (route === 'index') {
if (rest.length || !entry.source?.partitions)
return json(404, { error: 'not_partitioned' });
const { available, reason } = availability.get(id);
if (!available) return json(403, { error: reason });
return index(entry);
}
const isDataset = route === 'data' && rest.length === 0;
const isTile = route === 'tile';
if (
!routed(entry) ||
(isDataset && entry.kind !== 'dataset') ||
(isTile && entry.kind !== 'tiles') ||
(!isDataset && !isTile)
)
return json(404, { error: 'not_proxied' });
const { available, reason } = availability.get(id);
if (!available) return json(403, { error: reason });
return isDataset ? data(entry, url.searchParams) : tile(entry, rest);
};
}
+256
View File
@@ -0,0 +1,256 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createLibraryHandler } from './proxy.js';
import { libraryEntry } from './catalog.js';
import {
libraryViewBox,
parseLibraryViewBox,
validateLibraryCatalog,
} from './manifest.js';
const base = {
family: 'imagery',
provider: 'Fixture',
description: 'Fixture entry.',
maintainer: 'test',
coverage: { global: true },
licence: { name: 'Public domain', url: 'https://example.org/licence' },
attribution: { text: 'Fixture' },
status: 'community',
};
const { entries, errors } = validateLibraryCatalog([
{
...base,
id: 'direct-tiles',
kind: 'tiles',
title: 'Direct tiles',
source: {
format: 'bbox',
url: 'https://maps.example.org/export?bbox={bbox-epsg-3857}&f=image',
fetch: 'browser-direct',
cacheSeconds: 900,
},
},
{
...base,
id: 'hidden-tiles',
kind: 'tiles',
title: 'Hidden tiles',
source: {
format: 'xyz',
url: 'https://tiles.example.org/{z}/{x}/{y}.png',
fetch: 'browser-direct',
},
},
{
...base,
id: 'direct-points',
kind: 'dataset',
family: 'events',
title: 'Direct points',
source: {
format: 'geojson',
url: 'https://data.example.org/points.json',
fetch: 'browser-direct',
},
},
]);
assert.deepEqual(errors, []);
const view = libraryEntry('fema-usa-structures');
const png = () =>
new Response(new Uint8Array([137, 80, 78, 71]), {
headers: { 'content-type': 'image/png' },
});
const get = (handle, path) =>
handle(new Request(`https://app.example/api/library${path}`));
test('a hidden entry is listed as not offered and no route serves it', async () => {
let calls = 0;
const handle = createLibraryHandler({
entries,
listed: (entry) => entry.id !== 'hidden-tiles',
proxied: () => true,
fetchImpl: async () => {
calls++;
return png();
},
});
const catalog = await (await get(handle, '/catalog')).json();
assert.deepEqual(
catalog.entries.find(({ id }) => id === 'hidden-tiles'),
{
id: 'hidden-tiles',
available: false,
reason: 'operator-disabled',
listed: false,
},
);
assert.equal((await get(handle, '/tile/hidden-tiles/1/0/0')).status, 404);
assert.equal(calls, 0);
});
test('a routed browser-direct entry is served from its manifest and cacheable for cacheSeconds', async () => {
const calls = [];
const handle = createLibraryHandler({
entries,
proxied: (entry) => entry.id === 'direct-tiles',
keep: false,
fetchImpl: async (url, options) => {
calls.push({ url, options });
return png();
},
});
const catalog = await (await get(handle, '/catalog')).json();
assert.equal(
catalog.entries.find(({ id }) => id === 'direct-tiles').fetch,
'proxied',
);
assert.equal(
catalog.entries.find(({ id }) => id === 'direct-points').fetch,
undefined,
);
const tile = await get(handle, '/tile/direct-tiles/1/1/0?url=https://evil');
assert.equal(tile.status, 200);
assert.equal(tile.headers.get('content-type'), 'image/png');
assert.equal(tile.headers.get('cache-control'), 'public, max-age=900');
assert.equal(calls.length, 1);
assert.match(
calls[0].url,
/^https:\/\/maps\.example\.org\/export\?bbox=0\.000,0\.000,20037508\.343,20037508\.343&f=image$/,
);
assert.equal(calls[0].options.redirect, 'error');
// Not kept in memory: a shared cache in front holds it.
await get(handle, '/tile/direct-tiles/1/1/0');
assert.equal(calls.length, 2);
// An entry the server does not route stays the browser's to fetch.
assert.equal((await get(handle, '/data/direct-points')).status, 404);
});
test('a routed view-bounded entry needs a snapped box and queries only it', async () => {
const calls = [];
const handle = createLibraryHandler({
entries: [view],
proxied: () => true,
fetchImpl: async (url) => {
calls.push(new URL(url));
return new Response(
JSON.stringify({ type: 'FeatureCollection', features: [] }),
);
},
});
assert.equal((await get(handle, '/data/fema-usa-structures')).status, 400);
assert.equal(
(await get(handle, '/data/fema-usa-structures?bbox=-77.03,38.89,-77,38.9'))
.status,
400,
);
assert.equal(calls.length, 0);
const box = libraryViewBox([-77.031, 38.889, -77.02, 38.9]);
const ok = await get(handle, `/data/fema-usa-structures?bbox=${box}`);
assert.equal(ok.status, 200);
assert.equal(
ok.headers.get('cache-control'),
`public, max-age=${view.source.cacheSeconds}`,
);
assert.equal(calls.length, 1);
assert.equal(
calls[0].searchParams.get('geometry'),
box.map((value) => value.toFixed(6)).join(','),
);
});
test('a view box snaps out to two grid cells a side, with one spelling', () => {
const asked = [-77.031, 38.889, -77.02, 38.9];
const box = libraryViewBox(asked);
assert.ok(box[0] <= asked[0] && box[1] <= asked[1]);
assert.ok(box[2] >= asked[2] && box[3] >= asked[3]);
assert.equal(box[2] - box[0], box[3] - box[1]);
// Nearby views name the same box.
assert.deepEqual(libraryViewBox([-77.0305, 38.8895, -77.0205, 38.8995]), box);
assert.deepEqual(parseLibraryViewBox(box.join(',')), box);
assert.equal(
parseLibraryViewBox(box.map((v) => v.toFixed(8)).join(',')),
null,
);
assert.equal(parseLibraryViewBox('-77,38,-76,39.5'), null);
assert.equal(parseLibraryViewBox('0,0,4,4'), null);
assert.equal(libraryViewBox([0, 0, 3, 3]), null);
});
test('a tile path has one spelling: no leading zeros, no empty segments, inside the grid', async () => {
const calls = [];
const xyz = entries.find(({ id }) => id === 'hidden-tiles');
const handle = createLibraryHandler({
entries: [xyz],
proxied: () => true,
keep: false,
fetchImpl: async (url) => {
calls.push(url);
return png();
},
});
for (const path of [
'/tile/hidden-tiles/02/1/1',
'/tile/hidden-tiles/2/01/1',
'/tile/hidden-tiles/2/1/4',
'/tile/hidden-tiles/2/1/1.5',
'/tile/hidden-tiles/23/1/1',
])
assert.equal((await get(handle, path)).status, 400, path);
for (const path of [
'/tile/hidden-tiles//2/1/1',
'/tile/hidden-tiles/2/1/1/',
'//tile/hidden-tiles/2/1/1',
])
assert.equal((await get(handle, path)).status, 404, path);
assert.equal(calls.length, 0);
assert.equal((await get(handle, '/tile/hidden-tiles/2/1/3')).status, 200);
assert.equal(calls.length, 1);
});
test('a partitioned dataset reads its index from the given fetch, never builds one, and stays under the subrequest limit', async () => {
const epc = libraryEntry('epc-england-wales');
const box = libraryViewBox(epc.source.view.sample);
const upstream = [];
const indexes = [];
const handler = (fetchIndex, subrequestLimit = Infinity) =>
createLibraryHandler({
entries: [epc],
proxied: () => true,
keep: false,
fetchIndex,
subrequestLimit,
fetchImpl: async (url) => {
upstream.push(String(url));
return new Response('', { status: 500 });
},
});
// No index to be had: refused clearly, the publisher never asked.
const missing = await get(
handler(async (entry) => {
indexes.push(entry.id);
return new Response('', { status: 404 });
}),
`/data/epc-england-wales?bbox=${box}`,
);
assert.equal(missing.status, 503);
assert.deepEqual(await missing.json(), { error: 'index_unavailable' });
assert.deepEqual(indexes, ['epc-england-wales']);
assert.deepEqual(upstream, []);
// An index naming partitions in view: reading them would pass the limit.
const partitions = Array.from({ length: 8 }, (_, i) => ({
url: `${epc.source.partitions.keyBase}addresscloud/epc/geoparquet-local-authority/p${i}.parquet`,
bbox: box,
}));
const over = await get(
handler(async () => Response.json({ partitions, complete: true }), 3),
`/data/epc-england-wales?bbox=${box}`,
);
assert.equal(over.status, 503);
assert.deepEqual(await over.json(), { error: 'subrequest_limit' });
// The index fetch and two upstream reads; the third read was refused.
assert.equal(upstream.length, 2);
});
+147 -45
View File
@@ -4,12 +4,13 @@
*
* Browser-direct entries are fetched from their upstream without cookies or
* a referrer; proxied entries go through `/api/library`, which names entries
* by id only. A static build with no server still serves every browser-direct
* entry that is not operator opt-in.
* by id only. A server may hide entries and route browser-direct ones through
* itself; its catalog route says which. A static build with no server still
* serves every browser-direct entry that is not operator opt-in.
*/
import { loadLibraryDataset, boundLibraryFeatures } from './formats.js';
import { readResponseTextCapped } from '../sources/httpBody.js';
import { BBOX_TOKEN } from './manifest.js';
import { BBOX_TOKEN, libraryViewBox } from './manifest.js';
/** Partitions read per view before giving up, in parallel batches. */
const MAX_PARTITION_READS_PER_VIEW = 8;
@@ -150,26 +151,66 @@ export function staticLibraryAvailability(entries) {
);
}
/** Every entry unavailable and left out of the list. */
function unlisted(entries) {
return new Map(
entries.map(({ id }) => [
id,
{ available: false, reason: 'server-unavailable', listed: false },
]),
);
}
/**
* @param {{ entries: object[], fetchImpl?: typeof fetch, base?: string }} options
* @param {object} options
* @param {object[]} options.entries
* @param {typeof fetch} [options.fetchImpl]
* @param {string|null} [options.base] The library server's path; null for
* none (a server reading upstream itself).
* @param {Storage|null} [options.storage] Where partition indexes and
* learned extents are kept between visits (the page's localStorage); none
* by default.
* @param {(request: { entry: object, tile: object, url: string, signal: AbortSignal }) => Promise<import('../render/contract/globe').TileFetch>} [options.fetchTile]
* Fetches a tiles entry's tile in place of the renderer (the imagery
* spec's `fetchTile`): for a host that keeps tiles in its own cache.
* @param {(request: { entry: object, url: string, signal?: AbortSignal }) => Promise<Response>} [options.fetchData]
* Fetches a proxied dataset's answer from the library server in place of
* `fetchImpl`.
* @param {boolean} [options.requireCatalog] Offer only what the server's
* catalog names: while it does not answer, nothing is offered (and
* `getAvailability` says `retry`), rather than every browser-direct entry.
* @param {Map<string, object[]>} [options.partitionCache] Decoded GeoParquet
* partitions, most recent last, shared by sources given the same map; at
* most four are kept.
* @param {(entry: object, init: { signal?: AbortSignal }) => Promise<Response>} [options.fetchIndex]
* Fetches a partitioned entry's index (`{ partitions, complete }`) in
* place of the library server's `/index/<id>`.
* @param {boolean} [options.buildIndex] Build a partition index here when
* no complete one can be fetched; false uses an incomplete one or fails.
*/
export function createLibrarySource({
entries,
fetchImpl = (...args) => globalThis.fetch(...args),
base = LIBRARY_API_BASE,
storage = (() => {
try {
return globalThis.localStorage ?? null;
} catch {
return null;
}
})(),
fetchTile = null,
fetchData = null,
requireCatalog = false,
partitionCache = new Map(),
fetchIndex = null,
buildIndex = true,
storage = null,
now = () => Date.now(),
loadParquet = () => import('./parquet.js'),
}) {
const known = new Set(entries.map(({ id }) => id));
/** Entries the library server routes through itself (its catalog says). */
const routed = new Set();
// What the last catalog answer offered; with `requireCatalog`, nothing
// else is ever loaded.
const offered = new Set();
const isProxied = (entry) =>
entry.source.fetch === 'proxied' || routed.has(entry.id);
const indexes = new Map();
const partitionCache = new Map();
/**
* A partitioned entry's index: from this app's server (built once and
@@ -209,11 +250,14 @@ export function createLibrarySource({
: null;
let partial = null;
try {
const response = await fetchImpl(
`${base}/index/${encodeURIComponent(entry.id)}`,
{ headers: { Accept: 'application/json' } },
);
if (response.ok) {
const response = fetchIndex
? await fetchIndex(entry, {})
: base
? await fetchImpl(`${base}/index/${encodeURIComponent(entry.id)}`, {
headers: { Accept: 'application/json' },
})
: null;
if (response?.ok) {
const body = await response.json();
if (Array.isArray(body?.partitions)) {
if (body.complete !== false)
@@ -229,6 +273,10 @@ export function createLibrarySource({
now() - storedComplete.savedAt < entry.source.cacheSeconds * 1000
)
return { partitions: storedComplete.partitions, complete: true };
if (!buildIndex) {
if (partial) return { partitions: partial, complete: false };
throw Object.assign(new Error('index_unavailable'), { status: 503 });
}
// The server's index is incomplete, or there is no server: build here.
const { buildPartitionIndex } = await loadParquet();
const built = await buildPartitionIndex(entry, { fetchImpl });
@@ -366,6 +414,11 @@ export function createLibrarySource({
* @returns {Promise<{ server: boolean, availability: Map<string, { available: boolean, reason: string|null }> }>}
*/
async function getAvailability({ signal } = {}) {
if (!base)
return {
server: false,
availability: staticLibraryAvailability(entries),
};
try {
const response = await fetchImpl(`${base}/catalog`, {
signal,
@@ -373,21 +426,42 @@ export function createLibrarySource({
});
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const body = await response.json();
const availability = staticLibraryAvailability(entries);
for (const row of Array.isArray(body?.entries) ? body.entries : []) {
if (!Array.isArray(body?.entries)) throw new Error('no catalog');
// A required catalog is an allowlist: what it does not name is not
// offered.
const availability = requireCatalog
? unlisted(entries)
: staticLibraryAvailability(entries);
routed.clear();
offered.clear();
for (const row of body.entries) {
if (!known.has(row?.id)) continue;
// An entry the server does not list is not offered at all.
const listed = row.listed !== false;
const available = listed && row.available === true;
availability.set(row.id, {
available: row.available === true,
reason: row.available === true ? null : String(row.reason || ''),
available,
reason: available
? null
: String(row.reason || (listed ? '' : 'operator-disabled')),
...(listed ? {} : { listed: false }),
});
if (listed && row.fetch === 'proxied') routed.add(row.id);
if (available) offered.add(row.id);
}
return { server: true, availability };
} catch (error) {
if (signal?.aborted) throw error;
return {
server: false,
availability: staticLibraryAvailability(entries),
};
// Without the server's word, a required catalog offers nothing (the
// caller asks again later); otherwise the static rule applies.
routed.clear();
offered.clear();
return requireCatalog
? { server: false, availability: unlisted(entries), retry: true }
: {
server: false,
availability: staticLibraryAvailability(entries),
};
}
}
@@ -399,6 +473,37 @@ export function createLibrarySource({
* @returns {Promise<{ features: object[], truncated: boolean }>}
*/
async function loadDataset(entry, { signal, bbox = null } = {}) {
if (requireCatalog && !offered.has(entry.id))
throw new Error(`${entry.id} is not offered by the library server`);
if (isProxied(entry)) {
// A view-bounded entry asks for its snapped box (`libraryViewBox`),
// so nearby views share one answer.
let query = '';
if (entry.source.view) {
const box = bbox && libraryViewBox(bbox);
if (!box) return { features: [], truncated: false };
query = `?bbox=${box.join(',')}`;
}
const url = `${base}/data/${encodeURIComponent(entry.id)}${query}`;
const response = fetchData
? await fetchData({ entry, url, signal })
: await fetchImpl(url, {
signal,
headers: { Accept: 'application/json' },
});
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const body = await response.json();
// The server already bounded the set; bound again so a page never
// trusts more than its manifest allows.
const bounded = boundLibraryFeatures(body?.features || [], {
maxFeatures: entry.source.maxFeatures,
display: entry.display,
});
return {
features: bounded.features,
truncated: Boolean(body?.truncated) || bounded.truncated,
};
}
if (entry.source.format === 'geoparquet') {
if (entry.source.partitions) {
if (!bbox) return { features: [], truncated: false };
@@ -417,24 +522,6 @@ export function createLibrarySource({
);
return { features: bounded.features, truncated: bounded.truncated };
}
if (entry.source.fetch === 'proxied') {
const response = await fetchImpl(
`${base}/data/${encodeURIComponent(entry.id)}`,
{ signal, headers: { Accept: 'application/json' } },
);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const body = await response.json();
// The server already bounded the set; bound again so a page never
// trusts more than its manifest allows.
const bounded = boundLibraryFeatures(body?.features || [], {
maxFeatures: entry.source.maxFeatures,
display: entry.display,
});
return {
features: bounded.features,
truncated: Boolean(body?.truncated) || bounded.truncated,
};
}
const collection = await loadLibraryDataset(
entry,
async (url, budget) => {
@@ -461,7 +548,7 @@ export function createLibrarySource({
/** The tile URL template the imagery provider requests. */
function tileTemplate(entry) {
if (entry.source.fetch === 'proxied')
if (isProxied(entry))
return `${base}/tile/${encodeURIComponent(entry.id)}/{z}/{x}/{y}`;
// Cesium fills a tile's projected corners itself.
return entry.source.url.replaceAll(
@@ -470,5 +557,20 @@ export function createLibrarySource({
);
}
return { getAvailability, loadDataset, tileTemplate };
/**
* The imagery spec's `fetchTile` for a tiles entry: the host's, told which
* entry asks; null to let the renderer fetch.
*/
function tileFetch(entry) {
if (typeof fetchTile !== 'function') return null;
return (request) => fetchTile({ ...request, entry });
}
return {
getAvailability,
loadDataset,
tileTemplate,
tileFetch,
requireCatalog,
};
}
+110
View File
@@ -0,0 +1,110 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createLibrarySource } from './source.js';
import { libraryEntry } from './catalog.js';
import { libraryViewBox } from './manifest.js';
import { libraryImagerySpec } from './layers.js';
const tiles = libraryEntry('usgs-topo');
const view = libraryEntry('fema-usa-structures');
const entries = [tiles, view];
function serverSaying(rows, calls = []) {
return async (url, options) => {
calls.push({ url: String(url), options });
if (String(url).endsWith('/catalog'))
return Response.json({ entries: rows });
return Response.json({ features: [], truncated: false });
};
}
test('the server can hide an entry and route a browser-direct one through itself', async () => {
const source = createLibrarySource({
entries,
storage: null,
fetchImpl: serverSaying([
{ id: tiles.id, available: true, reason: null, fetch: 'proxied' },
{
id: view.id,
available: false,
reason: 'operator-disabled',
listed: false,
},
]),
});
assert.equal(source.tileTemplate(tiles), tiles.source.url);
const { availability } = await source.getAvailability();
assert.deepEqual(availability.get(view.id), {
available: false,
reason: 'operator-disabled',
listed: false,
});
assert.equal(availability.get(tiles.id).available, true);
assert.equal(
source.tileTemplate(tiles),
`/api/library/tile/${tiles.id}/{z}/{x}/{y}`,
);
});
test('a routed view-bounded dataset asks the server for its snapped box, through the host fetch', async () => {
const calls = [];
const asked = [];
const source = createLibrarySource({
entries,
storage: null,
fetchImpl: serverSaying(
[{ id: view.id, available: true, reason: null, fetch: 'proxied' }],
calls,
),
fetchData: async ({ entry, url }) => {
asked.push({ entry: entry.id, url });
return Response.json({ features: [], truncated: true });
},
});
await source.getAvailability();
const bbox = [-77.031, 38.889, -77.02, 38.9];
const result = await source.loadDataset(view, { bbox });
assert.equal(result.truncated, true);
assert.deepEqual(asked, [
{
entry: view.id,
url: `/api/library/data/${view.id}?bbox=${libraryViewBox(bbox).join(',')}`,
},
]);
// No box, nothing asked.
assert.deepEqual(await source.loadDataset(view), {
features: [],
truncated: false,
});
assert.equal(asked.length, 1);
assert.equal(calls.length, 1);
});
test("a host's tile fetch reaches the imagery spec, told the entry", async () => {
const seen = [];
const source = createLibrarySource({
entries,
storage: null,
fetchTile: async ({ entry, url }) => {
seen.push([entry.id, url]);
return { missing: true };
},
});
const spec = libraryImagerySpec(
tiles,
source.tileTemplate(tiles),
source.tileFetch(tiles),
);
assert.deepEqual(await spec.fetchTile({ tile: {}, url: 'u' }), {
missing: true,
});
assert.deepEqual(seen, [[tiles.id, 'u']]);
assert.equal(
'fetchTile' in libraryImagerySpec(tiles, tiles.source.url),
false,
);
assert.equal(
createLibrarySource({ entries, storage: null }).tileFetch(tiles),
null,
);
});