From 9554d6ea71f7da548d31eb78d812b32f90e15a81 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Tue, 17 Nov 2026 23:34:00 -0800 Subject: [PATCH] Serve the Data Library from any server, and let a server shape it (#1807) A server can answer the library with createLibraryHandler, a fetch handler, and shape what it offers: hide entries (listed: false) or route browser-direct ones through itself (fetch: proxied). View-bounded entries ask for a grid-snapped box, and GeoParquet is read on the server within it. Applications can pass their own tile and data fetchers (createApplicationTools({library: {fetchTile, fetchData}})), and with library.requireCatalog a catalog that does not answer offers nothing and is asked again later. Tile paths have one spelling: integer z/x/y without leading zeros, within the grid and the entry's zooms. A server's index and partition reads are bounded and counted against its subrequest limit. --- changelog.d/1807-library-server-routing.md | 9 + docs/CURRENT-STATE.md | 11 + docs/DATA-LIBRARY.md | 47 ++ package.json | 1 + scripts/package-boundaries.json | 114 +++++ server/providers/library.js | 391 ++------------- src/app/tools.js | 10 + src/library/controller.js | 68 ++- src/library/controller.test.mjs | 100 +++- src/library/layers.js | 11 +- src/library/manifest.js | 52 +- src/library/proxy.js | 533 +++++++++++++++++++++ src/library/proxy.test.mjs | 256 ++++++++++ src/library/source.js | 192 ++++++-- src/library/source.test.mjs | 110 +++++ 15 files changed, 1510 insertions(+), 395 deletions(-) create mode 100644 changelog.d/1807-library-server-routing.md create mode 100644 src/library/proxy.js create mode 100644 src/library/proxy.test.mjs create mode 100644 src/library/source.test.mjs diff --git a/changelog.d/1807-library-server-routing.md b/changelog.d/1807-library-server-routing.md new file mode 100644 index 0000000000..58296e4f33 --- /dev/null +++ b/changelog.d/1807-library-server-routing.md @@ -0,0 +1,9 @@ +- The Data Library's proxy is a fetch handler any server can mount + (`createLibraryHandler`, `sources/library`). A server can hide catalog + entries or route browser-direct ones through its proxy, view-bounded entries + ask it for a grid-snapped box, answers are cacheable for `cacheSeconds`, and + an application can fetch library tiles and datasets through its own cache + (`createApplicationTools({ library })`) and offer only what the server's + catalog names, nothing while it does not answer (`requireCatalog`). Tile + paths and view boxes have one spelling, and partitioned reads can be capped + per process and per request (samehkhamis, #1807). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index e6d297611d..731ea7bcae 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -589,6 +589,17 @@ keyless routes use the keyless gate so they work behind a reverse proxy. Operators opt entries in with `GEV_LIBRARY_ENABLE` and remove them with `GEV_LIBRARY_DISABLE`. A static build with no server still serves every browser-direct, non-opt-in entry; proxied ones say they need the app's server. +The proxy is a fetch handler (`createLibraryHandler`, exported as +`gods-eye-view/sources/library`) that the dev server mounts behind its gates. +Another server can mount it with `listed` (hide entries: the catalog marks them +`listed: false` and Browse leaves them out) and `proxied` (route browser-direct +entries through it: the catalog marks them `fetch: "proxied"`). A routed +view-bounded entry asks for its box snapped to a power-of-two grid +(`libraryViewBox`, `?bbox=` on `/data/`); GeoParquet entries are read on the +server then. Tile, data and complete index answers carry `public, +max-age=` (`private` when keyed). `createApplicationTools` +takes `library.fetchTile` and `library.fetchData` so an application can fetch +library tiles and proxied datasets through its own cache. The Browse dialog is a `dialog[data-panel-surface]`: modal, labelled, search focused on open, Escape and backdrop dismiss and return focus to Browse, and it diff --git a/docs/DATA-LIBRARY.md b/docs/DATA-LIBRARY.md index c549e7b4d2..897eb8e0a7 100644 --- a/docs/DATA-LIBRARY.md +++ b/docs/DATA-LIBRARY.md @@ -167,6 +167,53 @@ code. It reaches only the manifest's host, refuses redirects, caps bodies, caches per `cacheSeconds` and never returns upstream error text. The browser names an entry id, never a URL, so the proxy cannot be pointed anywhere else. +## Serving the library from another server + +The proxy is also a fetch handler, `createLibraryHandler` from +`gods-eye-view/sources/library` (`src/library/proxy.js`): a `Request` in, a +`Response` out, for a server other than the dev server (a Worker, say). Its +options decide what the server offers: + +- `listed(entry)` false hides an entry: `/catalog` marks it `listed: false`, + Browse leaves it out entirely, and no route serves it. +- `proxied(entry)` true routes a `browser-direct` entry through the handler: + `/catalog` marks it `fetch: "proxied"` and the browser asks the server for + its tiles and data. A view-bounded entry then asks `/data/?bbox=…` for + the camera's box snapped out to a power-of-two grid (`libraryViewBox`), so + nearby views share one answer; any other box is refused. +- `keep: false` keeps nothing in the process's memory, for a server with a + shared cache in front. Answers carry `Cache-Control: public, + max-age=` (`private` for a keyed entry). +- `partitionCache` is the map of decoded GeoParquet partitions (at most four + are kept); pass one map per process to cap them there. +- `fetchIndex(entry)` reads a partitioned entry's index, for example the + handler's own `/index/` through a shared cache. A `/data` request never + builds an index itself: without a complete or partial one it answers 503 + `index_unavailable`. +- `subrequestLimit` caps the upstream requests one request may make (a host's + per-request limit); a request that would need more answers 503 + `subrequest_limit` instead. + +Every resource has one spelling: tile paths are `/tile////` with +integers without leading zeros, `x` and `y` below `2^z` and `z` within the +entry's zoom range; a path with an empty segment or a trailing slash is +refused; a view box is accepted only in `libraryViewBox`'s exact spelling. A +shared cache keyed on the path (and a `/data` request's `bbox`) therefore +holds each answer once. + +Same-site gates and rate limits stay with the server that mounts it. An +application can also fetch the library's tiles and proxied datasets itself, +to keep them in its own cache: `createApplicationTools({ library: { fetchTile, +fetchData } })`. `fetchTile` is the imagery spec's tile fetch, told which +entry asks; `fetchData` answers a proxied dataset URL. + +`library: { requireCatalog: true }` makes the server's `/catalog` an +allowlist: the library offers only what it names, and while it does not +answer (an error status, a timeout, a page that is not the catalog) it offers +nothing at all, rather than every entry the browser could fetch itself. It is +asked again after 30 seconds, then at doubling intervals up to ten minutes, +and the viewer's saved entries come back once it answers. + ## Operator switches | Variable | Effect | diff --git a/package.json b/package.json index b085926e78..f4eb8082fd 100644 --- a/package.json +++ b/package.json @@ -293,6 +293,7 @@ "./sources/reference": "./src/sources/reference.js", "./layers/submarine-cables/source": "./src/layers/submarineCables/bundledSource.js", "./sources/http-body": "./src/sources/httpBody.js", + "./sources/library": "./src/library/proxy.js", "./sources/overpass": "./src/sources/overpass.js", "./search/nominatim": "./src/search/nominatim.js", "./sources/nominatim": "./src/sources/nominatim.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 82eb04879a..2fb079afb2 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -7082,6 +7082,120 @@ "modules": ["src/sources/httpBody.js", "src/i18n/marker.js"], "external": [] }, + "library-proxy": { + "exports": ["./sources/library"], + "modules": [ + "src/library/proxy.js", + "src/library/catalog.js", + "src/library/manifest.js", + "src/data/libraryLayerIds.js", + "src/library/ledger.json", + "src/library/entries/ak-statewide-imagery.json", + "src/library/entries/ar-statewide-imagery.json", + "src/library/entries/az-statewide-imagery.json", + "src/library/entries/amsterdam-traffic-cameras.json", + "src/library/entries/bayern-dop40.json", + "src/library/entries/bismarck-mandan-nd.json", + "src/library/entries/bkg-topplus-open.json", + "src/library/entries/cctv-florida-fl511.json", + "src/library/entries/cctv-lithuania.json", + "src/library/entries/cctv-nyc-dot.json", + "src/library/entries/ct-statewide-imagery.json", + "src/library/entries/dakota-county-mn.json", + "src/library/entries/dc-statewide-imagery.json", + "src/library/entries/de-statewide-imagery.json", + "src/library/entries/ee-maa-amet-ortofoto.json", + "src/library/entries/eia-power-plants.json", + "src/library/entries/epa-superfund-sites.json", + "src/library/entries/epa-toxic-release-facilities.json", + "src/library/entries/epc-england-wales.json", + "src/library/entries/fema-usa-structures.json", + "src/library/entries/gbif-occurrence-density.json", + "src/library/entries/gdacs-disaster-alerts.json", + "src/library/entries/grand-forks-nd.json", + "src/library/entries/gsi-japan-map.json", + "src/library/entries/gsi-japan-photo.json", + "src/library/entries/gvp-holocene-volcanoes.json", + "src/library/entries/ia-statewide-imagery.json", + "src/library/entries/id-statewide-imagery.json", + "src/library/entries/ign-orthophotos-france.json", + "src/library/entries/il-statewide-imagery.json", + "src/library/entries/imf-portwatch-chokepoints.json", + "src/library/entries/imf-portwatch-disruptions.json", + "src/library/entries/in-statewide-imagery.json", + "src/library/entries/jrc-surface-water.json", + "src/library/entries/kartverket-admin-units.json", + "src/library/entries/kartverket-cadastre.json", + "src/library/entries/kartverket-depth.json", + "src/library/entries/kartverket-hillshade.json", + "src/library/entries/kartverket-nautical-charts.json", + "src/library/entries/kartverket-terrain-relief.json", + "src/library/entries/kartverket-topo-grey.json", + "src/library/entries/kartverket-topo.json", + "src/library/entries/kartverket-toporaster.json", + "src/library/entries/ks-statewide-imagery.json", + "src/library/entries/ky-statewide-imagery.json", + "src/library/entries/lake-county-in.json", + "src/library/entries/lawrence-ks.json", + "src/library/entries/lee-county-fl.json", + "src/library/entries/loudoun-county-va.json", + "src/library/entries/louisville-lojic-ky.json", + "src/library/entries/ma-statewide-imagery.json", + "src/library/entries/macrostrat-geology.json", + "src/library/entries/mapterhorn-terrain.json", + "src/library/entries/maris-ms.json", + "src/library/entries/md-statewide-imagery.json", + "src/library/entries/me-statewide-imagery.json", + "src/library/entries/mi-statewide-imagery.json", + "src/library/entries/mountain-village-co.json", + "src/library/entries/ms-statewide-imagery.json", + "src/library/entries/mt-statewide-imagery.json", + "src/library/entries/naip.json", + "src/library/entries/nasa-black-marble.json", + "src/library/entries/nasa-eonet-events.json", + "src/library/entries/nc-statewide-imagery.json", + "src/library/entries/nctcog-tx.json", + "src/library/entries/nd-statewide-imagery.json", + "src/library/entries/new-orleans-rpc-la.json", + "src/library/entries/nh-statewide-imagery.json", + "src/library/entries/nj-statewide-imagery.json", + "src/library/entries/noaa-tide-stations.json", + "src/library/entries/nrw-dop.json", + "src/library/entries/nsw-imagery.json", + "src/library/entries/ntad-national-bridges.json", + "src/library/entries/nwa-benton-washington-ar.json", + "src/library/entries/nws-active-alerts.json", + "src/library/entries/ny-statewide-imagery.json", + "src/library/entries/oakland-county-mi.json", + "src/library/entries/oh-statewide-imagery.json", + "src/library/entries/or-statewide-imagery.json", + "src/library/entries/pa-statewide-imagery.json", + "src/library/entries/pl-ortofotomapa.json", + "src/library/entries/pnoa-ma.json", + "src/library/entries/portland-metro-or.json", + "src/library/entries/pt-dgt-ortos.json", + "src/library/entries/ri-statewide-imagery.json", + "src/library/entries/swisstopo-swissimage.json", + "src/library/entries/tx-statewide-imagery.json", + "src/library/entries/union-county-oh.json", + "src/library/entries/usgs-hydrography.json", + "src/library/entries/usgs-topo.json", + "src/library/entries/va-statewide-imagery.json", + "src/library/entries/vt-statewide-imagery.json", + "src/library/entries/wallonie-ortho.json", + "src/library/entries/wi-statewide-imagery.json", + "src/library/entries/wri-global-power-plants.json", + "src/library/entries/wv-statewide-imagery.json", + "src/library/formats.js", + "src/library/parquet.js", + "src/sources/httpBody.js", + "src/i18n/marker.js", + "src/library/source.js", + "src/keySetupCore.mjs", + "src/localRequestGate.mjs" + ], + "external": ["hyparquet", "fzstd"] + }, "overpass-lexer": { "exports": ["./sources/overpass"], "modules": ["src/sources/overpass.js"], diff --git a/server/providers/library.js b/server/providers/library.js index b0c17ebef5..822d912a3e 100644 --- a/server/providers/library.js +++ b/server/providers/library.js @@ -1,103 +1,25 @@ import { LIBRARY_ENTRIES } from '../../src/library/catalog.js'; -import { loadLibraryDataset } from '../../src/library/formats.js'; -import { buildPartitionIndex } from '../../src/library/parquet.js'; -import { - libraryUpstreamUrl, - parseLibraryIdList, - libraryTileUrl, -} from '../../src/library/manifest.js'; -import { KEY_SETUP_KEYS } from '../../src/keySetupCore.mjs'; -import { - readResponseBytesCapped, - readResponseTextCapped, - coalesceProxyRequest, -} from './common/http.js'; +import { createLibraryHandler } from '../../src/library/proxy.js'; import { makeRateLimiter, clientKey } from './common/rate-limit.js'; import { admitSameSite, admitSameSiteKeyless } from './common/same-site.js'; -const DATASET_TIMEOUT_MS = 30_000; -const TILE_TIMEOUT_MS = 10_000; -/** Each listing page or footer read while building a partition index. */ -const INDEX_TIMEOUT_MS = 20_000; -/** How long an incomplete partition index is served before a rebuild. */ -const PARTIAL_INDEX_MS = 5 * 60_000; -const TILE_MAX_BYTES = 2 * 1024 * 1024; -const TILE_CACHE_LIMIT = 512; -const IN_FLIGHT_LIMIT = 64; +export { + libraryAvailability, + libraryKeyValue, + parseLibraryTilePath, +} from '../../src/library/proxy.js'; + +/** `content-type` as `Content-Type`: a fetch Response's names, as Node writes them. */ +const headerName = (name) => + name.replace( + /(^|-)([a-z])/g, + (_, dash, letter) => dash + letter.toUpperCase(), + ); /** - * Availability of every entry on this instance, decided by the operator's - * environment alone: `GEV_LIBRARY_DISABLE` removes entries, opt-in entries - * need `GEV_LIBRARY_ENABLE` (or, for a pack, its host's own switch), and a - * keyed entry needs its key. The answer names a reason, never a secret. - * @param {object[]} entries - * @param {Record} env - * @returns {Map} - */ -export function libraryAvailability(entries, env) { - const enabled = parseLibraryIdList(env.GEV_LIBRARY_ENABLE); - const disabled = parseLibraryIdList(env.GEV_LIBRARY_DISABLE); - const result = new Map(); - for (const entry of entries) { - let reason = null; - if (disabled.has(entry.id)) reason = 'operator-disabled'; - else if (entry.kind === 'pack') { - if (String(env[entry.pack.env] || '').trim() !== '1') - reason = 'operator-opt-in'; - } else if (entry.licence.operatorOptIn && !enabled.has(entry.id)) - reason = 'operator-opt-in'; - else if (entry.source.key && !libraryKeyValue(entry.source.key, env)) - reason = 'key-missing'; - result.set(entry.id, { available: reason === null, reason }); - } - return result; -} - -/** The configured value for a manifest key, through the key registry. */ -export function libraryKeyValue(key, env) { - const registered = KEY_SETUP_KEYS.find(({ id }) => id === key?.id); - for (const name of registered?.envVars || []) { - const value = String(env[name] || '').trim(); - if (value) return value; - } - return null; -} - -function withKey(url, key, env, headers) { - if (!key) return url; - const value = libraryKeyValue(key, env); - if (!value) throw Object.assign(new Error('key_missing'), { status: 503 }); - if (key.in === 'header') { - headers[key.name] = value; - return url; - } - const keyed = new URL(url); - keyed.searchParams.set(key.name, value); - return keyed.href; -} - -/** - * Parse `////` against a tiles entry's zoom range. - * @returns {{ z: number, x: number, y: number } | null} - */ -export function parseLibraryTilePath(parts, source) { - if (parts.length !== 3 || !parts.every((part) => /^\d{1,8}$/.test(part))) - return null; - const [z, x, y] = parts.map(Number); - const minZoom = source.minZoom ?? 0; - const maxZoom = source.maxZoom ?? 22; - if (z < minZoom || z > maxZoom) return null; - const span = 2 ** z; - if (x >= span || y >= span) return null; - return { z, x, y }; -} - -/** - * The Data Library's one server route. Proxied datasets and tiles reach their - * manifest's fixed upstream only: the browser names an entry id, never a URL, - * so the allowlist is exactly the catalog. Redirects are refused, bodies are - * capped while streaming, keys come from this server's environment and - * upstream error text is never returned. + * The Data Library's one server route on the dev and preview servers: the + * portable handler (`src/library/proxy.js`) behind this server's same-site + * gate and rate limits. */ export function libraryProxy({ entries = LIBRARY_ENTRIES, @@ -106,11 +28,7 @@ export function libraryProxy({ now = () => Date.now(), } = {}) { const byId = new Map(entries.map((entry) => [entry.id, entry])); - const datasets = new Map(); - const tiles = new Map(); - const indexes = new Map(); - const partialIndexes = new Map(); - const inFlight = new Map(); + const handle = createLibraryHandler({ entries, env, fetchImpl, now }); const allowData = makeRateLimiter({ windowMs: 60_000, max: 30, @@ -122,261 +40,54 @@ export function libraryProxy({ globalMax: 6000, }); - async function upstream(url, entry, headers, timeoutMs) { - // Defense in depth: every derived request stays on the manifest's host. - const target = libraryUpstreamUrl(url); - const declared = libraryUpstreamUrl(entry.source.url); - if (!target || !declared || target.host !== declared.host) - throw Object.assign(new Error('off_manifest_host'), { status: 502 }); - const signal = AbortSignal.timeout(timeoutMs); - const response = await fetchImpl(url, { - signal, - redirect: 'error', - headers: { Accept: '*/*', ...headers }, - }); - if (!response.ok) { - await response.body?.cancel?.().catch?.(() => {}); - throw Object.assign(new Error('upstream_unavailable'), { status: 502 }); - } - return { response, signal }; - } - - async function fetchDataset(entry) { - const headers = {}; - const collection = await loadLibraryDataset(entry, async (url, budget) => { - const keyed = withKey(url, entry.source.key, env, headers); - const { response, signal } = await upstream( - keyed, - entry, - headers, - DATASET_TIMEOUT_MS, - ); - const text = await readResponseTextCapped( - response, - budget.remainingBytes, - signal, - ); - budget.remainingBytes -= new TextEncoder().encode(text).byteLength; - return text; - }); - return { ...collection, fetchedAt: now() }; - } - - async function cached(store, key, ttlMs, load, limit = Infinity) { - const previous = store.get(key); - if (previous && now() - previous.savedAt < ttlMs) - return { value: previous.value, stale: false }; - if (!inFlight.has(key) && inFlight.size >= IN_FLIGHT_LIMIT) - throw Object.assign(new Error('busy'), { status: 429 }); - try { - const { promise } = coalesceProxyRequest(inFlight, key, async () => { - const value = await load(); - store.delete(key); - store.set(key, { value, savedAt: now() }); - while (store.size > limit) store.delete(store.keys().next().value); - return value; - }); - return { value: await promise, stale: false }; - } catch (error) { - if (previous) return { value: previous.value, stale: true }; - throw error; - } - } - - function json(res, status, value, extra = {}) { + function refuse(res, status, error) { if (res.destroyed) return; res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', ...(status === 405 ? { Allow: 'GET' } : {}), ...(status === 429 ? { 'Retry-After': '60' } : {}), - ...extra, }); - res.end(JSON.stringify(value)); - } - - function gate(req, res, entry) { - // A keyed entry spends the operator's credential: refuse forwarded - // requests like the other cost-bearing routes. Keyless entries keep - // working behind a reverse proxy, like terrain heights. - return entry?.source?.key - ? admitSameSite(req, res) - : admitSameSiteKeyless(req, res); + res.end(JSON.stringify({ error })); } async function handler(req, res) { - if (req.method !== 'GET') - return json(res, 405, { error: 'method_not_allowed' }); + if (req.method !== 'GET') return refuse(res, 405, 'method_not_allowed'); const path = (req.url || '/').split('?')[0]; - const parts = path.split('/').filter(Boolean); - const availability = libraryAvailability(entries, env); - - if (parts.length === 1 && parts[0] === 'catalog') { - if (admitSameSiteKeyless(req, res)) return undefined; - return json(res, 200, { - entries: entries.map(({ id }) => ({ id, ...availability.get(id) })), - }); - } - - const [route, id, ...rest] = parts; + const [route, id] = path.split('/').filter(Boolean); const entry = byId.get(id); - if (!entry || !['data', 'tile', 'index'].includes(route)) - return json(res, 404, { error: 'unknown_entry' }); - - // A partitioned GeoParquet entry's index: which file covers where, read - // from each file's footer. Built once per cacheSeconds and shared, so - // visitors do not each list the bucket and read every footer. - if (route === 'index') { - if (rest.length || !entry.source?.partitions) - return json(res, 404, { error: 'not_partitioned' }); + if (route === 'catalog' || !entry) { if (admitSameSiteKeyless(req, res)) return undefined; - const { available, reason } = availability.get(id); - if (!available) return json(res, 403, { error: reason }); - if (!allowData(clientKey(req))) - return json(res, 429, { error: 'rate_limited' }); - // Only a complete index is cached for cacheSeconds. An incomplete one - // (footers that kept failing) is served as `complete: false` and kept - // for PARTIAL_INDEX_MS only, so the next request soon rebuilds it; a - // complete index from before keeps being served while rebuilds fail. - const key = `index:${id}`; - const partial = partialIndexes.get(key); - if (partial && now() - partial.savedAt < PARTIAL_INDEX_MS) { - const previous = indexes.get(key); - return json(res, 200, previous?.value ?? partial.value); - } - try { - const { value } = await cached( - indexes, - key, - entry.source.cacheSeconds * 1000, - async () => { - const built = await buildPartitionIndex(entry, { - fetchImpl: (url, options = {}) => - fetchImpl(url, { - ...options, - signal: - options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS), - }), - }); - const value = { - partitions: built.partitions, - complete: built.complete, - builtAt: now(), - }; - if (!built.complete) { - // Recorded here too: with an earlier complete index, cached() - // serves that one and swallows this error, and the rebuild - // still waits PARTIAL_INDEX_MS. - partialIndexes.set(key, { value, savedAt: now() }); - throw Object.assign(new Error('index_incomplete'), { - partialIndex: value, - }); - } - partialIndexes.delete(key); - return value; - }, - ); - return json(res, 200, value); - } catch (error) { - if (error?.partialIndex) return json(res, 200, error.partialIndex); - const status = error?.status === 429 ? 429 : 502; - return json(res, status, { error: 'library_upstream_unavailable' }); - } - } - const isDataset = route === 'data' && rest.length === 0; - const isTile = route === 'tile'; - if ( - entry.source.fetch !== 'proxied' || - (isDataset && entry.kind !== 'dataset') || - (isTile && entry.kind !== 'tiles') || - (!isDataset && !isTile) - ) - return json(res, 404, { error: 'not_proxied' }); - if (gate(req, res, entry)) return undefined; - const { available, reason } = availability.get(id); - if (!available) return json(res, 403, { error: reason }); - - const ttlMs = entry.source.cacheSeconds * 1000; - if (isDataset) { - if (!allowData(clientKey(req))) - return json(res, 429, { error: 'rate_limited' }); - try { - const { value, stale } = await cached( - datasets, - `data:${id}`, - ttlMs, - () => fetchDataset(entry), - ); - return json( - res, - 200, - stale ? { ...value, stale: true } : value, - stale ? { 'X-Data-Stale': 'true' } : {}, - ); - } catch (error) { - const status = [429, 503].includes(error?.status) ? error.status : 502; - return json(res, status, { - error: - status === 503 ? 'key_missing' : 'library_upstream_unavailable', - }); - } - } - - const tile = parseLibraryTilePath(rest, entry.source); - if (!tile) return json(res, 400, { error: 'invalid_tile' }); - if (!allowTiles(clientKey(req))) - return json(res, 429, { error: 'rate_limited' }); - try { - const key = `tile:${id}:${tile.z}/${tile.x}/${tile.y}`; - const { value } = await cached( - tiles, - key, - ttlMs, - async () => { - const headers = {}; - const url = withKey( - libraryTileUrl(entry.source.url, tile.z, tile.x, tile.y), - entry.source.key, - env, - headers, - ); - const { response, signal } = await upstream( - url, - entry, - headers, - TILE_TIMEOUT_MS, - ); - const type = String(response.headers.get('content-type') || '') - .split(';')[0] - .trim() - .toLowerCase(); - if (!/^image\/(png|jpeg|webp)$/.test(type)) { - await response.body?.cancel?.().catch?.(() => {}); - throw Object.assign(new Error('not_an_image'), { status: 502 }); - } - const bytes = await readResponseBytesCapped( - response, - TILE_MAX_BYTES, - signal, - ); - return { type, bytes }; - }, - TILE_CACHE_LIMIT, - ); - if (res.destroyed) return undefined; - res.writeHead(200, { - 'Content-Type': value.type, - 'Cache-Control': `private, max-age=${entry.source.cacheSeconds}`, - 'X-Content-Type-Options': 'nosniff', - }); - res.end(Buffer.from(value.bytes)); - return undefined; - } catch (error) { - const status = [429, 503].includes(error?.status) ? error.status : 502; - return json(res, status, { - error: status === 503 ? 'key_missing' : 'library_upstream_unavailable', - }); + } else { + // A keyed entry spends the operator's credential: refuse forwarded + // requests like the other cost-bearing routes. Keyless entries keep + // working behind a reverse proxy, like terrain heights. + if ( + entry.source?.key + ? admitSameSite(req, res) + : admitSameSiteKeyless(req, res) + ) + return undefined; + const allow = route === 'tile' ? allowTiles : allowData; + if (!allow(clientKey(req))) return refuse(res, 429, 'rate_limited'); } + const response = await handle( + new Request(`http://library.invalid/api/library${req.url || '/'}`, { + headers: { Accept: String(req.headers?.accept || '*/*') }, + }), + ); + if (res.destroyed) return undefined; + const headers = {}; + for (const [name, value] of response.headers) + headers[headerName(name)] = value; + const body = Buffer.from(await response.arrayBuffer()); + res.writeHead(response.status, headers); + res.end( + String(headers['Content-Type']).startsWith('application/json') + ? body.toString('utf8') + : body, + ); + return undefined; } return { diff --git a/src/app/tools.js b/src/app/tools.js index bc16a9af0b..86c67ff791 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -48,6 +48,10 @@ import { overlayHost } from './layers/overlayHost.js'; * Attach scene tools, rendering listeners and the application debug handle. * `scene` is what `createApplicationScene` returns; its `trimStaleCommands` * runs after each layer disable (a scene without one keeps that memory). + * `library.fetchTile` and `library.fetchData` fetch the Data Library's tiles + * and proxied datasets in place of the defaults (`createLibrarySource`). + * `library.requireCatalog` offers only what the server's library catalog + * names, and nothing while it does not answer. */ export function createApplicationTools({ scene, @@ -60,6 +64,7 @@ export function createApplicationTools({ onSceneDirector, sceneDataPacks, drawMarkActions = [], + library = {}, signal, defer, }) { @@ -158,6 +163,11 @@ export function createApplicationTools({ mapStackController, shareLink: styleManager.shareLinkManager, requestRender: governorRequestRender, + // An application's own tile and dataset fetches, when it keeps them in + // its own cache. + fetchTile: library.fetchTile ?? null, + fetchData: library.fetchData ?? null, + requireCatalog: library.requireCatalog === true, // Zoom to layer: an immediate navigation, so Cockpit can refuse it and // the current follow owner is released first (as for imported files). zoomTo: (extent) => diff --git a/src/library/controller.js b/src/library/controller.js index 200bbd07e2..f6773ca290 100644 --- a/src/library/controller.js +++ b/src/library/controller.js @@ -25,6 +25,10 @@ import { } from './layers.js'; import { isExplicitLayerStateOrigin } from '../data/layerState.js'; +/** A required catalog that did not answer is asked again after this, doubling. */ +const CATALOG_RETRY_MS = 30_000; +const CATALOG_RETRY_MAX_MS = 10 * 60_000; + function safeLocalStorage() { try { return globalThis.localStorage ?? null; @@ -44,6 +48,12 @@ function safeLocalStorage() { * @param {(reason: string) => void} [options.requestRender] * @param {(extent: object) => boolean} [options.zoomTo] Fly to an extent * through the shell's navigation authority; false when refused. + * @param {Function} [options.fetchTile] A tiles entry's tile fetch in place + * of the renderer's (`createLibrarySource`). + * @param {Function} [options.fetchData] A proxied dataset's fetch + * (`createLibrarySource`). + * @param {boolean} [options.requireCatalog] Offer only what the server's + * catalog names, nothing while it does not answer; it is asked again later. */ export function createDataLibrary({ entries = LIBRARY_ENTRIES, @@ -57,7 +67,19 @@ export function createDataLibrary({ requestRender = () => {}, zoomTo = null, storage = safeLocalStorage(), - source = createLibrarySource({ entries }), + fetchTile = null, + fetchData = null, + requireCatalog = false, + source = createLibrarySource({ + entries, + storage, + fetchTile, + fetchData, + requireCatalog, + }), + retryMs = CATALOG_RETRY_MS, + setTimer = (run, ms) => setTimeout(run, ms), + clearTimer = (timer) => clearTimeout(timer), }) { if (!dataManager?.registerLibraryLayer) throw new TypeError('A sealed layer manager is required'); @@ -80,6 +102,8 @@ export function createDataLibrary({ let destroyed = false; const listeners = new Set(); const pending = new Map(); + let retryTimer = null; + let retryDelay = retryMs; const notify = () => { for (const listener of [...listeners]) { @@ -315,6 +339,7 @@ export function createDataLibrary({ if (destroyed) return; availability = answer.availability; serverPresent = answer.server; + if (answer.retry) scheduleRetry(signal); const stored = readStoredLibrarySelection(storage, switchable); savedOn = stored.on; // A share opened in this tab while availability loaded wins over the @@ -342,6 +367,37 @@ export function createDataLibrary({ applySelection(next, linked ? 'share-restore' : 'local-restore'); } + /** + * Ask a required catalog again until it answers, then restore what the + * library holds as start-up would have. + */ + function scheduleRetry(signal) { + if (destroyed || retryTimer !== null || signal?.aborted) return; + retryTimer = setTimer(async () => { + retryTimer = null; + let answer; + try { + answer = await source.getAvailability({ signal }); + } catch { + return; + } + if (destroyed) return; + availability = answer.availability; + serverPresent = answer.server; + if (answer.retry) { + retryDelay = Math.min(retryDelay * 2, CATALOG_RETRY_MAX_MS); + scheduleRetry(signal); + notify(); + return; + } + for (const id of selection.added) { + const entry = byId.get(id); + if (entry && addable(id)) register(entry); + } + applySelection(selection, 'local-restore'); + }, retryDelay); + } + /** * Add an entry to the panel and switch it on. * @param {string} id @@ -460,7 +516,13 @@ export function createDataLibrary({ * @returns {Array<{ entry: object, available: boolean, reason: string|null, added: boolean, on: boolean, busy: boolean }>} */ function list() { - return entries.map((entry) => { + // A required catalog shows nothing until it answers, and an entry this + // instance's server does not list is not shown at all. + if (source.requireCatalog && !loaded) return []; + const shown = entries.filter( + (entry) => availability.get(entry.id)?.listed !== false, + ); + return shown.map((entry) => { const state = availability.get(entry.id) || { available: false, reason: started ? 'server-unavailable' : null, @@ -538,6 +600,8 @@ export function createDataLibrary({ destroy() { if (destroyed) return; destroyed = true; + if (retryTimer !== null) clearTimer(retryTimer); + retryTimer = null; unsubscribeManager?.(); shareLink?.setLibraryParamsEncoder?.(null); shareLink?.setLibraryShareListener?.(null); diff --git a/src/library/controller.test.mjs b/src/library/controller.test.mjs index 970d6c54ce..af6d656246 100644 --- a/src/library/controller.test.mjs +++ b/src/library/controller.test.mjs @@ -5,6 +5,8 @@ import { createDataLibrary } from './controller.js'; import { LIBRARY_ENTRIES, LIBRARY_LEDGER } from './catalog.js'; import { validateLibraryCatalog } from './manifest.js'; import { LIBRARY_STORAGE_KEY } from './selection.js'; +import { createLibrarySource } from './source.js'; +import { libraryLayerId } from '../data/libraryLayerIds.js'; const base = { family: 'imagery', @@ -502,7 +504,10 @@ test('entries this instance cannot serve still show as added and can be removed' assert.equal(row.added, true); assert.equal(library.summary().added, 1); assert.equal(await library.remove('night-lights'), true); - assert.deepEqual(JSON.parse(storage.store.get(LIBRARY_STORAGE_KEY)).added, []); + assert.deepEqual( + JSON.parse(storage.store.get(LIBRARY_STORAGE_KEY)).added, + [], + ); library.destroy(); }); @@ -530,10 +535,9 @@ test('a layer that fails to tear down stays in the library', async (t) => { assert.equal(await library.remove('night-lights'), false); assert.equal(await library.reset(), 0); assert.equal(manager.layers.has('lib-night-lights'), true); - assert.deepEqual( - JSON.parse(storage.store.get(LIBRARY_STORAGE_KEY)).added, - ['night-lights'], - ); + assert.deepEqual(JSON.parse(storage.store.get(LIBRARY_STORAGE_KEY)).added, [ + 'night-lights', + ]); assert.equal( library.list().find((row) => row.entry.id === 'night-lights').added, true, @@ -772,3 +776,89 @@ test('every shipped entry kind registers, view-bounded datasets included', async assert.equal(module.getDiagnostics().tooHigh, true); assert.equal(await library.add('nasa-black-marble'), true); }); + +test('an entry the server does not list is neither shown nor added', async (t) => { + const manager = sealedManager(t); + const library = createDataLibrary({ + entries, + ledger, + dataManager: manager, + services: {}, + storage: memoryStorage(), + initialHash: '', + source: source({ + 'sea-marks': { + available: false, + reason: 'operator-disabled', + listed: false, + }, + }), + }); + await library.start(); + assert.deepEqual( + library.list().map(({ entry }) => entry.id), + ['night-lights', 'fixture-pack'], + ); + assert.equal(library.summary().entries, 2); + assert.equal(await library.add('sea-marks'), false); +}); + +test('a required catalog that does not answer offers nothing and is asked again later', async (t) => { + const manager = sealedManager(t); + const calls = []; + let catalog = new Response('busy', { status: 503 }); + const timers = []; + const library = createDataLibrary({ + entries, + ledger, + dataManager: manager, + services: {}, + storage: memoryStorage( + JSON.stringify({ added: ['night-lights'], on: ['night-lights'] }), + ), + initialHash: '', + requireCatalog: true, + source: createLibrarySource({ + entries, + storage: null, + requireCatalog: true, + fetchImpl: async (url) => { + calls.push(String(url)); + return String(url).endsWith('/catalog') + ? catalog + : new Response('', { status: 500 }); + }, + }), + setTimer: (run, ms) => timers.push({ run, ms }), + clearTimer: () => {}, + }); + await library.start(); + assert.deepEqual(library.list(), []); + assert.equal(library.summary().entries, 0); + assert.equal(await library.add('night-lights'), false); + assert.deepEqual(calls, ['/api/library/catalog']); + assert.equal(timers.length, 1); + assert.equal(timers[0].ms, 30_000); + + // Still failing: asked again, later still. + catalog = new Response('', { status: 429 }); + await timers[0].run(); + assert.equal(timers.length, 2); + assert.equal(timers[1].ms, 60_000); + assert.deepEqual(library.list(), []); + + // Answered: what it names is offered, and the stored selection returns. + catalog = Response.json({ + entries: [{ id: 'night-lights', available: true, reason: null }], + }); + await timers[1].run(); + assert.equal(timers.length, 2); + assert.deepEqual( + library.list().map(({ entry }) => entry.id), + ['night-lights'], + ); + assert.ok(manager.layers.has(libraryLayerId('night-lights'))); + // No publisher was ever asked. + assert.ok(calls.every((url) => url === '/api/library/catalog')); + library.destroy(); +}); diff --git a/src/library/layers.js b/src/library/layers.js index 4fd95cd453..c2928b9a27 100644 --- a/src/library/layers.js +++ b/src/library/layers.js @@ -410,9 +410,11 @@ export const LIBRARY_AERIAL_Z = 10; * The globe port imagery spec for a tiles entry. * @param {object} entry Validated tiles entry. * @param {string} url Tile URL template. + * @param {import('../render/contract/globe').ImagerySpec['fetchTile']|null} [fetchTile] + * Fetches each tile in place of the renderer (`source.tileFetch`). * @returns {import('../render/contract/globe').ImagerySpec} */ -export function libraryImagerySpec(entry, url) { +export function libraryImagerySpec(entry, url, fetchTile = null) { const bbox = entry.source.bounded ? entry.coverage.bbox : null; const minZoom = entry.source.minZoom ?? 0; // A dense entry (`minimumTerrainLevelFor`) is hidden where the view is @@ -442,6 +444,7 @@ export function libraryImagerySpec(entry, url) { : {}), ...(entry.display.aerial ? { zOrder: LIBRARY_AERIAL_Z } : {}), ...(tileTransform ? { tileTransform } : {}), + ...(fetchTile ? { fetchTile } : {}), }; } @@ -563,7 +566,11 @@ export function createLibraryTileLayer( if (host.kind === 'tileset') markBaseTileset(tileset); try { imagery = globe.imagery.add( - libraryImagerySpec(entry, source.tileTemplate(entry)), + libraryImagerySpec( + entry, + source.tileTemplate(entry), + source.tileFetch?.(entry) ?? null, + ), ); } catch (error) { // A renderer without tile hooks cannot draw relief or clear the diff --git a/src/library/manifest.js b/src/library/manifest.js index 8b4595c85f..cd4b56c555 100644 --- a/src/library/manifest.js +++ b/src/library/manifest.js @@ -67,6 +67,55 @@ export function libraryTileUrl(template, z, x, y) { .replaceAll('{y}', String(y)); } +/** The finest and coarsest grid a proxied view box snaps to, in degrees. */ +const VIEW_GRID_MIN_DEG = 2 ** -10; +const VIEW_GRID_MAX_DEG = 1; + +/** + * The box a view-bounded entry asks a proxy for: the camera's box snapped + * out to a grid of two cells a side, the cell the smallest power of two + * (degrees) at least as large as the box. Nearby views name the same box, + * so a proxy and the caches in front of it see few distinct requests, and + * each box always holds the box it came from. + * @param {number[]} box `[west, south, east, north]` in degrees. + * @returns {number[]|null} The snapped box, or null for one too large. + */ +export function libraryViewBox([west, south, east, north]) { + const span = Math.max(east - west, north - south); + if (!(span > 0)) return null; + let cell = VIEW_GRID_MIN_DEG; + while (cell < span) cell *= 2; + if (cell > VIEW_GRID_MAX_DEG) return null; + const left = Math.floor(west / cell) * cell; + const bottom = Math.floor(south / cell) * cell; + return [left, bottom, left + 2 * cell, bottom + 2 * cell]; +} + +/** + * Parse a proxied view box (`west,south,east,north`) as + * {@link libraryViewBox} writes it: two grid cells a side, on the grid, and + * written exactly as `String` writes its numbers, so every box has one + * spelling. + * @param {string|null} text + * @returns {number[]|null} + */ +export function parseLibraryViewBox(text) { + if (typeof text !== 'string' || text.length > 128) return null; + const box = text.split(',').map(Number); + if (box.length !== 4 || !box.every(Number.isFinite)) return null; + if (box.join(',') !== text) return null; + const [west, south, east, north] = box; + const cell = (east - west) / 2; + if (north - south !== 2 * cell) return null; + if (!(cell >= VIEW_GRID_MIN_DEG && cell <= VIEW_GRID_MAX_DEG)) return null; + if (2 ** Math.round(Math.log2(cell)) !== cell) return null; + if (!Number.isInteger(west / cell) || !Number.isInteger(south / cell)) + return null; + if (west < -180 - 2 * cell || east > 180 + 2 * cell) return null; + if (south < -90 - 2 * cell || north > 90 + 2 * cell) return null; + return box; +} + /** How the browser reaches the upstream. */ export const FETCH_MODES = Object.freeze(['browser-direct', 'proxied']); @@ -338,7 +387,8 @@ function checkSource(entry, errors) { `source.view needs maxAltitudeM (${MIN_VIEW_ALTITUDE_M}–${MAX_VIEW_ALTITUDE_M}) and an optional sample bbox`, ); // View-bounded loading queries the camera's box; this build does that - // for ArcGIS layers and partitioned GeoParquet, fetched by the browser. + // for ArcGIS layers and partitioned GeoParquet, fetched by the browser + // (or, when a server routes the entry, by its proxy). if ( !['arcgis', 'geoparquet'].includes(source.format) || source.fetch !== 'browser-direct' diff --git a/src/library/proxy.js b/src/library/proxy.js new file mode 100644 index 0000000000..88cc315d72 --- /dev/null +++ b/src/library/proxy.js @@ -0,0 +1,533 @@ +/** + * The Data Library's server side as a fetch handler: a `Request` in, a + * `Response` out, so any server (Node, a Worker, a test) can answer the + * library routes the browser asks for. + * + * GET /catalog what this server offers, per entry + * GET /data/[?bbox=w,s,e,n] a dataset's features + * GET /tile//// one tile's image + * GET /index/ a partitioned GeoParquet's index + * + * The browser names an entry id, never a URL: every upstream request is built + * from that entry's manifest and stays on its host, so the allowlist is + * exactly the catalog. Redirects are refused, bodies are capped while + * streaming, keys come from the server's environment and upstream error text + * is never returned. + * + * A server may hide entries (`listed`: never offered, never served) and route + * browser-direct entries through itself (`proxied`); the catalog route tells + * the browser both. Same-site gates and rate limits belong to the server + * that mounts the handler. + */ +import { LIBRARY_ENTRIES } from './catalog.js'; +import { loadLibraryDataset } from './formats.js'; +import { buildPartitionIndex } from './parquet.js'; +import { + libraryTileUrl, + libraryUpstreamUrl, + parseLibraryIdList, + parseLibraryViewBox, +} from './manifest.js'; +import { createLibrarySource } from './source.js'; +import { KEY_SETUP_KEYS } from '../keySetupCore.mjs'; +import { + coalesceProxyRequest, + readResponseBytesCapped, + readResponseTextCapped, +} from '../sources/httpBody.js'; + +/** The catalog and its view grid, for a server that decides what it serves. */ +export { LIBRARY_ENTRIES } from './catalog.js'; +export { libraryViewBox, parseLibraryViewBox } from './manifest.js'; + +const DATASET_TIMEOUT_MS = 30_000; +const TILE_TIMEOUT_MS = 10_000; +/** Each listing page or footer read while building a partition index. */ +const INDEX_TIMEOUT_MS = 20_000; +/** How long an incomplete partition index is served before a rebuild. */ +const PARTIAL_INDEX_MS = 5 * 60_000; +const TILE_MAX_BYTES = 2 * 1024 * 1024; +const TILE_CACHE_LIMIT = 512; +const VIEW_CACHE_LIMIT = 64; +const IN_FLIGHT_LIMIT = 64; +const TILE_TYPES = /^image\/(png|jpeg|webp)$/; + +/** The configured value for a manifest key, through the key registry. */ +export function libraryKeyValue(key, env) { + const registered = KEY_SETUP_KEYS.find(({ id }) => id === key?.id); + for (const name of registered?.envVars || []) { + const value = String(env[name] || '').trim(); + if (value) return value; + } + return null; +} + +/** + * Availability of every entry on this instance, decided by the operator's + * environment alone: `GEV_LIBRARY_DISABLE` removes entries, opt-in entries + * need `GEV_LIBRARY_ENABLE` (or, for a pack, its host's own switch), and a + * keyed entry needs its key. The answer names a reason, never a secret. + * @param {object[]} entries + * @param {Record} env + * @returns {Map} + */ +export function libraryAvailability(entries, env) { + const enabled = parseLibraryIdList(env.GEV_LIBRARY_ENABLE); + const disabled = parseLibraryIdList(env.GEV_LIBRARY_DISABLE); + const result = new Map(); + for (const entry of entries) { + let reason = null; + if (disabled.has(entry.id)) reason = 'operator-disabled'; + else if (entry.kind === 'pack') { + if (String(env[entry.pack.env] || '').trim() !== '1') + reason = 'operator-opt-in'; + } else if (entry.licence.operatorOptIn && !enabled.has(entry.id)) + reason = 'operator-opt-in'; + else if (entry.source.key && !libraryKeyValue(entry.source.key, env)) + reason = 'key-missing'; + result.set(entry.id, { available: reason === null, reason }); + } + return result; +} + +/** + * Parse `////` against a tiles entry's zoom range. + * @returns {{ z: number, x: number, y: number } | null} + */ +export function parseLibraryTilePath(parts, source) { + if ( + parts.length !== 3 || + !parts.every((part) => /^(0|[1-9]\d{0,7})$/.test(part)) + ) + return null; + const [z, x, y] = parts.map(Number); + const minZoom = source.minZoom ?? 0; + const maxZoom = source.maxZoom ?? 22; + if (z < minZoom || z > maxZoom) return null; + const span = 2 ** z; + if (x >= span || y >= span) return null; + return { z, x, y }; +} + +function withKey(url, key, env, headers) { + if (!key) return url; + const value = libraryKeyValue(key, env); + if (!value) throw Object.assign(new Error('key_missing'), { status: 503 }); + if (key.in === 'header') { + headers[key.name] = value; + return url; + } + const keyed = new URL(url); + keyed.searchParams.set(key.name, value); + return keyed.href; +} + +/** A timeout that also ends with the caller's signal, when there is one. */ +function timeoutSignal(timeoutMs, signal) { + const timeout = AbortSignal.timeout(timeoutMs); + return signal && typeof AbortSignal.any === 'function' + ? AbortSignal.any([timeout, signal]) + : timeout; +} + +function json(status, value, headers = {}) { + return new Response(JSON.stringify(value), { + status, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + ...(status === 405 ? { Allow: 'GET' } : {}), + ...(status === 429 ? { 'Retry-After': '60' } : {}), + ...headers, + }, + }); +} + +/** + * How long a browser or shared cache may keep an entry's answer: its + * `cacheSeconds`, shared unless a key fetched it (a keyed answer is the + * operator's, kept by the browser alone). + */ +function cacheControl(entry) { + const scope = entry.source.key ? 'private' : 'public'; + return `${scope}, max-age=${entry.source.cacheSeconds}`; +} + +/** + * @param {object} [options] + * @param {object[]} [options.entries] The catalog. + * @param {Record} [options.env] Keys and the + * `GEV_LIBRARY_*` switches (`libraryAvailability`). + * @param {typeof fetch} [options.fetchImpl] + * @param {() => number} [options.now] + * @param {string} [options.base] Path the routes are mounted under. + * @param {(entry: object) => boolean} [options.listed] False hides an entry: + * the catalog leaves it out of the browser's list and no route serves it. + * @param {(entry: object) => boolean} [options.proxied] True routes a + * browser-direct entry through this handler. + * @param {boolean} [options.keep] Keep answers in this process's memory for + * `cacheSeconds` (and serve a stale dataset when a refresh fails). A server + * with a shared cache in front turns it off. + * @param {Map} [options.partitionCache] Decoded GeoParquet + * partitions, shared by every request this map is given to; at most four + * are kept (pass one map per process to cap them there). + * @param {(entry: object, init: { signal?: AbortSignal }) => Promise} [options.fetchIndex] + * Fetches a partitioned entry's index, e.g. this handler's own `/index/` + * through a shared cache. By default the index route is answered in + * process. A dataset request never builds an index itself. + * @param {number} [options.subrequestLimit] Most upstream requests one + * dataset or tile request may make (a host's per-request subrequest + * limit); a request that would need more is refused with 503 + * `subrequest_limit` before passing it. + * @returns {(request: Request) => Promise} + */ +export function createLibraryHandler({ + entries = LIBRARY_ENTRIES, + env = {}, + fetchImpl = (...args) => globalThis.fetch(...args), + now = () => Date.now(), + base = '/api/library', + listed = () => true, + proxied = () => false, + keep = true, + partitionCache = new Map(), + fetchIndex = null, + subrequestLimit = Infinity, +} = {}) { + const shown = entries.filter((entry) => listed(entry)); + const byId = new Map(shown.map((entry) => [entry.id, entry])); + const datasets = new Map(); + const views = new Map(); + const tiles = new Map(); + const indexes = new Map(); + const partialIndexes = new Map(); + const inFlight = new Map(); + + const routed = (entry) => + entry.kind !== 'pack' && + (entry.source.fetch === 'proxied' || proxied(entry) === true); + + /** The upstream request for `url`, on the manifest's host only. */ + async function upstream(entry, url, { timeoutMs, signal, headers = {} }) { + // Defense in depth: every derived request stays on the manifest's host. + const target = libraryUpstreamUrl(url); + const declared = libraryUpstreamUrl(entry.source.url); + if (!target || !declared || target.host !== declared.host) + throw Object.assign(new Error('off_manifest_host'), { status: 502 }); + const keyHeaders = {}; + const keyed = withKey(url, entry.source.key, env, keyHeaders); + const timed = timeoutSignal(timeoutMs, signal); + const response = await fetchImpl(keyed, { + signal: timed, + redirect: 'error', + headers: { Accept: '*/*', ...headers, ...keyHeaders }, + }); + return { response, signal: timed }; + } + + async function okUpstream(entry, url, options) { + const result = await upstream(entry, url, options); + if (!result.response.ok) { + await result.response.body?.cancel?.().catch?.(() => {}); + throw Object.assign(new Error('upstream_unavailable'), { status: 502 }); + } + return result; + } + + /** Counts one request's upstream fetches against `subrequestLimit`. */ + function subrequests() { + let used = 0; + return () => { + used += 1; + if (used > subrequestLimit) + throw Object.assign(new Error('subrequest_limit'), { status: 503 }); + }; + } + + /** + * A GeoParquet entry's reader for one request: the browser's own, fetching + * upstream from here, with its index fetched (never built) and decoded + * partitions shared through `partitionCache`. + */ + function readerFor(entry, count) { + const direct = { + ...entry, + source: { ...entry.source, fetch: 'browser-direct' }, + }; + const source = createLibrarySource({ + entries: [direct], + base: null, + storage: null, + now, + partitionCache, + buildIndex: false, + fetchIndex: async (_entry, init) => { + count(); + return fetchIndex ? fetchIndex(entry, init) : index(entry); + }, + fetchImpl: async (url, init = {}) => { + count(); + const range = new Headers(init.headers).get('range'); + const { response } = await upstream(entry, String(url), { + timeoutMs: DATASET_TIMEOUT_MS, + signal: init.signal, + headers: range ? { Range: range } : {}, + }); + return response; + }, + }); + return { entry: direct, source }; + } + + async function fetchDataset(entry, bbox) { + const count = subrequests(); + if (entry.source.format === 'geoparquet') { + const reader = readerFor(entry, count); + const result = await reader.source.loadDataset(reader.entry, { bbox }); + return { ...result, fetchedAt: now() }; + } + const collection = await loadLibraryDataset( + entry, + async (url, budget) => { + count(); + const { response, signal } = await okUpstream(entry, url, { + timeoutMs: DATASET_TIMEOUT_MS, + }); + const text = await readResponseTextCapped( + response, + budget.remainingBytes, + signal, + ); + budget.remainingBytes -= new TextEncoder().encode(text).byteLength; + return text; + }, + { bbox }, + ); + return { ...collection, fetchedAt: now() }; + } + + async function cached(store, key, ttlMs, load, limit = Infinity) { + const previous = keep ? store.get(key) : undefined; + if (previous && now() - previous.savedAt < ttlMs) + return { value: previous.value, stale: false }; + if (!inFlight.has(key) && inFlight.size >= IN_FLIGHT_LIMIT) + throw Object.assign(new Error('busy'), { status: 429 }); + try { + const { promise } = coalesceProxyRequest(inFlight, key, async () => { + const value = await load(); + if (keep) { + store.delete(key); + store.set(key, { value, savedAt: now() }); + while (store.size > limit) store.delete(store.keys().next().value); + } + return value; + }); + return { value: await promise, stale: false }; + } catch (error) { + if (previous) return { value: previous.value, stale: true }; + throw error; + } + } + + function failure(error) { + if (error?.status === 429) return json(429, { error: 'rate_limited' }); + if (error?.status === 503) + return json(503, { + error: ['subrequest_limit', 'index_unavailable'].includes(error.message) + ? error.message + : 'key_missing', + }); + return json(502, { error: 'library_upstream_unavailable' }); + } + + async function index(entry) { + // Only a complete index is kept for cacheSeconds. An incomplete one + // (footers that kept failing) is served as `complete: false` and kept + // for PARTIAL_INDEX_MS only, so the next request soon rebuilds it; a + // complete index from before keeps being served while rebuilds fail. + const key = `index:${entry.id}`; + const partial = partialIndexes.get(key); + if (partial && now() - partial.savedAt < PARTIAL_INDEX_MS) { + const previous = indexes.get(key); + return json(200, previous?.value ?? partial.value); + } + try { + const { value } = await cached( + indexes, + key, + entry.source.cacheSeconds * 1000, + async () => { + const count = subrequests(); + const built = await buildPartitionIndex(entry, { + fetchImpl: async (url, options = {}) => { + count(); + const range = new Headers(options.headers).get('range'); + const { response } = await upstream(entry, String(url), { + timeoutMs: INDEX_TIMEOUT_MS, + signal: options.signal, + headers: range ? { Range: range } : {}, + }); + return response; + }, + }); + const value = { + partitions: built.partitions, + complete: built.complete, + builtAt: now(), + }; + if (!built.complete) { + // Recorded here too: with an earlier complete index, cached() + // serves that one and swallows this error, and the rebuild + // still waits PARTIAL_INDEX_MS. + partialIndexes.set(key, { value, savedAt: now() }); + throw Object.assign(new Error('index_incomplete'), { + partialIndex: value, + }); + } + partialIndexes.delete(key); + return value; + }, + ); + return json(200, value, { 'Cache-Control': cacheControl(entry) }); + } catch (error) { + if (error?.partialIndex) return json(200, error.partialIndex); + return error?.status === 429 + ? json(429, { error: 'rate_limited' }) + : json(502, { error: 'library_upstream_unavailable' }); + } + } + + async function data(entry, params) { + const view = Boolean(entry.source.view); + const bboxText = params.get('bbox'); + const bbox = view ? parseLibraryViewBox(bboxText) : null; + if (view && !bbox) return json(400, { error: 'invalid_bbox' }); + if (!view && bboxText !== null) return json(400, { error: 'invalid_bbox' }); + try { + const { value, stale } = await cached( + view ? views : datasets, + view ? `view:${entry.id}:${bboxText}` : `data:${entry.id}`, + entry.source.cacheSeconds * 1000, + () => fetchDataset(entry, bbox), + view ? VIEW_CACHE_LIMIT : Infinity, + ); + return stale + ? json(200, { ...value, stale: true }, { 'X-Data-Stale': 'true' }) + : json(200, value, { 'Cache-Control': cacheControl(entry) }); + } catch (error) { + return failure(error); + } + } + + async function tile(entry, parts) { + const at = parseLibraryTilePath(parts, entry.source); + if (!at) return json(400, { error: 'invalid_tile' }); + try { + const { value } = await cached( + tiles, + `tile:${entry.id}:${at.z}/${at.x}/${at.y}`, + entry.source.cacheSeconds * 1000, + async () => { + const { response, signal } = await okUpstream( + entry, + libraryTileUrl(entry.source.url, at.z, at.x, at.y), + { timeoutMs: TILE_TIMEOUT_MS }, + ); + const type = String(response.headers.get('content-type') || '') + .split(';')[0] + .trim() + .toLowerCase(); + if (!TILE_TYPES.test(type)) { + await response.body?.cancel?.().catch?.(() => {}); + throw Object.assign(new Error('not_an_image'), { status: 502 }); + } + const bytes = await readResponseBytesCapped( + response, + TILE_MAX_BYTES, + signal, + ); + return { type, bytes }; + }, + TILE_CACHE_LIMIT, + ); + return new Response(value.bytes, { + status: 200, + headers: { + 'Content-Type': value.type, + 'Cache-Control': cacheControl(entry), + 'X-Content-Type-Options': 'nosniff', + }, + }); + } catch (error) { + return failure(error); + } + } + + return async function handle(request) { + if (request.method !== 'GET') + return json(405, { error: 'method_not_allowed' }); + const url = new URL(request.url); + const path = url.pathname.startsWith(base) + ? url.pathname.slice(base.length) + : url.pathname; + // One spelling per resource, so a cache keyed on the path holds each + // tile or answer once: no empty segments, no trailing slash. + const parts = path.replace(/^\//, '').split('/'); + if (parts.some((part) => part === '')) + return json(404, { error: 'unknown_entry' }); + const availability = libraryAvailability(shown, env); + + if (parts.length === 1 && parts[0] === 'catalog') { + return json(200, { + entries: [ + ...shown.map(({ id }) => { + const entry = byId.get(id); + return { + id, + ...availability.get(id), + ...(entry.kind !== 'pack' && + entry.source.fetch !== 'proxied' && + routed(entry) + ? { fetch: 'proxied' } + : {}), + }; + }), + ...entries + .filter((entry) => !byId.has(entry.id)) + .map(({ id }) => ({ + id, + available: false, + reason: 'operator-disabled', + listed: false, + })), + ], + }); + } + + const [route, id, ...rest] = parts; + const entry = byId.get(id); + if (!entry || !['data', 'tile', 'index'].includes(route)) + return json(404, { error: 'unknown_entry' }); + + if (route === 'index') { + if (rest.length || !entry.source?.partitions) + return json(404, { error: 'not_partitioned' }); + const { available, reason } = availability.get(id); + if (!available) return json(403, { error: reason }); + return index(entry); + } + + const isDataset = route === 'data' && rest.length === 0; + const isTile = route === 'tile'; + if ( + !routed(entry) || + (isDataset && entry.kind !== 'dataset') || + (isTile && entry.kind !== 'tiles') || + (!isDataset && !isTile) + ) + return json(404, { error: 'not_proxied' }); + const { available, reason } = availability.get(id); + if (!available) return json(403, { error: reason }); + return isDataset ? data(entry, url.searchParams) : tile(entry, rest); + }; +} diff --git a/src/library/proxy.test.mjs b/src/library/proxy.test.mjs new file mode 100644 index 0000000000..96a96217cb --- /dev/null +++ b/src/library/proxy.test.mjs @@ -0,0 +1,256 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createLibraryHandler } from './proxy.js'; +import { libraryEntry } from './catalog.js'; +import { + libraryViewBox, + parseLibraryViewBox, + validateLibraryCatalog, +} from './manifest.js'; + +const base = { + family: 'imagery', + provider: 'Fixture', + description: 'Fixture entry.', + maintainer: 'test', + coverage: { global: true }, + licence: { name: 'Public domain', url: 'https://example.org/licence' }, + attribution: { text: 'Fixture' }, + status: 'community', +}; +const { entries, errors } = validateLibraryCatalog([ + { + ...base, + id: 'direct-tiles', + kind: 'tiles', + title: 'Direct tiles', + source: { + format: 'bbox', + url: 'https://maps.example.org/export?bbox={bbox-epsg-3857}&f=image', + fetch: 'browser-direct', + cacheSeconds: 900, + }, + }, + { + ...base, + id: 'hidden-tiles', + kind: 'tiles', + title: 'Hidden tiles', + source: { + format: 'xyz', + url: 'https://tiles.example.org/{z}/{x}/{y}.png', + fetch: 'browser-direct', + }, + }, + { + ...base, + id: 'direct-points', + kind: 'dataset', + family: 'events', + title: 'Direct points', + source: { + format: 'geojson', + url: 'https://data.example.org/points.json', + fetch: 'browser-direct', + }, + }, +]); +assert.deepEqual(errors, []); +const view = libraryEntry('fema-usa-structures'); + +const png = () => + new Response(new Uint8Array([137, 80, 78, 71]), { + headers: { 'content-type': 'image/png' }, + }); +const get = (handle, path) => + handle(new Request(`https://app.example/api/library${path}`)); + +test('a hidden entry is listed as not offered and no route serves it', async () => { + let calls = 0; + const handle = createLibraryHandler({ + entries, + listed: (entry) => entry.id !== 'hidden-tiles', + proxied: () => true, + fetchImpl: async () => { + calls++; + return png(); + }, + }); + const catalog = await (await get(handle, '/catalog')).json(); + assert.deepEqual( + catalog.entries.find(({ id }) => id === 'hidden-tiles'), + { + id: 'hidden-tiles', + available: false, + reason: 'operator-disabled', + listed: false, + }, + ); + assert.equal((await get(handle, '/tile/hidden-tiles/1/0/0')).status, 404); + assert.equal(calls, 0); +}); + +test('a routed browser-direct entry is served from its manifest and cacheable for cacheSeconds', async () => { + const calls = []; + const handle = createLibraryHandler({ + entries, + proxied: (entry) => entry.id === 'direct-tiles', + keep: false, + fetchImpl: async (url, options) => { + calls.push({ url, options }); + return png(); + }, + }); + const catalog = await (await get(handle, '/catalog')).json(); + assert.equal( + catalog.entries.find(({ id }) => id === 'direct-tiles').fetch, + 'proxied', + ); + assert.equal( + catalog.entries.find(({ id }) => id === 'direct-points').fetch, + undefined, + ); + const tile = await get(handle, '/tile/direct-tiles/1/1/0?url=https://evil'); + assert.equal(tile.status, 200); + assert.equal(tile.headers.get('content-type'), 'image/png'); + assert.equal(tile.headers.get('cache-control'), 'public, max-age=900'); + assert.equal(calls.length, 1); + assert.match( + calls[0].url, + /^https:\/\/maps\.example\.org\/export\?bbox=0\.000,0\.000,20037508\.343,20037508\.343&f=image$/, + ); + assert.equal(calls[0].options.redirect, 'error'); + // Not kept in memory: a shared cache in front holds it. + await get(handle, '/tile/direct-tiles/1/1/0'); + assert.equal(calls.length, 2); + // An entry the server does not route stays the browser's to fetch. + assert.equal((await get(handle, '/data/direct-points')).status, 404); +}); + +test('a routed view-bounded entry needs a snapped box and queries only it', async () => { + const calls = []; + const handle = createLibraryHandler({ + entries: [view], + proxied: () => true, + fetchImpl: async (url) => { + calls.push(new URL(url)); + return new Response( + JSON.stringify({ type: 'FeatureCollection', features: [] }), + ); + }, + }); + assert.equal((await get(handle, '/data/fema-usa-structures')).status, 400); + assert.equal( + (await get(handle, '/data/fema-usa-structures?bbox=-77.03,38.89,-77,38.9')) + .status, + 400, + ); + assert.equal(calls.length, 0); + const box = libraryViewBox([-77.031, 38.889, -77.02, 38.9]); + const ok = await get(handle, `/data/fema-usa-structures?bbox=${box}`); + assert.equal(ok.status, 200); + assert.equal( + ok.headers.get('cache-control'), + `public, max-age=${view.source.cacheSeconds}`, + ); + assert.equal(calls.length, 1); + assert.equal( + calls[0].searchParams.get('geometry'), + box.map((value) => value.toFixed(6)).join(','), + ); +}); + +test('a view box snaps out to two grid cells a side, with one spelling', () => { + const asked = [-77.031, 38.889, -77.02, 38.9]; + const box = libraryViewBox(asked); + assert.ok(box[0] <= asked[0] && box[1] <= asked[1]); + assert.ok(box[2] >= asked[2] && box[3] >= asked[3]); + assert.equal(box[2] - box[0], box[3] - box[1]); + // Nearby views name the same box. + assert.deepEqual(libraryViewBox([-77.0305, 38.8895, -77.0205, 38.8995]), box); + assert.deepEqual(parseLibraryViewBox(box.join(',')), box); + assert.equal( + parseLibraryViewBox(box.map((v) => v.toFixed(8)).join(',')), + null, + ); + assert.equal(parseLibraryViewBox('-77,38,-76,39.5'), null); + assert.equal(parseLibraryViewBox('0,0,4,4'), null); + assert.equal(libraryViewBox([0, 0, 3, 3]), null); +}); + +test('a tile path has one spelling: no leading zeros, no empty segments, inside the grid', async () => { + const calls = []; + const xyz = entries.find(({ id }) => id === 'hidden-tiles'); + const handle = createLibraryHandler({ + entries: [xyz], + proxied: () => true, + keep: false, + fetchImpl: async (url) => { + calls.push(url); + return png(); + }, + }); + for (const path of [ + '/tile/hidden-tiles/02/1/1', + '/tile/hidden-tiles/2/01/1', + '/tile/hidden-tiles/2/1/4', + '/tile/hidden-tiles/2/1/1.5', + '/tile/hidden-tiles/23/1/1', + ]) + assert.equal((await get(handle, path)).status, 400, path); + for (const path of [ + '/tile/hidden-tiles//2/1/1', + '/tile/hidden-tiles/2/1/1/', + '//tile/hidden-tiles/2/1/1', + ]) + assert.equal((await get(handle, path)).status, 404, path); + assert.equal(calls.length, 0); + assert.equal((await get(handle, '/tile/hidden-tiles/2/1/3')).status, 200); + assert.equal(calls.length, 1); +}); + +test('a partitioned dataset reads its index from the given fetch, never builds one, and stays under the subrequest limit', async () => { + const epc = libraryEntry('epc-england-wales'); + const box = libraryViewBox(epc.source.view.sample); + const upstream = []; + const indexes = []; + const handler = (fetchIndex, subrequestLimit = Infinity) => + createLibraryHandler({ + entries: [epc], + proxied: () => true, + keep: false, + fetchIndex, + subrequestLimit, + fetchImpl: async (url) => { + upstream.push(String(url)); + return new Response('', { status: 500 }); + }, + }); + + // No index to be had: refused clearly, the publisher never asked. + const missing = await get( + handler(async (entry) => { + indexes.push(entry.id); + return new Response('', { status: 404 }); + }), + `/data/epc-england-wales?bbox=${box}`, + ); + assert.equal(missing.status, 503); + assert.deepEqual(await missing.json(), { error: 'index_unavailable' }); + assert.deepEqual(indexes, ['epc-england-wales']); + assert.deepEqual(upstream, []); + + // An index naming partitions in view: reading them would pass the limit. + const partitions = Array.from({ length: 8 }, (_, i) => ({ + url: `${epc.source.partitions.keyBase}addresscloud/epc/geoparquet-local-authority/p${i}.parquet`, + bbox: box, + })); + const over = await get( + handler(async () => Response.json({ partitions, complete: true }), 3), + `/data/epc-england-wales?bbox=${box}`, + ); + assert.equal(over.status, 503); + assert.deepEqual(await over.json(), { error: 'subrequest_limit' }); + // The index fetch and two upstream reads; the third read was refused. + assert.equal(upstream.length, 2); +}); diff --git a/src/library/source.js b/src/library/source.js index 1c6f544a69..7aae814fb2 100644 --- a/src/library/source.js +++ b/src/library/source.js @@ -4,12 +4,13 @@ * * Browser-direct entries are fetched from their upstream without cookies or * a referrer; proxied entries go through `/api/library`, which names entries - * by id only. A static build with no server still serves every browser-direct - * entry that is not operator opt-in. + * by id only. A server may hide entries and route browser-direct ones through + * itself; its catalog route says which. A static build with no server still + * serves every browser-direct entry that is not operator opt-in. */ import { loadLibraryDataset, boundLibraryFeatures } from './formats.js'; import { readResponseTextCapped } from '../sources/httpBody.js'; -import { BBOX_TOKEN } from './manifest.js'; +import { BBOX_TOKEN, libraryViewBox } from './manifest.js'; /** Partitions read per view before giving up, in parallel batches. */ const MAX_PARTITION_READS_PER_VIEW = 8; @@ -150,26 +151,66 @@ export function staticLibraryAvailability(entries) { ); } +/** Every entry unavailable and left out of the list. */ +function unlisted(entries) { + return new Map( + entries.map(({ id }) => [ + id, + { available: false, reason: 'server-unavailable', listed: false }, + ]), + ); +} + /** - * @param {{ entries: object[], fetchImpl?: typeof fetch, base?: string }} options + * @param {object} options + * @param {object[]} options.entries + * @param {typeof fetch} [options.fetchImpl] + * @param {string|null} [options.base] The library server's path; null for + * none (a server reading upstream itself). + * @param {Storage|null} [options.storage] Where partition indexes and + * learned extents are kept between visits (the page's localStorage); none + * by default. + * @param {(request: { entry: object, tile: object, url: string, signal: AbortSignal }) => Promise} [options.fetchTile] + * Fetches a tiles entry's tile in place of the renderer (the imagery + * spec's `fetchTile`): for a host that keeps tiles in its own cache. + * @param {(request: { entry: object, url: string, signal?: AbortSignal }) => Promise} [options.fetchData] + * Fetches a proxied dataset's answer from the library server in place of + * `fetchImpl`. + * @param {boolean} [options.requireCatalog] Offer only what the server's + * catalog names: while it does not answer, nothing is offered (and + * `getAvailability` says `retry`), rather than every browser-direct entry. + * @param {Map} [options.partitionCache] Decoded GeoParquet + * partitions, most recent last, shared by sources given the same map; at + * most four are kept. + * @param {(entry: object, init: { signal?: AbortSignal }) => Promise} [options.fetchIndex] + * Fetches a partitioned entry's index (`{ partitions, complete }`) in + * place of the library server's `/index/`. + * @param {boolean} [options.buildIndex] Build a partition index here when + * no complete one can be fetched; false uses an incomplete one or fails. */ export function createLibrarySource({ entries, fetchImpl = (...args) => globalThis.fetch(...args), base = LIBRARY_API_BASE, - storage = (() => { - try { - return globalThis.localStorage ?? null; - } catch { - return null; - } - })(), + fetchTile = null, + fetchData = null, + requireCatalog = false, + partitionCache = new Map(), + fetchIndex = null, + buildIndex = true, + storage = null, now = () => Date.now(), loadParquet = () => import('./parquet.js'), }) { const known = new Set(entries.map(({ id }) => id)); + /** Entries the library server routes through itself (its catalog says). */ + const routed = new Set(); + // What the last catalog answer offered; with `requireCatalog`, nothing + // else is ever loaded. + const offered = new Set(); + const isProxied = (entry) => + entry.source.fetch === 'proxied' || routed.has(entry.id); const indexes = new Map(); - const partitionCache = new Map(); /** * A partitioned entry's index: from this app's server (built once and @@ -209,11 +250,14 @@ export function createLibrarySource({ : null; let partial = null; try { - const response = await fetchImpl( - `${base}/index/${encodeURIComponent(entry.id)}`, - { headers: { Accept: 'application/json' } }, - ); - if (response.ok) { + const response = fetchIndex + ? await fetchIndex(entry, {}) + : base + ? await fetchImpl(`${base}/index/${encodeURIComponent(entry.id)}`, { + headers: { Accept: 'application/json' }, + }) + : null; + if (response?.ok) { const body = await response.json(); if (Array.isArray(body?.partitions)) { if (body.complete !== false) @@ -229,6 +273,10 @@ export function createLibrarySource({ now() - storedComplete.savedAt < entry.source.cacheSeconds * 1000 ) return { partitions: storedComplete.partitions, complete: true }; + if (!buildIndex) { + if (partial) return { partitions: partial, complete: false }; + throw Object.assign(new Error('index_unavailable'), { status: 503 }); + } // The server's index is incomplete, or there is no server: build here. const { buildPartitionIndex } = await loadParquet(); const built = await buildPartitionIndex(entry, { fetchImpl }); @@ -366,6 +414,11 @@ export function createLibrarySource({ * @returns {Promise<{ server: boolean, availability: Map }>} */ async function getAvailability({ signal } = {}) { + if (!base) + return { + server: false, + availability: staticLibraryAvailability(entries), + }; try { const response = await fetchImpl(`${base}/catalog`, { signal, @@ -373,21 +426,42 @@ export function createLibrarySource({ }); if (!response.ok) throw new Error(`HTTP ${response.status}`); const body = await response.json(); - const availability = staticLibraryAvailability(entries); - for (const row of Array.isArray(body?.entries) ? body.entries : []) { + if (!Array.isArray(body?.entries)) throw new Error('no catalog'); + // A required catalog is an allowlist: what it does not name is not + // offered. + const availability = requireCatalog + ? unlisted(entries) + : staticLibraryAvailability(entries); + routed.clear(); + offered.clear(); + for (const row of body.entries) { if (!known.has(row?.id)) continue; + // An entry the server does not list is not offered at all. + const listed = row.listed !== false; + const available = listed && row.available === true; availability.set(row.id, { - available: row.available === true, - reason: row.available === true ? null : String(row.reason || ''), + available, + reason: available + ? null + : String(row.reason || (listed ? '' : 'operator-disabled')), + ...(listed ? {} : { listed: false }), }); + if (listed && row.fetch === 'proxied') routed.add(row.id); + if (available) offered.add(row.id); } return { server: true, availability }; } catch (error) { if (signal?.aborted) throw error; - return { - server: false, - availability: staticLibraryAvailability(entries), - }; + // Without the server's word, a required catalog offers nothing (the + // caller asks again later); otherwise the static rule applies. + routed.clear(); + offered.clear(); + return requireCatalog + ? { server: false, availability: unlisted(entries), retry: true } + : { + server: false, + availability: staticLibraryAvailability(entries), + }; } } @@ -399,6 +473,37 @@ export function createLibrarySource({ * @returns {Promise<{ features: object[], truncated: boolean }>} */ async function loadDataset(entry, { signal, bbox = null } = {}) { + if (requireCatalog && !offered.has(entry.id)) + throw new Error(`${entry.id} is not offered by the library server`); + if (isProxied(entry)) { + // A view-bounded entry asks for its snapped box (`libraryViewBox`), + // so nearby views share one answer. + let query = ''; + if (entry.source.view) { + const box = bbox && libraryViewBox(bbox); + if (!box) return { features: [], truncated: false }; + query = `?bbox=${box.join(',')}`; + } + const url = `${base}/data/${encodeURIComponent(entry.id)}${query}`; + const response = fetchData + ? await fetchData({ entry, url, signal }) + : await fetchImpl(url, { + signal, + headers: { Accept: 'application/json' }, + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const body = await response.json(); + // The server already bounded the set; bound again so a page never + // trusts more than its manifest allows. + const bounded = boundLibraryFeatures(body?.features || [], { + maxFeatures: entry.source.maxFeatures, + display: entry.display, + }); + return { + features: bounded.features, + truncated: Boolean(body?.truncated) || bounded.truncated, + }; + } if (entry.source.format === 'geoparquet') { if (entry.source.partitions) { if (!bbox) return { features: [], truncated: false }; @@ -417,24 +522,6 @@ export function createLibrarySource({ ); return { features: bounded.features, truncated: bounded.truncated }; } - if (entry.source.fetch === 'proxied') { - const response = await fetchImpl( - `${base}/data/${encodeURIComponent(entry.id)}`, - { signal, headers: { Accept: 'application/json' } }, - ); - if (!response.ok) throw new Error(`HTTP ${response.status}`); - const body = await response.json(); - // The server already bounded the set; bound again so a page never - // trusts more than its manifest allows. - const bounded = boundLibraryFeatures(body?.features || [], { - maxFeatures: entry.source.maxFeatures, - display: entry.display, - }); - return { - features: bounded.features, - truncated: Boolean(body?.truncated) || bounded.truncated, - }; - } const collection = await loadLibraryDataset( entry, async (url, budget) => { @@ -461,7 +548,7 @@ export function createLibrarySource({ /** The tile URL template the imagery provider requests. */ function tileTemplate(entry) { - if (entry.source.fetch === 'proxied') + if (isProxied(entry)) return `${base}/tile/${encodeURIComponent(entry.id)}/{z}/{x}/{y}`; // Cesium fills a tile's projected corners itself. return entry.source.url.replaceAll( @@ -470,5 +557,20 @@ export function createLibrarySource({ ); } - return { getAvailability, loadDataset, tileTemplate }; + /** + * The imagery spec's `fetchTile` for a tiles entry: the host's, told which + * entry asks; null to let the renderer fetch. + */ + function tileFetch(entry) { + if (typeof fetchTile !== 'function') return null; + return (request) => fetchTile({ ...request, entry }); + } + + return { + getAvailability, + loadDataset, + tileTemplate, + tileFetch, + requireCatalog, + }; } diff --git a/src/library/source.test.mjs b/src/library/source.test.mjs new file mode 100644 index 0000000000..479ab38052 --- /dev/null +++ b/src/library/source.test.mjs @@ -0,0 +1,110 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createLibrarySource } from './source.js'; +import { libraryEntry } from './catalog.js'; +import { libraryViewBox } from './manifest.js'; +import { libraryImagerySpec } from './layers.js'; + +const tiles = libraryEntry('usgs-topo'); +const view = libraryEntry('fema-usa-structures'); +const entries = [tiles, view]; + +function serverSaying(rows, calls = []) { + return async (url, options) => { + calls.push({ url: String(url), options }); + if (String(url).endsWith('/catalog')) + return Response.json({ entries: rows }); + return Response.json({ features: [], truncated: false }); + }; +} + +test('the server can hide an entry and route a browser-direct one through itself', async () => { + const source = createLibrarySource({ + entries, + storage: null, + fetchImpl: serverSaying([ + { id: tiles.id, available: true, reason: null, fetch: 'proxied' }, + { + id: view.id, + available: false, + reason: 'operator-disabled', + listed: false, + }, + ]), + }); + assert.equal(source.tileTemplate(tiles), tiles.source.url); + const { availability } = await source.getAvailability(); + assert.deepEqual(availability.get(view.id), { + available: false, + reason: 'operator-disabled', + listed: false, + }); + assert.equal(availability.get(tiles.id).available, true); + assert.equal( + source.tileTemplate(tiles), + `/api/library/tile/${tiles.id}/{z}/{x}/{y}`, + ); +}); + +test('a routed view-bounded dataset asks the server for its snapped box, through the host fetch', async () => { + const calls = []; + const asked = []; + const source = createLibrarySource({ + entries, + storage: null, + fetchImpl: serverSaying( + [{ id: view.id, available: true, reason: null, fetch: 'proxied' }], + calls, + ), + fetchData: async ({ entry, url }) => { + asked.push({ entry: entry.id, url }); + return Response.json({ features: [], truncated: true }); + }, + }); + await source.getAvailability(); + const bbox = [-77.031, 38.889, -77.02, 38.9]; + const result = await source.loadDataset(view, { bbox }); + assert.equal(result.truncated, true); + assert.deepEqual(asked, [ + { + entry: view.id, + url: `/api/library/data/${view.id}?bbox=${libraryViewBox(bbox).join(',')}`, + }, + ]); + // No box, nothing asked. + assert.deepEqual(await source.loadDataset(view), { + features: [], + truncated: false, + }); + assert.equal(asked.length, 1); + assert.equal(calls.length, 1); +}); + +test("a host's tile fetch reaches the imagery spec, told the entry", async () => { + const seen = []; + const source = createLibrarySource({ + entries, + storage: null, + fetchTile: async ({ entry, url }) => { + seen.push([entry.id, url]); + return { missing: true }; + }, + }); + const spec = libraryImagerySpec( + tiles, + source.tileTemplate(tiles), + source.tileFetch(tiles), + ); + assert.deepEqual(await spec.fetchTile({ tile: {}, url: 'u' }), { + missing: true, + }); + assert.deepEqual(seen, [[tiles.id, 'u']]); + assert.equal( + 'fetchTile' in libraryImagerySpec(tiles, tiles.source.url), + false, + ); + assert.equal( + createLibrarySource({ entries, storage: null }).tileFetch(tiles), + null, + ); +});