gods-eye-view/server/providers/library.js

import { LIBRARY_ENTRIES } from '../../src/library/catalog.js';
import { loadLibraryDataset } from '../../src/library/formats.js';
import { buildPartitionIndex } from '../../src/library/parquet.js';
import {
  libraryUpstreamUrl,
  parseLibraryIdList,
  libraryTileUrl,
} from '../../src/library/manifest.js';
import { KEY_SETUP_KEYS } from '../../src/keySetupCore.mjs';
import {
  readResponseBytesCapped,
  readResponseTextCapped,
  coalesceProxyRequest,
} from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSite, admitSameSiteKeyless } from './common/same-site.js';

const DATASET_TIMEOUT_MS = 30_000;
const TILE_TIMEOUT_MS = 10_000;
/** Each listing page or footer read while building a partition index. */
const INDEX_TIMEOUT_MS = 20_000;
const TILE_MAX_BYTES = 2 * 1024 * 1024;
const TILE_CACHE_LIMIT = 512;
const IN_FLIGHT_LIMIT = 64;

/**
 * Availability of every entry on this instance, decided by the operator's
 * environment alone: `GEV_LIBRARY_DISABLE` removes entries, opt-in entries
 * need `GEV_LIBRARY_ENABLE` (or, for a pack, its host's own switch), and a
 * keyed entry needs its key. The answer names a reason, never a secret.
 * @param {object[]} entries
 * @param {Record<string, string|undefined>} env
 * @returns {Map<string, { available: boolean, reason: string|null }>}
 */
export function libraryAvailability(entries, env) {
  const enabled = parseLibraryIdList(env.GEV_LIBRARY_ENABLE);
  const disabled = parseLibraryIdList(env.GEV_LIBRARY_DISABLE);
  const result = new Map();
  for (const entry of entries) {
    let reason = null;
    if (disabled.has(entry.id)) reason = 'operator-disabled';
    else if (entry.kind === 'pack') {
      if (String(env[entry.pack.env] || '').trim() !== '1')
        reason = 'operator-opt-in';
    } else if (entry.licence.operatorOptIn && !enabled.has(entry.id))
      reason = 'operator-opt-in';
    else if (entry.source.key && !libraryKeyValue(entry.source.key, env))
      reason = 'key-missing';
    result.set(entry.id, { available: reason === null, reason });
  }
  return result;
}

/** The configured value for a manifest key, through the key registry. */
export function libraryKeyValue(key, env) {
  const registered = KEY_SETUP_KEYS.find(({ id }) => id === key?.id);
  for (const name of registered?.envVars || []) {
    const value = String(env[name] || '').trim();
    if (value) return value;
  }
  return null;
}

function withKey(url, key, env, headers) {
  if (!key) return url;
  const value = libraryKeyValue(key, env);
  if (!value) throw Object.assign(new Error('key_missing'), { status: 503 });
  if (key.in === 'header') {
    headers[key.name] = value;
    return url;
  }
  const keyed = new URL(url);
  keyed.searchParams.set(key.name, value);
  return keyed.href;
}

/**
 * Parse `/<id>/<z>/<x>/<y>` against a tiles entry's zoom range.
 * @returns {{ z: number, x: number, y: number } | null}
 */
export function parseLibraryTilePath(parts, source) {
  if (parts.length !== 3 || !parts.every((part) => /^\d{1,8}$/.test(part)))
    return null;
  const [z, x, y] = parts.map(Number);
  const minZoom = source.minZoom ?? 0;
  const maxZoom = source.maxZoom ?? 22;
  if (z < minZoom || z > maxZoom) return null;
  const span = 2 ** z;
  if (x >= span || y >= span) return null;
  return { z, x, y };
}

/**
 * The Data Library's one server route. Proxied datasets and tiles reach their
 * manifest's fixed upstream only: the browser names an entry id, never a URL,
 * so the allowlist is exactly the catalog. Redirects are refused, bodies are
 * capped while streaming, keys come from this server's environment and
 * upstream error text is never returned.
 */
export function libraryProxy({
  entries = LIBRARY_ENTRIES,
  env = process.env,
  fetchImpl = (...args) => globalThis.fetch(...args),
  now = () => Date.now(),
} = {}) {
  const byId = new Map(entries.map((entry) => [entry.id, entry]));
  const datasets = new Map();
  const tiles = new Map();
  const indexes = new Map();
  const inFlight = new Map();
  const allowData = makeRateLimiter({
    windowMs: 60_000,
    max: 30,
    globalMax: 600,
  });
  const allowTiles = makeRateLimiter({
    windowMs: 60_000,
    max: 600,
    globalMax: 6000,
  });

  async function upstream(url, entry, headers, timeoutMs) {
    // Defense in depth: every derived request stays on the manifest's host.
    const target = libraryUpstreamUrl(url);
    const declared = libraryUpstreamUrl(entry.source.url);
    if (!target || !declared || target.host !== declared.host)
      throw Object.assign(new Error('off_manifest_host'), { status: 502 });
    const signal = AbortSignal.timeout(timeoutMs);
    const response = await fetchImpl(url, {
      signal,
      redirect: 'error',
      headers: { Accept: '*/*', ...headers },
    });
    if (!response.ok) {
      await response.body?.cancel?.().catch?.(() => {});
      throw Object.assign(new Error('upstream_unavailable'), { status: 502 });
    }
    return { response, signal };
  }

  async function fetchDataset(entry) {
    const headers = {};
    const collection = await loadLibraryDataset(entry, async (url, budget) => {
      const keyed = withKey(url, entry.source.key, env, headers);
      const { response, signal } = await upstream(
        keyed,
        entry,
        headers,
        DATASET_TIMEOUT_MS,
      );
      const text = await readResponseTextCapped(
        response,
        budget.remainingBytes,
        signal,
      );
      budget.remainingBytes -= new TextEncoder().encode(text).byteLength;
      return text;
    });
    return { ...collection, fetchedAt: now() };
  }

  async function cached(store, key, ttlMs, load, limit = Infinity) {
    const previous = store.get(key);
    if (previous && now() - previous.savedAt < ttlMs)
      return { value: previous.value, stale: false };
    if (!inFlight.has(key) && inFlight.size >= IN_FLIGHT_LIMIT)
      throw Object.assign(new Error('busy'), { status: 429 });
    try {
      const { promise } = coalesceProxyRequest(inFlight, key, async () => {
        const value = await load();
        store.delete(key);
        store.set(key, { value, savedAt: now() });
        while (store.size > limit) store.delete(store.keys().next().value);
        return value;
      });
      return { value: await promise, stale: false };
    } catch (error) {
      if (previous) return { value: previous.value, stale: true };
      throw error;
    }
  }

  function json(res, status, value, extra = {}) {
    if (res.destroyed) return;
    res.writeHead(status, {
      'Content-Type': 'application/json',
      'Cache-Control': 'no-store',
      ...(status === 405 ? { Allow: 'GET' } : {}),
      ...(status === 429 ? { 'Retry-After': '60' } : {}),
      ...extra,
    });
    res.end(JSON.stringify(value));
  }

  function gate(req, res, entry) {
    // A keyed entry spends the operator's credential: refuse forwarded
    // requests like the other cost-bearing routes. Keyless entries keep
    // working behind a reverse proxy, like terrain heights.
    return entry?.source?.key
      ? admitSameSite(req, res)
      : admitSameSiteKeyless(req, res);
  }

  async function handler(req, res) {
    if (req.method !== 'GET')
      return json(res, 405, { error: 'method_not_allowed' });
    const path = (req.url || '/').split('?')[0];
    const parts = path.split('/').filter(Boolean);
    const availability = libraryAvailability(entries, env);

    if (parts.length === 1 && parts[0] === 'catalog') {
      if (admitSameSiteKeyless(req, res)) return undefined;
      return json(res, 200, {
        entries: entries.map(({ id }) => ({ id, ...availability.get(id) })),
      });
    }

    const [route, id, ...rest] = parts;
    const entry = byId.get(id);
    if (!entry || !['data', 'tile', 'index'].includes(route))
      return json(res, 404, { error: 'unknown_entry' });

    // A partitioned GeoParquet entry's index: which file covers where, read
    // from each file's footer. Built once per cacheSeconds and shared, so
    // visitors do not each list the bucket and read every footer.
    if (route === 'index') {
      if (rest.length || !entry.source?.partitions)
        return json(res, 404, { error: 'not_partitioned' });
      if (admitSameSiteKeyless(req, res)) return undefined;
      const { available, reason } = availability.get(id);
      if (!available) return json(res, 403, { error: reason });
      if (!allowData(clientKey(req)))
        return json(res, 429, { error: 'rate_limited' });
      try {
        const { value } = await cached(
          indexes,
          `index:${id}`,
          entry.source.cacheSeconds * 1000,
          async () => ({
            partitions: await buildPartitionIndex(entry, {
              fetchImpl: (url, options = {}) =>
                fetchImpl(url, {
                  ...options,
                  signal:
                    options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS),
                }),
            }),
            builtAt: now(),
          }),
        );
        return json(res, 200, value);
      } catch (error) {
        const status = error?.status === 429 ? 429 : 502;
        return json(res, status, { error: 'library_upstream_unavailable' });
      }
    }
    const isDataset = route === 'data' && rest.length === 0;
    const isTile = route === 'tile';
    if (
      entry.source.fetch !== 'proxied' ||
      (isDataset && entry.kind !== 'dataset') ||
      (isTile && entry.kind !== 'tiles') ||
      (!isDataset && !isTile)
    )
      return json(res, 404, { error: 'not_proxied' });
    if (gate(req, res, entry)) return undefined;
    const { available, reason } = availability.get(id);
    if (!available) return json(res, 403, { error: reason });

    const ttlMs = entry.source.cacheSeconds * 1000;
    if (isDataset) {
      if (!allowData(clientKey(req)))
        return json(res, 429, { error: 'rate_limited' });
      try {
        const { value, stale } = await cached(
          datasets,
          `data:${id}`,
          ttlMs,
          () => fetchDataset(entry),
        );
        return json(
          res,
          200,
          stale ? { ...value, stale: true } : value,
          stale ? { 'X-Data-Stale': 'true' } : {},
        );
      } catch (error) {
        const status = [429, 503].includes(error?.status) ? error.status : 502;
        return json(res, status, {
          error:
            status === 503 ? 'key_missing' : 'library_upstream_unavailable',
        });
      }
    }

    const tile = parseLibraryTilePath(rest, entry.source);
    if (!tile) return json(res, 400, { error: 'invalid_tile' });
    if (!allowTiles(clientKey(req)))
      return json(res, 429, { error: 'rate_limited' });
    try {
      const key = `tile:${id}:${tile.z}/${tile.x}/${tile.y}`;
      const { value } = await cached(
        tiles,
        key,
        ttlMs,
        async () => {
          const headers = {};
          const url = withKey(
            libraryTileUrl(entry.source.url, tile.z, tile.x, tile.y),
            entry.source.key,
            env,
            headers,
          );
          const { response, signal } = await upstream(
            url,
            entry,
            headers,
            TILE_TIMEOUT_MS,
          );
          const type = String(response.headers.get('content-type') || '')
            .split(';')[0]
            .trim()
            .toLowerCase();
          if (!/^image\/(png|jpeg|webp)$/.test(type)) {
            await response.body?.cancel?.().catch?.(() => {});
            throw Object.assign(new Error('not_an_image'), { status: 502 });
          }
          const bytes = await readResponseBytesCapped(
            response,
            TILE_MAX_BYTES,
            signal,
          );
          return { type, bytes };
        },
        TILE_CACHE_LIMIT,
      );
      if (res.destroyed) return undefined;
      res.writeHead(200, {
        'Content-Type': value.type,
        'Cache-Control': `private, max-age=${entry.source.cacheSeconds}`,
        'X-Content-Type-Options': 'nosniff',
      });
      res.end(Buffer.from(value.bytes));
      return undefined;
    } catch (error) {
      const status = [429, 503].includes(error?.status) ? error.status : 502;
      return json(res, status, {
        error: status === 503 ? 'key_missing' : 'library_upstream_unavailable',
      });
    }
  }

  return {
    name: 'data-library',
    handler,
    configureServer({ middlewares }) {
      middlewares.use('/api/library', handler);
    },
    configurePreviewServer({ middlewares }) {
      middlewares.use('/api/library', handler);
    },
  };
}