Merge origin/main into the renderer-contract branch: the procedural fleet models drawn through the renderer's model sets

Aircraft, military, local ADS-B, vessel, transit and rail models come from
the fleet model set, loaded up front and drawn life size through the draw
port's models sets; no layer touches the viewer or Cesium outside
src/render/cesium/. The models capability gains glTF bytes (gltf), a
per-axis stretch (axisScale), a colour for tinted parts (maskColor) and a
cut below the origin (clip: 'belowOrigin'), with conformance checks.

Main's rail layer and Data Library are ported to the ports; the CCTV
footprint probe keeps the globe-terrain clamp through the query port.
This commit is contained in:
Sameh Khamis committed 2026-11-13 14:14:42 -08:00
commit e3865f462f
533 files changed
+55738 -2518

No files matched your search

+392
View File
@@ -0,0 +1,392 @@
import { LIBRARY_ENTRIES } from '../../src/library/catalog.js';
import { loadLibraryDataset } from '../../src/library/formats.js';
import { buildPartitionIndex } from '../../src/library/parquet.js';
import {
libraryUpstreamUrl,
parseLibraryIdList,
libraryTileUrl,
} from '../../src/library/manifest.js';
import { KEY_SETUP_KEYS } from '../../src/keySetupCore.mjs';
import {
readResponseBytesCapped,
readResponseTextCapped,
coalesceProxyRequest,
} from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSite, admitSameSiteKeyless } from './common/same-site.js';
const DATASET_TIMEOUT_MS = 30_000;
const TILE_TIMEOUT_MS = 10_000;
/** Each listing page or footer read while building a partition index. */
const INDEX_TIMEOUT_MS = 20_000;
/** How long an incomplete partition index is served before a rebuild. */
const PARTIAL_INDEX_MS = 5 * 60_000;
const TILE_MAX_BYTES = 2 * 1024 * 1024;
const TILE_CACHE_LIMIT = 512;
const IN_FLIGHT_LIMIT = 64;
/**
* Availability of every entry on this instance, decided by the operator's
* environment alone: `GEV_LIBRARY_DISABLE` removes entries, opt-in entries
* need `GEV_LIBRARY_ENABLE` (or, for a pack, its host's own switch), and a
* keyed entry needs its key. The answer names a reason, never a secret.
* @param {object[]} entries
* @param {Record<string, string|undefined>} env
* @returns {Map<string, { available: boolean, reason: string|null }>}
*/
export function libraryAvailability(entries, env) {
const enabled = parseLibraryIdList(env.GEV_LIBRARY_ENABLE);
const disabled = parseLibraryIdList(env.GEV_LIBRARY_DISABLE);
const result = new Map();
for (const entry of entries) {
let reason = null;
if (disabled.has(entry.id)) reason = 'operator-disabled';
else if (entry.kind === 'pack') {
if (String(env[entry.pack.env] || '').trim() !== '1')
reason = 'operator-opt-in';
} else if (entry.licence.operatorOptIn && !enabled.has(entry.id))
reason = 'operator-opt-in';
else if (entry.source.key && !libraryKeyValue(entry.source.key, env))
reason = 'key-missing';
result.set(entry.id, { available: reason === null, reason });
}
return result;
}
/** The configured value for a manifest key, through the key registry. */
export function libraryKeyValue(key, env) {
const registered = KEY_SETUP_KEYS.find(({ id }) => id === key?.id);
for (const name of registered?.envVars || []) {
const value = String(env[name] || '').trim();
if (value) return value;
}
return null;
}
function withKey(url, key, env, headers) {
if (!key) return url;
const value = libraryKeyValue(key, env);
if (!value) throw Object.assign(new Error('key_missing'), { status: 503 });
if (key.in === 'header') {
headers[key.name] = value;
return url;
}
const keyed = new URL(url);
keyed.searchParams.set(key.name, value);
return keyed.href;
}
/**
* Parse `/<id>/<z>/<x>/<y>` against a tiles entry's zoom range.
* @returns {{ z: number, x: number, y: number } | null}
*/
export function parseLibraryTilePath(parts, source) {
if (parts.length !== 3 || !parts.every((part) => /^\d{1,8}$/.test(part)))
return null;
const [z, x, y] = parts.map(Number);
const minZoom = source.minZoom ?? 0;
const maxZoom = source.maxZoom ?? 22;
if (z < minZoom || z > maxZoom) return null;
const span = 2 ** z;
if (x >= span || y >= span) return null;
return { z, x, y };
}
/**
* The Data Library's one server route. Proxied datasets and tiles reach their
* manifest's fixed upstream only: the browser names an entry id, never a URL,
* so the allowlist is exactly the catalog. Redirects are refused, bodies are
* capped while streaming, keys come from this server's environment and
* upstream error text is never returned.
*/
export function libraryProxy({
entries = LIBRARY_ENTRIES,
env = process.env,
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
} = {}) {
const byId = new Map(entries.map((entry) => [entry.id, entry]));
const datasets = new Map();
const tiles = new Map();
const indexes = new Map();
const partialIndexes = new Map();
const inFlight = new Map();
const allowData = makeRateLimiter({
windowMs: 60_000,
max: 30,
globalMax: 600,
});
const allowTiles = makeRateLimiter({
windowMs: 60_000,
max: 600,
globalMax: 6000,
});
async function upstream(url, entry, headers, timeoutMs) {
// Defense in depth: every derived request stays on the manifest's host.
const target = libraryUpstreamUrl(url);
const declared = libraryUpstreamUrl(entry.source.url);
if (!target || !declared || target.host !== declared.host)
throw Object.assign(new Error('off_manifest_host'), { status: 502 });
const signal = AbortSignal.timeout(timeoutMs);
const response = await fetchImpl(url, {
signal,
redirect: 'error',
headers: { Accept: '*/*', ...headers },
});
if (!response.ok) {
await response.body?.cancel?.().catch?.(() => {});
throw Object.assign(new Error('upstream_unavailable'), { status: 502 });
}
return { response, signal };
}
async function fetchDataset(entry) {
const headers = {};
const collection = await loadLibraryDataset(entry, async (url, budget) => {
const keyed = withKey(url, entry.source.key, env, headers);
const { response, signal } = await upstream(
keyed,
entry,
headers,
DATASET_TIMEOUT_MS,
);
const text = await readResponseTextCapped(
response,
budget.remainingBytes,
signal,
);
budget.remainingBytes -= new TextEncoder().encode(text).byteLength;
return text;
});
return { ...collection, fetchedAt: now() };
}
async function cached(store, key, ttlMs, load, limit = Infinity) {
const previous = store.get(key);
if (previous && now() - previous.savedAt < ttlMs)
return { value: previous.value, stale: false };
if (!inFlight.has(key) && inFlight.size >= IN_FLIGHT_LIMIT)
throw Object.assign(new Error('busy'), { status: 429 });
try {
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
const value = await load();
store.delete(key);
store.set(key, { value, savedAt: now() });
while (store.size > limit) store.delete(store.keys().next().value);
return value;
});
return { value: await promise, stale: false };
} catch (error) {
if (previous) return { value: previous.value, stale: true };
throw error;
}
}
function json(res, status, value, extra = {}) {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...(status === 429 ? { 'Retry-After': '60' } : {}),
...extra,
});
res.end(JSON.stringify(value));
}
function gate(req, res, entry) {
// A keyed entry spends the operator's credential: refuse forwarded
// requests like the other cost-bearing routes. Keyless entries keep
// working behind a reverse proxy, like terrain heights.
return entry?.source?.key
? admitSameSite(req, res)
: admitSameSiteKeyless(req, res);
}
async function handler(req, res) {
if (req.method !== 'GET')
return json(res, 405, { error: 'method_not_allowed' });
const path = (req.url || '/').split('?')[0];
const parts = path.split('/').filter(Boolean);
const availability = libraryAvailability(entries, env);
if (parts.length === 1 && parts[0] === 'catalog') {
if (admitSameSiteKeyless(req, res)) return undefined;
return json(res, 200, {
entries: entries.map(({ id }) => ({ id, ...availability.get(id) })),
});
}
const [route, id, ...rest] = parts;
const entry = byId.get(id);
if (!entry || !['data', 'tile', 'index'].includes(route))
return json(res, 404, { error: 'unknown_entry' });
// A partitioned GeoParquet entry's index: which file covers where, read
// from each file's footer. Built once per cacheSeconds and shared, so
// visitors do not each list the bucket and read every footer.
if (route === 'index') {
if (rest.length || !entry.source?.partitions)
return json(res, 404, { error: 'not_partitioned' });
if (admitSameSiteKeyless(req, res)) return undefined;
const { available, reason } = availability.get(id);
if (!available) return json(res, 403, { error: reason });
if (!allowData(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
// Only a complete index is cached for cacheSeconds. An incomplete one
// (footers that kept failing) is served as `complete: false` and kept
// for PARTIAL_INDEX_MS only, so the next request soon rebuilds it; a
// complete index from before keeps being served while rebuilds fail.
const key = `index:${id}`;
const partial = partialIndexes.get(key);
if (partial && now() - partial.savedAt < PARTIAL_INDEX_MS) {
const previous = indexes.get(key);
return json(res, 200, previous?.value ?? partial.value);
}
try {
const { value } = await cached(
indexes,
key,
entry.source.cacheSeconds * 1000,
async () => {
const built = await buildPartitionIndex(entry, {
fetchImpl: (url, options = {}) =>
fetchImpl(url, {
...options,
signal:
options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS),
}),
});
const value = {
partitions: built.partitions,
complete: built.complete,
builtAt: now(),
};
if (!built.complete) {
// Recorded here too: with an earlier complete index, cached()
// serves that one and swallows this error, and the rebuild
// still waits PARTIAL_INDEX_MS.
partialIndexes.set(key, { value, savedAt: now() });
throw Object.assign(new Error('index_incomplete'), {
partialIndex: value,
});
}
partialIndexes.delete(key);
return value;
},
);
return json(res, 200, value);
} catch (error) {
if (error?.partialIndex) return json(res, 200, error.partialIndex);
const status = error?.status === 429 ? 429 : 502;
return json(res, status, { error: 'library_upstream_unavailable' });
}
}
const isDataset = route === 'data' && rest.length === 0;
const isTile = route === 'tile';
if (
entry.source.fetch !== 'proxied' ||
(isDataset && entry.kind !== 'dataset') ||
(isTile && entry.kind !== 'tiles') ||
(!isDataset && !isTile)
)
return json(res, 404, { error: 'not_proxied' });
if (gate(req, res, entry)) return undefined;
const { available, reason } = availability.get(id);
if (!available) return json(res, 403, { error: reason });
const ttlMs = entry.source.cacheSeconds * 1000;
if (isDataset) {
if (!allowData(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
try {
const { value, stale } = await cached(
datasets,
`data:${id}`,
ttlMs,
() => fetchDataset(entry),
);
return json(
res,
200,
stale ? { ...value, stale: true } : value,
stale ? { 'X-Data-Stale': 'true' } : {},
);
} catch (error) {
const status = [429, 503].includes(error?.status) ? error.status : 502;
return json(res, status, {
error:
status === 503 ? 'key_missing' : 'library_upstream_unavailable',
});
}
}
const tile = parseLibraryTilePath(rest, entry.source);
if (!tile) return json(res, 400, { error: 'invalid_tile' });
if (!allowTiles(clientKey(req)))
return json(res, 429, { error: 'rate_limited' });
try {
const key = `tile:${id}:${tile.z}/${tile.x}/${tile.y}`;
const { value } = await cached(
tiles,
key,
ttlMs,
async () => {
const headers = {};
const url = withKey(
libraryTileUrl(entry.source.url, tile.z, tile.x, tile.y),
entry.source.key,
env,
headers,
);
const { response, signal } = await upstream(
url,
entry,
headers,
TILE_TIMEOUT_MS,
);
const type = String(response.headers.get('content-type') || '')
.split(';')[0]
.trim()
.toLowerCase();
if (!/^image\/(png|jpeg|webp)$/.test(type)) {
await response.body?.cancel?.().catch?.(() => {});
throw Object.assign(new Error('not_an_image'), { status: 502 });
}
const bytes = await readResponseBytesCapped(
response,
TILE_MAX_BYTES,
signal,
);
return { type, bytes };
},
TILE_CACHE_LIMIT,
);
if (res.destroyed) return undefined;
res.writeHead(200, {
'Content-Type': value.type,
'Cache-Control': `private, max-age=${entry.source.cacheSeconds}`,
'X-Content-Type-Options': 'nosniff',
});
res.end(Buffer.from(value.bytes));
return undefined;
} catch (error) {
const status = [429, 503].includes(error?.status) ? error.status : 502;
return json(res, status, {
error: status === 503 ? 'key_missing' : 'library_upstream_unavailable',
});
}
}
return {
name: 'data-library',
handler,
configureServer({ middlewares }) {
middlewares.use('/api/library', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/library', handler);
},
};
}
+381
View File
@@ -0,0 +1,381 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
libraryAvailability,
libraryProxy,
parseLibraryTilePath,
} from './library.js';
import { validateLibraryCatalog } from '../../src/library/manifest.js';
import { libraryEntry } from '../../src/library/catalog.js';
const base = {
family: 'events',
provider: 'Fixture',
description: 'Fixture entry.',
maintainer: 'test',
coverage: { global: true },
licence: { name: 'Public domain', url: 'https://example.org/licence' },
attribution: { text: 'Fixture' },
status: 'community',
};
const { entries, errors } = validateLibraryCatalog([
{
...base,
id: 'proxied-points',
kind: 'dataset',
title: 'Proxied points',
source: {
format: 'geojson',
url: 'https://data.example.org/points.json',
fetch: 'proxied',
maxBytes: 4096,
},
display: { label: 'title' },
},
{
...base,
id: 'direct-points',
kind: 'dataset',
title: 'Direct points',
source: {
format: 'geojson',
url: 'https://data.example.org/direct.json',
fetch: 'browser-direct',
},
},
{
...base,
id: 'keyed-points',
kind: 'dataset',
title: 'Keyed points',
source: {
format: 'geojson',
url: 'https://keyed.example.org/points.json',
fetch: 'proxied',
key: { id: 'firms', in: 'query', name: 'MAP_KEY' },
},
},
{
...base,
id: 'opt-in-points',
kind: 'dataset',
title: 'Opt-in points',
source: {
format: 'geojson',
url: 'https://data.example.org/opt.json',
fetch: 'proxied',
},
licence: { ...base.licence, operatorOptIn: true },
},
{
...base,
id: 'proxied-tiles',
kind: 'tiles',
title: 'Proxied tiles',
source: {
format: 'xyz',
url: 'https://tiles.example.org/{z}/{x}/{y}.png',
fetch: 'proxied',
minZoom: 2,
maxZoom: 5,
},
},
{
...base,
id: 'fixture-pack',
kind: 'pack',
family: 'cameras',
title: 'Fixture cameras',
source: { format: 'host', url: 'https://cams.example.org/' },
pack: { host: 'cctv', env: 'CCTV_FIXTURE_ENABLED' },
licence: { ...base.licence, operatorOptIn: true },
},
]);
assert.deepEqual(errors, []);
const collection = (features) =>
JSON.stringify({ type: 'FeatureCollection', features });
const point = (title) => ({
type: 'Feature',
geometry: { type: 'Point', coordinates: [1, 2] },
properties: { title },
});
function install({ env = {}, fetchImpl, now, catalog = entries } = {}) {
const plugin = libraryProxy({ entries: catalog, env, fetchImpl, now });
let handler;
plugin.configurePreviewServer({
middlewares: {
use(path, callback) {
assert.equal(path, '/api/library');
handler = callback;
},
},
});
return async (url, headers = {}) => {
const res = {
headers: {},
writeHead(status, values) {
this.status = status;
this.headers = values;
},
setHeader(name, value) {
this.headers[name] = value;
},
end(body) {
this.raw = body;
try {
this.body = JSON.parse(body);
} catch {
this.body = null;
}
},
set statusCode(value) {
this.status = value;
},
};
await handler(
{ url, method: 'GET', headers, socket: { remoteAddress: '127.0.0.1' } },
res,
);
return res;
};
}
test('availability is decided by the operator environment and names no secret', () => {
const none = libraryAvailability(entries, {});
assert.deepEqual(none.get('proxied-points'), {
available: true,
reason: null,
});
assert.equal(none.get('keyed-points').reason, 'key-missing');
assert.equal(none.get('opt-in-points').reason, 'operator-opt-in');
assert.equal(none.get('fixture-pack').reason, 'operator-opt-in');
const configured = libraryAvailability(entries, {
FIRMS_MAP_KEY: 'secret',
GEV_LIBRARY_ENABLE: 'opt-in-points',
GEV_LIBRARY_DISABLE: 'proxied-points',
CCTV_FIXTURE_ENABLED: '1',
});
assert.equal(configured.get('keyed-points').available, true);
assert.equal(configured.get('opt-in-points').available, true);
assert.equal(configured.get('proxied-points').reason, 'operator-disabled');
assert.equal(configured.get('fixture-pack').available, true);
assert.doesNotMatch(JSON.stringify([...configured]), /secret/);
});
test('the catalog route lists every entry with its availability', async () => {
const request = install();
const res = await request('/catalog');
assert.equal(res.status, 200);
assert.equal(res.body.entries.length, entries.length);
assert.deepEqual(
res.body.entries.find(({ id }) => id === 'opt-in-points'),
{ id: 'opt-in-points', available: false, reason: 'operator-opt-in' },
);
});
test('a proxied dataset reaches only its manifest URL, refuses redirects and is normalized', async () => {
const calls = [];
const request = install({
now: () => 42,
fetchImpl: async (url, options) => {
calls.push({ url, options });
return new Response(collection([point('Storm'), { geometry: null }]));
},
});
const res = await request('/data/proxied-points?url=https://evil.example');
assert.equal(res.status, 200);
assert.equal(calls.length, 1);
assert.equal(calls[0].url, 'https://data.example.org/points.json');
assert.equal(calls[0].options.redirect, 'error');
assert.ok(calls[0].options.signal instanceof AbortSignal);
assert.equal(res.body.features.length, 1);
assert.equal(res.body.features[0].properties.name, 'Storm');
assert.equal(res.body.dropped, 1);
assert.equal(res.body.fetchedAt, 42);
// A fresh cache answers without another upstream request.
await request('/data/proxied-points');
assert.equal(calls.length, 1);
});
test('browser-direct, unknown, opt-in and wrong-kind entries are not proxied', async () => {
let calls = 0;
const request = install({
fetchImpl: async () => {
calls++;
return new Response(collection([]));
},
});
assert.equal((await request('/data/direct-points')).status, 404);
assert.equal((await request('/data/nope-nope')).status, 404);
assert.equal((await request('/data/proxied-tiles')).status, 404);
assert.equal((await request('/tile/proxied-points/2/0/0')).status, 404);
const optIn = await request('/data/opt-in-points');
assert.equal(optIn.status, 403);
assert.equal(optIn.body.error, 'operator-opt-in');
assert.equal(calls, 0);
});
test('a keyed entry gets the operator key server-side and refuses forwarded requests', async () => {
const calls = [];
const request = install({
env: { FIRMS_MAP_KEY: 'k3y' },
fetchImpl: async (url) => {
calls.push(new URL(url));
return new Response(collection([point('Fire')]));
},
});
const forwarded = await request('/data/keyed-points', {
'x-forwarded-for': '203.0.113.9',
});
assert.equal(forwarded.status, 403);
assert.equal(calls.length, 0);
const res = await request('/data/keyed-points');
assert.equal(res.status, 200);
assert.equal(calls[0].searchParams.get('MAP_KEY'), 'k3y');
assert.doesNotMatch(res.raw, /k3y/);
});
test('an oversized or failing upstream answers 502 without its text, then serves stale', async () => {
let clock = 0;
let mode = 'ok';
const request = install({
now: () => clock,
fetchImpl: async () => {
if (mode === 'big') return new Response('x'.repeat(8192));
if (mode === 'fail')
return new Response('secret upstream text', { status: 500 });
return new Response(collection([point('A')]));
},
});
mode = 'big';
const big = await request('/data/proxied-points');
assert.equal(big.status, 502);
assert.doesNotMatch(big.raw, /x{20}/);
mode = 'ok';
assert.equal((await request('/data/proxied-points')).status, 200);
clock = 10 * 3600 * 1000;
mode = 'fail';
const stale = await request('/data/proxied-points');
assert.equal(stale.status, 200);
assert.equal(stale.body.stale, true);
assert.equal(stale.headers['X-Data-Stale'], 'true');
assert.doesNotMatch(stale.raw, /secret upstream text/);
});
test('cross-site browser requests are refused before any upstream request', async () => {
let calls = 0;
const request = install({
fetchImpl: async () => {
calls++;
return new Response(collection([]));
},
});
const res = await request('/data/proxied-points', {
host: 'localhost:5173',
origin: 'https://evil.example',
'sec-fetch-site': 'cross-site',
});
assert.equal(res.status, 403);
assert.equal(calls, 0);
});
test('tiles: bounded z/x/y, image types only, cached', async () => {
const calls = [];
const request = install({
fetchImpl: async (url) => {
calls.push(url);
if (url.includes('/3/'))
return new Response('<html>', {
headers: { 'content-type': 'text/html' },
});
return new Response(new Uint8Array([137, 80, 78, 71]), {
headers: { 'content-type': 'image/png' },
});
},
});
assert.equal((await request('/tile/proxied-tiles/1/0/0')).status, 400);
assert.equal((await request('/tile/proxied-tiles/2/4/0')).status, 400);
assert.equal((await request('/tile/proxied-tiles/2/a/0')).status, 400);
const ok = await request('/tile/proxied-tiles/2/1/3');
assert.equal(ok.status, 200);
assert.equal(ok.headers['Content-Type'], 'image/png');
assert.equal(calls[0], 'https://tiles.example.org/2/1/3.png');
await request('/tile/proxied-tiles/2/1/3');
assert.equal(calls.length, 1);
assert.equal((await request('/tile/proxied-tiles/3/1/1')).status, 502);
});
test('tile paths are parsed against the zoom range', () => {
const source = { minZoom: 0, maxZoom: 3 };
assert.deepEqual(parseLibraryTilePath(['3', '7', '7'], source), {
z: 3,
x: 7,
y: 7,
});
assert.equal(parseLibraryTilePath(['3', '8', '0'], source), null);
assert.equal(parseLibraryTilePath(['4', '0', '0'], source), null);
assert.equal(parseLibraryTilePath(['1', '0'], source), null);
});
test('only GET is served', async () => {
const plugin = libraryProxy({ entries, env: {} });
const res = {
writeHead(status, headers) {
this.status = status;
this.headers = headers;
},
end() {},
};
await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res);
assert.equal(res.status, 405);
});
test('a partition index is cached only when complete; a complete one outlives failed rebuilds', async () => {
const epc = libraryEntry('epc-england-wales');
let clock = 0;
let footers = 'ok';
let listings = 0;
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
const fetchImpl = async (url) => {
if (url.includes('list-type')) {
listings += 1;
return new Response(listing);
}
return footers === 'ok'
? new Response(new Uint8Array(100))
: new Response('', { status: 503 });
};
const request = install({ catalog: [epc], now: () => clock, fetchImpl });
const first = await request('/index/epc-england-wales');
assert.equal(first.status, 200);
assert.equal(first.body.complete, true);
const builtAt = first.body.builtAt;
// Past cacheSeconds the rebuild fails: the complete index is still served,
// and no rebuild runs again until the partial pause has passed.
clock += (epc.source.cacheSeconds + 1) * 1000;
footers = 'fail';
const kept = await request('/index/epc-england-wales');
assert.equal(kept.body.complete, true);
assert.equal(kept.body.builtAt, builtAt);
const listed = listings;
await request('/index/epc-england-wales');
assert.equal(listings, listed);
// A server that never had a complete index serves the partial one as such,
// and rebuilds after the pause.
const fresh = install({ catalog: [epc], now: () => clock, fetchImpl });
const partial = await fresh('/index/epc-england-wales');
assert.equal(partial.status, 200);
assert.equal(partial.body.complete, false);
const before = listings;
await fresh('/index/epc-england-wales');
assert.equal(listings, before);
clock += 6 * 60_000;
footers = 'ok';
const rebuilt = await fresh('/index/epc-england-wales');
assert.equal(rebuilt.body.complete, true);
assert.equal(listings, before + 1);
});
+4
View File
@@ -16,6 +16,7 @@ import { radioBrowserProxy } from './radio.js';
import { gbfsProxy } from './gbfs.js';
import { localReceiversProxy } from './local-receivers.js';
import { transitProxy } from './transit.js';
import { railProxy } from './rail.js';
import { adsbLolProxy } from './aircraft/adsb-lol.js';
import { aisLiveProxy } from './vessels/ais-live.js';
import { trackBackfillProxies } from './aircraft/tracks.js';
@@ -35,6 +36,7 @@ import { inaturalistProxy } from './inaturalist/observations.js';
import { acartiaProxy } from './acartia.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
import { libraryProxy } from './library.js';
/**
* Construct the local provider plugins in their established order.
@@ -71,6 +73,7 @@ function localProviderPlugins({
gbfsProxy(),
localReceiversProxy(),
transitProxy(),
railProxy(),
adsbLolProxy(),
aisLiveProxy(),
trackBackfillProxies(),
@@ -83,6 +86,7 @@ function localProviderPlugins({
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
libraryProxy(),
ghostMapsProxy(),
shipwrecksProxy(),
droneAirspaceProxy(),
+45 -16
View File
@@ -72,6 +72,14 @@ class UpstreamRateLimitError extends Error {
}
}
/** Thrown when the routing service failed or sent no JSON; answered as 502. */
class UpstreamUnavailableError extends Error {
constructor() {
super('routing service unavailable');
this.name = 'UpstreamUnavailableError';
}
}
/**
* Run one outbound request no sooner than the shared gate allows.
* @template T
@@ -167,18 +175,26 @@ async function fetchRoute({
`${base.replace(/\/$/, '')}/route/v1/${osrmProfile}/${coords}` +
`?overview=full&geometries=geojson&alternatives=false&steps=${withSteps ? 'true' : 'false'}`;
const controller = new AbortController();
// The 12 s budget includes queue wait, so Directions (15 s) hears back.
const timer = setTimeout(() => controller.abort(), 12000);
let osrm;
try {
const upstreamRes = await throughUpstreamGate(() =>
fetchImpl(upstream, {
signal: controller.signal,
// The endpoint is configured above; a redirect is the one way out of
// it, so it is refused rather than followed.
redirect: 'error',
headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
}),
);
const upstreamRes = await throughUpstreamGate(async () => {
// Expiry before dispatch reflects our backlog, not an upstream outage.
if (controller.signal.aborted)
throw new RouteBusyError(_upstreamIntervalMs);
try {
return await fetchImpl(upstream, {
signal: controller.signal,
// The endpoint is configured above; a redirect is the one way out of
// it, so it is refused rather than followed.
redirect: 'error',
headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
});
} catch {
throw new UpstreamUnavailableError();
}
});
if (upstreamRes.status === 429) {
await cancelBody(upstreamRes);
const retryAfter = Number(upstreamRes.headers.get('retry-after'));
@@ -186,20 +202,28 @@ async function fetchRoute({
Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : 5,
);
}
if (!upstreamRes.ok) {
// OSRM answers 400 for a route it cannot find; other failures are outages.
if (upstreamRes.status === 400) {
await cancelBody(upstreamRes);
return { payload: null, error: 'no route found' };
}
if (!upstreamRes.ok) {
await cancelBody(upstreamRes);
throw new UpstreamUnavailableError();
}
const ctype = upstreamRes.headers.get('content-type') || '';
if (!ctype.includes('json')) {
await cancelBody(upstreamRes);
return { payload: null, error: 'no route found' };
throw new UpstreamUnavailableError();
}
// Read and parse failures are outages; later ones stay "route proxy error".
try {
osrm = JSON.parse(
await readResponseTextCapped(upstreamRes, ROUTE_MAX_RESPONSE_BYTES),
);
} catch {
throw new UpstreamUnavailableError();
}
const text = await readResponseTextCapped(
upstreamRes,
ROUTE_MAX_RESPONSE_BYTES,
);
osrm = JSON.parse(text);
} finally {
clearTimeout(timer);
}
@@ -355,6 +379,11 @@ export function installRouteMiddleware(
upstreamError.retryAfterSec,
);
}
if (upstreamError instanceof UpstreamUnavailableError) {
res.writeHead(502, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ ok: false, error: upstreamError.message }));
return;
}
throw upstreamError;
}
if (error || !payload) return fail(error || 'no route found');
+285
View File
@@ -0,0 +1,285 @@
import { open, stat } from 'node:fs/promises';
import path from 'node:path';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { fileURLToPath } from 'node:url';
import { createRailService } from '../../src/sources/railService.js';
import { RAIL_FEED_ID_PATTERN } from '../../src/data/railFeeds.js';
/**
* Where development and preview look for path and schedule packs when
* RAIL_PATHS_DIR is unset: the default output of scripts/build-rail-paths.mjs.
*/
export const DEFAULT_RAIL_PATHS_DIR = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
'../../.cache/rail-paths',
);
/**
* Pack bytes the development server keeps in memory, across path and
* schedule packs: a few of the largest national packs.
*/
export const RAIL_PACK_CACHE_MAX_BYTES = 64 * 1024 * 1024;
/**
* A bounded cache of pack files' bytes, by path, least recently read first
* out. An entry is used only while the file's modification time and size
* are the ones it was read with, so a rebuilt pack is read again; a file
* larger than the whole byte budget is never kept. Reads of the same file at
* the same time share one read.
* @param {number} [maxBytes]
* @returns {{load: (file: string, info: {mtimeMs: number, size: number},
* read: () => Promise<Buffer>) => Promise<Buffer>, bytes: () => number}}
*/
export function createRailPackCache(maxBytes = RAIL_PACK_CACHE_MAX_BYTES) {
/** @type {Map<string, {mtimeMs: number, size: number, bytes: Buffer}>} */
const entries = new Map();
/** @type {Map<string, Promise<Buffer>>} */
const pending = new Map();
let total = 0;
const drop = (file) => {
const entry = entries.get(file);
if (!entry) return;
entries.delete(file);
total -= entry.bytes.byteLength;
};
return {
async load(file, info, read) {
const entry = entries.get(file);
if (entry && entry.mtimeMs === info.mtimeMs && entry.size === info.size) {
// Most recently read last: the first entry is the next one out.
entries.delete(file);
entries.set(file, entry);
return entry.bytes;
}
const version = `${file}|${info.mtimeMs}|${info.size}`;
const reading = pending.get(version);
if (reading) return reading;
const promise = (async () => {
const bytes = await read();
drop(file);
if (bytes.byteLength <= maxBytes) {
entries.set(file, { mtimeMs: info.mtimeMs, size: info.size, bytes });
total += bytes.byteLength;
for (const [oldest] of entries) {
if (total <= maxBytes) break;
drop(oldest);
}
}
return bytes;
})().finally(() => pending.delete(version));
pending.set(version, promise);
return promise;
},
bytes: () => total,
};
}
/**
* A pack reader for the rail service that reads `<dir>/<feedId><suffix>`.
* Ids are re-checked against the registry's id pattern, so a name can never
* leave the directory; a missing file is "no pack", and a file over the cap is
* refused without being read (an error with `code: 'PACK_TOO_LARGE'`). The
* pack is returned as the file's bytes, never decoded to text; with a
* `cache` (createRailPackCache), an unchanged file is read once.
* @param {string} dir
* @param {string} suffix `.json` (path packs) or `.schedule.json`.
* @param {ReturnType<typeof createRailPackCache>|null} [cache]
* @returns {(feedId: string, options?: {maxBytes?: number}) =>
* Promise<Buffer|null>}
*/
function createFsPackReader(dir, suffix, cache = null) {
const root = path.resolve(dir);
return async (feedId, { maxBytes = Infinity } = {}) => {
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
return null;
const file = path.join(root, `${feedId}${suffix}`);
let handle;
try {
handle = await open(file, 'r');
} catch (error) {
if (error?.code === 'ENOENT') return null;
throw error;
}
try {
const info = await handle.stat();
if (!info.isFile()) return null;
if (info.size > maxBytes) {
const error = new Error(`pack is ${info.size} bytes`);
error.code = 'PACK_TOO_LARGE';
throw error;
}
return cache
? await cache.load(file, info, () => handle.readFile())
: await handle.readFile();
} finally {
await handle.close();
}
};
}
/**
* A `readPathPack` that reads `<dir>/<feedId>.json`.
* @param {string} dir
* @param {{cache?: ReturnType<typeof createRailPackCache>|null}} [options]
*/
export function createFsPathPackReader(dir, { cache = null } = {}) {
return createFsPackReader(dir, '.json', cache);
}
/**
* A `readSchedulePack` that reads `<dir>/<feedId>.schedule.json`.
* @param {string} dir
* @param {{cache?: ReturnType<typeof createRailPackCache>|null}} [options]
*/
export function createFsSchedulePackReader(dir, { cache = null } = {}) {
return createFsPackReader(dir, '.schedule.json', cache);
}
/**
* A `hasSchedulePack` for the rail service: whether
* `<dir>/<feedId>.schedule.json` is a file. Only the directory entry is
* read, never the pack; a missing file is false, any other error is thrown
* (the service then lists the feed, so the read shows the failure).
* @param {string} dir
* @returns {(feedId: string) => Promise<boolean>}
*/
export function createFsSchedulePackProbe(dir) {
const root = path.resolve(dir);
return async (feedId) => {
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
return false;
try {
return (await stat(path.join(root, `${feedId}.schedule.json`))).isFile();
} catch (error) {
if (error?.code === 'ENOENT') return false;
throw error;
}
};
}
/**
* A pack version for the rail service: the modification time and size of
* `<dir>/<feedId><suffix>`, or null when there is no such file. Only the
* directory entry is read; a rebuilt pack has a new version.
* @param {string} dir
* @param {string} suffix
* @returns {(feedId: string) => Promise<string|null>}
*/
function createFsPackVersion(dir, suffix) {
const root = path.resolve(dir);
return async (feedId) => {
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
return null;
try {
const info = await stat(path.join(root, `${feedId}${suffix}`));
return info.isFile() ? `${info.mtimeMs}:${info.size}` : null;
} catch (error) {
if (error?.code === 'ENOENT') return null;
throw error;
}
};
}
/**
* A `schedulePackVersion` for the rail service: the modification time and
* size of `<dir>/<feedId>.schedule.json`, or null when there is no such
* file.
* @param {string} dir
* @returns {(feedId: string) => Promise<string|null>}
*/
export function createFsSchedulePackVersion(dir) {
return createFsPackVersion(dir, '.schedule.json');
}
/**
* A `pathPackVersion` for the rail service: the same for
* `<dir>/<feedId>.json`.
* @param {string} dir
* @returns {(feedId: string) => Promise<string|null>}
*/
export function createFsPathPackVersion(dir) {
return createFsPackVersion(dir, '.json');
}
/**
* Write a service Response to a Node response, streaming its body: a pack's
* bytes go to the socket as they are, with no text decode and no copy.
* @param {Response} response
* @param {import('node:http').ServerResponse} res
* @returns {Promise<void>}
*/
export async function sendRailResponse(response, res) {
res.writeHead(response.status, Object.fromEntries(response.headers));
if (!response.body) {
res.end();
return;
}
await pipeline(Readable.fromWeb(response.body), res);
}
/** Connect the reusable rail request service to development and preview. */
export function railProxy(options = {}) {
const pathsDir =
options.pathsDir || process.env.RAIL_PATHS_DIR || DEFAULT_RAIL_PATHS_DIR;
// One byte budget for both kinds of pack.
const cache = createRailPackCache();
const service = createRailService({
...options,
readPathPack:
options.readPathPack || createFsPathPackReader(pathsDir, { cache }),
readSchedulePack:
options.readSchedulePack ||
createFsSchedulePackReader(pathsDir, { cache }),
// Probe the same directory only when its fs reader is the one in use.
hasSchedulePack:
options.hasSchedulePack ||
(options.readSchedulePack ? null : createFsSchedulePackProbe(pathsDir)),
schedulePackVersion:
options.schedulePackVersion ||
(options.readSchedulePack ? null : createFsSchedulePackVersion(pathsDir)),
pathPackVersion:
options.pathPackVersion ||
(options.readPathPack ? null : createFsPathPackVersion(pathsDir)),
});
// A throw here would escape into the HTTP server's event loop and end the
// whole dev server (Node exits on an unhandled rejection or a throwing
// 'close' listener), so failures stay inside the rail routes.
function closeService() {
try {
service.close();
} catch (error) {
console.warn('[rail] provider close failed', error);
}
}
function install(server) {
server.middlewares.use('/api/rail', async (req, res) => {
try {
const response = await service.handle({
url: `http://localhost/api/rail${req.url || '/'}`,
method: req.method,
// Read for If-None-Match on a pack.
headers: req.headers,
});
await sendRailResponse(response, res);
} catch (error) {
// An answer already under way (a viewer that left mid-pack) has
// nothing to add: the stream is closed and that is all.
if (res.headersSent) {
res.destroy();
return;
}
console.warn('[rail] request failed', error);
res.writeHead(500, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Rail provider error' }));
}
});
server.httpServer?.once('close', closeService);
}
return {
name: 'rail-proxy',
closeBundle: closeService,
configureServer: install,
configurePreviewServer: install,
};
}
+28
View File
@@ -57,8 +57,13 @@ export function ingestAisStreamEnvelope(envelope) {
message.ReportB?.Valid === false
? undefined
: message.ReportB?.ShipType,
Dimension:
message.ReportB?.Valid === false
? undefined
: message.ReportB?.Dimension,
}
: message;
const dimensions = aisDimensions(fields.Dimension);
// Static reports may be partial (including Class B's separate A/B parts).
// Keep known values in the cache as well as in the current live row.
const staticData = {
@@ -84,6 +89,8 @@ export function ingestAisStreamEnvelope(envelope) {
staticCode(live?.imo) ||
staticCode(previous.imo) ||
'',
length: dimensions.length ?? live?.length ?? previous.length ?? null,
beam: dimensions.beam ?? live?.beam ?? previous.beam ?? null,
};
_aisStreamStatic.set(mmsi, staticData);
mergeAisStaticIntoLiveVessel(mmsi, staticData);
@@ -113,6 +120,8 @@ export function ingestAisStreamEnvelope(envelope) {
imo: stringValue(message.ImoNumber ?? message.IMO ?? staticData.imo),
type: vesselTypeFromAis(message, staticData),
destination: stringValue(message.Destination ?? staticData.destination),
length: staticData.length ?? null,
beam: staticData.beam ?? null,
speed: normalizedSpeedOverGround(message.Sog ?? message.SOG),
course: normalizedCourseOverGround(message.Cog ?? message.COG),
heading: normalizedHeading(message.TrueHeading ?? message.Heading),
@@ -236,6 +245,25 @@ function mergeAisStaticIntoLiveVessel(mmsi, staticData) {
if (staticData.type) existing.type = staticData.type;
if (staticData.destination) existing.destination = staticData.destination;
if (staticData.imo) existing.imo = staticData.imo;
if (staticData.length) existing.length = staticData.length;
if (staticData.beam) existing.beam = staticData.beam;
}
/**
* A static report's hull size from its AIS dimensions: the antenna's
* distances to bow (A), stern (B), port (C) and starboard (D), in metres.
* Zero is AIS's "not available", so a side missing its two parts has none.
* @param {{A?: number, B?: number, C?: number, D?: number}|undefined} dimension
* @returns {{length: number|null, beam: number|null}}
*/
function aisDimensions(dimension) {
const part = (value) => {
const number = numberValue(value);
return number !== null && number > 0 ? number : 0;
};
const length = part(dimension?.A) + part(dimension?.B);
const beam = part(dimension?.C) + part(dimension?.D);
return { length: length > 0 ? length : null, beam: beam > 0 ? beam : null };
}
function vesselNameFromAis(metadata, message, staticData = {}) {