mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 08:12:24 +00:00
Merge origin/main into the renderer-contract branch: the procedural fleet models drawn through the renderer's model sets
Aircraft, military, local ADS-B, vessel, transit and rail models come from the fleet model set, loaded up front and drawn life size through the draw port's models sets; no layer touches the viewer or Cesium outside src/render/cesium/. The models capability gains glTF bytes (gltf), a per-axis stretch (axisScale), a colour for tinted parts (maskColor) and a cut below the origin (clip: 'belowOrigin'), with conformance checks. Main's rail layer and Data Library are ported to the ports; the CCTV footprint probe keeps the globe-terrain clamp through the query port.
This commit is contained in:
commit
e3865f462f
533 files changed
+55738
-2518
No files matched your search
@@ -0,0 +1,392 @@
|
||||
import { LIBRARY_ENTRIES } from '../../src/library/catalog.js';
|
||||
import { loadLibraryDataset } from '../../src/library/formats.js';
|
||||
import { buildPartitionIndex } from '../../src/library/parquet.js';
|
||||
import {
|
||||
libraryUpstreamUrl,
|
||||
parseLibraryIdList,
|
||||
libraryTileUrl,
|
||||
} from '../../src/library/manifest.js';
|
||||
import { KEY_SETUP_KEYS } from '../../src/keySetupCore.mjs';
|
||||
import {
|
||||
readResponseBytesCapped,
|
||||
readResponseTextCapped,
|
||||
coalesceProxyRequest,
|
||||
} from './common/http.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { admitSameSite, admitSameSiteKeyless } from './common/same-site.js';
|
||||
|
||||
const DATASET_TIMEOUT_MS = 30_000;
|
||||
const TILE_TIMEOUT_MS = 10_000;
|
||||
/** Each listing page or footer read while building a partition index. */
|
||||
const INDEX_TIMEOUT_MS = 20_000;
|
||||
/** How long an incomplete partition index is served before a rebuild. */
|
||||
const PARTIAL_INDEX_MS = 5 * 60_000;
|
||||
const TILE_MAX_BYTES = 2 * 1024 * 1024;
|
||||
const TILE_CACHE_LIMIT = 512;
|
||||
const IN_FLIGHT_LIMIT = 64;
|
||||
|
||||
/**
|
||||
* Availability of every entry on this instance, decided by the operator's
|
||||
* environment alone: `GEV_LIBRARY_DISABLE` removes entries, opt-in entries
|
||||
* need `GEV_LIBRARY_ENABLE` (or, for a pack, its host's own switch), and a
|
||||
* keyed entry needs its key. The answer names a reason, never a secret.
|
||||
* @param {object[]} entries
|
||||
* @param {Record<string, string|undefined>} env
|
||||
* @returns {Map<string, { available: boolean, reason: string|null }>}
|
||||
*/
|
||||
export function libraryAvailability(entries, env) {
|
||||
const enabled = parseLibraryIdList(env.GEV_LIBRARY_ENABLE);
|
||||
const disabled = parseLibraryIdList(env.GEV_LIBRARY_DISABLE);
|
||||
const result = new Map();
|
||||
for (const entry of entries) {
|
||||
let reason = null;
|
||||
if (disabled.has(entry.id)) reason = 'operator-disabled';
|
||||
else if (entry.kind === 'pack') {
|
||||
if (String(env[entry.pack.env] || '').trim() !== '1')
|
||||
reason = 'operator-opt-in';
|
||||
} else if (entry.licence.operatorOptIn && !enabled.has(entry.id))
|
||||
reason = 'operator-opt-in';
|
||||
else if (entry.source.key && !libraryKeyValue(entry.source.key, env))
|
||||
reason = 'key-missing';
|
||||
result.set(entry.id, { available: reason === null, reason });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** The configured value for a manifest key, through the key registry. */
|
||||
export function libraryKeyValue(key, env) {
|
||||
const registered = KEY_SETUP_KEYS.find(({ id }) => id === key?.id);
|
||||
for (const name of registered?.envVars || []) {
|
||||
const value = String(env[name] || '').trim();
|
||||
if (value) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function withKey(url, key, env, headers) {
|
||||
if (!key) return url;
|
||||
const value = libraryKeyValue(key, env);
|
||||
if (!value) throw Object.assign(new Error('key_missing'), { status: 503 });
|
||||
if (key.in === 'header') {
|
||||
headers[key.name] = value;
|
||||
return url;
|
||||
}
|
||||
const keyed = new URL(url);
|
||||
keyed.searchParams.set(key.name, value);
|
||||
return keyed.href;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `/<id>/<z>/<x>/<y>` against a tiles entry's zoom range.
|
||||
* @returns {{ z: number, x: number, y: number } | null}
|
||||
*/
|
||||
export function parseLibraryTilePath(parts, source) {
|
||||
if (parts.length !== 3 || !parts.every((part) => /^\d{1,8}$/.test(part)))
|
||||
return null;
|
||||
const [z, x, y] = parts.map(Number);
|
||||
const minZoom = source.minZoom ?? 0;
|
||||
const maxZoom = source.maxZoom ?? 22;
|
||||
if (z < minZoom || z > maxZoom) return null;
|
||||
const span = 2 ** z;
|
||||
if (x >= span || y >= span) return null;
|
||||
return { z, x, y };
|
||||
}
|
||||
|
||||
/**
|
||||
* The Data Library's one server route. Proxied datasets and tiles reach their
|
||||
* manifest's fixed upstream only: the browser names an entry id, never a URL,
|
||||
* so the allowlist is exactly the catalog. Redirects are refused, bodies are
|
||||
* capped while streaming, keys come from this server's environment and
|
||||
* upstream error text is never returned.
|
||||
*/
|
||||
export function libraryProxy({
|
||||
entries = LIBRARY_ENTRIES,
|
||||
env = process.env,
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
} = {}) {
|
||||
const byId = new Map(entries.map((entry) => [entry.id, entry]));
|
||||
const datasets = new Map();
|
||||
const tiles = new Map();
|
||||
const indexes = new Map();
|
||||
const partialIndexes = new Map();
|
||||
const inFlight = new Map();
|
||||
const allowData = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 30,
|
||||
globalMax: 600,
|
||||
});
|
||||
const allowTiles = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 600,
|
||||
globalMax: 6000,
|
||||
});
|
||||
|
||||
async function upstream(url, entry, headers, timeoutMs) {
|
||||
// Defense in depth: every derived request stays on the manifest's host.
|
||||
const target = libraryUpstreamUrl(url);
|
||||
const declared = libraryUpstreamUrl(entry.source.url);
|
||||
if (!target || !declared || target.host !== declared.host)
|
||||
throw Object.assign(new Error('off_manifest_host'), { status: 502 });
|
||||
const signal = AbortSignal.timeout(timeoutMs);
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: { Accept: '*/*', ...headers },
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw Object.assign(new Error('upstream_unavailable'), { status: 502 });
|
||||
}
|
||||
return { response, signal };
|
||||
}
|
||||
|
||||
async function fetchDataset(entry) {
|
||||
const headers = {};
|
||||
const collection = await loadLibraryDataset(entry, async (url, budget) => {
|
||||
const keyed = withKey(url, entry.source.key, env, headers);
|
||||
const { response, signal } = await upstream(
|
||||
keyed,
|
||||
entry,
|
||||
headers,
|
||||
DATASET_TIMEOUT_MS,
|
||||
);
|
||||
const text = await readResponseTextCapped(
|
||||
response,
|
||||
budget.remainingBytes,
|
||||
signal,
|
||||
);
|
||||
budget.remainingBytes -= new TextEncoder().encode(text).byteLength;
|
||||
return text;
|
||||
});
|
||||
return { ...collection, fetchedAt: now() };
|
||||
}
|
||||
|
||||
async function cached(store, key, ttlMs, load, limit = Infinity) {
|
||||
const previous = store.get(key);
|
||||
if (previous && now() - previous.savedAt < ttlMs)
|
||||
return { value: previous.value, stale: false };
|
||||
if (!inFlight.has(key) && inFlight.size >= IN_FLIGHT_LIMIT)
|
||||
throw Object.assign(new Error('busy'), { status: 429 });
|
||||
try {
|
||||
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
|
||||
const value = await load();
|
||||
store.delete(key);
|
||||
store.set(key, { value, savedAt: now() });
|
||||
while (store.size > limit) store.delete(store.keys().next().value);
|
||||
return value;
|
||||
});
|
||||
return { value: await promise, stale: false };
|
||||
} catch (error) {
|
||||
if (previous) return { value: previous.value, stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function json(res, status, value, extra = {}) {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...(status === 429 ? { 'Retry-After': '60' } : {}),
|
||||
...extra,
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
}
|
||||
|
||||
function gate(req, res, entry) {
|
||||
// A keyed entry spends the operator's credential: refuse forwarded
|
||||
// requests like the other cost-bearing routes. Keyless entries keep
|
||||
// working behind a reverse proxy, like terrain heights.
|
||||
return entry?.source?.key
|
||||
? admitSameSite(req, res)
|
||||
: admitSameSiteKeyless(req, res);
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
if (req.method !== 'GET')
|
||||
return json(res, 405, { error: 'method_not_allowed' });
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
const parts = path.split('/').filter(Boolean);
|
||||
const availability = libraryAvailability(entries, env);
|
||||
|
||||
if (parts.length === 1 && parts[0] === 'catalog') {
|
||||
if (admitSameSiteKeyless(req, res)) return undefined;
|
||||
return json(res, 200, {
|
||||
entries: entries.map(({ id }) => ({ id, ...availability.get(id) })),
|
||||
});
|
||||
}
|
||||
|
||||
const [route, id, ...rest] = parts;
|
||||
const entry = byId.get(id);
|
||||
if (!entry || !['data', 'tile', 'index'].includes(route))
|
||||
return json(res, 404, { error: 'unknown_entry' });
|
||||
|
||||
// A partitioned GeoParquet entry's index: which file covers where, read
|
||||
// from each file's footer. Built once per cacheSeconds and shared, so
|
||||
// visitors do not each list the bucket and read every footer.
|
||||
if (route === 'index') {
|
||||
if (rest.length || !entry.source?.partitions)
|
||||
return json(res, 404, { error: 'not_partitioned' });
|
||||
if (admitSameSiteKeyless(req, res)) return undefined;
|
||||
const { available, reason } = availability.get(id);
|
||||
if (!available) return json(res, 403, { error: reason });
|
||||
if (!allowData(clientKey(req)))
|
||||
return json(res, 429, { error: 'rate_limited' });
|
||||
// Only a complete index is cached for cacheSeconds. An incomplete one
|
||||
// (footers that kept failing) is served as `complete: false` and kept
|
||||
// for PARTIAL_INDEX_MS only, so the next request soon rebuilds it; a
|
||||
// complete index from before keeps being served while rebuilds fail.
|
||||
const key = `index:${id}`;
|
||||
const partial = partialIndexes.get(key);
|
||||
if (partial && now() - partial.savedAt < PARTIAL_INDEX_MS) {
|
||||
const previous = indexes.get(key);
|
||||
return json(res, 200, previous?.value ?? partial.value);
|
||||
}
|
||||
try {
|
||||
const { value } = await cached(
|
||||
indexes,
|
||||
key,
|
||||
entry.source.cacheSeconds * 1000,
|
||||
async () => {
|
||||
const built = await buildPartitionIndex(entry, {
|
||||
fetchImpl: (url, options = {}) =>
|
||||
fetchImpl(url, {
|
||||
...options,
|
||||
signal:
|
||||
options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS),
|
||||
}),
|
||||
});
|
||||
const value = {
|
||||
partitions: built.partitions,
|
||||
complete: built.complete,
|
||||
builtAt: now(),
|
||||
};
|
||||
if (!built.complete) {
|
||||
// Recorded here too: with an earlier complete index, cached()
|
||||
// serves that one and swallows this error, and the rebuild
|
||||
// still waits PARTIAL_INDEX_MS.
|
||||
partialIndexes.set(key, { value, savedAt: now() });
|
||||
throw Object.assign(new Error('index_incomplete'), {
|
||||
partialIndex: value,
|
||||
});
|
||||
}
|
||||
partialIndexes.delete(key);
|
||||
return value;
|
||||
},
|
||||
);
|
||||
return json(res, 200, value);
|
||||
} catch (error) {
|
||||
if (error?.partialIndex) return json(res, 200, error.partialIndex);
|
||||
const status = error?.status === 429 ? 429 : 502;
|
||||
return json(res, status, { error: 'library_upstream_unavailable' });
|
||||
}
|
||||
}
|
||||
const isDataset = route === 'data' && rest.length === 0;
|
||||
const isTile = route === 'tile';
|
||||
if (
|
||||
entry.source.fetch !== 'proxied' ||
|
||||
(isDataset && entry.kind !== 'dataset') ||
|
||||
(isTile && entry.kind !== 'tiles') ||
|
||||
(!isDataset && !isTile)
|
||||
)
|
||||
return json(res, 404, { error: 'not_proxied' });
|
||||
if (gate(req, res, entry)) return undefined;
|
||||
const { available, reason } = availability.get(id);
|
||||
if (!available) return json(res, 403, { error: reason });
|
||||
|
||||
const ttlMs = entry.source.cacheSeconds * 1000;
|
||||
if (isDataset) {
|
||||
if (!allowData(clientKey(req)))
|
||||
return json(res, 429, { error: 'rate_limited' });
|
||||
try {
|
||||
const { value, stale } = await cached(
|
||||
datasets,
|
||||
`data:${id}`,
|
||||
ttlMs,
|
||||
() => fetchDataset(entry),
|
||||
);
|
||||
return json(
|
||||
res,
|
||||
200,
|
||||
stale ? { ...value, stale: true } : value,
|
||||
stale ? { 'X-Data-Stale': 'true' } : {},
|
||||
);
|
||||
} catch (error) {
|
||||
const status = [429, 503].includes(error?.status) ? error.status : 502;
|
||||
return json(res, status, {
|
||||
error:
|
||||
status === 503 ? 'key_missing' : 'library_upstream_unavailable',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const tile = parseLibraryTilePath(rest, entry.source);
|
||||
if (!tile) return json(res, 400, { error: 'invalid_tile' });
|
||||
if (!allowTiles(clientKey(req)))
|
||||
return json(res, 429, { error: 'rate_limited' });
|
||||
try {
|
||||
const key = `tile:${id}:${tile.z}/${tile.x}/${tile.y}`;
|
||||
const { value } = await cached(
|
||||
tiles,
|
||||
key,
|
||||
ttlMs,
|
||||
async () => {
|
||||
const headers = {};
|
||||
const url = withKey(
|
||||
libraryTileUrl(entry.source.url, tile.z, tile.x, tile.y),
|
||||
entry.source.key,
|
||||
env,
|
||||
headers,
|
||||
);
|
||||
const { response, signal } = await upstream(
|
||||
url,
|
||||
entry,
|
||||
headers,
|
||||
TILE_TIMEOUT_MS,
|
||||
);
|
||||
const type = String(response.headers.get('content-type') || '')
|
||||
.split(';')[0]
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (!/^image\/(png|jpeg|webp)$/.test(type)) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw Object.assign(new Error('not_an_image'), { status: 502 });
|
||||
}
|
||||
const bytes = await readResponseBytesCapped(
|
||||
response,
|
||||
TILE_MAX_BYTES,
|
||||
signal,
|
||||
);
|
||||
return { type, bytes };
|
||||
},
|
||||
TILE_CACHE_LIMIT,
|
||||
);
|
||||
if (res.destroyed) return undefined;
|
||||
res.writeHead(200, {
|
||||
'Content-Type': value.type,
|
||||
'Cache-Control': `private, max-age=${entry.source.cacheSeconds}`,
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
});
|
||||
res.end(Buffer.from(value.bytes));
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
const status = [429, 503].includes(error?.status) ? error.status : 502;
|
||||
return json(res, status, {
|
||||
error: status === 503 ? 'key_missing' : 'library_upstream_unavailable',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'data-library',
|
||||
handler,
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/library', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/library', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
libraryAvailability,
|
||||
libraryProxy,
|
||||
parseLibraryTilePath,
|
||||
} from './library.js';
|
||||
import { validateLibraryCatalog } from '../../src/library/manifest.js';
|
||||
import { libraryEntry } from '../../src/library/catalog.js';
|
||||
|
||||
const base = {
|
||||
family: 'events',
|
||||
provider: 'Fixture',
|
||||
description: 'Fixture entry.',
|
||||
maintainer: 'test',
|
||||
coverage: { global: true },
|
||||
licence: { name: 'Public domain', url: 'https://example.org/licence' },
|
||||
attribution: { text: 'Fixture' },
|
||||
status: 'community',
|
||||
};
|
||||
const { entries, errors } = validateLibraryCatalog([
|
||||
{
|
||||
...base,
|
||||
id: 'proxied-points',
|
||||
kind: 'dataset',
|
||||
title: 'Proxied points',
|
||||
source: {
|
||||
format: 'geojson',
|
||||
url: 'https://data.example.org/points.json',
|
||||
fetch: 'proxied',
|
||||
maxBytes: 4096,
|
||||
},
|
||||
display: { label: 'title' },
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'direct-points',
|
||||
kind: 'dataset',
|
||||
title: 'Direct points',
|
||||
source: {
|
||||
format: 'geojson',
|
||||
url: 'https://data.example.org/direct.json',
|
||||
fetch: 'browser-direct',
|
||||
},
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'keyed-points',
|
||||
kind: 'dataset',
|
||||
title: 'Keyed points',
|
||||
source: {
|
||||
format: 'geojson',
|
||||
url: 'https://keyed.example.org/points.json',
|
||||
fetch: 'proxied',
|
||||
key: { id: 'firms', in: 'query', name: 'MAP_KEY' },
|
||||
},
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'opt-in-points',
|
||||
kind: 'dataset',
|
||||
title: 'Opt-in points',
|
||||
source: {
|
||||
format: 'geojson',
|
||||
url: 'https://data.example.org/opt.json',
|
||||
fetch: 'proxied',
|
||||
},
|
||||
licence: { ...base.licence, operatorOptIn: true },
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'proxied-tiles',
|
||||
kind: 'tiles',
|
||||
title: 'Proxied tiles',
|
||||
source: {
|
||||
format: 'xyz',
|
||||
url: 'https://tiles.example.org/{z}/{x}/{y}.png',
|
||||
fetch: 'proxied',
|
||||
minZoom: 2,
|
||||
maxZoom: 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
...base,
|
||||
id: 'fixture-pack',
|
||||
kind: 'pack',
|
||||
family: 'cameras',
|
||||
title: 'Fixture cameras',
|
||||
source: { format: 'host', url: 'https://cams.example.org/' },
|
||||
pack: { host: 'cctv', env: 'CCTV_FIXTURE_ENABLED' },
|
||||
licence: { ...base.licence, operatorOptIn: true },
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(errors, []);
|
||||
|
||||
const collection = (features) =>
|
||||
JSON.stringify({ type: 'FeatureCollection', features });
|
||||
const point = (title) => ({
|
||||
type: 'Feature',
|
||||
geometry: { type: 'Point', coordinates: [1, 2] },
|
||||
properties: { title },
|
||||
});
|
||||
|
||||
function install({ env = {}, fetchImpl, now, catalog = entries } = {}) {
|
||||
const plugin = libraryProxy({ entries: catalog, env, fetchImpl, now });
|
||||
let handler;
|
||||
plugin.configurePreviewServer({
|
||||
middlewares: {
|
||||
use(path, callback) {
|
||||
assert.equal(path, '/api/library');
|
||||
handler = callback;
|
||||
},
|
||||
},
|
||||
});
|
||||
return async (url, headers = {}) => {
|
||||
const res = {
|
||||
headers: {},
|
||||
writeHead(status, values) {
|
||||
this.status = status;
|
||||
this.headers = values;
|
||||
},
|
||||
setHeader(name, value) {
|
||||
this.headers[name] = value;
|
||||
},
|
||||
end(body) {
|
||||
this.raw = body;
|
||||
try {
|
||||
this.body = JSON.parse(body);
|
||||
} catch {
|
||||
this.body = null;
|
||||
}
|
||||
},
|
||||
set statusCode(value) {
|
||||
this.status = value;
|
||||
},
|
||||
};
|
||||
await handler(
|
||||
{ url, method: 'GET', headers, socket: { remoteAddress: '127.0.0.1' } },
|
||||
res,
|
||||
);
|
||||
return res;
|
||||
};
|
||||
}
|
||||
|
||||
test('availability is decided by the operator environment and names no secret', () => {
|
||||
const none = libraryAvailability(entries, {});
|
||||
assert.deepEqual(none.get('proxied-points'), {
|
||||
available: true,
|
||||
reason: null,
|
||||
});
|
||||
assert.equal(none.get('keyed-points').reason, 'key-missing');
|
||||
assert.equal(none.get('opt-in-points').reason, 'operator-opt-in');
|
||||
assert.equal(none.get('fixture-pack').reason, 'operator-opt-in');
|
||||
const configured = libraryAvailability(entries, {
|
||||
FIRMS_MAP_KEY: 'secret',
|
||||
GEV_LIBRARY_ENABLE: 'opt-in-points',
|
||||
GEV_LIBRARY_DISABLE: 'proxied-points',
|
||||
CCTV_FIXTURE_ENABLED: '1',
|
||||
});
|
||||
assert.equal(configured.get('keyed-points').available, true);
|
||||
assert.equal(configured.get('opt-in-points').available, true);
|
||||
assert.equal(configured.get('proxied-points').reason, 'operator-disabled');
|
||||
assert.equal(configured.get('fixture-pack').available, true);
|
||||
assert.doesNotMatch(JSON.stringify([...configured]), /secret/);
|
||||
});
|
||||
|
||||
test('the catalog route lists every entry with its availability', async () => {
|
||||
const request = install();
|
||||
const res = await request('/catalog');
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.body.entries.length, entries.length);
|
||||
assert.deepEqual(
|
||||
res.body.entries.find(({ id }) => id === 'opt-in-points'),
|
||||
{ id: 'opt-in-points', available: false, reason: 'operator-opt-in' },
|
||||
);
|
||||
});
|
||||
|
||||
test('a proxied dataset reaches only its manifest URL, refuses redirects and is normalized', async () => {
|
||||
const calls = [];
|
||||
const request = install({
|
||||
now: () => 42,
|
||||
fetchImpl: async (url, options) => {
|
||||
calls.push({ url, options });
|
||||
return new Response(collection([point('Storm'), { geometry: null }]));
|
||||
},
|
||||
});
|
||||
const res = await request('/data/proxied-points?url=https://evil.example');
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].url, 'https://data.example.org/points.json');
|
||||
assert.equal(calls[0].options.redirect, 'error');
|
||||
assert.ok(calls[0].options.signal instanceof AbortSignal);
|
||||
assert.equal(res.body.features.length, 1);
|
||||
assert.equal(res.body.features[0].properties.name, 'Storm');
|
||||
assert.equal(res.body.dropped, 1);
|
||||
assert.equal(res.body.fetchedAt, 42);
|
||||
// A fresh cache answers without another upstream request.
|
||||
await request('/data/proxied-points');
|
||||
assert.equal(calls.length, 1);
|
||||
});
|
||||
|
||||
test('browser-direct, unknown, opt-in and wrong-kind entries are not proxied', async () => {
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: async () => {
|
||||
calls++;
|
||||
return new Response(collection([]));
|
||||
},
|
||||
});
|
||||
assert.equal((await request('/data/direct-points')).status, 404);
|
||||
assert.equal((await request('/data/nope-nope')).status, 404);
|
||||
assert.equal((await request('/data/proxied-tiles')).status, 404);
|
||||
assert.equal((await request('/tile/proxied-points/2/0/0')).status, 404);
|
||||
const optIn = await request('/data/opt-in-points');
|
||||
assert.equal(optIn.status, 403);
|
||||
assert.equal(optIn.body.error, 'operator-opt-in');
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
test('a keyed entry gets the operator key server-side and refuses forwarded requests', async () => {
|
||||
const calls = [];
|
||||
const request = install({
|
||||
env: { FIRMS_MAP_KEY: 'k3y' },
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(new URL(url));
|
||||
return new Response(collection([point('Fire')]));
|
||||
},
|
||||
});
|
||||
const forwarded = await request('/data/keyed-points', {
|
||||
'x-forwarded-for': '203.0.113.9',
|
||||
});
|
||||
assert.equal(forwarded.status, 403);
|
||||
assert.equal(calls.length, 0);
|
||||
const res = await request('/data/keyed-points');
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(calls[0].searchParams.get('MAP_KEY'), 'k3y');
|
||||
assert.doesNotMatch(res.raw, /k3y/);
|
||||
});
|
||||
|
||||
test('an oversized or failing upstream answers 502 without its text, then serves stale', async () => {
|
||||
let clock = 0;
|
||||
let mode = 'ok';
|
||||
const request = install({
|
||||
now: () => clock,
|
||||
fetchImpl: async () => {
|
||||
if (mode === 'big') return new Response('x'.repeat(8192));
|
||||
if (mode === 'fail')
|
||||
return new Response('secret upstream text', { status: 500 });
|
||||
return new Response(collection([point('A')]));
|
||||
},
|
||||
});
|
||||
mode = 'big';
|
||||
const big = await request('/data/proxied-points');
|
||||
assert.equal(big.status, 502);
|
||||
assert.doesNotMatch(big.raw, /x{20}/);
|
||||
mode = 'ok';
|
||||
assert.equal((await request('/data/proxied-points')).status, 200);
|
||||
clock = 10 * 3600 * 1000;
|
||||
mode = 'fail';
|
||||
const stale = await request('/data/proxied-points');
|
||||
assert.equal(stale.status, 200);
|
||||
assert.equal(stale.body.stale, true);
|
||||
assert.equal(stale.headers['X-Data-Stale'], 'true');
|
||||
assert.doesNotMatch(stale.raw, /secret upstream text/);
|
||||
});
|
||||
|
||||
test('cross-site browser requests are refused before any upstream request', async () => {
|
||||
let calls = 0;
|
||||
const request = install({
|
||||
fetchImpl: async () => {
|
||||
calls++;
|
||||
return new Response(collection([]));
|
||||
},
|
||||
});
|
||||
const res = await request('/data/proxied-points', {
|
||||
host: 'localhost:5173',
|
||||
origin: 'https://evil.example',
|
||||
'sec-fetch-site': 'cross-site',
|
||||
});
|
||||
assert.equal(res.status, 403);
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
test('tiles: bounded z/x/y, image types only, cached', async () => {
|
||||
const calls = [];
|
||||
const request = install({
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(url);
|
||||
if (url.includes('/3/'))
|
||||
return new Response('<html>', {
|
||||
headers: { 'content-type': 'text/html' },
|
||||
});
|
||||
return new Response(new Uint8Array([137, 80, 78, 71]), {
|
||||
headers: { 'content-type': 'image/png' },
|
||||
});
|
||||
},
|
||||
});
|
||||
assert.equal((await request('/tile/proxied-tiles/1/0/0')).status, 400);
|
||||
assert.equal((await request('/tile/proxied-tiles/2/4/0')).status, 400);
|
||||
assert.equal((await request('/tile/proxied-tiles/2/a/0')).status, 400);
|
||||
const ok = await request('/tile/proxied-tiles/2/1/3');
|
||||
assert.equal(ok.status, 200);
|
||||
assert.equal(ok.headers['Content-Type'], 'image/png');
|
||||
assert.equal(calls[0], 'https://tiles.example.org/2/1/3.png');
|
||||
await request('/tile/proxied-tiles/2/1/3');
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal((await request('/tile/proxied-tiles/3/1/1')).status, 502);
|
||||
});
|
||||
|
||||
test('tile paths are parsed against the zoom range', () => {
|
||||
const source = { minZoom: 0, maxZoom: 3 };
|
||||
assert.deepEqual(parseLibraryTilePath(['3', '7', '7'], source), {
|
||||
z: 3,
|
||||
x: 7,
|
||||
y: 7,
|
||||
});
|
||||
assert.equal(parseLibraryTilePath(['3', '8', '0'], source), null);
|
||||
assert.equal(parseLibraryTilePath(['4', '0', '0'], source), null);
|
||||
assert.equal(parseLibraryTilePath(['1', '0'], source), null);
|
||||
});
|
||||
|
||||
test('only GET is served', async () => {
|
||||
const plugin = libraryProxy({ entries, env: {} });
|
||||
const res = {
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
this.headers = headers;
|
||||
},
|
||||
end() {},
|
||||
};
|
||||
await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res);
|
||||
assert.equal(res.status, 405);
|
||||
});
|
||||
|
||||
test('a partition index is cached only when complete; a complete one outlives failed rebuilds', async () => {
|
||||
const epc = libraryEntry('epc-england-wales');
|
||||
let clock = 0;
|
||||
let footers = 'ok';
|
||||
let listings = 0;
|
||||
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
|
||||
const fetchImpl = async (url) => {
|
||||
if (url.includes('list-type')) {
|
||||
listings += 1;
|
||||
return new Response(listing);
|
||||
}
|
||||
return footers === 'ok'
|
||||
? new Response(new Uint8Array(100))
|
||||
: new Response('', { status: 503 });
|
||||
};
|
||||
const request = install({ catalog: [epc], now: () => clock, fetchImpl });
|
||||
const first = await request('/index/epc-england-wales');
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(first.body.complete, true);
|
||||
const builtAt = first.body.builtAt;
|
||||
|
||||
// Past cacheSeconds the rebuild fails: the complete index is still served,
|
||||
// and no rebuild runs again until the partial pause has passed.
|
||||
clock += (epc.source.cacheSeconds + 1) * 1000;
|
||||
footers = 'fail';
|
||||
const kept = await request('/index/epc-england-wales');
|
||||
assert.equal(kept.body.complete, true);
|
||||
assert.equal(kept.body.builtAt, builtAt);
|
||||
const listed = listings;
|
||||
await request('/index/epc-england-wales');
|
||||
assert.equal(listings, listed);
|
||||
|
||||
// A server that never had a complete index serves the partial one as such,
|
||||
// and rebuilds after the pause.
|
||||
const fresh = install({ catalog: [epc], now: () => clock, fetchImpl });
|
||||
const partial = await fresh('/index/epc-england-wales');
|
||||
assert.equal(partial.status, 200);
|
||||
assert.equal(partial.body.complete, false);
|
||||
const before = listings;
|
||||
await fresh('/index/epc-england-wales');
|
||||
assert.equal(listings, before);
|
||||
clock += 6 * 60_000;
|
||||
footers = 'ok';
|
||||
const rebuilt = await fresh('/index/epc-england-wales');
|
||||
assert.equal(rebuilt.body.complete, true);
|
||||
assert.equal(listings, before + 1);
|
||||
});
|
||||
@@ -16,6 +16,7 @@ import { radioBrowserProxy } from './radio.js';
|
||||
import { gbfsProxy } from './gbfs.js';
|
||||
import { localReceiversProxy } from './local-receivers.js';
|
||||
import { transitProxy } from './transit.js';
|
||||
import { railProxy } from './rail.js';
|
||||
import { adsbLolProxy } from './aircraft/adsb-lol.js';
|
||||
import { aisLiveProxy } from './vessels/ais-live.js';
|
||||
import { trackBackfillProxies } from './aircraft/tracks.js';
|
||||
@@ -35,6 +36,7 @@ import { inaturalistProxy } from './inaturalist/observations.js';
|
||||
import { acartiaProxy } from './acartia.js';
|
||||
import { cycloneProxy } from './cyclones.js';
|
||||
import { windProxy } from './wind.js';
|
||||
import { libraryProxy } from './library.js';
|
||||
|
||||
/**
|
||||
* Construct the local provider plugins in their established order.
|
||||
@@ -71,6 +73,7 @@ function localProviderPlugins({
|
||||
gbfsProxy(),
|
||||
localReceiversProxy(),
|
||||
transitProxy(),
|
||||
railProxy(),
|
||||
adsbLolProxy(),
|
||||
aisLiveProxy(),
|
||||
trackBackfillProxies(),
|
||||
@@ -83,6 +86,7 @@ function localProviderPlugins({
|
||||
mapillaryProxy(),
|
||||
firePerimetersProxy(),
|
||||
gnssIntegrityProxy(),
|
||||
libraryProxy(),
|
||||
ghostMapsProxy(),
|
||||
shipwrecksProxy(),
|
||||
droneAirspaceProxy(),
|
||||
|
||||
@@ -72,6 +72,14 @@ class UpstreamRateLimitError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** Thrown when the routing service failed or sent no JSON; answered as 502. */
|
||||
class UpstreamUnavailableError extends Error {
|
||||
constructor() {
|
||||
super('routing service unavailable');
|
||||
this.name = 'UpstreamUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one outbound request no sooner than the shared gate allows.
|
||||
* @template T
|
||||
@@ -167,18 +175,26 @@ async function fetchRoute({
|
||||
`${base.replace(/\/$/, '')}/route/v1/${osrmProfile}/${coords}` +
|
||||
`?overview=full&geometries=geojson&alternatives=false&steps=${withSteps ? 'true' : 'false'}`;
|
||||
const controller = new AbortController();
|
||||
// The 12 s budget includes queue wait, so Directions (15 s) hears back.
|
||||
const timer = setTimeout(() => controller.abort(), 12000);
|
||||
let osrm;
|
||||
try {
|
||||
const upstreamRes = await throughUpstreamGate(() =>
|
||||
fetchImpl(upstream, {
|
||||
signal: controller.signal,
|
||||
// The endpoint is configured above; a redirect is the one way out of
|
||||
// it, so it is refused rather than followed.
|
||||
redirect: 'error',
|
||||
headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
|
||||
}),
|
||||
);
|
||||
const upstreamRes = await throughUpstreamGate(async () => {
|
||||
// Expiry before dispatch reflects our backlog, not an upstream outage.
|
||||
if (controller.signal.aborted)
|
||||
throw new RouteBusyError(_upstreamIntervalMs);
|
||||
try {
|
||||
return await fetchImpl(upstream, {
|
||||
signal: controller.signal,
|
||||
// The endpoint is configured above; a redirect is the one way out of
|
||||
// it, so it is refused rather than followed.
|
||||
redirect: 'error',
|
||||
headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
|
||||
});
|
||||
} catch {
|
||||
throw new UpstreamUnavailableError();
|
||||
}
|
||||
});
|
||||
if (upstreamRes.status === 429) {
|
||||
await cancelBody(upstreamRes);
|
||||
const retryAfter = Number(upstreamRes.headers.get('retry-after'));
|
||||
@@ -186,20 +202,28 @@ async function fetchRoute({
|
||||
Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : 5,
|
||||
);
|
||||
}
|
||||
if (!upstreamRes.ok) {
|
||||
// OSRM answers 400 for a route it cannot find; other failures are outages.
|
||||
if (upstreamRes.status === 400) {
|
||||
await cancelBody(upstreamRes);
|
||||
return { payload: null, error: 'no route found' };
|
||||
}
|
||||
if (!upstreamRes.ok) {
|
||||
await cancelBody(upstreamRes);
|
||||
throw new UpstreamUnavailableError();
|
||||
}
|
||||
const ctype = upstreamRes.headers.get('content-type') || '';
|
||||
if (!ctype.includes('json')) {
|
||||
await cancelBody(upstreamRes);
|
||||
return { payload: null, error: 'no route found' };
|
||||
throw new UpstreamUnavailableError();
|
||||
}
|
||||
// Read and parse failures are outages; later ones stay "route proxy error".
|
||||
try {
|
||||
osrm = JSON.parse(
|
||||
await readResponseTextCapped(upstreamRes, ROUTE_MAX_RESPONSE_BYTES),
|
||||
);
|
||||
} catch {
|
||||
throw new UpstreamUnavailableError();
|
||||
}
|
||||
const text = await readResponseTextCapped(
|
||||
upstreamRes,
|
||||
ROUTE_MAX_RESPONSE_BYTES,
|
||||
);
|
||||
osrm = JSON.parse(text);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
@@ -355,6 +379,11 @@ export function installRouteMiddleware(
|
||||
upstreamError.retryAfterSec,
|
||||
);
|
||||
}
|
||||
if (upstreamError instanceof UpstreamUnavailableError) {
|
||||
res.writeHead(502, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ ok: false, error: upstreamError.message }));
|
||||
return;
|
||||
}
|
||||
throw upstreamError;
|
||||
}
|
||||
if (error || !payload) return fail(error || 'no route found');
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
import { open, stat } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createRailService } from '../../src/sources/railService.js';
|
||||
import { RAIL_FEED_ID_PATTERN } from '../../src/data/railFeeds.js';
|
||||
|
||||
/**
|
||||
* Where development and preview look for path and schedule packs when
|
||||
* RAIL_PATHS_DIR is unset: the default output of scripts/build-rail-paths.mjs.
|
||||
*/
|
||||
export const DEFAULT_RAIL_PATHS_DIR = path.resolve(
|
||||
path.dirname(fileURLToPath(import.meta.url)),
|
||||
'../../.cache/rail-paths',
|
||||
);
|
||||
|
||||
/**
|
||||
* Pack bytes the development server keeps in memory, across path and
|
||||
* schedule packs: a few of the largest national packs.
|
||||
*/
|
||||
export const RAIL_PACK_CACHE_MAX_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* A bounded cache of pack files' bytes, by path, least recently read first
|
||||
* out. An entry is used only while the file's modification time and size
|
||||
* are the ones it was read with, so a rebuilt pack is read again; a file
|
||||
* larger than the whole byte budget is never kept. Reads of the same file at
|
||||
* the same time share one read.
|
||||
* @param {number} [maxBytes]
|
||||
* @returns {{load: (file: string, info: {mtimeMs: number, size: number},
|
||||
* read: () => Promise<Buffer>) => Promise<Buffer>, bytes: () => number}}
|
||||
*/
|
||||
export function createRailPackCache(maxBytes = RAIL_PACK_CACHE_MAX_BYTES) {
|
||||
/** @type {Map<string, {mtimeMs: number, size: number, bytes: Buffer}>} */
|
||||
const entries = new Map();
|
||||
/** @type {Map<string, Promise<Buffer>>} */
|
||||
const pending = new Map();
|
||||
let total = 0;
|
||||
const drop = (file) => {
|
||||
const entry = entries.get(file);
|
||||
if (!entry) return;
|
||||
entries.delete(file);
|
||||
total -= entry.bytes.byteLength;
|
||||
};
|
||||
return {
|
||||
async load(file, info, read) {
|
||||
const entry = entries.get(file);
|
||||
if (entry && entry.mtimeMs === info.mtimeMs && entry.size === info.size) {
|
||||
// Most recently read last: the first entry is the next one out.
|
||||
entries.delete(file);
|
||||
entries.set(file, entry);
|
||||
return entry.bytes;
|
||||
}
|
||||
const version = `${file}|${info.mtimeMs}|${info.size}`;
|
||||
const reading = pending.get(version);
|
||||
if (reading) return reading;
|
||||
const promise = (async () => {
|
||||
const bytes = await read();
|
||||
drop(file);
|
||||
if (bytes.byteLength <= maxBytes) {
|
||||
entries.set(file, { mtimeMs: info.mtimeMs, size: info.size, bytes });
|
||||
total += bytes.byteLength;
|
||||
for (const [oldest] of entries) {
|
||||
if (total <= maxBytes) break;
|
||||
drop(oldest);
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
})().finally(() => pending.delete(version));
|
||||
pending.set(version, promise);
|
||||
return promise;
|
||||
},
|
||||
bytes: () => total,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A pack reader for the rail service that reads `<dir>/<feedId><suffix>`.
|
||||
* Ids are re-checked against the registry's id pattern, so a name can never
|
||||
* leave the directory; a missing file is "no pack", and a file over the cap is
|
||||
* refused without being read (an error with `code: 'PACK_TOO_LARGE'`). The
|
||||
* pack is returned as the file's bytes, never decoded to text; with a
|
||||
* `cache` (createRailPackCache), an unchanged file is read once.
|
||||
* @param {string} dir
|
||||
* @param {string} suffix `.json` (path packs) or `.schedule.json`.
|
||||
* @param {ReturnType<typeof createRailPackCache>|null} [cache]
|
||||
* @returns {(feedId: string, options?: {maxBytes?: number}) =>
|
||||
* Promise<Buffer|null>}
|
||||
*/
|
||||
function createFsPackReader(dir, suffix, cache = null) {
|
||||
const root = path.resolve(dir);
|
||||
return async (feedId, { maxBytes = Infinity } = {}) => {
|
||||
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
|
||||
return null;
|
||||
const file = path.join(root, `${feedId}${suffix}`);
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(file, 'r');
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const info = await handle.stat();
|
||||
if (!info.isFile()) return null;
|
||||
if (info.size > maxBytes) {
|
||||
const error = new Error(`pack is ${info.size} bytes`);
|
||||
error.code = 'PACK_TOO_LARGE';
|
||||
throw error;
|
||||
}
|
||||
return cache
|
||||
? await cache.load(file, info, () => handle.readFile())
|
||||
: await handle.readFile();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A `readPathPack` that reads `<dir>/<feedId>.json`.
|
||||
* @param {string} dir
|
||||
* @param {{cache?: ReturnType<typeof createRailPackCache>|null}} [options]
|
||||
*/
|
||||
export function createFsPathPackReader(dir, { cache = null } = {}) {
|
||||
return createFsPackReader(dir, '.json', cache);
|
||||
}
|
||||
|
||||
/**
|
||||
* A `readSchedulePack` that reads `<dir>/<feedId>.schedule.json`.
|
||||
* @param {string} dir
|
||||
* @param {{cache?: ReturnType<typeof createRailPackCache>|null}} [options]
|
||||
*/
|
||||
export function createFsSchedulePackReader(dir, { cache = null } = {}) {
|
||||
return createFsPackReader(dir, '.schedule.json', cache);
|
||||
}
|
||||
|
||||
/**
|
||||
* A `hasSchedulePack` for the rail service: whether
|
||||
* `<dir>/<feedId>.schedule.json` is a file. Only the directory entry is
|
||||
* read, never the pack; a missing file is false, any other error is thrown
|
||||
* (the service then lists the feed, so the read shows the failure).
|
||||
* @param {string} dir
|
||||
* @returns {(feedId: string) => Promise<boolean>}
|
||||
*/
|
||||
export function createFsSchedulePackProbe(dir) {
|
||||
const root = path.resolve(dir);
|
||||
return async (feedId) => {
|
||||
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
|
||||
return false;
|
||||
try {
|
||||
return (await stat(path.join(root, `${feedId}.schedule.json`))).isFile();
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return false;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A pack version for the rail service: the modification time and size of
|
||||
* `<dir>/<feedId><suffix>`, or null when there is no such file. Only the
|
||||
* directory entry is read; a rebuilt pack has a new version.
|
||||
* @param {string} dir
|
||||
* @param {string} suffix
|
||||
* @returns {(feedId: string) => Promise<string|null>}
|
||||
*/
|
||||
function createFsPackVersion(dir, suffix) {
|
||||
const root = path.resolve(dir);
|
||||
return async (feedId) => {
|
||||
if (typeof feedId !== 'string' || !RAIL_FEED_ID_PATTERN.test(feedId))
|
||||
return null;
|
||||
try {
|
||||
const info = await stat(path.join(root, `${feedId}${suffix}`));
|
||||
return info.isFile() ? `${info.mtimeMs}:${info.size}` : null;
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A `schedulePackVersion` for the rail service: the modification time and
|
||||
* size of `<dir>/<feedId>.schedule.json`, or null when there is no such
|
||||
* file.
|
||||
* @param {string} dir
|
||||
* @returns {(feedId: string) => Promise<string|null>}
|
||||
*/
|
||||
export function createFsSchedulePackVersion(dir) {
|
||||
return createFsPackVersion(dir, '.schedule.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* A `pathPackVersion` for the rail service: the same for
|
||||
* `<dir>/<feedId>.json`.
|
||||
* @param {string} dir
|
||||
* @returns {(feedId: string) => Promise<string|null>}
|
||||
*/
|
||||
export function createFsPathPackVersion(dir) {
|
||||
return createFsPackVersion(dir, '.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a service Response to a Node response, streaming its body: a pack's
|
||||
* bytes go to the socket as they are, with no text decode and no copy.
|
||||
* @param {Response} response
|
||||
* @param {import('node:http').ServerResponse} res
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function sendRailResponse(response, res) {
|
||||
res.writeHead(response.status, Object.fromEntries(response.headers));
|
||||
if (!response.body) {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
await pipeline(Readable.fromWeb(response.body), res);
|
||||
}
|
||||
|
||||
/** Connect the reusable rail request service to development and preview. */
|
||||
export function railProxy(options = {}) {
|
||||
const pathsDir =
|
||||
options.pathsDir || process.env.RAIL_PATHS_DIR || DEFAULT_RAIL_PATHS_DIR;
|
||||
// One byte budget for both kinds of pack.
|
||||
const cache = createRailPackCache();
|
||||
const service = createRailService({
|
||||
...options,
|
||||
readPathPack:
|
||||
options.readPathPack || createFsPathPackReader(pathsDir, { cache }),
|
||||
readSchedulePack:
|
||||
options.readSchedulePack ||
|
||||
createFsSchedulePackReader(pathsDir, { cache }),
|
||||
// Probe the same directory only when its fs reader is the one in use.
|
||||
hasSchedulePack:
|
||||
options.hasSchedulePack ||
|
||||
(options.readSchedulePack ? null : createFsSchedulePackProbe(pathsDir)),
|
||||
schedulePackVersion:
|
||||
options.schedulePackVersion ||
|
||||
(options.readSchedulePack ? null : createFsSchedulePackVersion(pathsDir)),
|
||||
pathPackVersion:
|
||||
options.pathPackVersion ||
|
||||
(options.readPathPack ? null : createFsPathPackVersion(pathsDir)),
|
||||
});
|
||||
// A throw here would escape into the HTTP server's event loop and end the
|
||||
// whole dev server (Node exits on an unhandled rejection or a throwing
|
||||
// 'close' listener), so failures stay inside the rail routes.
|
||||
function closeService() {
|
||||
try {
|
||||
service.close();
|
||||
} catch (error) {
|
||||
console.warn('[rail] provider close failed', error);
|
||||
}
|
||||
}
|
||||
function install(server) {
|
||||
server.middlewares.use('/api/rail', async (req, res) => {
|
||||
try {
|
||||
const response = await service.handle({
|
||||
url: `http://localhost/api/rail${req.url || '/'}`,
|
||||
method: req.method,
|
||||
// Read for If-None-Match on a pack.
|
||||
headers: req.headers,
|
||||
});
|
||||
await sendRailResponse(response, res);
|
||||
} catch (error) {
|
||||
// An answer already under way (a viewer that left mid-pack) has
|
||||
// nothing to add: the stream is closed and that is all.
|
||||
if (res.headersSent) {
|
||||
res.destroy();
|
||||
return;
|
||||
}
|
||||
console.warn('[rail] request failed', error);
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Rail provider error' }));
|
||||
}
|
||||
});
|
||||
server.httpServer?.once('close', closeService);
|
||||
}
|
||||
return {
|
||||
name: 'rail-proxy',
|
||||
closeBundle: closeService,
|
||||
configureServer: install,
|
||||
configurePreviewServer: install,
|
||||
};
|
||||
}
|
||||
@@ -57,8 +57,13 @@ export function ingestAisStreamEnvelope(envelope) {
|
||||
message.ReportB?.Valid === false
|
||||
? undefined
|
||||
: message.ReportB?.ShipType,
|
||||
Dimension:
|
||||
message.ReportB?.Valid === false
|
||||
? undefined
|
||||
: message.ReportB?.Dimension,
|
||||
}
|
||||
: message;
|
||||
const dimensions = aisDimensions(fields.Dimension);
|
||||
// Static reports may be partial (including Class B's separate A/B parts).
|
||||
// Keep known values in the cache as well as in the current live row.
|
||||
const staticData = {
|
||||
@@ -84,6 +89,8 @@ export function ingestAisStreamEnvelope(envelope) {
|
||||
staticCode(live?.imo) ||
|
||||
staticCode(previous.imo) ||
|
||||
'',
|
||||
length: dimensions.length ?? live?.length ?? previous.length ?? null,
|
||||
beam: dimensions.beam ?? live?.beam ?? previous.beam ?? null,
|
||||
};
|
||||
_aisStreamStatic.set(mmsi, staticData);
|
||||
mergeAisStaticIntoLiveVessel(mmsi, staticData);
|
||||
@@ -113,6 +120,8 @@ export function ingestAisStreamEnvelope(envelope) {
|
||||
imo: stringValue(message.ImoNumber ?? message.IMO ?? staticData.imo),
|
||||
type: vesselTypeFromAis(message, staticData),
|
||||
destination: stringValue(message.Destination ?? staticData.destination),
|
||||
length: staticData.length ?? null,
|
||||
beam: staticData.beam ?? null,
|
||||
speed: normalizedSpeedOverGround(message.Sog ?? message.SOG),
|
||||
course: normalizedCourseOverGround(message.Cog ?? message.COG),
|
||||
heading: normalizedHeading(message.TrueHeading ?? message.Heading),
|
||||
@@ -236,6 +245,25 @@ function mergeAisStaticIntoLiveVessel(mmsi, staticData) {
|
||||
if (staticData.type) existing.type = staticData.type;
|
||||
if (staticData.destination) existing.destination = staticData.destination;
|
||||
if (staticData.imo) existing.imo = staticData.imo;
|
||||
if (staticData.length) existing.length = staticData.length;
|
||||
if (staticData.beam) existing.beam = staticData.beam;
|
||||
}
|
||||
|
||||
/**
|
||||
* A static report's hull size from its AIS dimensions: the antenna's
|
||||
* distances to bow (A), stern (B), port (C) and starboard (D), in metres.
|
||||
* Zero is AIS's "not available", so a side missing its two parts has none.
|
||||
* @param {{A?: number, B?: number, C?: number, D?: number}|undefined} dimension
|
||||
* @returns {{length: number|null, beam: number|null}}
|
||||
*/
|
||||
function aisDimensions(dimension) {
|
||||
const part = (value) => {
|
||||
const number = numberValue(value);
|
||||
return number !== null && number > 0 ? number : 0;
|
||||
};
|
||||
const length = part(dimension?.A) + part(dimension?.B);
|
||||
const beam = part(dimension?.C) + part(dimension?.D);
|
||||
return { length: length > 0 ? length : null, beam: beam > 0 ? beam : null };
|
||||
}
|
||||
|
||||
function vesselNameFromAis(metadata, message, staticData = {}) {
|
||||
|
||||
Reference in new issue
Block a user