mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
refactor: separate transit sources and request service
This commit is contained in:
Sameh Khamis committed
1 parent
e284a9cd34
commit
3155e4e0bf
26 files changed
+1444
-1157
No files matched your search
@@ -1,69 +1,10 @@
|
||||
export {
|
||||
readResponseTextCapped,
|
||||
readResponseJsonCapped,
|
||||
readResponseBytesCapped,
|
||||
coalesceProxyRequest,
|
||||
} from '../../../src/sources/httpBody.js';
|
||||
|
||||
/**
|
||||
* Read a fetch() Response body as bytes with the same hard cap as
|
||||
* readResponseTextCapped — for protobuf upstreams (GTFS-Realtime).
|
||||
* Throws { code:'RESPONSE_TOO_LARGE' }.
|
||||
*/
|
||||
export async function readResponseBytesCapped(response, maxBytes) {
|
||||
const tooLarge = () => {
|
||||
const err = new Error('Upstream response too large');
|
||||
err.code = 'RESPONSE_TOO_LARGE';
|
||||
return err;
|
||||
};
|
||||
const declared = Number(response.headers.get('content-length'));
|
||||
if (Number.isFinite(declared) && declared > maxBytes) throw tooLarge();
|
||||
const reader = response.body?.getReader?.();
|
||||
if (!reader) {
|
||||
const bytes = new Uint8Array(await response.arrayBuffer());
|
||||
if (bytes.byteLength > maxBytes) throw tooLarge();
|
||||
return bytes;
|
||||
}
|
||||
const chunks = [];
|
||||
let total = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
total += value.byteLength;
|
||||
if (total > maxBytes) {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
throw tooLarge();
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
const out = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
out.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the existing promise for a cache key, or create one and remove it
|
||||
* only when that exact promise settles.
|
||||
*/
|
||||
export function coalesceProxyRequest(inFlight, key, create) {
|
||||
const existing = inFlight.get(key);
|
||||
if (existing) return { promise: existing, shared: true };
|
||||
let promise;
|
||||
promise = Promise.resolve()
|
||||
.then(create)
|
||||
.finally(() => {
|
||||
if (inFlight.get(key) === promise) inFlight.delete(key);
|
||||
});
|
||||
inFlight.set(key, promise);
|
||||
return { promise, shared: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a fetch Response body as text while enforcing a hard byte cap during
|
||||
* the read — so a malicious or buggy upstream that streams an unbounded body
|
||||
|
||||
@@ -1,38 +1,5 @@
|
||||
/**
|
||||
* Minimal fixed-window per-key rate limiter for the dev proxies. Not a hard
|
||||
* security boundary (dev-only), just a backstop so a runaway client can't hammer
|
||||
* the public Overpass / OSRM mirrors or exhaust this process.
|
||||
*/
|
||||
const RATE_LIMITER_MAX_KEYS = 2000;
|
||||
|
||||
export function makeRateLimiter({ windowMs, max, globalMax }) {
|
||||
const hits = new Map(); // key -> number[] (timestamps within window)
|
||||
let globalTimes = []; // all hits in window, for the global backstop
|
||||
return function allow(key) {
|
||||
const now = Date.now();
|
||||
globalTimes = globalTimes.filter((t) => now - t < windowMs);
|
||||
if (globalMax && globalTimes.length >= globalMax) return false; // global backstop
|
||||
const recent = (hits.get(key) || []).filter((t) => now - t < windowMs);
|
||||
if (recent.length >= max) {
|
||||
hits.set(key, recent);
|
||||
return false;
|
||||
}
|
||||
recent.push(now);
|
||||
hits.set(key, recent);
|
||||
globalTimes.push(now);
|
||||
// Hard key cap so a key-rotating caller can't grow the map without bound.
|
||||
if (hits.size > RATE_LIMITER_MAX_KEYS) {
|
||||
const oldest = hits.keys().next().value;
|
||||
if (oldest !== undefined) hits.delete(oldest);
|
||||
}
|
||||
if (hits.size > 256) {
|
||||
for (const [k, v] of hits) {
|
||||
if (!v.length || now - v[v.length - 1] > windowMs) hits.delete(k);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
}
|
||||
import { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
export { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
|
||||
/**
|
||||
* Opt-in per-IP rate limiter for the cost-bearing API proxies (OpenAI / Google).
|
||||
|
||||
+17
-420
@@ -1,427 +1,24 @@
|
||||
import { createTransitHistory } from './transitHistory.js';
|
||||
/**
|
||||
* GTFS-Realtime transit proxy for the Transit data layer.
|
||||
*
|
||||
* Only URLs registered in `src/data/transitFeeds.js` are ever fetched — the
|
||||
* browser names a registered id, never a URL. Pure request/cache mechanics
|
||||
* live in `src/data/transitProxy.js` so the offline suite covers them.
|
||||
*/
|
||||
import { createTransitService } from '../../src/sources/transitService.js';
|
||||
export { fetchTransitFeed } from '../../src/sources/transitService.js';
|
||||
|
||||
import {
|
||||
coalesceProxyRequest,
|
||||
readResponseBytesCapped,
|
||||
} from './common/http.js';
|
||||
import { makeRateLimiter } from './common/rate-limit.js';
|
||||
import { publicTransitCatalog } from '../../src/data/transitFeeds.js';
|
||||
import {
|
||||
TRANSIT_ADMISSION_MAX_GLOBAL,
|
||||
TRANSIT_ADMISSION_MAX_PER_FEED,
|
||||
TRANSIT_ADMISSION_WINDOW_MS,
|
||||
TRANSIT_MAX_REDIRECTS,
|
||||
TRANSIT_PROXY_MAX_BODY_BYTES,
|
||||
TRANSIT_PROXY_TIMEOUT_MS,
|
||||
TRANSIT_PROXY_TTL_MS,
|
||||
TransitFeedShapeError,
|
||||
buildTransitSnapshot,
|
||||
isAcceptableTransitUpstreamUrl,
|
||||
isTransitRedirectStatus,
|
||||
nextTransitBackoffMs,
|
||||
resolveTransitRoute,
|
||||
transitCacheState,
|
||||
transitRedirectDecision,
|
||||
transitResponseHeaders,
|
||||
transitUpstreamHeaders,
|
||||
} from '../../src/data/transitProxy.js';
|
||||
|
||||
/**
|
||||
* Fetch a registered feed, resolving each redirect hop BEFORE following it.
|
||||
*
|
||||
* `redirect: 'follow'` hands the hop to the runtime and only lets us inspect
|
||||
* where it landed — by which point a disallowed host has already been
|
||||
* contacted and has already answered. Each hop is therefore requested with
|
||||
* `redirect: 'manual'` and its `Location` checked against the feed's own https
|
||||
* origin first, the rule the CCTV frame proxy applies to camera images.
|
||||
*
|
||||
* @param {object} feed Registry entry.
|
||||
* @param {AbortSignal} signal Abort signal for the whole hop chain.
|
||||
* @param {typeof fetch} [fetchImpl] Injected for tests.
|
||||
* @param {{etag?: string|null, lastModified?: string|null}} [validators] Cached validators.
|
||||
* @returns {Promise<{response: Response, finalUrl: string}>}
|
||||
*/
|
||||
export async function fetchTransitFeed(
|
||||
feed,
|
||||
signal,
|
||||
fetchImpl = fetch,
|
||||
validators = null,
|
||||
) {
|
||||
let current = feed.url;
|
||||
for (let hop = 0; hop <= TRANSIT_MAX_REDIRECTS; hop += 1) {
|
||||
const response = await fetchImpl(current, {
|
||||
signal,
|
||||
headers: transitUpstreamHeaders(feed, validators),
|
||||
redirect: 'manual',
|
||||
/** Connect the reusable transit request service to development and preview. */
|
||||
export function transitProxy(options = {}) {
|
||||
const service = createTransitService(options);
|
||||
function install(server) {
|
||||
server.middlewares.use('/api/transit', async (req, res) => {
|
||||
const response = await service.handle({
|
||||
url: `http://localhost/api/transit${req.url || '/'}`,
|
||||
method: req.method,
|
||||
});
|
||||
res.writeHead(response.status, Object.fromEntries(response.headers));
|
||||
res.end(await response.text());
|
||||
});
|
||||
// Only a real redirect is followed. A 304 is the successful answer to our
|
||||
// own conditional request and has no Location; sending it round this loop
|
||||
// fails a feed that is simply unchanged.
|
||||
if (!isTransitRedirectStatus(response?.status)) {
|
||||
return { response, finalUrl: current };
|
||||
}
|
||||
const decision = transitRedirectDecision(
|
||||
feed.url,
|
||||
current,
|
||||
response.headers.get('location'),
|
||||
);
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
if (!decision.ok) throw new Error(`upstream ${decision.reason}`);
|
||||
if (hop === TRANSIT_MAX_REDIRECTS) {
|
||||
throw new Error('upstream redirected too many times');
|
||||
}
|
||||
current = decision.url;
|
||||
}
|
||||
throw new Error('upstream redirected too many times');
|
||||
}
|
||||
|
||||
/**
|
||||
* Vite plugin: GTFS-Realtime VehiclePositions proxy for the Transit layer.
|
||||
*
|
||||
* GET /api/transit/feeds → public catalog (coverage + credit)
|
||||
* GET /api/transit/vehicles/<feedId> → decoded snapshot as JSON
|
||||
*
|
||||
* Only URLs in `src/data/transitFeeds.js` are ever fetched — the browser
|
||||
* names a registered id, never a URL (SECURITY.md). Redirects are resolved hop
|
||||
* by hop and must stay on the feed's own https origin. Bytes are capped at
|
||||
* TRANSIT_PROXY_MAX_BODY_BYTES and decoded server-side, so the browser never
|
||||
* parses protobuf. Per feed: 15 s memory cache, single-flight refresh, and
|
||||
* serve-stale-on-failure for up to 10 minutes (the launch-library pattern).
|
||||
* No disk cache — transit positions are worthless after a few minutes.
|
||||
*
|
||||
* Two backstops sit in front of every upstream request. A fixed-window
|
||||
* admission limiter bounds how often this process may ask any one operator,
|
||||
* and all of them together, no matter how many browser tabs are open. A
|
||||
* per-feed cooldown ladder then keeps a feed that is genuinely down from being
|
||||
* re-asked once per poll for a whole session: consecutive failures push the
|
||||
* next permitted attempt out to five minutes, and one success resets it.
|
||||
*
|
||||
* @param {{fetchImpl?: typeof fetch}} [options]
|
||||
* @returns {object} Vite plugin.
|
||||
*/
|
||||
export function transitProxy({ fetchImpl = fetch } = {}) {
|
||||
/** @type {Map<string, {at:number, body:string, host:string}>} feedId → snapshot */
|
||||
const cache = new Map();
|
||||
const history = createTransitHistory();
|
||||
const admitHistory = makeRateLimiter({
|
||||
windowMs: 60000,
|
||||
max: 30,
|
||||
globalMax: 120,
|
||||
});
|
||||
let closed = false;
|
||||
const inFlight = new Map();
|
||||
const controllers = new Set();
|
||||
/** @type {Map<string, {failures:number, nextAttemptAt:number, reason:string}>} */
|
||||
const cooldown = new Map();
|
||||
const admit = makeRateLimiter({
|
||||
windowMs: TRANSIT_ADMISSION_WINDOW_MS,
|
||||
max: TRANSIT_ADMISSION_MAX_PER_FEED,
|
||||
globalMax: TRANSIT_ADMISSION_MAX_GLOBAL,
|
||||
});
|
||||
const ttlSeconds = Math.ceil(TRANSIT_PROXY_TTL_MS / 1000);
|
||||
|
||||
function send(res, status, body, headers) {
|
||||
res.writeHead(status, headers);
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
function noteFailure(feedId, reason, now) {
|
||||
const previous = cooldown.get(feedId);
|
||||
const failures = (previous?.failures || 0) + 1;
|
||||
const wait = nextTransitBackoffMs(failures);
|
||||
cooldown.set(feedId, { failures, nextAttemptAt: now + wait, reason });
|
||||
return wait;
|
||||
}
|
||||
|
||||
async function refresh(feed, now) {
|
||||
const controller = new AbortController();
|
||||
controllers.add(controller);
|
||||
const timeout = setTimeout(
|
||||
() => controller.abort(),
|
||||
TRANSIT_PROXY_TIMEOUT_MS,
|
||||
);
|
||||
const previous = cache.get(feed.id);
|
||||
try {
|
||||
const { response, finalUrl } = await fetchTransitFeed(
|
||||
feed,
|
||||
controller.signal,
|
||||
fetchImpl,
|
||||
previous
|
||||
? { etag: previous.etag, lastModified: previous.lastModified }
|
||||
: null,
|
||||
);
|
||||
if (closed) throw new Error('Transit provider closed');
|
||||
if (!isAcceptableTransitUpstreamUrl(finalUrl)) {
|
||||
throw new Error('upstream is not https');
|
||||
}
|
||||
// Nothing changed upstream: keep the snapshot we have and restart its
|
||||
// freshness window rather than asking for a body we already hold.
|
||||
if (response.status === 304 && previous) {
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
// `at` restarts the freshness window; `contactedAt` records that the
|
||||
// operator answered just now. They are different facts: the body still
|
||||
// carries the fetch time of the payload it holds, and reading that as
|
||||
// "when did we last hear from them" would age a feed that is answering
|
||||
// every single request into silence.
|
||||
const revalidated = { ...previous, at: now, contactedAt: now };
|
||||
cache.set(feed.id, revalidated);
|
||||
cooldown.delete(feed.id);
|
||||
return revalidated;
|
||||
}
|
||||
if (!response.ok) {
|
||||
const error = new Error(`upstream HTTP ${response.status}`);
|
||||
error.upstreamStatus = response.status;
|
||||
throw error;
|
||||
}
|
||||
const bytes = await readResponseBytesCapped(
|
||||
response,
|
||||
TRANSIT_PROXY_MAX_BODY_BYTES,
|
||||
);
|
||||
const snapshot = buildTransitSnapshot(feed, bytes, now);
|
||||
if (closed) throw new Error('Transit provider closed');
|
||||
history.ingest(feed, snapshot.vehicles, Date.now());
|
||||
const entry = {
|
||||
at: snapshot.fetchedAt,
|
||||
contactedAt: snapshot.fetchedAt,
|
||||
body: JSON.stringify(snapshot),
|
||||
host: new URL(finalUrl).hostname,
|
||||
etag: response.headers.get('etag'),
|
||||
lastModified: response.headers.get('last-modified'),
|
||||
};
|
||||
cache.set(feed.id, entry);
|
||||
cooldown.delete(feed.id);
|
||||
return entry;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
controller.abort();
|
||||
controllers.delete(controller);
|
||||
}
|
||||
}
|
||||
|
||||
function install(middlewares) {
|
||||
middlewares.use('/api/transit', async (req, res) => {
|
||||
if (req.method !== 'GET') {
|
||||
send(
|
||||
res,
|
||||
405,
|
||||
JSON.stringify({ error: 'Method Not Allowed' }),
|
||||
transitResponseHeaders('NONE'),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const route = resolveTransitRoute(req.url);
|
||||
if (!route) {
|
||||
send(res, 404, JSON.stringify({ error: 'Unknown transit feed' }), {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (route.route === 'feeds') {
|
||||
send(res, 200, JSON.stringify({ feeds: publicTransitCatalog() }), {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Cache-Control': 'public, max-age=3600',
|
||||
});
|
||||
return;
|
||||
}
|
||||
const { feed } = route;
|
||||
const now = Date.now();
|
||||
if (route.route === 'trail') {
|
||||
if (!admitHistory(feed.id)) {
|
||||
send(
|
||||
res,
|
||||
429,
|
||||
JSON.stringify({ error: 'Transit history read limit reached' }),
|
||||
{
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
'Retry-After': '60',
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
send(
|
||||
res,
|
||||
200,
|
||||
JSON.stringify(history.get(feed.id, route.vehicleId, now)),
|
||||
{ 'Content-Type': 'application/json', 'Cache-Control': 'no-store' },
|
||||
);
|
||||
return;
|
||||
}
|
||||
const cached = cache.get(feed.id);
|
||||
const state = transitCacheState(cached, now);
|
||||
if (state === 'fresh') {
|
||||
send(
|
||||
res,
|
||||
200,
|
||||
cached.body,
|
||||
transitResponseHeaders('HIT', cached.host, cached.contactedAt),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Answer without contacting the operator: the cached snapshot when there
|
||||
// is one, otherwise an honest "unavailable, come back in N seconds".
|
||||
const answerWithoutUpstream = (reason, retryAfterS) => {
|
||||
if (state === 'stale' && cached) {
|
||||
// The contact time stays as old as it is: a feed being served from
|
||||
// cache during an outage has NOT answered, and saying otherwise
|
||||
// would keep a dead feed on the globe.
|
||||
send(
|
||||
res,
|
||||
200,
|
||||
cached.body,
|
||||
transitResponseHeaders(
|
||||
'STALE-ERROR',
|
||||
cached.host,
|
||||
cached.contactedAt,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
send(
|
||||
res,
|
||||
503,
|
||||
JSON.stringify({
|
||||
error: 'Transit feed unavailable',
|
||||
feedId: feed.id,
|
||||
retryInSec: retryAfterS,
|
||||
}),
|
||||
{
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
'X-GEV-Cache': 'NONE',
|
||||
'Retry-After': String(retryAfterS),
|
||||
'X-Transit-Backoff': reason,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const waiting = cooldown.get(feed.id);
|
||||
if (waiting && waiting.nextAttemptAt > now && !inFlight.has(feed.id)) {
|
||||
answerWithoutUpstream(
|
||||
'cooldown',
|
||||
Math.max(1, Math.ceil((waiting.nextAttemptAt - now) / 1000)),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!inFlight.has(feed.id) && !admit(feed.id)) {
|
||||
answerWithoutUpstream('rate-limited', ttlSeconds);
|
||||
return;
|
||||
}
|
||||
|
||||
const request = coalesceProxyRequest(inFlight, feed.id, () =>
|
||||
refresh(feed, now),
|
||||
);
|
||||
try {
|
||||
const fresh = await request.promise;
|
||||
send(
|
||||
res,
|
||||
200,
|
||||
fresh.body,
|
||||
transitResponseHeaders(
|
||||
request.shared ? 'INFLIGHT' : 'MISS',
|
||||
fresh.host,
|
||||
fresh.contactedAt,
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
const differential = error instanceof TransitFeedShapeError;
|
||||
// A shape fault condemns what we already hold. The cached copy was
|
||||
// decoded under the assumption this feed is a full snapshot, and that
|
||||
// assumption has just been shown false — leaving it in place would let
|
||||
// the very next request serve it back as a 200.
|
||||
if (differential) cache.delete(feed.id);
|
||||
if (!request.shared) {
|
||||
const wait = noteFailure(
|
||||
feed.id,
|
||||
error?.message || String(error),
|
||||
Date.now(),
|
||||
);
|
||||
console.warn(
|
||||
`[transit-proxy] ${feed.id} unavailable: ${error?.message || error} — next attempt in ${Math.round(wait / 1000)}s`,
|
||||
);
|
||||
}
|
||||
// A differential feed is a shape fault, not an outage: serving the old
|
||||
// snapshot would hide a feed this decoder must not render at all.
|
||||
if (state === 'stale' && cached && !differential) {
|
||||
// The contact time is the last SUCCESSFUL one. This request failed,
|
||||
// so it does not advance — otherwise a dead feed would look like one
|
||||
// answering on time.
|
||||
send(
|
||||
res,
|
||||
200,
|
||||
cached.body,
|
||||
transitResponseHeaders(
|
||||
'STALE-ERROR',
|
||||
cached.host,
|
||||
cached.contactedAt,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const pending = cooldown.get(feed.id);
|
||||
const retryAfterS = pending
|
||||
? Math.max(1, Math.ceil((pending.nextAttemptAt - Date.now()) / 1000))
|
||||
: ttlSeconds;
|
||||
send(
|
||||
res,
|
||||
differential ||
|
||||
error?.code === 'RESPONSE_TOO_LARGE' ||
|
||||
Number.isInteger(error?.upstreamStatus)
|
||||
? 502
|
||||
: 504,
|
||||
JSON.stringify({
|
||||
error: differential
|
||||
? 'Transit feed is differential and is not supported'
|
||||
: 'Transit feed unavailable',
|
||||
feedId: feed.id,
|
||||
retryInSec: retryAfterS,
|
||||
}),
|
||||
{
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
'X-GEV-Cache': 'NONE',
|
||||
'Retry-After': String(retryAfterS),
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function close() {
|
||||
closed = true;
|
||||
for (const controller of controllers) controller.abort();
|
||||
controllers.clear();
|
||||
history.clear();
|
||||
cache.clear();
|
||||
cooldown.clear();
|
||||
server.httpServer?.once('close', service.close);
|
||||
}
|
||||
return {
|
||||
closeBundle: close,
|
||||
name: 'transit-proxy',
|
||||
configureServer(server) {
|
||||
install(server.middlewares);
|
||||
server.httpServer?.once('close', close);
|
||||
},
|
||||
configurePreviewServer(server) {
|
||||
install(server.middlewares);
|
||||
server.httpServer?.once('close', close);
|
||||
},
|
||||
closeBundle: service.close,
|
||||
configureServer: install,
|
||||
configurePreviewServer: install,
|
||||
};
|
||||
}
|
||||
@@ -1,493 +1,5 @@
|
||||
import { validTransitIdentifier } from '../../src/data/transitProxy.js';
|
||||
export { validTransitIdentifier } from '../../src/data/transitProxy.js';
|
||||
/** Bounded, process-local operator position history. No upstream or scene access. */
|
||||
export const TRANSIT_HISTORY_LIMITS = Object.freeze({
|
||||
retentionMs: 900000,
|
||||
fixes: 128,
|
||||
feedBytes: 16 * 1024 * 1024,
|
||||
processBytes: 64 * 1024 * 1024,
|
||||
keys: 15000,
|
||||
identifierBytes: 256,
|
||||
responseBytes: 32768,
|
||||
});
|
||||
const FIX_BYTES = 32,
|
||||
CHUNK = 8,
|
||||
EPOCH_SLOTS = 8,
|
||||
EPOCH_BYTES = 544;
|
||||
const META_BYTES = 64;
|
||||
const encoder = new TextEncoder(),
|
||||
decoder = new TextDecoder();
|
||||
// All stores in this process share the hard process allocation ceiling.
|
||||
const processStorage = { bytes: 0, keys: 0, stores: new Set() };
|
||||
|
||||
function contextOf(record, mode) {
|
||||
const trip = typeof record.tripId === 'string' ? record.tripId : '';
|
||||
const route = typeof record.routeId === 'string' ? record.routeId : '';
|
||||
if (encoder.encode(trip).length > 256 || encoder.encode(route).length > 256)
|
||||
return null;
|
||||
return { trip, route, mode: mode || 'unknown' };
|
||||
}
|
||||
function contextAt(entry, slot) {
|
||||
const offset = 0,
|
||||
view = entry.contexts[slot];
|
||||
const bytes = new Uint8Array(view.buffer);
|
||||
const length = view.getUint16(offset + 4),
|
||||
routeLength = view.getUint16(offset + 6);
|
||||
return {
|
||||
id: view.getUint32(offset),
|
||||
trip: decoder.decode(bytes.subarray(offset + 16, offset + 16 + length)),
|
||||
route: decoder.decode(
|
||||
bytes.subarray(offset + 272, offset + 272 + routeLength),
|
||||
),
|
||||
mode: decoder.decode(
|
||||
bytes.subarray(offset + 528, offset + 528 + view.getUint8(offset + 8)),
|
||||
),
|
||||
};
|
||||
}
|
||||
function storeContext(entry, context, id) {
|
||||
const slot = (id - 1) % EPOCH_SLOTS,
|
||||
offset = 0;
|
||||
const trip = encoder.encode(context.trip),
|
||||
route = encoder.encode(context.route),
|
||||
mode = encoder.encode(context.mode).subarray(0, 16);
|
||||
const view = entry.contexts[slot],
|
||||
bytes = new Uint8Array(view.buffer);
|
||||
view.setUint32(offset, id);
|
||||
view.setUint16(offset + 4, trip.length);
|
||||
view.setUint16(offset + 6, route.length);
|
||||
view.setUint8(offset + 8, mode.length);
|
||||
bytes.set(trip, offset + 16);
|
||||
bytes.set(route, offset + 272);
|
||||
bytes.set(mode, offset + 528);
|
||||
}
|
||||
function offset(entry, index) {
|
||||
return ((entry.head + index) % entry.capacity) * FIX_BYTES;
|
||||
}
|
||||
function read(entry, index) {
|
||||
const at = offset(entry, index),
|
||||
v = entry.fixes;
|
||||
return [
|
||||
v.getFloat64(at),
|
||||
v.getFloat64(at + 8),
|
||||
v.getFloat64(at + 16),
|
||||
v.getUint32(at + 24),
|
||||
v.getUint32(at + 28),
|
||||
];
|
||||
}
|
||||
function firstTime(entry) {
|
||||
return entry.count ? entry.fixes.getFloat64(offset(entry, 0)) : Infinity;
|
||||
}
|
||||
function drop(entry) {
|
||||
entry.head = (entry.head + 1) % entry.capacity;
|
||||
entry.count--;
|
||||
entry.truncated = true;
|
||||
}
|
||||
|
||||
// Indexed eviction queues keep removal and receipt-order updates logarithmic.
|
||||
function evictionQueue() {
|
||||
const items = [],
|
||||
positions = new Map();
|
||||
const less = (a, b) =>
|
||||
a.observedAt < b.observedAt ||
|
||||
(a.observedAt === b.observedAt && a.key < b.key);
|
||||
function swap(a, b) {
|
||||
[items[a], items[b]] = [items[b], items[a]];
|
||||
positions.set(items[a], a);
|
||||
positions.set(items[b], b);
|
||||
}
|
||||
function repair(index) {
|
||||
while (index > 0) {
|
||||
const parent = (index - 1) >>> 1;
|
||||
if (!less(items[index], items[parent])) break;
|
||||
swap(index, parent);
|
||||
index = parent;
|
||||
}
|
||||
for (;;) {
|
||||
let child = index * 2 + 1;
|
||||
if (child >= items.length) break;
|
||||
if (child + 1 < items.length && less(items[child + 1], items[child]))
|
||||
child++;
|
||||
if (!less(items[child], items[index])) break;
|
||||
swap(index, child);
|
||||
index = child;
|
||||
}
|
||||
}
|
||||
function remove(entry) {
|
||||
const index = positions.get(entry);
|
||||
if (index === undefined) return;
|
||||
positions.delete(entry);
|
||||
const last = items.pop();
|
||||
if (index < items.length) {
|
||||
items[index] = last;
|
||||
positions.set(last, index);
|
||||
repair(index);
|
||||
}
|
||||
}
|
||||
return {
|
||||
remove,
|
||||
add(entry) {
|
||||
remove(entry);
|
||||
positions.set(entry, items.length);
|
||||
items.push(entry);
|
||||
repair(items.length - 1);
|
||||
},
|
||||
oldest(except) {
|
||||
if (items[0] !== except) return items[0] || null;
|
||||
if (!items[1]) return null;
|
||||
return items[2] && less(items[2], items[1]) ? items[2] : items[1];
|
||||
},
|
||||
get size() {
|
||||
return items.length;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createTransitHistory({
|
||||
limits = {},
|
||||
storage = processStorage,
|
||||
sweep = true,
|
||||
} = {}) {
|
||||
const policy = { ...TRANSIT_HISTORY_LIMITS };
|
||||
for (const [key, limit] of Object.entries(limits))
|
||||
if (key in policy && Number.isFinite(limit) && limit > 0)
|
||||
policy[key] = Math.min(policy[key], Math.floor(limit));
|
||||
const entries = new Map(),
|
||||
feedBytes = new Map(),
|
||||
eviction = evictionQueue(),
|
||||
feedEviction = new Map();
|
||||
storage.feedBytes ||= new Map();
|
||||
storage.lossUntil ||= new Map();
|
||||
let protectedEntries = null;
|
||||
function queue(entry) {
|
||||
if (!feedEviction.has(entry.feed))
|
||||
feedEviction.set(entry.feed, evictionQueue());
|
||||
eviction.add(entry);
|
||||
feedEviction.get(entry.feed).add(entry);
|
||||
}
|
||||
function unqueue(entry) {
|
||||
eviction.remove(entry);
|
||||
const feedQueue = feedEviction.get(entry.feed);
|
||||
feedQueue?.remove(entry);
|
||||
if (feedQueue?.size === 0) feedEviction.delete(entry.feed);
|
||||
}
|
||||
function recordLoss(feed, until) {
|
||||
storage.lossUntil.set(
|
||||
feed,
|
||||
Math.max(storage.lossUntil.get(feed) || 0, until),
|
||||
);
|
||||
}
|
||||
let prunedAt = -Infinity;
|
||||
let allocatedBytes = 0,
|
||||
timer = null,
|
||||
closed = false;
|
||||
function account(feed, delta) {
|
||||
allocatedBytes += delta;
|
||||
storage.bytes += delta;
|
||||
storage.feedBytes.set(feed, (storage.feedBytes.get(feed) || 0) + delta);
|
||||
if (storage.feedBytes.get(feed) === 0) storage.feedBytes.delete(feed);
|
||||
feedBytes.set(feed, (feedBytes.get(feed) || 0) + delta);
|
||||
if (feedBytes.get(feed) === 0) feedBytes.delete(feed);
|
||||
}
|
||||
function remove(entry) {
|
||||
unqueue(entry);
|
||||
entries.delete(entry.key);
|
||||
storage.keys--;
|
||||
account(entry.feed, -entry.bytes);
|
||||
}
|
||||
function canAllocate(feed, bytes) {
|
||||
return (
|
||||
storage.bytes + bytes <= policy.processBytes &&
|
||||
(storage.feedBytes.get(feed) || 0) + bytes <= policy.feedBytes
|
||||
);
|
||||
}
|
||||
function resize(entry, capacity) {
|
||||
const next = new Uint8Array(capacity * FIX_BYTES),
|
||||
before = new Uint8Array(entry.fixes.buffer);
|
||||
for (let i = 0; i < entry.count; i++)
|
||||
next.set(
|
||||
before.subarray(offset(entry, i), offset(entry, i) + FIX_BYTES),
|
||||
i * FIX_BYTES,
|
||||
);
|
||||
const delta = next.byteLength - entry.fixes.byteLength;
|
||||
entry.fixes = new DataView(next.buffer);
|
||||
entry.head = 0;
|
||||
entry.capacity = capacity;
|
||||
entry.bytes += delta;
|
||||
account(entry.feed, delta);
|
||||
}
|
||||
function prune(now) {
|
||||
if (now === prunedAt) return;
|
||||
prunedAt = now;
|
||||
for (const [feed, until] of storage.lossUntil)
|
||||
if (until <= now) storage.lossUntil.delete(feed);
|
||||
for (const entry of entries.values()) {
|
||||
while (entry.count && firstTime(entry) < now - policy.retentionMs)
|
||||
drop(entry);
|
||||
if (!entry.count || now - entry.observedAt >= policy.retentionMs)
|
||||
remove(entry);
|
||||
else if (
|
||||
entry.capacity > CHUNK &&
|
||||
entry.count <= entry.capacity / 2 &&
|
||||
canAllocate(
|
||||
entry.feed,
|
||||
Math.max(CHUNK, Math.ceil(entry.count / CHUNK) * CHUNK) * FIX_BYTES,
|
||||
)
|
||||
)
|
||||
resize(entry, Math.max(CHUNK, Math.ceil(entry.count / CHUNK) * CHUNK));
|
||||
}
|
||||
}
|
||||
function oldest(feed = null, except = null) {
|
||||
return (feed ? feedEviction.get(feed) : eviction)?.oldest(except) || null;
|
||||
}
|
||||
function evictOne(feed = null, except = null) {
|
||||
const entry = oldest(feed, except);
|
||||
if (!entry) return false;
|
||||
recordLoss(entry.feed, entry.observedAt + policy.retentionMs);
|
||||
if (entry.count > CHUNK) {
|
||||
const target = Math.max(CHUNK, entry.capacity - CHUNK);
|
||||
if (!canAllocate(entry.feed, target * FIX_BYTES)) {
|
||||
remove(entry);
|
||||
return true;
|
||||
}
|
||||
while (entry.count > target) drop(entry);
|
||||
resize(entry, target);
|
||||
} else remove(entry);
|
||||
return true;
|
||||
}
|
||||
const api = {
|
||||
prune,
|
||||
evictOne,
|
||||
oldest,
|
||||
get allocatedBytes() {
|
||||
return allocatedBytes;
|
||||
},
|
||||
};
|
||||
storage.stores.add(api);
|
||||
function reserve(feed, bytes, except = null, newKey = false) {
|
||||
while ((storage.feedBytes.get(feed) || 0) + bytes > policy.feedBytes) {
|
||||
let owner = null,
|
||||
candidate = null;
|
||||
for (const store of storage.stores) {
|
||||
const entry = store.oldest(feed, except);
|
||||
if (entry && (!candidate || entry.observedAt < candidate.observedAt)) {
|
||||
owner = store;
|
||||
candidate = entry;
|
||||
}
|
||||
}
|
||||
if (!owner || !owner.evictOne(feed, except)) return false;
|
||||
}
|
||||
while (
|
||||
storage.bytes + bytes > policy.processBytes ||
|
||||
(newKey && storage.keys >= policy.keys)
|
||||
) {
|
||||
let owner = null,
|
||||
candidate = null;
|
||||
for (const store of storage.stores) {
|
||||
const entry = store.oldest(null, except);
|
||||
if (entry && (!candidate || entry.observedAt < candidate.observedAt)) {
|
||||
owner = store;
|
||||
candidate = entry;
|
||||
}
|
||||
}
|
||||
if (!owner || !owner.evictOne(null, except)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
function ingest(feed, records, now) {
|
||||
if (
|
||||
closed ||
|
||||
feed.historyRetention !== true ||
|
||||
!validTransitIdentifier(feed.id)
|
||||
)
|
||||
return;
|
||||
prune(now);
|
||||
// Pin all existing snapshot members before admitting any new key. A later
|
||||
// row must not lose its retained path to an earlier row in the same feed.
|
||||
protectedEntries = new Set();
|
||||
for (const record of records) {
|
||||
const entry = entries.get(`${feed.id}\u0000${record.id}`);
|
||||
if (entry) {
|
||||
protectedEntries.add(entry);
|
||||
unqueue(entry);
|
||||
}
|
||||
}
|
||||
try {
|
||||
for (const record of records) {
|
||||
if (
|
||||
!validTransitIdentifier(record.id) ||
|
||||
!Number.isFinite(record.lat) ||
|
||||
Math.abs(record.lat) > 90 ||
|
||||
!Number.isFinite(record.lon) ||
|
||||
Math.abs(record.lon) > 180
|
||||
)
|
||||
continue;
|
||||
const t = record.timestamp * 1000;
|
||||
if (
|
||||
!Number.isFinite(t) ||
|
||||
t < now - policy.retentionMs ||
|
||||
t > now + 10000
|
||||
)
|
||||
continue;
|
||||
const mode =
|
||||
feed.routeMode?.(record.routeId) || feed.defaultMode || 'unknown';
|
||||
const context = contextOf(record, mode);
|
||||
if (!context) continue;
|
||||
const key = `${feed.id}\u0000${record.id}`;
|
||||
let entry = entries.get(key);
|
||||
if (entry) {
|
||||
const last = read(entry, entry.count - 1);
|
||||
if (t <= last[0]) continue;
|
||||
} else {
|
||||
const encodedKey = encoder.encode(key),
|
||||
bytes =
|
||||
META_BYTES +
|
||||
EPOCH_BYTES +
|
||||
encodedKey.byteLength +
|
||||
CHUNK * FIX_BYTES;
|
||||
if (!reserve(feed.id, bytes, null, true)) {
|
||||
recordLoss(feed.id, now + policy.retentionMs);
|
||||
continue;
|
||||
}
|
||||
entry = {
|
||||
key,
|
||||
encodedKey,
|
||||
feed: feed.id,
|
||||
vehicle: record.id,
|
||||
metadata: new DataView(new ArrayBuffer(META_BYTES)),
|
||||
contexts: [new DataView(new ArrayBuffer(EPOCH_BYTES))],
|
||||
fixes: new DataView(new ArrayBuffer(CHUNK * FIX_BYTES)),
|
||||
capacity: CHUNK,
|
||||
head: 0,
|
||||
count: 0,
|
||||
epoch: 0,
|
||||
observedAt: now,
|
||||
truncated: (storage.lossUntil.get(feed.id) || 0) > now,
|
||||
bytes,
|
||||
};
|
||||
entries.set(key, entry);
|
||||
protectedEntries.add(entry);
|
||||
storage.keys++;
|
||||
account(feed.id, bytes);
|
||||
}
|
||||
let flags =
|
||||
record.timestampSource === 'vehicle'
|
||||
? 1
|
||||
: record.timestampSource === 'header'
|
||||
? 2
|
||||
: 4;
|
||||
const previous = entry.epoch
|
||||
? contextAt(entry, (entry.epoch - 1) % EPOCH_SLOTS)
|
||||
: null;
|
||||
if (
|
||||
!previous ||
|
||||
previous.trip !== context.trip ||
|
||||
previous.route !== context.route ||
|
||||
previous.mode !== context.mode
|
||||
) {
|
||||
const slot = entry.epoch % EPOCH_SLOTS;
|
||||
if (!entry.contexts[slot]) {
|
||||
if (!reserve(feed.id, EPOCH_BYTES, entry)) {
|
||||
entry.truncated = true;
|
||||
recordLoss(feed.id, now + policy.retentionMs);
|
||||
continue;
|
||||
}
|
||||
entry.contexts[slot] = new DataView(new ArrayBuffer(EPOCH_BYTES));
|
||||
entry.bytes += EPOCH_BYTES;
|
||||
account(feed.id, EPOCH_BYTES);
|
||||
}
|
||||
entry.epoch++;
|
||||
// Expiring epoch metadata also expires its positions; never label a
|
||||
// retained point with a reused context slot.
|
||||
const expiredEpoch = entry.epoch - EPOCH_SLOTS;
|
||||
while (entry.count && read(entry, 0)[4] <= expiredEpoch) drop(entry);
|
||||
storeContext(entry, context, entry.epoch);
|
||||
flags |= 16;
|
||||
}
|
||||
if (entry.count === entry.capacity) {
|
||||
const next = Math.min(policy.fixes, entry.capacity + CHUNK);
|
||||
if (
|
||||
next > entry.capacity &&
|
||||
reserve(feed.id, next * FIX_BYTES, entry)
|
||||
)
|
||||
resize(entry, next);
|
||||
else drop(entry);
|
||||
}
|
||||
const at = offset(entry, entry.count++),
|
||||
v = entry.fixes;
|
||||
v.setFloat64(at, t);
|
||||
v.setFloat64(at + 8, record.lat);
|
||||
v.setFloat64(at + 16, record.lon);
|
||||
v.setUint32(at + 24, flags);
|
||||
v.setUint32(at + 28, entry.epoch);
|
||||
entry.observedAt = now;
|
||||
}
|
||||
} finally {
|
||||
for (const entry of protectedEntries) queue(entry);
|
||||
protectedEntries = null;
|
||||
}
|
||||
}
|
||||
function get(feedId, vehicleId, now) {
|
||||
prune(now);
|
||||
const entry = entries.get(`${feedId}\u0000${vehicleId}`);
|
||||
const fixes = [],
|
||||
epochs = [];
|
||||
if (entry) {
|
||||
const used = new Set();
|
||||
for (let i = 0; i < entry.count; i++) {
|
||||
const fix = read(entry, i);
|
||||
fixes.push(fix);
|
||||
used.add(fix[4]);
|
||||
}
|
||||
for (const id of used)
|
||||
epochs.push(contextAt(entry, (id - 1) % EPOCH_SLOTS));
|
||||
}
|
||||
const payload = {
|
||||
version: 1,
|
||||
feedId,
|
||||
vehicleId,
|
||||
oldestT: fixes[0]?.[0] ?? null,
|
||||
newestT: fixes.at(-1)?.[0] ?? null,
|
||||
truncated:
|
||||
entry?.truncated ||
|
||||
(!entry && (storage.lossUntil.get(feedId) || 0) > now),
|
||||
epochs,
|
||||
fixes,
|
||||
};
|
||||
// The normal bound is smaller; keep a byte guard for worst-case Unicode.
|
||||
while (
|
||||
encoder.encode(JSON.stringify(payload)).length > policy.responseBytes &&
|
||||
payload.fixes.length
|
||||
) {
|
||||
payload.fixes.shift();
|
||||
payload.oldestT = payload.fixes[0]?.[0] ?? null;
|
||||
payload.truncated = true;
|
||||
const used = new Set(payload.fixes.map((f) => f[4]));
|
||||
payload.epochs = payload.epochs.filter((e) => used.has(e.id));
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
function clear() {
|
||||
if (timer) clearInterval(timer);
|
||||
timer = null;
|
||||
closed = true;
|
||||
for (const entry of entries.values()) remove(entry);
|
||||
storage.stores.delete(api);
|
||||
}
|
||||
if (sweep) {
|
||||
timer = setInterval(() => prune(Date.now()), 60000);
|
||||
timer.unref?.();
|
||||
}
|
||||
return {
|
||||
ingest,
|
||||
get,
|
||||
prune,
|
||||
clear,
|
||||
diagnostics: () => ({
|
||||
allocatedBytes,
|
||||
processBytes: storage.bytes,
|
||||
keys: entries.size,
|
||||
feedBytes: Object.fromEntries(feedBytes),
|
||||
sweepActive: timer !== null,
|
||||
}),
|
||||
};
|
||||
}
|
||||
export {
|
||||
createTransitHistory,
|
||||
TRANSIT_HISTORY_LIMITS,
|
||||
validTransitIdentifier,
|
||||
} from '../../src/sources/transitHistoryStore.js';
|
||||
Reference in new issue
Block a user