feat(rail): add a Rail layer with keyless live trains

Live trains from 30 keyless operator feeds: GPS positions from 29 and
Irish Rail's station reports, shown as the operators publish them. 28
work out of the box; Renfe long and medium distance and KTMB appear once
their pack is built, because their vehicles name no route.

Packs are opt-in (scripts/build-rail-paths.mjs): the operator's own
route lines, stop names, next stops and trip-update delays for live
trains. Nothing downloads when the app starts, no train is placed from a
timetable, and no request carries a deployer's key.
This commit is contained in:
Denis Groshev committed 2026-11-11 16:06:22 +01:00
1 parent 14db73f201
commit 830c080803
114 files changed
+26639 -26

No files matched your search

+527
View File
@@ -0,0 +1,527 @@
import {
FIX_FLAGS,
createLagReservoir,
destroyTrack,
readFix,
} from '../../data/contactPlayback.js';
import { createRailPathPack } from '../../data/railPaths.js';
import {
initializePlayback,
recordFix,
syncPlayback,
updatePlayback,
} from '../transit/movement.js';
import {
FEED_EVICT_AFTER_MS,
FEED_STALE_AFTER_MS,
FUTURE_STAMP_TOLERANCE_MS,
MAX_TRAINS_TOTAL,
MISSED_POLLS_TO_DROP,
isLiveRailPositionSource,
isStaleTrainFix,
playbackModeFor,
railTrainKey,
trainReportTimeMs,
} from './policy.js';
import { bindTrainPath, followFix, rebindTrainPath } from './pathBinding.js';
import { scheduleRetryDelayMs } from './schedules.js';
import { createDuplicateFilter } from './dedupe.js';
/**
* Fetching rail feeds through the proxy and turning snapshots into trains.
* The aging rules are the Transit layer's: a train is drawn from its own
* report time, a feed is judged by when its operator last answered, and both
* a quiet train and a quiet feed end in removal rather than a frozen marker.
*
* Only live positions are drawn (isLiveRailPositionSource). A feed whose
* positions fail is shown as unavailable: its trains are never placed from
* its timetable, and those already drawn age out as any quiet train does.
* @param {object} context
* @returns {object}
*/
export function createIngestion({ state, services, parts, source }) {
const { governorRequestRender } = services.render;
const { registerDynamicCredit, railFeedCredit } = services.credits;
// One physical train in several loaded feeds is drawn once (dedupe.js).
const duplicates = createDuplicateFilter({ state, parts });
function feedStatus(feedId) {
let status = state._feedStatus.get(feedId);
if (!status) {
status = {
count: 0,
lastUpdate: null,
upstreamAt: null,
error: null,
retryInSec: null,
stale: false,
pollSeq: 0,
loading: false,
lag: null,
/** New trains its last poll listed that MAX_TRAINS_TOTAL left out. */
capped: 0,
};
state._feedStatus.set(feedId, status);
}
return status;
}
function removeTrain(key) {
const entry = state._trains.get(key);
if (!entry) return;
if (state._selectedKey === key) parts.selection.clearSelection();
if (entry.marker) state._markers?.remove(entry.marker);
entry.marker = null;
state._moving.delete(entry);
parts.rendering.cancelWake(entry);
destroyTrack(entry.track);
state._trains.delete(key);
state._revision += 1;
// A copy it hid may now be the one to draw.
duplicates.scheduleRefresh();
}
/**
* Remove a feed's trains, except `keepKey` when given (the selected train,
* whose feed is held for it alone: viewport.js).
* @param {string} feedId
* @param {string|null} [keepKey]
*/
function removeFeedTrains(feedId, keepKey = null) {
for (const [key, entry] of state._trains) {
if (entry.feedId === feedId && key !== keepKey) removeTrain(key);
}
}
/**
* Drop trains whose report aged out and feeds whose operator went quiet.
* @param {number} nowMs
* @returns {number} Trains removed.
*/
function sweepAgedTrains(nowMs) {
let removed = 0;
const deadFeeds = new Set();
for (const [feedId, status] of state._feedStatus) {
if (
Number.isFinite(status.upstreamAt) &&
nowMs - status.upstreamAt > FEED_EVICT_AFTER_MS
)
deadFeeds.add(feedId);
}
for (const [key, entry] of state._trains) {
if (
deadFeeds.has(entry.feedId) ||
isStaleTrainFix(entry.record, nowMs, entry.fetchedAt)
) {
removeTrain(key);
removed += 1;
}
}
for (const feedId of deadFeeds) {
const status = state._feedStatus.get(feedId);
if (status) {
status.count = 0;
status.stale = true;
}
}
if (removed > 0) {
parts.rendering.syncRenderHold();
governorRequestRender('rail-evict');
state._dataManager?.refreshLayerStats?.();
}
return removed;
}
/**
* Fetch a feed's path pack once. A 404 means the feed has no paths and its
* trains keep straight-line motion; any other failure (a 503 from a pack
* the server cannot read now, say) is retried on the schedule packs'
* stepped delay, 30 s doubling to ten minutes (scheduleRetryDelayMs).
* @param {object} feed Registry entry.
*/
function ensurePaths(feed) {
const known = state._pathPacks.get(feed.id);
if (known && !(known.status === 'none' && Date.now() >= known.retryAt))
return;
if (typeof source.requestPaths !== 'function') {
state._pathPacks.set(feed.id, {
status: 'none',
pack: null,
retryAt: Infinity,
});
return;
}
const slot = {
status: 'loading',
pack: null,
retryAt: Infinity,
/** Failed loads in a row. */
failures: known?.failures ?? 0,
};
state._pathPacks.set(feed.id, slot);
void (async () => {
try {
const response = await source.requestPaths(feed.id);
if (!response.ok) {
const error = new Error(`rail paths HTTP ${response.status}`);
error.status = response.status;
throw error;
}
const raw = await response.json();
const pack = createRailPathPack(raw);
if (pack.feedId !== feed.id)
throw new Error('rail path pack is for another feed');
if (state._pathPacks.get(feed.id) !== slot) return;
slot.pack = pack;
slot.status = 'ready';
bindFeedTrains(feed.id);
// The faint network is drawn from the same pack, built once here.
parts.routes.buildNetwork(feed.id, raw);
} catch (error) {
slot.status = 'none';
slot.failures += 1;
slot.retryAt =
error?.status === 404
? Infinity
: Date.now() + scheduleRetryDelayMs(slot.failures);
if (error?.status !== 404)
console.warn(
`[Data:Rail] ${feed.id} paths unavailable:`,
error?.message || error,
);
}
})();
}
/**
* Keep a train on its railway line: bind it when its trip or route changes,
* follow each new report, and move it to another of its route's shapes (or
* back to straight lines) when a report lands off the bound one.
*/
function followPath(entry, record, fix) {
const pack = state._pathPacks.get(entry.feedId)?.pack;
if (!pack) return;
const ids = { tripId: record.tripId || '', routeId: record.routeId || '' };
const key = `${ids.tripId}\n${ids.routeId}`;
if (entry.pathKey !== key) {
entry.pathKey = key;
entry.path = bindTrainPath(pack, ids, fix);
return;
}
if (!entry.path || followFix(entry.path, fix)) return;
entry.path = rebindTrainPath(pack, ids, entry.path, fix);
}
/** Bind the trains a feed already shows once its pack arrives. */
function bindFeedTrains(feedId) {
const newest = {};
for (const entry of state._trains.values()) {
if (entry.feedId !== feedId || entry.pathKey !== undefined) continue;
if (!entry.track?.count) continue;
readFix(entry.track, entry.track.count - 1, newest);
followPath(entry, entry.record, newest);
parts.rendering.schedulePlayback(entry);
}
governorRequestRender('rail-paths');
}
/** A playback fix for one record, with the time it was really reported. */
function fixFor(record, fetchedAt) {
const stamped = trainReportTimeMs(record, fetchedAt);
const reportAt =
stamped > fetchedAt + FUTURE_STAMP_TOLERANCE_MS ? fetchedAt : stamped;
return {
t: reportAt,
lat: record.lat,
lon: record.lon,
bearingDeg: Number.isFinite(record.bearing) ? record.bearing : NaN,
flags:
stamped !== reportAt
? FIX_FLAGS.RECEIPT_TIME
: record.timestampSource === 'vehicle'
? FIX_FLAGS.VEHICLE_TIME
: record.timestampSource === 'header'
? FIX_FLAGS.FEED_TIME
: FIX_FLAGS.RECEIPT_TIME,
};
}
/** Record a new report for a train already drawn. */
function advanceTrain(entry, record, fix, observation) {
const heard = recordFix(entry, fix, observation);
if (!heard.accepted) return heard;
followPath(entry, record, fix);
if (entry.category !== record.category || entry.railMode !== record.mode) {
entry.category = record.category;
entry.railMode = record.mode;
entry.mode = playbackModeFor(record.mode);
parts.rendering.paint(entry);
}
updatePlayback(entry, observation.wallNowMs, observation.monoNowMs);
parts.rendering.schedulePlayback(entry);
return heard;
}
/** Create and draw a train the layer has not seen before. */
function addTrain(key, feed, record, fix, observation, status) {
const entry = {
key,
feedId: feed.id,
marker: null,
category: record.category,
railMode: record.mode,
mode: playbackModeFor(record.mode),
// A route the classifier named is a fact about the train, so its
// speed ceiling applies; the movement module reads this flag.
modeInferred: false,
courseDeg: Number.isFinite(record.bearing) ? record.bearing : null,
courseAt: observation.wallNowMs,
courseEvalAt: observation.monoNowMs,
record,
fetchedAt: observation.fetchedAt,
pollSeq: status.pollSeq,
/** Binding to a railway line (pathBinding.js), null for straight lines. */
path: null,
/** Trip and route the binding was made for; undefined until tried. */
pathKey: undefined,
};
initializePlayback(entry, state._historyBudget, status.lag);
const first = recordFix(entry, fix, observation);
if (!first.accepted) {
destroyTrack(entry.track);
return null;
}
followPath(entry, record, fix);
syncPlayback(entry, observation.wallNowMs, observation.monoNowMs);
entry.marker = parts.rendering.addMarker(entry);
state._trains.set(key, entry);
parts.rendering.schedulePlayback(entry);
return entry;
}
function applySnapshot(feed, snapshot, { stale, contactedAt = null }) {
const now = Date.now();
const fetchedAt = Math.min(
Number.isFinite(snapshot?.fetchedAt) ? snapshot.fetchedAt : now,
now,
);
const contacted = Math.min(
Number.isFinite(contactedAt) ? contactedAt : fetchedAt,
now,
);
const status = feedStatus(feed.id);
status.upstreamAt = Math.max(contacted, fetchedAt);
const upstreamAgeMs = now - status.upstreamAt;
status.loading = false;
status.error = null;
status.retryInSec = null;
status.stale = stale === true || upstreamAgeMs > FEED_STALE_AFTER_MS;
if (upstreamAgeMs > FEED_EVICT_AFTER_MS) {
removeFeedTrains(feed.id);
status.count = 0;
parts.rendering.syncRenderHold();
state._dataManager?.refreshLayerStats?.();
return;
}
status.pollSeq += 1;
if (!status.lag) status.lag = createLagReservoir();
const replayed = status.snapshotAt === fetchedAt || stale === true;
status.snapshotAt = fetchedAt;
const monoNow = performance.now();
// A feed held only for the selected train (viewport.js) carries it alone,
// or a train its operator now lists on the same trip.
const only =
state._heldFeedId === feed.id
? state._trains.get(state._selectedKey)
: null;
let seen = 0;
// The cap counts the trains drawn: a copy another feed draws better
// (dedupe.js) is hidden and does not count.
let held = 0;
for (const entry of state._trains.values())
if (entry.duplicate !== true) held += 1;
let capped = 0;
for (const record of snapshot.trains || []) {
// Only a reported position is drawn, never one from a timetable.
if (!isLiveRailPositionSource(record?.positionSource)) continue;
if (isStaleTrainFix(record, now, fetchedAt)) continue;
const key = railTrainKey(feed.id, record.id);
if (
only &&
key !== only.key &&
!(only.record?.tripId && record.tripId === only.record.tripId)
)
continue;
const fix = fixFor(record, fetchedAt);
const observation = {
receivedAt: now,
monoNowMs: monoNow,
wallNowMs: now,
fetchedAt,
replayed,
modeInferred: false,
context: {
trip: record.tripId || '',
route: record.routeId || '',
mode: playbackModeFor(record.mode),
},
};
let entry = state._trains.get(key);
if (entry) {
const heard = advanceTrain(entry, record, fix, observation);
entry.pollSeq = status.pollSeq;
seen += 1;
if (!heard.accepted) continue;
} else {
if (held >= MAX_TRAINS_TOTAL) {
capped += 1;
continue;
}
entry = addTrain(key, feed, record, fix, observation, status);
if (!entry) continue;
held += 1;
seen += 1;
}
entry.record = record;
entry.fetchedAt = fetchedAt;
}
// Said in the layer's stats as a partial snapshot (controls.js).
status.capped = capped;
if (capped > 0 && !state._capWarned) {
state._capWarned = true;
console.warn(
`[Data:Rail] train cap ${MAX_TRAINS_TOTAL} reached; ${capped} trains of ${feed.id} are not drawn`,
);
}
for (const [key, entry] of state._trains) {
if (
entry.feedId === feed.id &&
status.pollSeq - entry.pollSeq >= MISSED_POLLS_TO_DROP
)
removeTrain(key);
}
sweepAgedTrains(now);
duplicates.scheduleRefresh();
status.count = seen;
status.lastUpdate = fetchedAt;
state._lastUpdate = Math.max(state._lastUpdate || 0, fetchedAt);
state._error = null;
if (state._viewer)
registerDynamicCredit(state._viewer, railFeedCredit(feed));
if (state._trains.get(state._selectedKey)?.feedId === feed.id)
parts.selection.refreshSelectedCard(true);
parts.rendering.warmFloors(feed.id);
state._revision += 1;
parts.rendering.syncRenderHold();
governorRequestRender('rail-poll');
state._dataManager?.refreshLayerStats?.();
parts.controls.notify();
}
/**
* A feed whose live positions failed (or that its server has none for):
* its row says it is unavailable. Nothing takes the place of its trains;
* those already drawn go as their reports age (sweepAgedTrains).
* @param {object} feed
* @param {object} status
* @param {{retryInSec?: number}|null} [error]
*/
function markUnavailable(feed, status, error = null) {
// i18n-ignore: English data, translated on display (ui/statusText.js)
status.error = `${feed.name} feed unavailable`;
status.retryInSec = Number.isFinite(error?.retryInSec)
? error.retryInSec
: null;
status.loading = false;
state._error = status.error;
sweepAgedTrains(Date.now());
state._dataManager?.refreshLayerStats?.();
}
/**
* Poll one feed. A request already in flight is reused rather than aborted,
* so enable() and the manager's first update() share one request.
* @param {object} feed Registry entry.
* @param {number} generation Enable generation the poll belongs to.
* @returns {Promise<void>}
*/
function pollFeed(feed, generation) {
if (!state._enabled || generation !== state._generation)
return Promise.resolve();
ensurePaths(feed);
const status = feedStatus(feed.id);
const existing = state._inFlight.get(feed.id);
if (existing) return existing.promise;
const controller = new AbortController();
status.loading = status.count === 0;
const promise = (async () => {
try {
const response = await source.requestSnapshot(feed.id, {
signal: controller.signal,
});
if (!response.ok) {
const detail = await response.json().catch(() => null);
const error = new Error(`rail proxy HTTP ${response.status}`);
error.status = response.status;
if (Number.isFinite(detail?.retryInSec))
error.retryInSec = detail.retryInSec;
throw error;
}
const snapshot = await response.json();
if (
!state._enabled ||
generation !== state._generation ||
!state._activeFeeds.has(feed.id)
)
return;
// An answer whose positions are not live (anything but GPS or
// station reports) has no live data for this feed: none of its
// trains are drawn, and the feed shows unavailable.
if (!isLiveRailPositionSource(snapshot?.positions)) {
console.warn(
`[Data:Rail] ${feed.id} has no live positions on this server`,
);
markUnavailable(feed, status);
return;
}
const contact = Number.parseInt(
response.headers.get('x-rail-contact') || '',
10,
);
applySnapshot(feed, snapshot, {
stale: response.headers.get('x-gev-cache') === 'STALE-ERROR',
contactedAt: Number.isFinite(contact) ? contact : null,
});
} catch (error) {
if (error?.name === 'AbortError') return;
if (generation !== state._generation) return;
console.warn(
`[Data:Rail] ${feed.id} poll failed:`,
error?.message || error,
);
markUnavailable(feed, status, error);
} finally {
if (state._inFlight.get(feed.id)?.controller === controller)
state._inFlight.delete(feed.id);
}
})();
state._inFlight.set(feed.id, { controller, promise });
return promise;
}
function abortAllInFlight() {
for (const { controller } of state._inFlight.values()) controller.abort();
state._inFlight.clear();
}
return {
feedStatus,
removeTrain,
removeFeedTrains,
sweepAgedTrains,
applySnapshot,
pollFeed,
abortAllInFlight,
duplicates,
};
}