Merge remote-tracking branch 'origin/main' into fix/tle-catalog-matching

src/sources/gbfsStations.js

@@ -25,6 +25,7 @@ export function normalizeGbfsBool(value, fallback = true) {
  * @returns {number|null}
  */
 export function toNonNegativeInteger(value, fallback = null) {
+  if (value == null || value === '') return fallback;
   const n = Number(value);
   if (!Number.isFinite(n) || n < 0) return fallback;
   return Math.round(n);

src/sources/httpBody.js

@@ -55,7 +55,9 @@ export async function readResponseJsonCapped(response, maxBytes, signal) {
 
 /**
  * Read a fetch() Response body as bytes with the same hard cap as
- * readResponseTextCapped — for protobuf upstreams (GTFS-Realtime).
+ * readResponseTextCapped — for protobuf and tile upstreams (GTFS-Realtime,
+ * TomTom flow). Same early Content-Length rejection, running cap while
+ * streaming and cancellation on abort, so callers can treat the two alike.
  * Throws { code:'RESPONSE_TOO_LARGE' }.
  */
 export async function readResponseBytesCapped(response, maxBytes, signal) {

src/sources/httpBody.test.mjs

@@ -0,0 +1,133 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readResponseBytesCapped } from './httpBody.js';
+
+/**
+ * A Response that streams `chunk` forever until the reader cancels it. The
+ * zero high-water mark keeps the stream from pulling eagerly at construction,
+ * so `pulls()` counts only what a reader actually asked for.
+ */
+function endlessResponse(chunk, onCancel, headers = {}) {
+  let pulls = 0;
+  const stream = new ReadableStream(
+    {
+      pull(controller) {
+        pulls += 1;
+        controller.enqueue(chunk);
+      },
+      cancel: onCancel,
+    },
+    { highWaterMark: 0 },
+  );
+  return { response: new Response(stream, { headers }), pulls: () => pulls };
+}
+
+function chunkedResponse(chunks, headers = {}) {
+  const stream = new ReadableStream(
+    {
+      start(controller) {
+        for (const chunk of chunks) controller.enqueue(chunk);
+        controller.close();
+      },
+    },
+    { highWaterMark: 0 },
+  );
+  return new Response(stream, { headers });
+}
+
+const bytes = (n, fill) => new Uint8Array(n).fill(fill);
+
+test('returns the exact bytes of a body that fits, untouched by text decoding', async () => {
+  // 0xFF and 0xFE are invalid UTF-8 lead bytes; a text path would mangle them.
+  const payload = Uint8Array.from([0xff, 0xfe, 0x00, 0x7f, 0x80, 0x01]);
+  const buf = await readResponseBytesCapped(chunkedResponse([payload]), 1024);
+  assert.ok(buf instanceof Uint8Array);
+  assert.deepEqual([...buf], [...payload]);
+});
+
+test('joins chunks in order and reports the total length', async () => {
+  const buf = await readResponseBytesCapped(
+    chunkedResponse([bytes(3, 1), bytes(2, 2), bytes(1, 3)]),
+    1024,
+  );
+  assert.deepEqual([...buf], [1, 1, 1, 2, 2, 3]);
+});
+
+test('refuses on a declared Content-Length without pulling the body', async () => {
+  let cancelled = false;
+  const { response, pulls } = endlessResponse(
+    bytes(512, 0),
+    () => {
+      cancelled = true;
+    },
+    { 'content-length': '4096' },
+  );
+  await assert.rejects(readResponseBytesCapped(response, 1024), {
+    code: 'RESPONSE_TOO_LARGE',
+  });
+  assert.equal(pulls(), 0);
+  assert.ok(cancelled);
+});
+
+test('refuses a chunked body the moment it passes the cap and stops pulling', async () => {
+  let cancelled = false;
+  const { response, pulls } = endlessResponse(bytes(512, 0), () => {
+    cancelled = true;
+  });
+  await assert.rejects(readResponseBytesCapped(response, 1024), {
+    code: 'RESPONSE_TOO_LARGE',
+  });
+  // Three 512-byte chunks is the first total over 1024; an uncapped read of
+  // this never-ending stream would not terminate at all.
+  assert.ok(pulls() <= 3, `pulled ${pulls()} chunks`);
+  assert.ok(cancelled);
+});
+
+test('accepts a body exactly at the cap', async () => {
+  const buf = await readResponseBytesCapped(
+    chunkedResponse([bytes(512, 7), bytes(512, 7)]),
+    1024,
+  );
+  assert.equal(buf.byteLength, 1024);
+});
+
+test('cancels a body still streaming when the signal aborts, and never returns a partial one', async () => {
+  const controller = new AbortController();
+  let cancelled = false;
+  let pulls = 0;
+  const stream = new ReadableStream(
+    {
+      pull(ctrl) {
+        pulls += 1;
+        ctrl.enqueue(bytes(64, 0));
+        if (pulls === 2) controller.abort();
+      },
+      cancel() {
+        cancelled = true;
+      },
+    },
+    { highWaterMark: 0 },
+  );
+  await assert.rejects(
+    readResponseBytesCapped(new Response(stream), 1 << 20, controller.signal),
+  );
+  assert.ok(cancelled);
+  assert.ok(pulls <= 3, `pulled ${pulls} chunks after abort`);
+});
+
+test('refuses immediately when the signal is already aborted', async () => {
+  const controller = new AbortController();
+  controller.abort();
+  const { response, pulls } = endlessResponse(bytes(8, 0), () => {});
+  await assert.rejects(
+    readResponseBytesCapped(response, 1024, controller.signal),
+  );
+  assert.equal(pulls(), 0);
+});
+
+test('falls back to arrayBuffer() for a body with no stream reader', async () => {
+  const response = new Response(bytes(16, 9));
+  const buf = await readResponseBytesCapped(response, 1024);
+  assert.equal(buf.byteLength, 16);
+  assert.equal(buf[0], 9);
+});

src/sources/live/aircraft.js

@@ -7,10 +7,12 @@ import {
   finite,
 } from './contract.js';
 
+const openSkyId = (row) =>
+  Array.isArray(row) ? cleanText(row[0]).toLowerCase() : '';
+
 /** Normalize the OpenSky state-vector route, including its regional fallback. */
 export function normalizeOpenSkyAircraft(row) {
-  if (!Array.isArray(row)) return null;
-  const id = cleanText(row[0]).toLowerCase();
+  const id = openSkyId(row);
   if (!id || !coordinates(row[6], row[5])) return null;
   return {
     id,
@@ -77,6 +79,16 @@ export function normalizeReadsbAircraft(row, snapshotTimeMs) {
   };
 }
 
+/**
+ * OpenSky allows null longitude/latitude for a contact that is still
+ * broadcasting. Strict nulls also reject rows truncated before the position.
+ */
+function openSkyUnpositionedId(row) {
+  return Array.isArray(row) && row[5] === null && row[6] === null
+    ? openSkyId(row)
+    : '';
+}
+
 export function openSkySnapshot(
   payload,
   {
@@ -96,6 +108,7 @@ export function openSkySnapshot(
       return record;
     },
     'OpenSky',
+    openSkyUnpositionedId,
   );
   const observedAtMs = epoch(payload?.time, 1000);
   const ageMs = observedAtMs == null ? null : Math.max(0, now - observedAtMs);

src/sources/live/contract.js

@@ -53,17 +53,30 @@ export function coordinates(latitude, longitude) {
   );
 }
 
-/** Admit a snapshot atomically; an all-invalid nonempty feed is unavailable. */
-export function admitRecords(rows, normalize, label) {
+/**
+ * Admit a snapshot atomically; an all-invalid nonempty feed is unavailable.
+ * `unpositionedId(row)` names a rejected row that is a valid contact without
+ * a position: it is returned in `retainIds` and does not make the snapshot
+ * incomplete.
+ */
+export function admitRecords(rows, normalize, label, unpositionedId) {
   if (!Array.isArray(rows))
     throw new LiveSourceError('malformed', `Malformed ${label} response`);
   const records = [];
   const ids = new Set();
+  const retainIds = new Set();
+  let unpositioned = 0;
   for (const row of rows) {
     const record = normalize(row);
     if (record && !ids.has(record.id)) {
       records.push(record);
       ids.add(record.id);
+    } else if (!record) {
+      const id = unpositionedId?.(row);
+      if (id) {
+        retainIds.add(id);
+        unpositioned += 1;
+      }
     }
   }
   if (rows.length && !records.length) {
@@ -71,7 +84,8 @@ export function admitRecords(rows, normalize, label) {
   }
   return {
     records,
-    complete: records.length === rows.length,
+    retainIds,
+    complete: records.length + unpositioned === rows.length,
     rejectedCount: rows.length - records.length,
   };
 }

src/sources/live/contract.test.mjs

@@ -67,6 +67,50 @@ test('mixed-invalid and duplicate admissions are explicitly incomplete; all-inva
   assert.equal(openSkySnapshot({ states: [] }).complete, true);
 });
 
+test('position-less OpenSky rows are seen, not missing; malformed rows stay incomplete', () => {
+  // OpenSky's spec allows null longitude/latitude; every live /states/all
+  // carries such rows.
+  const unpositioned = ['DEF456', ...aircraft.slice(1)];
+  unpositioned[5] = null;
+  unpositioned[6] = null;
+  const snapshot = openSkySnapshot({ states: [aircraft, unpositioned] });
+  assert.equal(snapshot.records.length, 1);
+  assert.equal(snapshot.complete, true);
+  assert.deepEqual([...snapshot.retainIds], ['def456']);
+  assert.equal(snapshot.rejectedCount, 1);
+  const malformed = [...aircraft];
+  malformed[5] = 181;
+  const garbled = openSkySnapshot({
+    states: [aircraft, unpositioned, malformed],
+  });
+  assert.equal(garbled.complete, false);
+  assert.deepEqual([...garbled.retainIds], ['def456']);
+  // No usable id, only half a position, a truncated row, or no row at all.
+  for (const row of [
+    [' ', ...unpositioned.slice(1)],
+    Object.assign([...unpositioned], { 5: -97 }),
+    ['DEF456'],
+    null,
+  ])
+    assert.equal(openSkySnapshot({ states: [aircraft, row] }).complete, false);
+  // Repeated position-less rows stay complete.
+  const repeated = openSkySnapshot({
+    states: [aircraft, unpositioned, unpositioned],
+  });
+  assert.equal(repeated.complete, true);
+  // A feed with no positions at all still cannot prove absence.
+  assert.throws(() => openSkySnapshot({ states: [unpositioned] }), {
+    code: 'malformed',
+  });
+  // Military readsb admission keeps its existing semantics.
+  const military = readsbSnapshot(
+    { ac: [{ hex: 'abc123', lat: 30, lon: -97 }, { hex: 'def456' }] },
+    { observedAtMs: null },
+  );
+  assert.equal(military.complete, false);
+  assert.equal(military.retainIds.size, 0);
+});
+
 test('readsb units and position/contact ages are normalized once against the source epoch', () => {
   const row = normalizeReadsbAircraft(
     {

src/sources/reference.js

@@ -1,4 +1,5 @@
 import { createUsgsEarthquakeSource } from '../layers/earthquakes/source.js';
+import { createEcccAirQualitySource } from '../layers/airQuality/source.js';
 import { createWfigsPerimeterSource } from '../layers/perimeters/source.js';
 import { createBundledCableSource } from '../layers/submarineCables/bundledSource.js';
 import { createAdsbGnssSource } from '../layers/gnss/source.js';
@@ -7,6 +8,7 @@ import { createAdsbGnssSource } from '../layers/gnss/source.js';
 export function createReferenceSources() {
   return {
     earthquakes: createUsgsEarthquakeSource(),
+    'air-quality': createEcccAirQualitySource(),
     'fire-perimeters': createWfigsPerimeterSource(),
     cables: createBundledCableSource(),
     'gnss-integrity': createAdsbGnssSource(),

src/sources/reference.test.mjs

@@ -14,15 +14,18 @@ test('reference factories retain compatibility without starting acquisition or s
   const second = createReferenceSources();
   assert.deepEqual(Object.keys(first), [
     'earthquakes',
+    'air-quality',
     'fire-perimeters',
     'cables',
     'gnss-integrity',
   ]);
   assert.notEqual(first.earthquakes, second.earthquakes);
+  assert.notEqual(first['air-quality'], second['air-quality']);
   assert.notEqual(first['fire-perimeters'], second['fire-perimeters']);
   assert.notEqual(first.cables, second.cables);
   assert.notEqual(first['gnss-integrity'], second['gnss-integrity']);
   assert.equal(typeof first.earthquakes.getSnapshot, 'function');
+  assert.equal(typeof first['air-quality'].getSnapshot, 'function');
   assert.equal(typeof first['fire-perimeters'].getSnapshot, 'function');
   assert.equal(typeof first.cables.fetch, 'function');
   assert.equal(typeof first['gnss-integrity'].getSnapshot, 'function');

src/sources/tle.js

@@ -28,8 +28,36 @@ export function parseTleText(text) {
   return result;
 }
 
+/**
+ * The catalog number an Alpha-5 field stands for, or null for any other field.
+ *
+ * Since the catalog passed 99999 (July 2026) a TLE writes the numbers above it
+ * in Alpha-5: a letter A–Z, skipping I and O, stands for 10–33 in the
+ * ten-thousands place, so 100000 is "A0000" and 339999 is "Z9999".
+ */
+function alpha5CatalogNumber(field) {
+  const letter = field[0];
+  if (
+    letter >= 'A' &&
+    letter <= 'Z' &&
+    letter !== 'I' &&
+    letter !== 'O' &&
+    /^\d{4}$/.test(field.slice(1))
+  ) {
+    let tens = letter.charCodeAt(0) - 65 + 10;
+    if (letter > 'I') tens -= 1;
+    if (letter > 'O') tens -= 1;
+    return tens * 10000 + Number(field.slice(1));
+  }
+  return null;
+}
+
 /** The NORAD catalog number from a TLE line, or null. */
 export function tleCatalogNumber(line) {
-  const number = Number.parseInt(String(line).slice(2, 7), 10);
+  const field = String(line).slice(2, 7);
+  const alpha5 = alpha5CatalogNumber(field);
+  if (alpha5 !== null) return alpha5;
+  const number = Number.parseInt(field, 10);
   return Number.isInteger(number) ? number : null;
 }
+

src/sources/transitHistoryStore.js

@@ -328,8 +328,12 @@ export function createTransitHistory({
           t > now + 10000
         )
           continue;
+        const byVehicle = feed.vehicleMode?.(record.id);
         const mode =
-          feed.routeMode?.(record.routeId) || feed.defaultMode || 'unknown';
+          (byVehicle !== 'unknown' && byVehicle) ||
+          feed.routeMode?.(record.routeId) ||
+          feed.defaultMode ||
+          'unknown';
         const context = contextOf(record, mode);
         if (!context) continue;
         const key = `${feed.id}\u0000${record.id}`;