Merge pull request #1110 from Rohilalala/fix/flights-unpositioned-rows-complete

fix(flights): don't treat OpenSky's position-less rows as a partial snapshot

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+- Flights age out vanished aircraft again. OpenSky's routine position-less rows
+  no longer mark every snapshot partial, which had kept vanished and landed
+  aircraft at full opacity for up to five minutes; those rows retain only their
+  own aircraft, and a likely-landed one still leaves on the landed limit.
+
 - Add WSDOT highway cameras (Washington State) as a keyless CCTV source pack:
   WSDOT's official Travel Center camera layer, with frames registered only on
   WSDOT's own image host (partner-hosted cameras such as Oregon DOT and City of

docs/CURRENT-STATE.md

@@ -872,7 +872,13 @@ addition to the locally accumulated trail, never a promise of complete coverage.
 
 Snapshot coverage, completeness and freshness are separate fields. A partially
 admitted aircraft snapshot retains absent contacts for up to five minutes before
-the usual missed-poll eviction. Unknown snapshot times remain unknown in stats. An invalid nonempty
+the usual missed-poll eviction. An OpenSky row with an ICAO24 address but null
+longitude and latitude is a contact without a position, not a malformed row: it
+leaves the snapshot complete and is listed in `retainIds`, so civil flights
+retain that aircraft under the same five-minute bound, unless it is likely
+landed, while other absent aircraft fade and leave on the missed-poll limits. A
+nonempty snapshot with no positioned rows is still malformed. Military readsb
+admission is unchanged. Unknown snapshot times remain unknown in stats. An invalid nonempty
 snapshot retains the previous display. Empty vessel refreshes retain the existing
 first-connect grace and warm-data behavior. Known source failures have bounded
 messages; arbitrary HTTP response bodies are not surfaced as diagnostics.

src/layers/flights/records.js

@@ -275,10 +275,12 @@ export class FlightRecords {
     return { icao24, prevMeta, meta, groundFlipped, fixEpochMs };
   }
 
-  absence(id, { complete, likelyLanded }) {
-    // Partial admissions do not prove absence, but retention remains bounded.
+  absence(id, { complete, retainIds, likelyLanded }) {
+    // Partial admissions and position-less rows do not prove absence, but
+    // retention remains bounded; a likely-landed position-less contact
+    // still takes the landed limit.
     if (
-      !complete &&
+      (!complete || (retainIds?.has(id) && !likelyLanded)) &&
       Date.now() - (this.data.get(id)?.observedReceiptMs ?? 0) < 300000
     )
       return 'retain';

src/layers/flights/records.test.mjs

@@ -107,6 +107,30 @@ test('ground transitions preserve last render height while a surface cell is col
   assert.equal(next.meta.wasAirborne, true);
 });
 
+test('complete snapshots age out vanished aircraft and retain position-less ones unless landed', () => {
+  const store = records();
+  for (const id of ['abc123', 'def456', 'ghi789'])
+    store.receive(observation({ id, reference: id }), view());
+  // abc123 and ghi789 are still listed without a position; def456 vanished.
+  const absent = (id, likelyLanded = false) =>
+    store.absence(id, {
+      complete: true,
+      retainIds: new Set(['abc123', 'ghi789']),
+      likelyLanded,
+    });
+  assert.deepEqual(
+    [absent('def456'), absent('def456'), absent('def456')],
+    ['stale', 'stale', 'remove'],
+  );
+  assert.equal(absent('abc123'), 'retain');
+  assert.equal(store.missingPolls.has('abc123'), false);
+  // A landed contact that lost its position takes the landed fast-cull.
+  assert.equal(absent('ghi789', true), 'remove');
+  // Position-less retention stays bounded like partial admission.
+  store.data.get('abc123').observedReceiptMs = Date.now() - 300001;
+  assert.equal(absent('abc123'), 'stale');
+});
+
 test('civil emergency status follows the latest poll instead of sticking', () => {
   const store = records();
   const first = store.receive(observation({ squawk: '7700' }), view());

src/layers/flights/snapshotRenderer.js

@@ -243,6 +243,7 @@ export function createFlightSnapshotRenderer({
       if (currentIcaos.has(icao24)) continue;
       const absence = records.absence(icao24, {
         complete: snapshot.complete,
+        retainIds: snapshot.retainIds,
         likelyLanded: queries._likelyLanded(icao24),
       });
       if (absence === 'retain') continue;

src/sources/live/aircraft.js

@@ -7,10 +7,12 @@ import {
   finite,
 } from './contract.js';
 
+const openSkyId = (row) =>
+  Array.isArray(row) ? cleanText(row[0]).toLowerCase() : '';
+
 /** Normalize the OpenSky state-vector route, including its regional fallback. */
 export function normalizeOpenSkyAircraft(row) {
-  if (!Array.isArray(row)) return null;
-  const id = cleanText(row[0]).toLowerCase();
+  const id = openSkyId(row);
   if (!id || !coordinates(row[6], row[5])) return null;
   return {
     id,
@@ -77,6 +79,16 @@ export function normalizeReadsbAircraft(row, snapshotTimeMs) {
   };
 }
 
+/**
+ * OpenSky allows null longitude/latitude for a contact that is still
+ * broadcasting. Strict nulls also reject rows truncated before the position.
+ */
+function openSkyUnpositionedId(row) {
+  return Array.isArray(row) && row[5] === null && row[6] === null
+    ? openSkyId(row)
+    : '';
+}
+
 export function openSkySnapshot(
   payload,
   {
@@ -96,6 +108,7 @@ export function openSkySnapshot(
       return record;
     },
     'OpenSky',
+    openSkyUnpositionedId,
   );
   const observedAtMs = epoch(payload?.time, 1000);
   const ageMs = observedAtMs == null ? null : Math.max(0, now - observedAtMs);

src/sources/live/contract.js

@@ -53,17 +53,30 @@ export function coordinates(latitude, longitude) {
   );
 }
 
-/** Admit a snapshot atomically; an all-invalid nonempty feed is unavailable. */
-export function admitRecords(rows, normalize, label) {
+/**
+ * Admit a snapshot atomically; an all-invalid nonempty feed is unavailable.
+ * `unpositionedId(row)` names a rejected row that is a valid contact without
+ * a position: it is returned in `retainIds` and does not make the snapshot
+ * incomplete.
+ */
+export function admitRecords(rows, normalize, label, unpositionedId) {
   if (!Array.isArray(rows))
     throw new LiveSourceError('malformed', `Malformed ${label} response`);
   const records = [];
   const ids = new Set();
+  const retainIds = new Set();
+  let unpositioned = 0;
   for (const row of rows) {
     const record = normalize(row);
     if (record && !ids.has(record.id)) {
       records.push(record);
       ids.add(record.id);
+    } else if (!record) {
+      const id = unpositionedId?.(row);
+      if (id) {
+        retainIds.add(id);
+        unpositioned += 1;
+      }
     }
   }
   if (rows.length && !records.length) {
@@ -71,7 +84,8 @@ export function admitRecords(rows, normalize, label) {
   }
   return {
     records,
-    complete: records.length === rows.length,
+    retainIds,
+    complete: records.length + unpositioned === rows.length,
     rejectedCount: rows.length - records.length,
   };
 }

src/sources/live/contract.test.mjs

@@ -67,6 +67,50 @@ test('mixed-invalid and duplicate admissions are explicitly incomplete; all-inva
   assert.equal(openSkySnapshot({ states: [] }).complete, true);
 });
 
+test('position-less OpenSky rows are seen, not missing; malformed rows stay incomplete', () => {
+  // OpenSky's spec allows null longitude/latitude; every live /states/all
+  // carries such rows.
+  const unpositioned = ['DEF456', ...aircraft.slice(1)];
+  unpositioned[5] = null;
+  unpositioned[6] = null;
+  const snapshot = openSkySnapshot({ states: [aircraft, unpositioned] });
+  assert.equal(snapshot.records.length, 1);
+  assert.equal(snapshot.complete, true);
+  assert.deepEqual([...snapshot.retainIds], ['def456']);
+  assert.equal(snapshot.rejectedCount, 1);
+  const malformed = [...aircraft];
+  malformed[5] = 181;
+  const garbled = openSkySnapshot({
+    states: [aircraft, unpositioned, malformed],
+  });
+  assert.equal(garbled.complete, false);
+  assert.deepEqual([...garbled.retainIds], ['def456']);
+  // No usable id, only half a position, a truncated row, or no row at all.
+  for (const row of [
+    [' ', ...unpositioned.slice(1)],
+    Object.assign([...unpositioned], { 5: -97 }),
+    ['DEF456'],
+    null,
+  ])
+    assert.equal(openSkySnapshot({ states: [aircraft, row] }).complete, false);
+  // Repeated position-less rows stay complete.
+  const repeated = openSkySnapshot({
+    states: [aircraft, unpositioned, unpositioned],
+  });
+  assert.equal(repeated.complete, true);
+  // A feed with no positions at all still cannot prove absence.
+  assert.throws(() => openSkySnapshot({ states: [unpositioned] }), {
+    code: 'malformed',
+  });
+  // Military readsb admission keeps its existing semantics.
+  const military = readsbSnapshot(
+    { ac: [{ hex: 'abc123', lat: 30, lon: -97 }, { hex: 'def456' }] },
+    { observedAtMs: null },
+  );
+  assert.equal(military.complete, false);
+  assert.equal(military.retainIds.size, 0);
+});
+
 test('readsb units and position/contact ages are normalized once against the source epoch', () => {
   const row = normalizeReadsbAircraft(
     {