Merge pull request #1435 from Rohilalala/fix/d-military-positionless-rows

fix(military): don't treat adsb.lol's position-less rows as a partial snapshot

changelog.d/1435-military-positionless-rows.md

@@ -0,0 +1,5 @@
+- Military aircraft that leave the adsb.lol feed fade and are removed again.
+  The feed's routine position-less rows no longer mark every snapshot partial,
+  which kept vanished aircraft at full opacity for up to five minutes; those
+  rows keep only their own aircraft, and a likely-landed one still leaves on
+  the landed limit (Aditya, @Rohilalala, #1435).

docs/CURRENT-STATE.md

@@ -1197,12 +1197,12 @@ addition to the locally accumulated trail, never a promise of complete coverage.
 Snapshot coverage, completeness and freshness are separate fields. A partially
 admitted aircraft snapshot retains absent contacts for up to five minutes before
 the usual missed-poll eviction. An OpenSky row with an ICAO24 address but null
-longitude and latitude is a contact without a position, not a malformed row: it
+longitude and latitude, or a readsb row with a hex but no `lat`/`lon`, is a
-leaves the snapshot complete and is listed in `retainIds`, so civil flights
+contact without a position, not a malformed row: it leaves the snapshot complete
-retain that aircraft under the same five-minute bound, unless it is likely
+and is listed in `retainIds`, so civil and military flights retain that aircraft
-landed, while other absent aircraft fade and leave on the missed-poll limits. A
+under the same five-minute bound, unless it is likely landed, while other absent
-nonempty snapshot with no positioned rows is still malformed. Military readsb
+aircraft fade and leave on the missed-poll limits. A nonempty snapshot with no
-admission is unchanged. Unknown snapshot times remain unknown in stats. An invalid nonempty
+positioned rows is still malformed. Unknown snapshot times remain unknown in stats. An invalid nonempty
 snapshot retains the previous display. Empty vessel refreshes retain the existing
 first-connect grace and warm-data behavior. Known source failures have bounded
 messages; arbitrary HTTP response bodies are not surfaced as diagnostics.

src/layers/military/ownership.test.mjs

@@ -28,6 +28,29 @@ function services() {
   };
 }
 
+// The application's own scene services, as src/app/layers/militaryFlights.js
+// composes them; only the surface samplers and registry are stubbed.
+async function sceneServices() {
+  return {
+    picking: await import('../../data/pickRegistry.js'),
+    sprites: await import('../../data/spriteOrder.js'),
+    trails: await import('../../data/trailRenderer.js'),
+    aircraftPresentation: await import('../../data/tr3bRegistry.js'),
+    camera: await import('../../data/trackedCamera.js'),
+    labels: await import('../../data/detectionDraw.js'),
+    geoid: await import('../../data/geoid.js'),
+    focus: await import('../../data/focusDeemphasis.js'),
+    readout: await import('../../data/trackedReadout.js'),
+    context: await import('../../data/contextStore.js'),
+    render: await import('../../renderGovernor.js'),
+    recession: await import('../../data/aircraftRecession.js'),
+    groundFloor: await import('../../data/groundFloor.js'),
+    groundSnap: await import('../../data/groundSnap.js'),
+    meshFloor: { sampleMeshFloorCells: () => {} },
+    militaryRegistry: { registerMilitaryIcaos: () => {} },
+  };
+}
+
 test('military layer instances isolate policy and restoration state without requesting a source', async () => {
   let requests = 0;
   const source = {
@@ -122,26 +145,7 @@ test('a newly seen military contact appears at its delayed position, not ahead o
   // the next fleet tick jumped it back.
   const Cesium = await import('cesium');
   const { RENDER_DELAY_SEC } = await import('./policy.js');
-  // The application's own scene services, as src/app/layers/militaryFlights.js
+  const supplied = await sceneServices();
-  // composes them; only the surface samplers and registry are stubbed.
-  const supplied = {
-    picking: await import('../../data/pickRegistry.js'),
-    sprites: await import('../../data/spriteOrder.js'),
-    trails: await import('../../data/trailRenderer.js'),
-    aircraftPresentation: await import('../../data/tr3bRegistry.js'),
-    camera: await import('../../data/trackedCamera.js'),
-    labels: await import('../../data/detectionDraw.js'),
-    geoid: await import('../../data/geoid.js'),
-    focus: await import('../../data/focusDeemphasis.js'),
-    readout: await import('../../data/trackedReadout.js'),
-    context: await import('../../data/contextStore.js'),
-    render: await import('../../renderGovernor.js'),
-    recession: await import('../../data/aircraftRecession.js'),
-    groundFloor: await import('../../data/groundFloor.js'),
-    groundSnap: await import('../../data/groundSnap.js'),
-    meshFloor: { sampleMeshFloorCells: () => {} },
-    militaryRegistry: { registerMilitaryIcaos: () => {} },
-  };
   const fixLon = -97.6;
   const fixLat = 30.3;
   const speedMps = 200;
@@ -213,6 +217,55 @@ test('a newly seen military contact appears at its delayed position, not ahead o
   );
 });
 
+test('a vanished military contact fades while position-less rows keep theirs', async () => {
+  // About a third of a live adsb.lol /v2/mil fetch is contacts without a
+  // position. Those rows marked every snapshot partial, so an aircraft that
+  // left the feed stayed at full opacity for five minutes.
+  const Cesium = await import('cesium');
+  const { readsbSnapshot } = await import('../../sources/live/aircraft.js');
+  const nowMs = Date.now();
+  const row = (hex, lon) => ({
+    hex,
+    lat: 30.3,
+    lon,
+    alt_baro: 20_000,
+    gs: 300,
+    track: 90,
+    seen_pos: 1,
+  });
+  const polls = [
+    [row('ae0001', -97.6), row('ae0002', -96.6), row('ae0003', -95.6)],
+    [row('ae0001', -97.5), { hex: 'ae0003' }],
+  ];
+  // Every poll also lists a contact that never has a position.
+  for (const poll of polls) poll.push({ hex: 'ae0009', type: 'mode_s' });
+  const layer = createMilitaryFlightLayer({
+    services: await sceneServices(),
+    source: {
+      label: 'Fixture aircraft',
+      getSnapshot: async () =>
+        readsbSnapshot(
+          { ac: polls.shift() },
+          { observedAtMs: nowMs, now: nowMs },
+        ),
+    },
+  });
+  const viewer = { camera: { positionCartographic: null }, scene: {} };
+  layer.testing._setTrackedMilitaryRefreshStateForTest({
+    icao24: 'seed00',
+    entity: null,
+    meta: { rawLat: 0, rawLon: 0, onGround: false },
+    billboard: { show: false, position: Cesium.Cartesian3.fromDegrees(0, 0) },
+    billboardCollection: { show: true, add: (options) => ({ ...options }) },
+    viewer,
+    tracked: false,
+  });
+  await layer.update(viewer);
+  await layer.update(viewer);
+  assert.equal(layer.findByQuery('ae0002')?.stale, true, 'vanished: fading');
+  assert.equal(layer.findByQuery('ae0003')?.stale, false, 'still listed');
+});
+
 test('voice context reads a track that rounds to north as 0°, not 360°', () => {
   const stubs = services();
   stubs.aircraftPresentation = { tr3bTypeLabel: (id, type) => type };

src/layers/military/records.js

@@ -187,9 +187,12 @@ export class MilitaryFlightRecords {
       fixEpochMs: aircraft.positionTimeMs ?? observedAtMs,
     };
   }
-  absence(id, { complete, likelyLanded }) {
+  absence(id, { complete, retainIds, likelyLanded }) {
+    // Partial admissions and position-less rows do not prove absence, but
+    // retention remains bounded; a likely-landed position-less contact
+    // still takes the landed limit.
     if (
-      !complete &&
+      (!complete || (retainIds?.has(id) && !likelyLanded)) &&
       Date.now() - (this.data.get(id)?.observedReceiptMs ?? 0) < 300000
     )
       return 'retain';

src/layers/military/records.test.mjs

@@ -99,6 +99,30 @@ test('military records keep bounded partial retention and release metadata only
   );
 });
 
+test('complete military snapshots age out vanished aircraft and retain position-less ones unless landed', () => {
+  const records = store();
+  for (const id of ['aaa001', 'aaa002', 'aaa003'])
+    records.receive(observation({ id, reference: id }), context());
+  // aaa001 and aaa003 are still listed without a position; aaa002 vanished.
+  const absent = (id, likelyLanded = false) =>
+    records.absence(id, {
+      complete: true,
+      retainIds: new Set(['aaa001', 'aaa003']),
+      likelyLanded,
+    });
+  assert.deepEqual(
+    [absent('aaa002'), absent('aaa002'), absent('aaa002')],
+    ['stale', 'stale', 'remove'],
+  );
+  assert.equal(absent('aaa001'), 'retain');
+  assert.equal(records.missingPolls.has('aaa001'), false);
+  // A landed contact that lost its position takes the landed fast-cull.
+  assert.equal(absent('aaa003', true), 'remove');
+  // Position-less retention stays bounded like partial admission.
+  records.data.get('aaa001').observedReceiptMs = Date.now() - 300001;
+  assert.equal(absent('aaa001'), 'stale');
+});
+
 test('military emergency status follows the latest poll instead of sticking', () => {
   const records = store();
   const first = records.receive(

src/layers/military/snapshotRenderer.js

@@ -259,6 +259,7 @@ export function createMilitarySnapshotRenderer({
       if (currentIcaos.has(icao24)) continue;
       const absence = records.absence(icao24, {
         complete: snapshot.complete,
+        retainIds: snapshot.retainIds,
         likelyLanded: queries._likelyLanded(icao24),
       });
       if (absence === 'retain') continue;

src/sources/live/aircraft.js

@@ -40,9 +40,11 @@ export function normalizeOpenSkyAircraft(row) {
   };
 }
 
+const readsbId = (row) => cleanText(row?.hex).toLowerCase();
+
 /** Normalize readsb observations. The caller supplies the source snapshot epoch. */
 export function normalizeReadsbAircraft(row, snapshotTimeMs) {
-  const id = cleanText(row?.hex).toLowerCase();
+  const id = readsbId(row);
   const latitude = finite(row?.lat),
     longitude = finite(row?.lon);
   if (!id || !coordinates(latitude, longitude)) return null;
@@ -129,6 +131,14 @@ export function openSkySnapshot(
   };
 }
 
+/**
+ * readsb omits lat/lon for a contact heard without a current position. Both
+ * must be absent; a half or unreadable position still marks the row malformed.
+ */
+function readsbUnpositionedId(row) {
+  return row?.lat == null && row?.lon == null ? readsbId(row) : '';
+}
+
 export function readsbSnapshot(
   payload,
   {
@@ -143,6 +153,7 @@ export function readsbSnapshot(
     payload?.ac,
     (row) => normalizeReadsbAircraft(row, observedAtMs),
     'adsb.lol',
+    readsbUnpositionedId,
   );
   const ageMs = observedAtMs == null ? null : Math.max(0, now - observedAtMs);
   return {

src/sources/live/contract.test.mjs

@@ -102,13 +102,34 @@ test('position-less OpenSky rows are seen, not missing; malformed rows stay inco
   assert.throws(() => openSkySnapshot({ states: [unpositioned] }), {
     code: 'malformed',
   });
-  // Military readsb admission keeps its existing semantics.
+});
-  const military = readsbSnapshot(
+
-    { ac: [{ hex: 'abc123', lat: 30, lon: -97 }, { hex: 'def456' }] },
+test('position-less readsb rows are seen, not missing; malformed rows stay incomplete', () => {
-    { observedAtMs: null },
+  // readsb omits lat/lon for a contact heard without a current position; a
-  );
+  // live adsb.lol /v2/mil fetch had 82 such rows of 239.
-  assert.equal(military.complete, false);
+  const positioned = { hex: 'ae638d', lat: 20.3, lon: -158.5, seen_pos: 0.2 };
-  assert.equal(military.retainIds.size, 0);
+  const heard = { hex: 'ADFAC3', type: 'mode_s', rr_lat: 19.9, seen: 24.2 };
+  const lastPosition = {
+    hex: '3b752d',
+    lastPosition: { lat: 45.3, lon: 5.98, seen_pos: 61.56 },
+  };
+  const read = (ac) => readsbSnapshot({ ac }, { observedAtMs: null });
+  const snapshot = read([positioned, heard, lastPosition]);
+  assert.equal(snapshot.records.length, 1);
+  assert.equal(snapshot.complete, true);
+  assert.deepEqual([...snapshot.retainIds], ['adfac3', '3b752d']);
+  assert.equal(snapshot.rejectedCount, 2);
+  // No usable id, only half a position, an unreadable one, or no row at all.
+  for (const row of [
+    { ...heard, hex: ' ' },
+    { ...heard, lat: 30 },
+    { ...heard, lon: -97 },
+    { ...heard, lat: 'x', lon: 'y' },
+    null,
+  ])
+    assert.equal(read([positioned, heard, row]).complete, false);
+  // A feed with no positions at all still cannot prove absence.
+  assert.throws(() => read([heard]), { code: 'malformed' });
 });
 
 test('readsb units and position/contact ages are normalized once against the source epoch', () => {
@@ -346,7 +367,8 @@ test('classification identities include positionless aircraft without admitting
     snapshot.records.map((record) => record.id),
     ['def456'],
   );
-  assert.equal(snapshot.complete, false);
+  assert.equal(snapshot.complete, true);
+  assert.deepEqual([...snapshot.retainIds], ['abc123']);
 });
 
 test('identity lookup validates its response and honors body-parse cancellation', async () => {