diff --git a/CHANGELOG.md b/CHANGELOG.md index d6d11d09b..c13343cee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ Manifest paths are relative to the manifest, so a build served under a path prefix installs and launches inside that prefix. +- Cyclone advisories no longer go dark while NHC publishes an advisory ahead + of its nominal time: advisory and position times up to 90 minutes ahead of + the server clock are accepted (previously 5 minutes), so the whole feed is + not rejected for about half an hour each advisory cycle. + +- Flights age out vanished aircraft again. OpenSky's routine position-less rows + no longer mark every snapshot partial, which had kept vanished and landed + aircraft at full opacity for up to five minutes; those rows retain only their + own aircraft, and a likely-landed one still leaves on the landed limit. + - Add WSDOT highway cameras (Washington State) as a keyless CCTV source pack: WSDOT's official Travel Center camera layer, with frames registered only on WSDOT's own image host (partner-hosted cameras such as Oregon DOT and City of diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 7f026775f..889e26431 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -339,7 +339,12 @@ Cyclones combine fixed NHC/CPHC status and summary GIS endpoints through a bound same-origin provider. The status endpoint lacks browser CORS; each installation fetches keyless public sources at runtime, with no centralized ingestion service. The five-minute snapshot covers Atlantic and eastern/central North Pacific only. -Current positions, advisory issue time and position time stay distinct. Tracks, +Current positions, advisory issue time and position time stay distinct. NHC may +release an advisory before its nominal time (routinely up to about 30 minutes, +rarely about an hour), so advisory issue and position times up to and including +90 minutes ahead of the server clock are accepted; 90 minutes is this app's +tolerance, not an NHC limit. Later, non-canonical, or more than 12-hour-old +times still reject the status snapshot. Tracks, forecast lead-hour points and cones render only when all source parts match the status advisory; a newer status shows its position with geometry pending rather than relabeling older geometry. The cone represents forecast center uncertainty, @@ -872,7 +877,13 @@ addition to the locally accumulated trail, never a promise of complete coverage. Snapshot coverage, completeness and freshness are separate fields. A partially admitted aircraft snapshot retains absent contacts for up to five minutes before -the usual missed-poll eviction. Unknown snapshot times remain unknown in stats. An invalid nonempty +the usual missed-poll eviction. An OpenSky row with an ICAO24 address but null +longitude and latitude is a contact without a position, not a malformed row: it +leaves the snapshot complete and is listed in `retainIds`, so civil flights +retain that aircraft under the same five-minute bound, unless it is likely +landed, while other absent aircraft fade and leave on the missed-poll limits. A +nonempty snapshot with no positioned rows is still malformed. Military readsb +admission is unchanged. Unknown snapshot times remain unknown in stats. An invalid nonempty snapshot retains the previous display. Empty vessel refreshes retain the existing first-connect grace and warm-data behavior. Known source failures have bounded messages; arbitrary HTTP response bodies are not surfaced as diagnostics. diff --git a/server/providers/cyclones.js b/server/providers/cyclones.js index fede1aade..d580357ce 100644 --- a/server/providers/cyclones.js +++ b/server/providers/cyclones.js @@ -41,6 +41,9 @@ function number(value, min, max) { const n = Number(value); return Number.isFinite(n) && n >= min && n <= max ? n : null; } +// NHC stamps an advisory with its nominal time (e.g. 03:00Z) and publishes it +// up to about half an hour before; allow that, but not a clock far ahead. +const EARLY_ADVISORY_MS = 90 * 60_000; function iso(value, now) { if ( typeof value !== 'string' || @@ -50,7 +53,7 @@ function iso(value, now) { const n = Date.parse(value); if ( !Number.isFinite(n) || - n > now + 5 * 60_000 || + n > now + EARLY_ADVISORY_MS || now - n > 12 * HOUR || new Date(n).toISOString().replace('.000Z', 'Z') !== value.replace('.000Z', 'Z') diff --git a/src/data/cyclonesProxy.test.mjs b/src/data/cyclonesProxy.test.mjs index 04729d33a..23e84826c 100644 --- a/src/data/cyclonesProxy.test.mjs +++ b/src/data/cyclonesProxy.test.mjs @@ -162,6 +162,44 @@ test('malformed, duplicate, oversized, future and expired status is rejected rat assert.deepEqual(parseCycloneStatus({ activeStorms: [] }, NOW), []); }); +test('advisories published before their nominal time are accepted, far-future ones are not', () => { + // NHC wrote the 03:00Z advisory at 02:34Z. + const early = Date.UTC(2026, 8, 16, 2, 34); + assert.equal(parseCycloneStatus(status(), early)[0].issuedAt, TIME); + assert.throws( + () => parseCycloneStatus(status(), Date.UTC(2026, 8, 16, 1, 25)), + /invalid_cyclone_data/, + ); +}); + +test('advisory and position times are bounded at 90 minutes ahead and 12 hours old', () => { + const MINUTE = 60_000; + const at = (offsetMs) => new Date(NOW + offsetMs).toISOString(); + const fields = { + issuedAt: (value) => + status({ + forecastAdvisory: { + advNum: '010', + issuance: value, + url: 'https://www.nhc.noaa.gov/text/MIATCMEP5.shtml', + }, + }), + positionAt: (value) => status({ lastUpdate: value }), + }; + for (const [field, payload] of Object.entries(fields)) { + for (const offset of [90 * MINUTE, -12 * 60 * MINUTE]) { + const [storm] = parseCycloneStatus(payload(at(offset)), NOW); + assert.equal(storm[field], at(offset), `${field} ${offset}`); + } + for (const offset of [90 * MINUTE + 1, -12 * 60 * MINUTE - 1]) + assert.throws( + () => parseCycloneStatus(payload(at(offset)), NOW), + /invalid_cyclone_data/, + `${field} ${offset}`, + ); + } +}); + test('coherent geometry is retained with holes/dateline while older advisory geometry is omitted', () => { const storms = parseCycloneStatus(status(), NOW); const parts = collections(); diff --git a/src/layers/flights/records.js b/src/layers/flights/records.js index 2b6f48dfb..709cd69ed 100644 --- a/src/layers/flights/records.js +++ b/src/layers/flights/records.js @@ -275,10 +275,12 @@ export class FlightRecords { return { icao24, prevMeta, meta, groundFlipped, fixEpochMs }; } - absence(id, { complete, likelyLanded }) { - // Partial admissions do not prove absence, but retention remains bounded. + absence(id, { complete, retainIds, likelyLanded }) { + // Partial admissions and position-less rows do not prove absence, but + // retention remains bounded; a likely-landed position-less contact + // still takes the landed limit. if ( - !complete && + (!complete || (retainIds?.has(id) && !likelyLanded)) && Date.now() - (this.data.get(id)?.observedReceiptMs ?? 0) < 300000 ) return 'retain'; diff --git a/src/layers/flights/records.test.mjs b/src/layers/flights/records.test.mjs index fff25b8b9..638c6f843 100644 --- a/src/layers/flights/records.test.mjs +++ b/src/layers/flights/records.test.mjs @@ -107,6 +107,30 @@ test('ground transitions preserve last render height while a surface cell is col assert.equal(next.meta.wasAirborne, true); }); +test('complete snapshots age out vanished aircraft and retain position-less ones unless landed', () => { + const store = records(); + for (const id of ['abc123', 'def456', 'ghi789']) + store.receive(observation({ id, reference: id }), view()); + // abc123 and ghi789 are still listed without a position; def456 vanished. + const absent = (id, likelyLanded = false) => + store.absence(id, { + complete: true, + retainIds: new Set(['abc123', 'ghi789']), + likelyLanded, + }); + assert.deepEqual( + [absent('def456'), absent('def456'), absent('def456')], + ['stale', 'stale', 'remove'], + ); + assert.equal(absent('abc123'), 'retain'); + assert.equal(store.missingPolls.has('abc123'), false); + // A landed contact that lost its position takes the landed fast-cull. + assert.equal(absent('ghi789', true), 'remove'); + // Position-less retention stays bounded like partial admission. + store.data.get('abc123').observedReceiptMs = Date.now() - 300001; + assert.equal(absent('abc123'), 'stale'); +}); + test('civil emergency status follows the latest poll instead of sticking', () => { const store = records(); const first = store.receive(observation({ squawk: '7700' }), view()); diff --git a/src/layers/flights/snapshotRenderer.js b/src/layers/flights/snapshotRenderer.js index f6e74b125..ec6c2933b 100644 --- a/src/layers/flights/snapshotRenderer.js +++ b/src/layers/flights/snapshotRenderer.js @@ -243,6 +243,7 @@ export function createFlightSnapshotRenderer({ if (currentIcaos.has(icao24)) continue; const absence = records.absence(icao24, { complete: snapshot.complete, + retainIds: snapshot.retainIds, likelyLanded: queries._likelyLanded(icao24), }); if (absence === 'retain') continue; diff --git a/src/sources/live/aircraft.js b/src/sources/live/aircraft.js index ebc27f559..1af1f6b47 100644 --- a/src/sources/live/aircraft.js +++ b/src/sources/live/aircraft.js @@ -7,10 +7,12 @@ import { finite, } from './contract.js'; +const openSkyId = (row) => + Array.isArray(row) ? cleanText(row[0]).toLowerCase() : ''; + /** Normalize the OpenSky state-vector route, including its regional fallback. */ export function normalizeOpenSkyAircraft(row) { - if (!Array.isArray(row)) return null; - const id = cleanText(row[0]).toLowerCase(); + const id = openSkyId(row); if (!id || !coordinates(row[6], row[5])) return null; return { id, @@ -77,6 +79,16 @@ export function normalizeReadsbAircraft(row, snapshotTimeMs) { }; } +/** + * OpenSky allows null longitude/latitude for a contact that is still + * broadcasting. Strict nulls also reject rows truncated before the position. + */ +function openSkyUnpositionedId(row) { + return Array.isArray(row) && row[5] === null && row[6] === null + ? openSkyId(row) + : ''; +} + export function openSkySnapshot( payload, { @@ -96,6 +108,7 @@ export function openSkySnapshot( return record; }, 'OpenSky', + openSkyUnpositionedId, ); const observedAtMs = epoch(payload?.time, 1000); const ageMs = observedAtMs == null ? null : Math.max(0, now - observedAtMs); diff --git a/src/sources/live/contract.js b/src/sources/live/contract.js index 2415ff43c..d99b55f0d 100644 --- a/src/sources/live/contract.js +++ b/src/sources/live/contract.js @@ -53,17 +53,30 @@ export function coordinates(latitude, longitude) { ); } -/** Admit a snapshot atomically; an all-invalid nonempty feed is unavailable. */ -export function admitRecords(rows, normalize, label) { +/** + * Admit a snapshot atomically; an all-invalid nonempty feed is unavailable. + * `unpositionedId(row)` names a rejected row that is a valid contact without + * a position: it is returned in `retainIds` and does not make the snapshot + * incomplete. + */ +export function admitRecords(rows, normalize, label, unpositionedId) { if (!Array.isArray(rows)) throw new LiveSourceError('malformed', `Malformed ${label} response`); const records = []; const ids = new Set(); + const retainIds = new Set(); + let unpositioned = 0; for (const row of rows) { const record = normalize(row); if (record && !ids.has(record.id)) { records.push(record); ids.add(record.id); + } else if (!record) { + const id = unpositionedId?.(row); + if (id) { + retainIds.add(id); + unpositioned += 1; + } } } if (rows.length && !records.length) { @@ -71,7 +84,8 @@ export function admitRecords(rows, normalize, label) { } return { records, - complete: records.length === rows.length, + retainIds, + complete: records.length + unpositioned === rows.length, rejectedCount: rows.length - records.length, }; } diff --git a/src/sources/live/contract.test.mjs b/src/sources/live/contract.test.mjs index 1bba6e28f..658db1596 100644 --- a/src/sources/live/contract.test.mjs +++ b/src/sources/live/contract.test.mjs @@ -67,6 +67,50 @@ test('mixed-invalid and duplicate admissions are explicitly incomplete; all-inva assert.equal(openSkySnapshot({ states: [] }).complete, true); }); +test('position-less OpenSky rows are seen, not missing; malformed rows stay incomplete', () => { + // OpenSky's spec allows null longitude/latitude; every live /states/all + // carries such rows. + const unpositioned = ['DEF456', ...aircraft.slice(1)]; + unpositioned[5] = null; + unpositioned[6] = null; + const snapshot = openSkySnapshot({ states: [aircraft, unpositioned] }); + assert.equal(snapshot.records.length, 1); + assert.equal(snapshot.complete, true); + assert.deepEqual([...snapshot.retainIds], ['def456']); + assert.equal(snapshot.rejectedCount, 1); + const malformed = [...aircraft]; + malformed[5] = 181; + const garbled = openSkySnapshot({ + states: [aircraft, unpositioned, malformed], + }); + assert.equal(garbled.complete, false); + assert.deepEqual([...garbled.retainIds], ['def456']); + // No usable id, only half a position, a truncated row, or no row at all. + for (const row of [ + [' ', ...unpositioned.slice(1)], + Object.assign([...unpositioned], { 5: -97 }), + ['DEF456'], + null, + ]) + assert.equal(openSkySnapshot({ states: [aircraft, row] }).complete, false); + // Repeated position-less rows stay complete. + const repeated = openSkySnapshot({ + states: [aircraft, unpositioned, unpositioned], + }); + assert.equal(repeated.complete, true); + // A feed with no positions at all still cannot prove absence. + assert.throws(() => openSkySnapshot({ states: [unpositioned] }), { + code: 'malformed', + }); + // Military readsb admission keeps its existing semantics. + const military = readsbSnapshot( + { ac: [{ hex: 'abc123', lat: 30, lon: -97 }, { hex: 'def456' }] }, + { observedAtMs: null }, + ); + assert.equal(military.complete, false); + assert.equal(military.retainIds.size, 0); +}); + test('readsb units and position/contact ages are normalized once against the source epoch', () => { const row = normalizeReadsbAircraft( {