From 84c1fef8d34dfd4dd556b6dca6bda9e3d2cf6b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Itallo=20Brand=C3=A3o?= <45429643+IMBrandao@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:15:34 -0300 Subject: [PATCH 1/5] fix(firms): stop requesting the retired Suomi NPP source NASA stopped delivering Suomi NPP data on 2026-11-02 (FIRMS news page). With a live MAP_KEY on 2026-11-04, VIIRS_SNPP_NRT came back ok with zero rows while NOAA-20, NOAA-21 and MODIS returned 57k, 49k and 11k, so the source only spent one transaction per refresh. The proxy now requests NOAA-20, NOAA-21 and MODIS NRT. The "N" satellite label stays, so cached or archived Suomi NPP rows still read SNPP. Tests count sources from the list instead of assuming four, and a regression test keeps the retired source out of the production list. Refs #849 Co-Authored-By: Claude Opus 5.5 --- DATA_SOURCES.md | 7 ++++--- docs/CURRENT-STATE.md | 5 +++++ server/providers/firms.js | 14 ++++++-------- src/data/firmsProxy.test.mjs | 18 ++++++++++++++---- src/tooling/environmentProviders.test.mjs | 4 ++-- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index cac0b5ab7..efee6b9ff 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -273,11 +273,12 @@ Bundled OSM-derived data is distributed under [ODbL 1.0](https://opendatacommons > We acknowledge the use of data and/or imagery from NASA's Fire Information for Resource Management System (FIRMS) (https://earthdata.nasa.gov/firms), part of NASA's Earth Observing System Data and Information System (EOSDIS). FIRMS active fires are **fetched live at runtime** (CC0 / U.S. public domain data): the -`/api/firms` server-side proxy merges the three VIIRS NRT sources (NOAA-20, NOAA-21, -Suomi-NPP) and MODIS NRT (Terra + Aqua), clamped to the trailing 24 h, cached 30 min to respect the shared MAP_KEY +`/api/firms` server-side proxy merges the two VIIRS NRT sources (NOAA-20, NOAA-21) +and MODIS NRT (Terra + Aqua), clamped to the trailing 24 h, cached 30 min to respect the shared MAP_KEY transaction quota. Requires a free `FIRMS_MAP_KEY` (https://firms.modaps.eosdis.nasa.gov/api/map_key/); the layer is empty without it. -The former bundled 2026-05-25 snapshot was removed 2026-07-16. +The former bundled 2026-05-25 snapshot was removed 2026-07-16. Suomi NPP (`VIIRS_SNPP_NRT`) +is no longer requested: NASA stopped delivering its data on 2026-11-02. ### Natural Earth physical regions (`natural_earth/`) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index e868dfc95..102efb997 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2325,6 +2325,11 @@ accessible name remains "Don't show this again". Control behavior is unchanged. ## FIRMS source status +The FIRMS proxy requests VIIRS NOAA-20, VIIRS NOAA-21 and MODIS NRT (Terra and +Aqua). It no longer requests VIIRS Suomi NPP: NASA stopped delivering Suomi NPP +data on 2026-11-02, and the source returned no rows while still costing one +MAP_KEY transaction per refresh. + The FIRMS CSV parser rejects impossible acquisition calendar dates before the proxy filters records to the trailing 24 hours, while retaining valid UTC acquisition times. The `get_active_fires` tool reports acquisition times as UTC ISO timestamps, including zero-padded hours and minutes; missing or invalid dates and times diff --git a/server/providers/firms.js b/server/providers/firms.js index 20a2f4bab..51ea4fc56 100644 --- a/server/providers/firms.js +++ b/server/providers/firms.js @@ -131,8 +131,8 @@ export async function fetchFirms( * NASA FIRMS live active-fire proxy with a memory + disk cache. * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2 * - * Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent - * satellites, no cross-source dedup) plus MODIS_NRT (combined Terra+Aqua, + * Merges two VIIRS NRT sources (NOAA-20, NOAA-21 — independent satellites, no + * cross-source dedup) plus MODIS_NRT (combined Terra+Aqua, * ~1 km, numeric 0-100 confidence), fetched sequentially (quota courtesy) * with `days=2` (`days=1` means "current UTC day", nearly empty just after * 00:00Z) and clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS @@ -141,6 +141,9 @@ export async function fetchFirms( * fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream * fetch across dev-server restarts. Pattern mirrors celestrakProxy. * + * VIIRS_SNPP_NRT is not requested: NASA stopped delivering Suomi NPP data on + * 2026-11-02, so that source only spent a transaction per refresh (#849). + * * Routes: * GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires} * GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions} @@ -153,12 +156,7 @@ export async function fetchFirms( export function firmsProxy() { const TTL_MS = 30 * 60_000; const STATUS_TTL_MS = 5 * 60_000; - const SOURCES = [ - 'VIIRS_NOAA20_NRT', - 'VIIRS_NOAA21_NRT', - 'VIIRS_SNPP_NRT', - 'MODIS_NRT', - ]; + const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'MODIS_NRT']; /** * Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about * 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a diff --git a/src/data/firmsProxy.test.mjs b/src/data/firmsProxy.test.mjs index 14668c825..135002ca7 100644 --- a/src/data/firmsProxy.test.mjs +++ b/src/data/firmsProxy.test.mjs @@ -21,7 +21,7 @@ assert.notEqual(start, -1, 'FIRMS refresh function exists'); const end = config.indexOf('\n }', start); assert.notEqual(end, -1, 'FIRMS refresh function closes'); const refreshSource = config.slice(start, end + 4); -const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT', 'MODIS_NRT']; +const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'MODIS_NRT']; const NOW = Date.UTC(2026, 8, 11, 12); const recent = { acqDate: '2026-09-11', acqTime: '1100' }; @@ -63,7 +63,7 @@ test('FIRMS keeps successful sources when another upstream fails', async () => { if (source === SOURCES[1]) throw new Error('upstream unavailable'); return [recent]; })('fixture'); - assert.equal(result.fires.length, 3); + assert.equal(result.fires.length, SOURCES.length - 1); assert.deepEqual(result.sources, SOURCES.map((source, index) => ({ source, count: index === 1 ? 0 : 1, ok: index !== 1, }))); @@ -77,7 +77,7 @@ test('FIRMS reports one failure if consuming a source throws before append', asy // Fault injection for aggregation; ordinary parsed CSV returns an array. return { length: 1, [Symbol.iterator]() { throw new Error('aggregation failed'); } }; })('fixture'); - assert.equal(result.fires.length, 3); + assert.equal(result.fires.length, SOURCES.length - 1); assert.deepEqual(result.sources, SOURCES.map((source, index) => ({ source, count: index === 0 ? 0 : 1, ok: index !== 0, }))); @@ -107,7 +107,7 @@ function wrappedAddressSelectionFailure() { { name: 'ConnectTimeoutError', code: 'UND_ERR_CONNECT_TIMEOUT', cause: inner }); return new TypeError('fetch failed', { cause }); } -const URL_WITH_KEY = 'https://firms.modaps.eosdis.nasa.gov/api/area/csv/SECRET/VIIRS_SNPP_NRT/world/2'; +const URL_WITH_KEY = 'https://firms.modaps.eosdis.nasa.gov/api/area/csv/SECRET/VIIRS_NOAA20_NRT/world/2'; test('FIRMS failures log the socket cause, never the keyed URL', () => { const described = describeFetchError(addressSelectionFailure()); @@ -307,3 +307,13 @@ test('every FIRMS upstream request goes through the IPv4 fallback', async () => assert.ok(calls[0][2] instanceof AbortSignal); assert.deepEqual(calls[1], ['read', response, calls[0][2]], 'the body read shares the request deadline'); }); + +test('FIRMS requests only the satellites NASA still delivers', () => { + const listed = config.match(/const SOURCES = \[([^\]]*)\];/); + assert.ok(listed, 'FIRMS source list exists'); + const production = [...listed[1].matchAll(/'([A-Z0-9_]+)'/g)].map((m) => m[1]); + assert.deepEqual(production, SOURCES, 'the test mirrors the production source list'); + // NASA stopped delivering Suomi NPP data on 2026-11-02 (#849): requesting it + // returns no rows and still spends a MAP_KEY transaction on every refresh. + assert.ok(!production.includes('VIIRS_SNPP_NRT')); +}); diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index da3cfc335..feca9432d 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -313,9 +313,9 @@ test('FIRMS retains a large successful source during partial failure and filters const first = json(await request()); assert.equal(first.count, 130001); assert.equal(first.sources.filter((s) => s.ok).length, 1); - assert.equal(calls, 4); + assert.equal(calls, first.sources.length); assert.equal(json(await request()).count, 130001); - assert.equal(calls, 4); + assert.equal(calls, first.sources.length); assert.deepEqual(json(await request('/status')).transactions, { used: 3, limit: 5000, From b342e405d7514505861ab26244f38496a59350cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Itallo=20Brand=C3=A3o?= <45429643+IMBrandao@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:20:09 -0300 Subject: [PATCH 2/5] docs(changelog): add fragment for #1552 Co-Authored-By: Claude Opus 5.5 --- changelog.d/1552-firms-drop-snpp.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/1552-firms-drop-snpp.md diff --git a/changelog.d/1552-firms-drop-snpp.md b/changelog.d/1552-firms-drop-snpp.md new file mode 100644 index 000000000..fcb375bda --- /dev/null +++ b/changelog.d/1552-firms-drop-snpp.md @@ -0,0 +1,4 @@ +- Stop requesting Suomi NPP in Active Fires. NASA stopped delivering its data + on November 2, 2026, so `VIIRS_SNPP_NRT` returned no rows while still costing + a MAP_KEY transaction on every refresh; the layer now merges VIIRS NOAA-20, + VIIRS NOAA-21 and MODIS (IMBrandao, #1552). From b37025414d90d6b696dcecd252c222972c00c557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Itallo=20Brand=C3=A3o?= <45429643+IMBrandao@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:57:27 -0300 Subject: [PATCH 3/5] test(firms): pin the refresh call count and read sources from production Review follow-up for #1552: - environmentProviders: assert exactly 3 upstream calls per refresh again, so a returning fourth source would fail the test. - firmsProxy: build the test's SOURCES from server/providers/firms.js, so there is one list instead of a mirrored copy. - firmsProxy: drop the unused fourth count entry. Co-Authored-By: Claude Opus 5.5 --- src/data/firmsProxy.test.mjs | 13 ++++++------- src/tooling/environmentProviders.test.mjs | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/data/firmsProxy.test.mjs b/src/data/firmsProxy.test.mjs index 135002ca7..dc9ed8144 100644 --- a/src/data/firmsProxy.test.mjs +++ b/src/data/firmsProxy.test.mjs @@ -21,7 +21,10 @@ assert.notEqual(start, -1, 'FIRMS refresh function exists'); const end = config.indexOf('\n }', start); assert.notEqual(end, -1, 'FIRMS refresh function closes'); const refreshSource = config.slice(start, end + 4); -const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'MODIS_NRT']; +// Read the source list from production so these tests share one source of truth. +const listed = config.match(/const SOURCES = \[([^\]]*)\];/); +assert.ok(listed, 'FIRMS source list exists'); +const SOURCES = [...listed[1].matchAll(/'([A-Z0-9_]+)'/g)].map((m) => m[1]); const NOW = Date.UTC(2026, 8, 11, 12); const recent = { acqDate: '2026-09-11', acqTime: '1100' }; @@ -54,7 +57,7 @@ test('FIRMS retains large sources in order and filters expired rows', async () = assert.equal(result.fires[199_999], large.at(-1)); assert.equal(result.fires.at(-1), last); assert.deepEqual(result.sources, SOURCES.map((source, index) => ({ - source, count: [200_000, 1, 0, 0][index], ok: true, + source, count: [200_000, 1, 0][index], ok: true, }))); }); @@ -309,11 +312,7 @@ test('every FIRMS upstream request goes through the IPv4 fallback', async () => }); test('FIRMS requests only the satellites NASA still delivers', () => { - const listed = config.match(/const SOURCES = \[([^\]]*)\];/); - assert.ok(listed, 'FIRMS source list exists'); - const production = [...listed[1].matchAll(/'([A-Z0-9_]+)'/g)].map((m) => m[1]); - assert.deepEqual(production, SOURCES, 'the test mirrors the production source list'); // NASA stopped delivering Suomi NPP data on 2026-11-02 (#849): requesting it // returns no rows and still spends a MAP_KEY transaction on every refresh. - assert.ok(!production.includes('VIIRS_SNPP_NRT')); + assert.ok(!SOURCES.includes('VIIRS_SNPP_NRT')); }); diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index feca9432d..22eb39ad3 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -313,9 +313,9 @@ test('FIRMS retains a large successful source during partial failure and filters const first = json(await request()); assert.equal(first.count, 130001); assert.equal(first.sources.filter((s) => s.ok).length, 1); - assert.equal(calls, first.sources.length); + assert.equal(calls, 3); assert.equal(json(await request()).count, 130001); - assert.equal(calls, first.sources.length); + assert.equal(calls, 3); assert.deepEqual(json(await request('/status')).transactions, { used: 3, limit: 5000, From b04fcf191cbaf781cde1db1ff08072dd819f1c5f Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Mon, 9 Nov 2026 21:38:07 +0400 Subject: [PATCH 4/5] docs(firms): count two VIIRS sources in the layer table and test note --- docs/CURRENT-STATE.md | 2 +- src/data/firmsInteraction.test.mjs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 102efb997..6346c1db6 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4152,7 +4152,7 @@ its criteria cannot be silently ignored. | Datacenters ▣ | OSM extract (bundled) | `src/data/infrastructure.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/infrastructure.js` | — | static | | Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | -| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | +| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×2 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | | Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, server memory cache 24 h) | | Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | | GNSS Integrity 📡 (`gnss-integrity`) | adsb.lol point snapshot, low navigation accuracy binned per 0.5° cell (GEV NIC/NACp threshold + gpsjam.org formula and bands, keyless, ODbL) | `src/layers/gnss/` via `src/app/layers/gnss.js` | `/api/gnss-integrity` | camera-driven (600 ms debounce, minimum 2.5 s between reads) + 60s poll (server cache 60 s; 30 min rolling window) | diff --git a/src/data/firmsInteraction.test.mjs b/src/data/firmsInteraction.test.mjs index d735df3f8..9931c8b32 100644 --- a/src/data/firmsInteraction.test.mjs +++ b/src/data/firmsInteraction.test.mjs @@ -119,9 +119,10 @@ test('fire detection key survives a refetch that renumbers the index', () => { }); test('two satellites over the same pixel at the same time stay distinct', () => { - // The proxy merges SNPP + NOAA-20 + NOAA-21 without dedup and their orbits - // overlap, so position and time alone are NOT unique. Collapsing them let a - // restored selection land on the other satellite's record. + // The proxy merges NOAA-20 + NOAA-21 without dedup and their orbits overlap, + // so position and time alone are NOT unique. Collapsing them let a restored + // selection land on the other satellite's record. Archived Suomi NPP (`N`) + // rows must stay distinct too. const base = makeFire({ satellite: 'N20' }); const twin = makeFire({ satellite: 'N21' }); const suomi = makeFire({ satellite: 'N' }); From fce5ee6381b948db2f263e1090a19b77d74dd164 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Mon, 9 Nov 2026 21:44:09 +0400 Subject: [PATCH 5/5] docs(firms): tighten the Suomi NPP notes --- DATA_SOURCES.md | 2 +- docs/CURRENT-STATE.md | 4 +--- server/providers/firms.js | 7 +++---- src/data/firmsProxy.test.mjs | 6 +++--- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 8ceb94e84..ba231df31 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -329,7 +329,7 @@ and MODIS NRT (Terra + Aqua), clamped to the trailing 24 h, cached 30 min to res transaction quota. Requires a free `FIRMS_MAP_KEY` (https://firms.modaps.eosdis.nasa.gov/api/map_key/); the layer is empty without it. The former bundled 2026-05-25 snapshot was removed 2026-07-16. Suomi NPP (`VIIRS_SNPP_NRT`) -is no longer requested: NASA stopped delivering its data on 2026-11-02. +is not requested: NASA stopped delivering it on 2026-11-02. ### UCS Satellite Database (`ucs_satellites/`) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 417e81551..05240d922 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2860,9 +2860,7 @@ reporting are unchanged. ## FIRMS source status The FIRMS proxy requests VIIRS NOAA-20, VIIRS NOAA-21 and MODIS NRT (Terra and -Aqua). It no longer requests VIIRS Suomi NPP: NASA stopped delivering Suomi NPP -data on 2026-11-02, and the source returned no rows while still costing one -MAP_KEY transaction per refresh. +Aqua). It skips VIIRS Suomi NPP, which NASA stopped delivering on 2026-11-02. The FIRMS CSV parser rejects impossible acquisition calendar dates before the proxy filters records to the trailing 24 hours, while retaining valid UTC acquisition times. The `get_active_fires` tool reports acquisition times as UTC ISO timestamps, diff --git a/server/providers/firms.js b/server/providers/firms.js index 51ea4fc56..d21dc296e 100644 --- a/server/providers/firms.js +++ b/server/providers/firms.js @@ -132,8 +132,8 @@ export async function fetchFirms( * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2 * * Merges two VIIRS NRT sources (NOAA-20, NOAA-21 — independent satellites, no - * cross-source dedup) plus MODIS_NRT (combined Terra+Aqua, - * ~1 km, numeric 0-100 confidence), fetched sequentially (quota courtesy) + * cross-source dedup) plus MODIS_NRT (combined Terra+Aqua, ~1 km, numeric + * 0-100 confidence), fetched sequentially (quota courtesy) * with `days=2` (`days=1` means "current UTC day", nearly empty just after * 00:00Z) and clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS * quota is 5,000 transactions / 10 min per MAP_KEY, so the cache is the point: @@ -141,8 +141,7 @@ export async function fetchFirms( * fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream * fetch across dev-server restarts. Pattern mirrors celestrakProxy. * - * VIIRS_SNPP_NRT is not requested: NASA stopped delivering Suomi NPP data on - * 2026-11-02, so that source only spent a transaction per refresh (#849). + * VIIRS_SNPP_NRT is left out: NASA stopped delivering Suomi NPP on 2026-11-02. * * Routes: * GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires} diff --git a/src/data/firmsProxy.test.mjs b/src/data/firmsProxy.test.mjs index dc9ed8144..71c0af4d6 100644 --- a/src/data/firmsProxy.test.mjs +++ b/src/data/firmsProxy.test.mjs @@ -21,7 +21,7 @@ assert.notEqual(start, -1, 'FIRMS refresh function exists'); const end = config.indexOf('\n }', start); assert.notEqual(end, -1, 'FIRMS refresh function closes'); const refreshSource = config.slice(start, end + 4); -// Read the source list from production so these tests share one source of truth. +// Read the list from production so these tests follow it. const listed = config.match(/const SOURCES = \[([^\]]*)\];/); assert.ok(listed, 'FIRMS source list exists'); const SOURCES = [...listed[1].matchAll(/'([A-Z0-9_]+)'/g)].map((m) => m[1]); @@ -312,7 +312,7 @@ test('every FIRMS upstream request goes through the IPv4 fallback', async () => }); test('FIRMS requests only the satellites NASA still delivers', () => { - // NASA stopped delivering Suomi NPP data on 2026-11-02 (#849): requesting it - // returns no rows and still spends a MAP_KEY transaction on every refresh. + // NASA stopped delivering Suomi NPP on 2026-11-02; asking for it only spends + // a MAP_KEY transaction. assert.ok(!SOURCES.includes('VIIRS_SNPP_NRT')); });