feat(fires): MODIS in Active Fires and a Fire Perimeters layer (#737)

Adds MODIS NRT to Active Fires and a keyless Fire Perimeters layer (NIFC WFIGS) with InciWeb incident links, served through a capped, cached /api/fire-perimeters proxy. Co-authored-by: Gustavo Beneduzi <[email protected]> Co-authored-by: James Cooke <[email protected]>
2026-11-05 02:44:02 +00:00 · 2026-09-23 21:21:52 -05:00
1 parent 4b56d0e917
commit f913066cf2
@@ -77,6 +77,11 @@ export const ANALYST_LAYERS = {
    text: ['name', 'operator', 'river', 'output'],
    flags: [],
  },
+  'fire-perimeters': {
+    numeric: ['acres', 'containedPct', 'personnel', 'costToDate'],
+    text: ['name', 'state', 'county', 'cause', 'behavior', 'complexity'],
+    flags: [],
+  },
 };

 const EARTH_R_KM = 6371;
@@ -86,6 +86,14 @@ export const DATA_CREDITS = [
      '<a href="https://gibs.earthdata.nasa.gov" target="_blank" rel="noopener">gibs.earthdata.nasa.gov</a> · ' +
      '<a href="https://lpdaac.usgs.gov/products/hlss30v002/" target="_blank" rel="noopener">HLS product page</a>',
  },
+  {
+    key: 'wfigs',
+    html:
+      'Wildfire perimeters: ' +
+      '<a href="https://data-nifc.opendata.arcgis.com/" target="_blank" rel="noopener">National Interagency Fire Center (WFIGS)</a>' +
+      ' · Incident information: ' +
+      '<a href="https://inciweb.wildfire.gov/" target="_blank" rel="noopener">InciWeb</a>',
+  },
  {
    key: 'overpass',
    html:
@@ -0,0 +1,410 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { firePerimetersProxy } from '../../server/providers/firePerimeters.js';
+
+const incidentHtml = `<meta property="og:updated_time" content="2026-09-23T11:32:31-04:00" />
+<th>Date of Origin</th>
+<td class="text-tabular text-left"><time datetime="2026-07-15T20:20:00Z">Origin</time></td>`;
+const incidentTimes = {
+  createdMs: Date.parse('2026-07-15T20:20:00Z'),
+  changedMs: Date.parse('2026-09-23T11:32:31-04:00'),
+};
+const incidentRedirect = () =>
+  new Response('redirect', {
+    status: 301,
+    headers: { location: 'http://inciweb.wildfire.gov/test-fire' },
+  });
+
+const feature = (id = 'fire') => ({
+  properties: { attr_UniqueFireIdentifier: id, poly_DateCurrent: 123 },
+  geometry: {
+    type: 'Polygon',
+    coordinates: [
+      [
+        [-108, 35],
+        [-107, 35],
+        [-107, 36],
+        [-108, 35],
+      ],
+    ],
+  },
+});
+function install(options = {}, hook = 'configureServer') {
+  let handler;
+  const plugin = firePerimetersProxy(options);
+  assert.equal(typeof plugin.configureServer, 'function');
+  assert.equal(typeof plugin.configurePreviewServer, 'function');
+  plugin[hook]({
+    middlewares: {
+      use(path, callback) {
+        assert.equal(path, '/api/fire-perimeters');
+        handler = callback;
+      },
+    },
+  });
+  return async (url = '/', method = 'GET', peer = 'local') => {
+    const res = {
+      writeHead(status, headers) {
+        this.status = status;
+        this.headers = headers;
+      },
+      end(body) {
+        this.body = JSON.parse(body);
+      },
+    };
+    await handler({ url, method, socket: { remoteAddress: peer } }, res);
+    return res;
+  };
+}
+
+for (const hook of ['configureServer', 'configurePreviewServer']) {
+  test(`${hook}: paging stops at five pages and normalizes the combined snapshot`, async () => {
+    const calls = [];
+    const request = install(
+      {
+        now: () => 1234,
+        fetchImpl: async (url, options) => {
+          calls.push(new URL(url));
+          assert.ok(options.signal instanceof AbortSignal);
+          assert.equal(options.redirect, 'error');
+          return Response.json({
+            features: [feature(String(calls.length))],
+            exceededTransferLimit: true,
+          });
+        },
+      },
+      hook,
+    );
+    const res = await request('/?url=https://invalid.example&resultOffset=999');
+    assert.equal(res.status, 200);
+    assert.equal(res.body.fetchedAt, 1234);
+    assert.equal(res.body.rows.length, 5);
+    assert.deepEqual(
+      calls.map((url) => url.searchParams.get('resultOffset')),
+      [null, '1', '2', '3', '4'],
+    );
+    assert.ok(calls.every((url) => url.hostname === 'services3.arcgis.com'));
+    assert.equal(calls[0].searchParams.get('maxAllowableOffset'), '0.001');
+  });
+}
+
+test('an explicit transfer-limit false stops paging; a fresh cache does not fetch', async () => {
+  let calls = 0;
+  const request = install({
+    fetchImpl: async () => {
+      calls++;
+      return Response.json({
+        features: [feature()],
+        exceededTransferLimit: false,
+      });
+    },
+  });
+  const first = await request();
+  assert.deepEqual((await request()).body, first.body);
+  assert.equal(calls, 1);
+});
+
+test('a failed refresh serves the last good perimeter snapshot as stale', async () => {
+  let clock = 0,
+    calls = 0;
+  const request = install({
+    now: () => clock,
+    fetchImpl: async () => {
+      if (++calls > 1) throw new Error('private upstream details');
+      return Response.json({ features: [feature()] });
+    },
+  });
+  await request();
+  clock = 300_000;
+  const res = await request();
+  assert.equal(res.status, 200);
+  assert.equal(res.body.stale, true);
+  assert.equal(res.body.fetchedAt, 0);
+  assert.equal(res.body.rows[0].stableId, 'fire');
+});
+
+test('oversized streamed and declared bodies produce sanitized 502 responses', async () => {
+  for (const declared of [true, false]) {
+    let cancelled = false;
+    const request = install({
+      fetchImpl: async () =>
+        new Response(
+          new ReadableStream({
+            pull(controller) {
+              controller.enqueue(new Uint8Array(1024 * 1024));
+            },
+            cancel() {
+              cancelled = true;
+            },
+          }),
+          {
+            headers: declared
+              ? { 'Content-Length': String(17 * 1024 * 1024) }
+              : {},
+          },
+        ),
+    });
+    const res = await request();
+    assert.equal(res.status, 502);
+    assert.deepEqual(res.body, { error: 'fire_perimeters_unavailable' });
+    assert.equal(cancelled, true);
+  }
+});
+
+test('malformed WFIGS data does not become a cached empty snapshot', async () => {
+  let calls = 0;
+  const request = install({
+    fetchImpl: async () => {
+      calls++;
+      return Response.json({ error: 'upstream detail' });
+    },
+  });
+  assert.equal((await request()).status, 502);
+  assert.equal((await request()).status, 502);
+  assert.equal(calls, 2);
+});
+
+test('concurrent requests share one upstream operation for every route', async () => {
+  for (const [path, payload] of [
+    ['/', { features: [] }],
+    ['/inciweb/index', []],
+    ['/inciweb/publication/42', {}],
+  ]) {
+    let release,
+      calls = 0;
+    const pending = new Promise((resolve) => {
+      release = resolve;
+    });
+    const request = install({
+      fetchImpl: async () => {
+        calls++;
+        await pending;
+        if (path.includes('/publication/'))
+          return calls === 1 ? incidentRedirect() : new Response(incidentHtml);
+        return Response.json(payload);
+      },
+    });
+    const responses = [request(path), request(path)];
+    await new Promise(setImmediate);
+    assert.equal(calls, 1);
+    release();
+    for (const res of await Promise.all(responses))
+      assert.equal(res.status, 200);
+    assert.equal(calls, path.includes('/publication/') ? 2 : 1);
+  }
+});
+
+test('bad publication ids and non-GET requests never fetch upstream', async () => {
+  let calls = 0;
+  const request = install({
+    fetchImpl: async () => {
+      calls++;
+      throw new Error('unexpected');
+    },
+  });
+  for (const id of ['', 'abc', '-1', '1234567890', '1/2', '%31', '1.0'])
+    assert.equal((await request(`/inciweb/publication/${id}`)).status, 400);
+  for (const path of ['/', '/inciweb/index', '/inciweb/publication/123']) {
+    for (const method of ['POST', 'HEAD', 'PUT', 'DELETE'])
+      assert.equal((await request(path, method)).status, 405);
+  }
+  assert.equal(calls, 0);
+});
+
+test('InciWeb index preserves the fixed POST, caches for one hour and serves stale arrays', async () => {
+  let clock = 0,
+    calls = 0;
+  const rows = [{ incident_id: '42', incident_title: 'Fire' }];
+  const request = install({
+    now: () => clock,
+    fetchImpl: async (url, options) => {
+      calls++;
+      assert.equal(url, 'https://inciweb.wildfire.gov/api/single-publication/');
+      assert.equal(options.method, 'POST');
+      assert.deepEqual(options.headers, { 'Content-Type': 'application/json' });
+      assert.deepEqual(JSON.parse(options.body), { title: '' });
+      if (calls > 1) throw new Error('offline');
+      return Response.json(rows);
+    },
+  });
+  assert.deepEqual((await request('/inciweb/index')).body, rows);
+  clock = 3_599_999;
+  assert.deepEqual((await request('/inciweb/index')).body, rows);
+  assert.equal(calls, 1);
+  clock++;
+  const stale = await request('/inciweb/index');
+  assert.deepEqual(stale.body, rows);
+  assert.equal(stale.headers['X-Data-Stale'], 'true');
+});
+
+test('publication timestamps, thirty-minute TTL and oldest-entry eviction', async () => {
+  let clock = 0;
+  const calls = [];
+  const request = install({
+    now: () => clock,
+    fetchImpl: async (url) => {
+      calls.push(url);
+      return url.includes('/node/')
+        ? incidentRedirect()
+        : new Response(incidentHtml);
+    },
+  });
+  const get = (id) =>
+    request(`/inciweb/publication/${id}`, 'GET', `peer-${id}`);
+  assert.deepEqual((await get(1)).body, incidentTimes);
+  clock = 1_799_999;
+  await get(1);
+  assert.equal(calls.length, 2);
+  clock++;
+  await get(1);
+  assert.equal(calls.length, 4);
+  for (let id = 2; id <= 257; id++) assert.equal((await get(id)).status, 200);
+  await get(2);
+  assert.equal(calls.length, 516);
+  await get(1);
+  assert.equal(calls.length, 518);
+  assert.equal(calls.at(-2), 'https://inciweb.wildfire.gov/node/1');
+});
+
+test('InciWeb index and publication reads enforce their smaller caps', async () => {
+  for (const [path, cap] of [
+    ['/inciweb/index', 4 * 1024 * 1024],
+    ['/inciweb/publication/1', 2 * 1024 * 1024],
+  ]) {
+    const request = install({
+      fetchImpl: async () =>
+        new Response('{}', { headers: { 'Content-Length': String(cap + 1) } }),
+    });
+    assert.equal((await request(path)).status, 502);
+  }
+});
+
+test('the per-client limit stops a loop while another client can read the cache', async () => {
+  let calls = 0;
+  const request = install({
+    fetchImpl: async () => {
+      calls++;
+      return Response.json({ features: [] });
+    },
+  });
+  for (let i = 0; i < 60; i++) assert.equal((await request()).status, 200);
+  assert.equal((await request()).status, 429);
+  assert.equal((await request('/', 'GET', 'another-client')).status, 200);
+  assert.equal(calls, 1);
+});
+
+test('publication redirects share a signal, cancel the redirect body and force HTTPS', async () => {
+  for (const status of [301, 302, 303, 307, 308]) {
+    const calls = [];
+    const redirect = new Response('redirect', {
+      status,
+      headers: {
+        location: 'http://inciweb.wildfire.gov/incident/test-fire#details',
+      },
+    });
+    const request = install({
+      fetchImpl: async (url, options) => {
+        calls.push({ url, options });
+        return calls.length === 1 ? redirect : new Response(incidentHtml);
+      },
+    });
+    assert.deepEqual(
+      (await request('/inciweb/publication/42')).body,
+      incidentTimes,
+    );
+    assert.equal(calls.length, 2);
+    assert.equal(redirect.bodyUsed, true);
+    assert.equal(calls[0].url, 'https://inciweb.wildfire.gov/node/42');
+    assert.equal(calls[0].options.redirect, 'manual');
+    assert.equal(
+      calls[1].url,
+      'https://inciweb.wildfire.gov/incident/test-fire',
+    );
+    assert.equal(calls[1].options.redirect, 'error');
+    assert.ok(calls[0].options.signal instanceof AbortSignal);
+    assert.equal(calls[0].options.signal, calls[1].options.signal);
+  }
+});
+
+test('unsafe publication redirect locations fail before a second fetch', async () => {
+  for (const location of [
+    'https://evil.example/x',
+    '/a/../x',
+    '/x?q=1',
+    '/x?',
+    '/%2e%2e/x',
+    '/a/b/c',
+    '',
+  ]) {
+    let calls = 0;
+    const request = install({
+      fetchImpl: async () => {
+        calls++;
+        return new Response('redirect', { status: 301, headers: { location } });
+      },
+    });
+    assert.equal((await request('/inciweb/publication/42')).status, 502);
+    assert.equal(calls, 1);
+  }
+});
+
+test('incident pages without valid timestamps fail closed and are not cached', async () => {
+  for (const html of [
+    '<html>No markers</html>',
+    incidentHtml.replaceAll(/2026-[^"]+/g, 'invalid'),
+  ]) {
+    let calls = 0;
+    const request = install({
+      fetchImpl: async () => {
+        calls++;
+        return calls % 2 ? incidentRedirect() : new Response(html);
+      },
+    });
+    assert.equal((await request('/inciweb/publication/42')).status, 502);
+    assert.equal((await request('/inciweb/publication/42')).status, 502);
+    assert.equal(calls, 4);
+  }
+});
+
+test('direct 200 incident pages work, including one missing timestamp', async () => {
+  for (const [html, expected] of [
+    [incidentHtml, incidentTimes],
+    [
+      incidentHtml.replace('og:updated_time', 'unrelated'),
+      { ...incidentTimes, changedMs: null },
+    ],
+    [
+      incidentHtml.replace('Date of Origin', 'Unrelated'),
+      { ...incidentTimes, createdMs: null },
+    ],
+  ]) {
+    let calls = 0;
+    const request = install({
+      fetchImpl: async () => {
+        calls++;
+        return new Response(html);
+      },
+    });
+    assert.deepEqual((await request('/inciweb/publication/42')).body, expected);
+    assert.equal(calls, 1);
+  }
+});
+
+test('publication HTML streams are capped at 2 MiB', async () => {
+  let cancelled = false;
+  const request = install({
+    fetchImpl: async () =>
+      new Response(
+        new ReadableStream({
+          pull(controller) {
+            controller.enqueue(new Uint8Array(1024 * 1024));
+          },
+          cancel() {
+            cancelled = true;
+          },
+        }),
+      ),
+  });
+  assert.equal((await request('/inciweb/publication/42')).status, 502);
+  assert.equal(cancelled, true);
+});
@@ -5,9 +5,15 @@
 * VIIRS header (confirmed live 2026-07-16):
 *   latitude,longitude,bright_ti4,scan,track,acq_date,acq_time,satellite,
 *   instrument,confidence,version,bright_ti5,frp,daynight
+ * MODIS header:
+ *   latitude,longitude,brightness,scan,track,acq_date,acq_time,satellite,
+ *   instrument,confidence,version,bright_t31,frp,daynight
 *
 * Quirks this module owns:
- * - `confidence` is CATEGORICAL for VIIRS (`l`/`n`/`h`) — passed through raw.
+ * - `confidence` is CATEGORICAL for VIIRS (`l`/`n`/`h`) and numeric 0-100 for
+ *   MODIS — passed through raw.
+ * - Brightness maps from `bright_ti4`/`brightness`; the secondary band maps
+ *   from `bright_ti5`/`bright_t31`.
 * - `acq_time` is NOT zero-padded ("45" = 00:45 UTC) — kept as-is in records;
 *   {@link acquisitionMsUtc} does the padding.
 * - Upstream errors come back as HTML or plain text ("Invalid MAP_KEY"),
@@ -48,6 +48,25 @@ test('fixture first row maps every field, categorical confidence preserved raw',
  assert.equal(first.instrument, 'VIIRS');
 });

+test('MODIS schema maps fields and preserves numeric confidence raw', () => {
+  const header =
+    'latitude,longitude,brightness,scan,track,acq_date,acq_time,satellite,instrument,confidence,version,bright_t31,frp,daynight';
+  const row =
+    '34.123,-117.456,321.5,1.2,0.8,2026-07-16,1045,Aqua,MODIS,78,6.1NRT,290.25,12.4,D';
+  const [record] = parseFirmsCsv(`${header}\n${row}\n`);
+  assert.equal(record.lat, 34.123);
+  assert.equal(record.lon, -117.456);
+  assert.equal(record.frp, 12.4);
+  assert.equal(record.brightness, 321.5);
+  assert.equal(record.brightnessTi5, 290.25);
+  assert.equal(record.daynight, 'D');
+  assert.equal(record.acqDate, '2026-07-16');
+  assert.equal(record.acqTime, '1045');
+  assert.equal(record.satellite, 'Aqua');
+  assert.equal(record.instrument, 'MODIS');
+  assert.equal(record.confidence, '78');
+});
+
 test('acquisitionMsUtc: "1006" = 10:06 UTC', () => {
  assert.equal(
    acquisitionMsUtc('2026-07-16', '1006'),
@@ -9,7 +9,7 @@ assert.notEqual(start, -1, 'FIRMS refresh function exists');
 const end = config.indexOf('\n  }', start);
 assert.notEqual(end, -1, 'FIRMS refresh function closes');
 const refreshSource = config.slice(start, end + 4);
-const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT'];
+const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT', 'MODIS_NRT'];
 const NOW = Date.UTC(2026, 8, 11, 12);
 const recent = { acqDate: '2026-09-11', acqTime: '1100' };

@@ -42,7 +42,7 @@ test('FIRMS retains large sources in order and filters expired rows', async () =
  assert.equal(result.fires[199_999], large.at(-1));
  assert.equal(result.fires.at(-1), last);
  assert.deepEqual(result.sources, SOURCES.map((source, index) => ({
-    source, count: [200_000, 1, 0][index], ok: true,
+    source, count: [200_000, 1, 0, 0][index], ok: true,
  })));
 });

@@ -51,7 +51,7 @@ test('FIRMS keeps successful sources when another upstream fails', async () => {
    if (source === SOURCES[1]) throw new Error('upstream unavailable');
    return [recent];
  })('fixture');
-  assert.equal(result.fires.length, 2);
+  assert.equal(result.fires.length, 3);
  assert.deepEqual(result.sources, SOURCES.map((source, index) => ({
    source, count: index === 1 ? 0 : 1, ok: index !== 1,
  })));
@@ -65,7 +65,7 @@ test('FIRMS reports one failure if consuming a source throws before append', asy
      // Fault injection for aggregation; ordinary parsed CSV returns an array.
      return { length: 1, [Symbol.iterator]() { throw new Error('aggregation failed'); } };
    })('fixture');
-  assert.equal(result.fires.length, 2);
+  assert.equal(result.fires.length, 3);
  assert.deepEqual(result.sources, SOURCES.map((source, index) => ({
    source, count: index === 0 ? 0 : 1, ok: index !== 0,
  })));
@@ -480,6 +480,11 @@ export const LAYER_STATE_REGISTRY = Object.freeze([
  }),
  Object.freeze({ id: 'directions', token: 'n', disposition: 'enabled-only' }),
  Object.freeze({ id: 'earthquakes', token: 'e', disposition: 'enabled-only' }),
+  Object.freeze({
+    id: 'fire-perimeters',
+    token: '2',
+    disposition: 'enabled-only',
+  }),
  Object.freeze({
    id: 'flights',
    token: 'f',
@@ -194,8 +194,8 @@ function encode(state) {

 test('production registry is exact, canonical, and rejects incomplete contracts', async () => {
  assert.equal(validateLayerStateRegistry(), true);
-  assert.equal(REGISTERED_LAYER_IDS.length, 27);
-  assert.equal(new Set(REGISTERED_LAYER_IDS).size, 27);
+  assert.equal(REGISTERED_LAYER_IDS.length, 28);
+  assert.equal(new Set(REGISTERED_LAYER_IDS).size, 28);
  assert.ok(REGISTERED_LAYER_IDS.includes('transit'));
  assert.deepEqual(REGISTERED_LAYER_IDS, [...REGISTERED_LAYER_IDS].sort());
  assert.throws(
@@ -2418,3 +2418,10 @@ test('the recent-imagery split is share-link only: never stored locally, and a s
    50,
  );
 });
+
+test('fire perimeters uses digit 2 without colliding with wind or recent imagery', () => {
+  const decoded = decodeLayerStateParams(new URLSearchParams('v=2&l=2.k.1'));
+  assert.deepEqual(decoded.enabledLayerIds, ['fire-perimeters', 'recent-imagery', 'wind']);
+  assert.equal(LAYER_STATE_REGISTRY.find(({ id }) => id === 'fire-perimeters').token, '2');
+  assert.deepEqual(decodeLayerStateParams(new URLSearchParams(encode(decoded))), decoded);
+});