Merge pull request #1078 from jeffblakely/fix/firms-ipv4-fallback

fix(firms): retry over IPv4 when every address fails to connect

changelog.d/1078-firms-ipv4-fallback.md

@@ -0,0 +1,4 @@
+- FIRMS fire data loads on hosts where IPv6 is unreachable and IPv4 is slow to
+  connect: a request whose every address fails to connect retries once over
+  IPv4, and FIRMS warnings name the underlying connection errors instead of
+  only `fetch failed` (jeffblakely, #1078).

docs/CURRENT-STATE.md

@@ -2114,6 +2114,19 @@ throws, that source reports failure without a contradictory success entry.
 The existing per-record append continues to support large feeds; sequential
 fetching, trailing-24-hour filtering and partial-success caching are unchanged.
 
+FIRMS requests retry once over IPv4 when every resolved address fails to
+connect. Node's network-family auto-selection gives each address a short
+attempt timeout, so on hosts where IPv6 is unreachable a slow IPv4 handshake
+was abandoned and the request reported only `fetch failed` (#68). The retry
+makes a single IPv4 connection within the same deadline and the same body
+cap, and does not follow redirects. HTTP errors, an expired request deadline
+and DNS failures are not retried; a per-address connect timeout, which Node 26
+reports as `ConnectTimeoutError`, is. A retry answered with a null-body status
+(204, 205, 304) or a status above 599 fails that source instead of crashing
+the server, and a non-OK response releases its connection at once. FIRMS
+warnings include the per-address connection errors and never the request URL,
+which embeds the MAP_KEY.
+
 ## Installations and map-source guidance
 
 - Keyless mapped installations use OpenFreeMap `landuse` military polygons,

server/providers/firms.js

@@ -1,9 +1,132 @@
 import path from 'node:path';
+import https from 'node:https';
 import { promises as fsp } from 'node:fs';
+import { Readable } from 'node:stream';
 
 import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
 import { readResponseTextCapped } from './common/http.js';
 
+const NULL_BODY_STATUSES = new Set([204, 205, 304]);
+
+/**
+ * The per-address connection errors behind a fetch failure, or null. Node 24
+ * puts the AggregateError directly on `err.cause`; Node 26's fetch wraps it in
+ * a ConnectTimeoutError one level deeper.
+ * @param {unknown} err
+ * @returns {?Array<unknown>}
+ */
+function connectAttemptErrors(err) {
+  let cause = err?.cause;
+  for (let depth = 0; cause && depth < 3; depth++) {
+    if (Array.isArray(cause.errors)) return cause.errors;
+    cause = cause.cause;
+  }
+  return null;
+}
+
+/**
+ * Describe a fetch failure with its socket-level cause. Node reports every
+ * connection failure as `fetch failed`; the cause carries the error code and
+ * the address that failed (never the URL, which embeds the MAP_KEY).
+ * @param {unknown} err
+ * @returns {string}
+ */
+export function describeFetchError(err) {
+  const message = err?.message || String(err);
+  const cause = err?.cause;
+  if (!cause) return message;
+  const attempts = connectAttemptErrors(err);
+  const parts = attempts?.length
+    ? attempts.map((inner) => inner?.message || inner?.code || String(inner))
+    : [cause.message || cause.code || String(cause)];
+  return `${message} (${parts.join('; ')})`;
+}
+
+/**
+ * Whether a fetch failed because every address it tried failed to connect.
+ * Node's network-family auto-selection gives each address a short attempt
+ * timeout, so a slow IPv4 handshake is abandoned in favour of an unreachable
+ * IPv6 address and the whole request fails (#68).
+ * @param {unknown} err
+ * @returns {boolean}
+ */
+export function isAddressSelectionFailure(err) {
+  return err instanceof TypeError && connectAttemptErrors(err)?.length > 0;
+}
+
+/**
+ * GET over IPv4 only, with one connection attempt bounded by the caller's
+ * signal rather than by the auto-selection attempt timeout.
+ * @param {string} url
+ * @param {{signal?: AbortSignal}} [options]
+ * @returns {Promise<Response>}
+ */
+export function fetchIpv4(url, { signal } = {}) {
+  return new Promise((resolve, reject) => {
+    const request = https.request(
+      url,
+      {
+        method: 'GET',
+        signal,
+        family: 4,
+        autoSelectFamily: false,
+        agent: false,
+      },
+      (response) => {
+        // Nothing thrown here may escape: this callback runs outside the
+        // promise, so an exception would be uncaught and crash the server.
+        try {
+          const status = response.statusCode || 500;
+          // A `Response` only represents statuses up to 599; Node never hands
+          // this callback a 1xx.
+          if (status > 599)
+            throw new Error(`FIRMS returned unsupported status ${status}`);
+          const headers = new Headers();
+          for (const [name, value] of Object.entries(response.headers)) {
+            if (Array.isArray(value))
+              value.forEach((item) => headers.append(name, item));
+            else if (value !== undefined) headers.set(name, String(value));
+          }
+          const nullBody = NULL_BODY_STATUSES.has(status);
+          const upstream = new Response(
+            nullBody ? null : Readable.toWeb(response),
+            { status, statusText: response.statusMessage || '', headers },
+          );
+          if (nullBody) response.resume();
+          resolve(upstream);
+        } catch (error) {
+          response.destroy();
+          reject(error);
+        }
+      },
+    );
+    request.on('error', reject);
+    request.end();
+  });
+}
+
+/**
+ * Fetch a FIRMS URL, retrying once over IPv4 when address selection fails.
+ * Other failures (HTTP status, timeout, DNS) are returned or thrown as is.
+ * @param {string} url
+ * @param {{signal?: AbortSignal}} options
+ * @param {{fetchImpl?: typeof fetch, fetchIpv4Impl?: typeof fetchIpv4, warn?: Function}} [deps]
+ * @returns {Promise<Response>}
+ */
+export async function fetchFirms(
+  url,
+  options,
+  { fetchImpl = fetch, fetchIpv4Impl = fetchIpv4, warn = console.warn } = {},
+) {
+  try {
+    return await fetchImpl(url, options);
+  } catch (err) {
+    if (!isAddressSelectionFailure(err) || options?.signal?.aborted) throw err;
+    warn(`[firms-proxy] ${describeFetchError(err)} — retrying over IPv4`);
+    return fetchIpv4Impl(url, options);
+  }
+}
+
 /**
  * NASA FIRMS live active-fire proxy with a memory + disk cache.
  * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
@@ -91,8 +214,12 @@ export function firmsProxy() {
   async function fetchSource(key, source) {
     const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
     const signal = AbortSignal.timeout(60_000);
-    const res = await fetch(url, { signal });
-    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    const res = await fetchFirms(url, { signal });
+    if (!res.ok) {
+      // Release the connection now rather than when the deadline fires.
+      await res.body?.cancel().catch(() => {});
+      throw new Error(`HTTP ${res.status}`);
+    }
     const records = parseFirmsCsv(
       await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
     );
@@ -121,7 +248,7 @@ export function firmsProxy() {
       } catch (err) {
         console.warn(
           `[firms-proxy] ${source} fetch failed:`,
-          err?.message || err,
+          describeFetchError(err),
         );
         sources.push({ source, count: 0, ok: false });
       }
@@ -156,8 +283,13 @@ export function firmsProxy() {
       statusInflight = (async () => {
         try {
           const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`;
-          const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
-          if (!res.ok) throw new Error(`HTTP ${res.status}`);
+          const res = await fetchFirms(url, {
+            signal: AbortSignal.timeout(10_000),
+          });
+          if (!res.ok) {
+            await res.body?.cancel().catch(() => {});
+            throw new Error(`HTTP ${res.status}`);
+          }
           const body = await res.json();
           const used = Number(body?.current_transactions);
           const limit = Number(body?.transaction_limit);
@@ -167,7 +299,7 @@ export function firmsProxy() {
         } catch (err) {
           console.warn(
             '[firms-proxy] mapkey status failed:',
-            err?.message || err,
+            describeFetchError(err),
           );
           return null;
         }

src/data/firmsProxy.test.mjs

@@ -1,7 +1,18 @@
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import test from 'node:test';
+import { EventEmitter } from 'node:events';
+import dns from 'node:dns/promises';
+import https from 'node:https';
+import net from 'node:net';
+import { Readable } from 'node:stream';
 import { filterTrailing24h, parseFirmsCsv } from './firmsCsv.js';
+import {
+  describeFetchError,
+  fetchFirms,
+  fetchIpv4,
+  isAddressSelectionFailure,
+} from '../../server/providers/firms.js';
 
 const config = fs.readFileSync(new URL('../../server/providers/firms.js', import.meta.url), 'utf8');
 const start = config.indexOf('  async function refreshUpstream(key) {');
@@ -17,8 +28,8 @@ const recent = { acqDate: '2026-09-11', acqTime: '1100' };
 // Inject only its upstream, clock and filter dependencies; keep its aggregation
 // and source-status code intact, including failures while consuming records.
 function createRefresh(fetchSource, filter = filterTrailing24h) {
-  return new Function('SOURCES', 'fetchSource', 'filterTrailing24h', 'Date', 'console',
-    `return (${refreshSource});`)(SOURCES, fetchSource, filter, { now: () => NOW }, { warn() {} });
+  return new Function('SOURCES', 'fetchSource', 'filterTrailing24h', 'describeFetchError', 'Date', 'console',
+    `return (${refreshSource});`)(SOURCES, fetchSource, filter, describeFetchError, { now: () => NOW }, { warn() {} });
 }
 
 test('FIRMS retains large sources in order and filters expired rows', async () => {
@@ -78,3 +89,216 @@ test('FIRMS distinguishes all-source failure from successful empty sources', asy
   assert.deepEqual(result.fires, []);
   assert.deepEqual(result.sources, SOURCES.map(source => ({ source, count: 0, ok: true })));
 });
+
+// #68: Node's network-family auto-selection gives each address a short attempt
+// timeout, abandons a slow IPv4 handshake for an unreachable IPv6 address, and
+// reports only `fetch failed`. This is the shape Node 24 throws for it.
+function addressSelectionFailure() {
+  const v4 = Object.assign(new Error('connect ETIMEDOUT 198.118.194.34:443'), { code: 'ETIMEDOUT' });
+  const v6 = Object.assign(new Error('connect ENETUNREACH 2001:4d0:241a:40c0::34:443'), { code: 'ENETUNREACH' });
+  const cause = Object.assign(new AggregateError([v4, v6]), { code: 'ETIMEDOUT' });
+  return new TypeError('fetch failed', { cause });
+}
+// Node 26's fetch wraps the same AggregateError in a ConnectTimeoutError.
+function wrappedAddressSelectionFailure() {
+  const inner = addressSelectionFailure().cause;
+  const cause = Object.assign(new Error('Connect Timeout Error (attempted addresses: 198.118.194.34:443, 2001:4d0:241a:40c0::34:443, timeout: 10000ms)'),
+    { name: 'ConnectTimeoutError', code: 'UND_ERR_CONNECT_TIMEOUT', cause: inner });
+  return new TypeError('fetch failed', { cause });
+}
+const URL_WITH_KEY = 'https://firms.modaps.eosdis.nasa.gov/api/area/csv/SECRET/VIIRS_SNPP_NRT/world/2';
+
+test('FIRMS failures log the socket cause, never the keyed URL', () => {
+  const described = describeFetchError(addressSelectionFailure());
+  assert.equal(described,
+    'fetch failed (connect ETIMEDOUT 198.118.194.34:443; connect ENETUNREACH 2001:4d0:241a:40c0::34:443)');
+  assert.ok(!described.includes('SECRET'));
+  assert.equal(describeFetchError(wrappedAddressSelectionFailure()), described, 'Node 26 reports the same attempts');
+  const single = new TypeError('fetch failed', { cause: Object.assign(new Error('getaddrinfo ENOTFOUND x'), { code: 'ENOTFOUND' }) });
+  assert.equal(describeFetchError(single), 'fetch failed (getaddrinfo ENOTFOUND x)');
+  assert.equal(describeFetchError(new Error('HTTP 503')), 'HTTP 503');
+});
+
+test('FIRMS retries once over IPv4 when every address fails to connect', async () => {
+  for (const shape of [addressSelectionFailure, wrappedAddressSelectionFailure]) {
+    const retried = [];
+    await fetchFirms(URL_WITH_KEY, {}, {
+      fetchImpl: async () => { throw shape(); },
+      fetchIpv4Impl: async (url) => { retried.push(url); return new Response('csv'); },
+      warn() {},
+    });
+    assert.deepEqual(retried, [URL_WITH_KEY], `${shape.name} is retried`);
+  }
+  const signal = new AbortController().signal;
+  const ok = new Response('csv');
+  const retried = [];
+  const warnings = [];
+  const response = await fetchFirms(URL_WITH_KEY, { signal }, {
+    fetchImpl: async () => { throw addressSelectionFailure(); },
+    fetchIpv4Impl: async (url, options) => { retried.push([url, options]); return ok; },
+    warn: (message) => warnings.push(message),
+  });
+  assert.equal(response, ok);
+  assert.deepEqual(retried, [[URL_WITH_KEY, { signal }]], 'one retry, same URL and deadline');
+  assert.equal(warnings.length, 1);
+  assert.match(warnings[0], /ETIMEDOUT 198\.118\.194\.34:443; .*ENETUNREACH .* retrying over IPv4/);
+  assert.ok(!warnings[0].includes('SECRET'));
+
+  const failure = new Error('connect ETIMEDOUT 198.118.194.34:443');
+  await assert.rejects(fetchFirms(URL_WITH_KEY, {}, {
+    fetchImpl: async () => { throw addressSelectionFailure(); },
+    fetchIpv4Impl: async () => { throw failure; },
+    warn() {},
+  }), (err) => err === failure, 'a failed retry reports its own error');
+});
+
+test('FIRMS does not retry anything but an address-selection failure', async () => {
+  const deps = (fetchImpl) => ({
+    fetchImpl,
+    fetchIpv4Impl: async () => assert.fail('must not retry'),
+    warn: () => assert.fail('must not warn'),
+  });
+  const serverError = new Response('down', { status: 503 });
+  assert.equal(await fetchFirms(URL_WITH_KEY, {}, deps(async () => serverError)), serverError);
+  for (const thrown of [
+    new TypeError('fetch failed', { cause: Object.assign(new Error('getaddrinfo ENOTFOUND x'), { code: 'ENOTFOUND' }) }),
+    new TypeError('fetch failed', { cause: new AggregateError([]) }),
+    new TypeError('fetch failed'),
+    new DOMException('The operation was aborted due to timeout', 'TimeoutError'),
+  ]) {
+    await assert.rejects(fetchFirms(URL_WITH_KEY, {}, deps(async () => { throw thrown; })), (err) => err === thrown);
+  }
+  const controller = new AbortController();
+  controller.abort();
+  const aborted = addressSelectionFailure();
+  await assert.rejects(fetchFirms(URL_WITH_KEY, { signal: controller.signal },
+    deps(async () => { throw aborted; })), (err) => err === aborted, 'an expired deadline is not retried');
+});
+
+test('the IPv4 retry connects to an IPv4 address only', async (t) => {
+  // `localhost` resolves to both ::1 and 127.0.0.1. Listen on both with the
+  // same port and record which one the retry reaches; the TLS handshake then
+  // fails against a plain socket, which is fine — only the connection matters.
+  const reached = [];
+  const listen = (host, port) => new Promise((resolve, reject) => {
+    const server = net.createServer((socket) => { reached.push(host); socket.destroy(); });
+    server.once('error', reject).listen(port, host, () => resolve(server));
+  });
+  const v4 = await listen('127.0.0.1', 0);
+  t.after(() => v4.close());
+  const { port } = v4.address();
+  let v6;
+  try {
+    v6 = await listen('::1', port);
+  } catch {
+    t.skip('no IPv6 loopback on this host, so IPv4-only cannot be told apart');
+    return;
+  }
+  t.after(() => v6.close());
+  const order = await dns.lookup('localhost', { all: true });
+  if (order[0]?.family !== 6) {
+    t.skip('localhost resolves IPv4 first here, so IPv4-only cannot be told apart');
+    return;
+  }
+  await assert.rejects(fetchIpv4(`https://localhost:${port}/`, { signal: AbortSignal.timeout(5_000) }));
+  assert.deepEqual(reached, ['127.0.0.1']);
+});
+
+// Answer the IPv4 retry with `status`, invoking the response callback
+// asynchronously as `https.request` does, outside the caller's promise.
+// The returned promise rejects on an uncaught exception, so a regression fails
+// the race at once instead of crashing the test runner.
+function mockIpv4Status(t, status, statusMessage = '') {
+  let crash;
+  const crashed = new Promise((_resolve, reject) => (crash = reject));
+  const onUncaught = (error) =>
+    crash(new Error(`uncaught exception: ${error.message}`));
+  process.on('uncaughtException', onUncaught);
+  t.after(() => process.off('uncaughtException', onUncaught));
+  t.mock.method(https, 'request', (_url, _options, callback) => {
+    const request = new EventEmitter();
+    request.end = () =>
+      setImmediate(() =>
+        callback(
+          Object.assign(Readable.from([]), {
+            statusCode: status,
+            statusMessage,
+            headers: { 'content-type': 'text/csv' },
+          }),
+        ),
+      );
+    return request;
+  });
+  return crashed;
+}
+
+for (const status of [204, 205, 304]) {
+  test(`the IPv4 retry answering null-body status ${status} resolves instead of crashing`, async (t) => {
+    const crashed = mockIpv4Status(t, status);
+    const response = await Promise.race([fetchIpv4(URL_WITH_KEY), crashed]);
+    assert.equal(response.status, status);
+    assert.equal(response.body, null);
+  });
+}
+
+for (const status of [600, 999]) {
+  test(`the IPv4 retry answering out-of-range status ${status} rejects instead of crashing`, async (t) => {
+    const crashed = mockIpv4Status(t, status);
+    await assert.rejects(
+      Promise.race([fetchIpv4(URL_WITH_KEY), crashed]),
+      new RegExp(`unsupported status ${status}`),
+    );
+  });
+}
+
+test('the IPv4 retry answering an invalid reason phrase rejects instead of crashing', async (t) => {
+  const crashed = mockIpv4Status(t, 200, 'OK\x7f');
+  await assert.rejects(Promise.race([fetchIpv4(URL_WITH_KEY), crashed]), TypeError);
+});
+
+// The fixtures above copy Node's error shape; check it against a real fetch so
+// a Node or undici change that stops matching fails here instead of silently
+// disabling the retry. A closed port on `localhost` refuses both addresses.
+test('a real all-addresses connect failure is recognised as address selection', async (t) => {
+  const probe = net.createServer();
+  try {
+    await new Promise((resolve, reject) => probe.once('error', reject).listen(0, '::1', resolve));
+  } catch {
+    t.skip('no IPv6 loopback on this host');
+    return;
+  }
+  const { port } = probe.address();
+  await new Promise((resolve) => probe.close(resolve));
+  const v4 = net.createServer();
+  await new Promise((resolve, reject) => v4.once('error', reject).listen(port, '127.0.0.1', resolve));
+  await new Promise((resolve) => v4.close(resolve));
+  const err = await fetch(`http://localhost:${port}/`).then(() => null, (error) => error);
+  assert.ok(err, 'a closed port refuses the connection');
+  assert.equal(isAddressSelectionFailure(err), true, describeFetchError(err));
+  assert.match(describeFetchError(err), /ECONNREFUSED .*; .*ECONNREFUSED/);
+});
+
+// #68 only stays fixed while both upstream calls go through fetchFirms; the
+// refresh harness injects fetchSource, so guard the call sites directly.
+test('every FIRMS upstream request goes through the IPv4 fallback', async () => {
+  assert.doesNotMatch(config, /await fetch\(/, 'no FIRMS request bypasses fetchFirms');
+  const fetchStart = config.indexOf('  async function fetchSource(key, source) {');
+  assert.notEqual(fetchStart, -1, 'FIRMS fetchSource exists');
+  const fetchSourceText = config.slice(fetchStart, config.indexOf('\n  }', fetchStart) + 4);
+  const calls = [];
+  const response = new Response('latitude,longitude\n');
+  const fetchSource = new Function('fetchFirms', 'readResponseTextCapped', 'parseFirmsCsv',
+    'FIRMS_MAX_RESPONSE_BYTES', 'AbortSignal', 'fetch', `return (${fetchSourceText});`)(
+    async (url, options) => { calls.push(['fetchFirms', url, options.signal]); return response; },
+    async (res, cap, signal) => { calls.push(['read', res, signal]); return 'csv'; },
+    () => [],
+    1024,
+    AbortSignal,
+    () => assert.fail('fetchSource must call fetchFirms, not fetch'),
+  );
+  assert.deepEqual(await fetchSource('SECRET', 'MODIS_NRT'), []);
+  assert.equal(calls.length, 2);
+  assert.equal(calls[0][0], 'fetchFirms');
+  assert.ok(calls[0][2] instanceof AbortSignal);
+  assert.deepEqual(calls[1], ['read', response, calls[0][2]], 'the body read shares the request deadline');
+});