From 166aef58db448788d58eb6db078055d1eef79af2 Mon Sep 17 00:00:00 2001 From: Jeff Blakely <7987380+jeffblakely@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:53:33 -0400 Subject: [PATCH] fix(firms): retry over IPv4 when every address fails to connect Node's network-family auto-selection gives each resolved address a short attempt timeout. On hosts where IPv6 is unreachable, a slow IPv4 handshake is abandoned for the IPv6 address, which fails at once, and the request reports only `fetch failed`. Retry such failures once with a single IPv4 connection within the same deadline, and log the per-address connection errors (never the keyed URL). Node 26 wraps the AggregateError one level deeper, so both shapes are recognised. Fixes #68 --- CHANGELOG.md | 5 ++ docs/CURRENT-STATE.md | 9 +++ server/providers/firms.js | 119 +++++++++++++++++++++++++++++++++-- src/data/firmsProxy.test.mjs | 118 +++++++++++++++++++++++++++++++++- 4 files changed, 245 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b24ac95ed..8901fa255 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +- FIRMS fire data loads on hosts where IPv6 is unreachable and IPv4 is slow to + connect: a request whose every address fails to connect retries once over + IPv4, and FIRMS warnings name the underlying connection errors instead of + only `fetch failed`. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 26d56e66b..bf26418ad 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1393,6 +1393,15 @@ throws, that source reports failure without a contradictory success entry. The existing per-record append continues to support large feeds; sequential fetching, trailing-24-hour filtering and partial-success caching are unchanged. +FIRMS requests retry once over IPv4 when every resolved address fails to +connect. Node's network-family auto-selection gives each address a short +attempt timeout, so on hosts where IPv6 is unreachable a slow IPv4 handshake +was abandoned and the request reported only `fetch failed` (#68). The retry +makes a single IPv4 connection within the same deadline; HTTP errors, +timeouts and DNS failures are not retried. FIRMS warnings include the +per-address connection errors and never the request URL, which embeds the +MAP_KEY. + ## Installations and map-source guidance - Keyless mapped installations use OpenFreeMap `landuse` military polygons, diff --git a/server/providers/firms.js b/server/providers/firms.js index 636968348..531c9a99f 100644 --- a/server/providers/firms.js +++ b/server/providers/firms.js @@ -1,8 +1,117 @@ import path from 'node:path'; +import https from 'node:https'; import { promises as fsp } from 'node:fs'; +import { Readable } from 'node:stream'; import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; +/** + * The per-address connection errors behind a fetch failure, or null. Node 24 + * puts the AggregateError directly on `err.cause`; Node 26's fetch wraps it in + * a ConnectTimeoutError one level deeper. + * @param {unknown} err + * @returns {?Array} + */ +function connectAttemptErrors(err) { + let cause = err?.cause; + for (let depth = 0; cause && depth < 3; depth++) { + if (Array.isArray(cause.errors)) return cause.errors; + cause = cause.cause; + } + return null; +} + +/** + * Describe a fetch failure with its socket-level cause. Node reports every + * connection failure as `fetch failed`; the cause carries the error code and + * the address that failed (never the URL, which embeds the MAP_KEY). + * @param {unknown} err + * @returns {string} + */ +export function describeFetchError(err) { + const message = err?.message || String(err); + const cause = err?.cause; + if (!cause) return message; + const attempts = connectAttemptErrors(err); + const parts = attempts?.length + ? attempts.map((inner) => inner?.message || inner?.code || String(inner)) + : [cause.message || cause.code || String(cause)]; + return `${message} (${parts.join('; ')})`; +} + +/** + * Whether a fetch failed because every address it tried failed to connect. + * Node's network-family auto-selection gives each address a short attempt + * timeout, so a slow IPv4 handshake is abandoned in favour of an unreachable + * IPv6 address and the whole request fails (#68). + * @param {unknown} err + * @returns {boolean} + */ +export function isAddressSelectionFailure(err) { + return err instanceof TypeError && connectAttemptErrors(err)?.length > 0; +} + +/** + * GET over IPv4 only, with one connection attempt bounded by the caller's + * signal rather than by the auto-selection attempt timeout. + * @param {string} url + * @param {{signal?: AbortSignal}} [options] + * @returns {Promise} + */ +export function fetchIpv4(url, { signal } = {}) { + return new Promise((resolve, reject) => { + const request = https.request( + url, + { + method: 'GET', + signal, + family: 4, + autoSelectFamily: false, + agent: false, + }, + (response) => { + const headers = new Headers(); + for (const [name, value] of Object.entries(response.headers)) { + if (Array.isArray(value)) + value.forEach((item) => headers.append(name, item)); + else if (value !== undefined) headers.set(name, String(value)); + } + resolve( + new Response(Readable.toWeb(response), { + status: response.statusCode || 500, + statusText: response.statusMessage || '', + headers, + }), + ); + }, + ); + request.on('error', reject); + request.end(); + }); +} + +/** + * Fetch a FIRMS URL, retrying once over IPv4 when address selection fails. + * Other failures (HTTP status, timeout, DNS) are returned or thrown as is. + * @param {string} url + * @param {{signal?: AbortSignal}} options + * @param {{fetchImpl?: typeof fetch, fetchIpv4Impl?: typeof fetchIpv4, warn?: Function}} [deps] + * @returns {Promise} + */ +export async function fetchFirms( + url, + options, + { fetchImpl = fetch, fetchIpv4Impl = fetchIpv4, warn = console.warn } = {}, +) { + try { + return await fetchImpl(url, options); + } catch (err) { + if (!isAddressSelectionFailure(err) || options?.signal?.aborted) throw err; + warn(`[firms-proxy] ${describeFetchError(err)} — retrying over IPv4`); + return fetchIpv4Impl(url, options); + } +} + /** * NASA FIRMS live active-fire proxy with a memory + disk cache. * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2 @@ -83,7 +192,7 @@ export function firmsProxy() { */ async function fetchSource(key, source) { const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; - const res = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + const res = await fetchFirms(url, { signal: AbortSignal.timeout(60_000) }); if (!res.ok) throw new Error(`HTTP ${res.status}`); const records = parseFirmsCsv(await res.text()); if (records === null) throw new Error('non-CSV upstream response'); @@ -111,7 +220,7 @@ export function firmsProxy() { } catch (err) { console.warn( `[firms-proxy] ${source} fetch failed:`, - err?.message || err, + describeFetchError(err), ); sources.push({ source, count: 0, ok: false }); } @@ -146,7 +255,9 @@ export function firmsProxy() { statusInflight = (async () => { try { const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`; - const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); + const res = await fetchFirms(url, { + signal: AbortSignal.timeout(10_000), + }); if (!res.ok) throw new Error(`HTTP ${res.status}`); const body = await res.json(); const used = Number(body?.current_transactions); @@ -157,7 +268,7 @@ export function firmsProxy() { } catch (err) { console.warn( '[firms-proxy] mapkey status failed:', - err?.message || err, + describeFetchError(err), ); return null; } diff --git a/src/data/firmsProxy.test.mjs b/src/data/firmsProxy.test.mjs index a54bd42fe..f24d93349 100644 --- a/src/data/firmsProxy.test.mjs +++ b/src/data/firmsProxy.test.mjs @@ -2,6 +2,12 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import test from 'node:test'; import { filterTrailing24h, parseFirmsCsv } from './firmsCsv.js'; +import net from 'node:net'; +import { + describeFetchError, + fetchFirms, + fetchIpv4, +} from '../../server/providers/firms.js'; const config = fs.readFileSync(new URL('../../server/providers/firms.js', import.meta.url), 'utf8'); const start = config.indexOf(' async function refreshUpstream(key) {'); @@ -17,8 +23,8 @@ const recent = { acqDate: '2026-09-11', acqTime: '1100' }; // Inject only its upstream, clock and filter dependencies; keep its aggregation // and source-status code intact, including failures while consuming records. function createRefresh(fetchSource, filter = filterTrailing24h) { - return new Function('SOURCES', 'fetchSource', 'filterTrailing24h', 'Date', 'console', - `return (${refreshSource});`)(SOURCES, fetchSource, filter, { now: () => NOW }, { warn() {} }); + return new Function('SOURCES', 'fetchSource', 'filterTrailing24h', 'describeFetchError', 'Date', 'console', + `return (${refreshSource});`)(SOURCES, fetchSource, filter, describeFetchError, { now: () => NOW }, { warn() {} }); } test('FIRMS retains large sources in order and filters expired rows', async () => { @@ -78,3 +84,111 @@ test('FIRMS distinguishes all-source failure from successful empty sources', asy assert.deepEqual(result.fires, []); assert.deepEqual(result.sources, SOURCES.map(source => ({ source, count: 0, ok: true }))); }); + +// #68: Node's network-family auto-selection gives each address a short attempt +// timeout, abandons a slow IPv4 handshake for an unreachable IPv6 address, and +// reports only `fetch failed`. This is the shape Node 24 throws for it. +function addressSelectionFailure() { + const v4 = Object.assign(new Error('connect ETIMEDOUT 198.118.194.34:443'), { code: 'ETIMEDOUT' }); + const v6 = Object.assign(new Error('connect ENETUNREACH 2001:4d0:241a:40c0::34:443'), { code: 'ENETUNREACH' }); + const cause = Object.assign(new AggregateError([v4, v6]), { code: 'ETIMEDOUT' }); + return new TypeError('fetch failed', { cause }); +} +// Node 26's fetch wraps the same AggregateError in a ConnectTimeoutError. +function wrappedAddressSelectionFailure() { + const inner = addressSelectionFailure().cause; + const cause = Object.assign(new Error('Connect Timeout Error (attempted addresses: 198.118.194.34:443, 2001:4d0:241a:40c0::34:443, timeout: 10000ms)'), + { name: 'ConnectTimeoutError', code: 'UND_ERR_CONNECT_TIMEOUT', cause: inner }); + return new TypeError('fetch failed', { cause }); +} +const URL_WITH_KEY = 'https://firms.modaps.eosdis.nasa.gov/api/area/csv/SECRET/VIIRS_SNPP_NRT/world/2'; + +test('FIRMS failures log the socket cause, never the keyed URL', () => { + const described = describeFetchError(addressSelectionFailure()); + assert.equal(described, + 'fetch failed (connect ETIMEDOUT 198.118.194.34:443; connect ENETUNREACH 2001:4d0:241a:40c0::34:443)'); + assert.ok(!described.includes('SECRET')); + assert.equal(describeFetchError(wrappedAddressSelectionFailure()), described, 'Node 26 reports the same attempts'); + const single = new TypeError('fetch failed', { cause: Object.assign(new Error('getaddrinfo ENOTFOUND x'), { code: 'ENOTFOUND' }) }); + assert.equal(describeFetchError(single), 'fetch failed (getaddrinfo ENOTFOUND x)'); + assert.equal(describeFetchError(new Error('HTTP 503')), 'HTTP 503'); +}); + +test('FIRMS retries once over IPv4 when every address fails to connect', async () => { + for (const shape of [addressSelectionFailure, wrappedAddressSelectionFailure]) { + const retried = []; + await fetchFirms(URL_WITH_KEY, {}, { + fetchImpl: async () => { throw shape(); }, + fetchIpv4Impl: async (url) => { retried.push(url); return new Response('csv'); }, + warn() {}, + }); + assert.deepEqual(retried, [URL_WITH_KEY], `${shape.name} is retried`); + } + const signal = new AbortController().signal; + const ok = new Response('csv'); + const retried = []; + const warnings = []; + const response = await fetchFirms(URL_WITH_KEY, { signal }, { + fetchImpl: async () => { throw addressSelectionFailure(); }, + fetchIpv4Impl: async (url, options) => { retried.push([url, options]); return ok; }, + warn: (message) => warnings.push(message), + }); + assert.equal(response, ok); + assert.deepEqual(retried, [[URL_WITH_KEY, { signal }]], 'one retry, same URL and deadline'); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /ETIMEDOUT 198\.118\.194\.34:443; .*ENETUNREACH .* retrying over IPv4/); + assert.ok(!warnings[0].includes('SECRET')); + + const failure = new Error('connect ETIMEDOUT 198.118.194.34:443'); + await assert.rejects(fetchFirms(URL_WITH_KEY, {}, { + fetchImpl: async () => { throw addressSelectionFailure(); }, + fetchIpv4Impl: async () => { throw failure; }, + warn() {}, + }), (err) => err === failure, 'a failed retry reports its own error'); +}); + +test('FIRMS does not retry anything but an address-selection failure', async () => { + const deps = (fetchImpl) => ({ + fetchImpl, + fetchIpv4Impl: async () => assert.fail('must not retry'), + warn: () => assert.fail('must not warn'), + }); + const serverError = new Response('down', { status: 503 }); + assert.equal(await fetchFirms(URL_WITH_KEY, {}, deps(async () => serverError)), serverError); + for (const thrown of [ + new TypeError('fetch failed', { cause: Object.assign(new Error('getaddrinfo ENOTFOUND x'), { code: 'ENOTFOUND' }) }), + new TypeError('fetch failed', { cause: new AggregateError([]) }), + new TypeError('fetch failed'), + new DOMException('The operation was aborted due to timeout', 'TimeoutError'), + ]) { + await assert.rejects(fetchFirms(URL_WITH_KEY, {}, deps(async () => { throw thrown; })), (err) => err === thrown); + } + const controller = new AbortController(); + controller.abort(); + const aborted = addressSelectionFailure(); + await assert.rejects(fetchFirms(URL_WITH_KEY, { signal: controller.signal }, + deps(async () => { throw aborted; })), (err) => err === aborted, 'an expired deadline is not retried'); +}); + +test('the IPv4 retry connects to an IPv4 address only', async (t) => { + // `localhost` resolves to both ::1 and 127.0.0.1. Listen on both with the + // same port and record which one the retry reaches; the TLS handshake then + // fails against a plain socket, which is fine — only the connection matters. + const reached = []; + const listen = (host, port) => new Promise((resolve, reject) => { + const server = net.createServer((socket) => { reached.push(host); socket.destroy(); }); + server.once('error', reject).listen(port, host, () => resolve(server)); + }); + const v4 = await listen('127.0.0.1', 0); + t.after(() => v4.close()); + const { port } = v4.address(); + let v6 = null; + try { + v6 = await listen('::1', port); + t.after(() => v6.close()); + } catch { + // No IPv6 loopback on this host; the IPv4 assertion still holds. + } + await assert.rejects(fetchIpv4(`https://localhost:${port}/`, { signal: AbortSignal.timeout(5_000) })); + assert.deepEqual(reached, ['127.0.0.1']); +});