diff --git a/changelog.d/1078-firms-ipv4-fallback.md b/changelog.d/1078-firms-ipv4-fallback.md new file mode 100644 index 000000000..ff4e60e1b --- /dev/null +++ b/changelog.d/1078-firms-ipv4-fallback.md @@ -0,0 +1,4 @@ +- FIRMS fire data loads on hosts where IPv6 is unreachable and IPv4 is slow to + connect: a request whose every address fails to connect retries once over + IPv4, and FIRMS warnings name the underlying connection errors instead of + only `fetch failed` (jeffblakely, #1078). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 52ad689aa..e1425bfc2 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2114,6 +2114,19 @@ throws, that source reports failure without a contradictory success entry. The existing per-record append continues to support large feeds; sequential fetching, trailing-24-hour filtering and partial-success caching are unchanged. +FIRMS requests retry once over IPv4 when every resolved address fails to +connect. Node's network-family auto-selection gives each address a short +attempt timeout, so on hosts where IPv6 is unreachable a slow IPv4 handshake +was abandoned and the request reported only `fetch failed` (#68). The retry +makes a single IPv4 connection within the same deadline and the same body +cap, and does not follow redirects. HTTP errors, an expired request deadline +and DNS failures are not retried; a per-address connect timeout, which Node 26 +reports as `ConnectTimeoutError`, is. A retry answered with a null-body status +(204, 205, 304) or a status above 599 fails that source instead of crashing +the server, and a non-OK response releases its connection at once. FIRMS +warnings include the per-address connection errors and never the request URL, +which embeds the MAP_KEY. + ## Installations and map-source guidance - Keyless mapped installations use OpenFreeMap `landuse` military polygons, diff --git a/server/providers/firms.js b/server/providers/firms.js index c73ca10ce..20a2f4bab 100644 --- a/server/providers/firms.js +++ b/server/providers/firms.js @@ -1,9 +1,132 @@ import path from 'node:path'; +import https from 'node:https'; import { promises as fsp } from 'node:fs'; +import { Readable } from 'node:stream'; import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; import { readResponseTextCapped } from './common/http.js'; +const NULL_BODY_STATUSES = new Set([204, 205, 304]); + +/** + * The per-address connection errors behind a fetch failure, or null. Node 24 + * puts the AggregateError directly on `err.cause`; Node 26's fetch wraps it in + * a ConnectTimeoutError one level deeper. + * @param {unknown} err + * @returns {?Array} + */ +function connectAttemptErrors(err) { + let cause = err?.cause; + for (let depth = 0; cause && depth < 3; depth++) { + if (Array.isArray(cause.errors)) return cause.errors; + cause = cause.cause; + } + return null; +} + +/** + * Describe a fetch failure with its socket-level cause. Node reports every + * connection failure as `fetch failed`; the cause carries the error code and + * the address that failed (never the URL, which embeds the MAP_KEY). + * @param {unknown} err + * @returns {string} + */ +export function describeFetchError(err) { + const message = err?.message || String(err); + const cause = err?.cause; + if (!cause) return message; + const attempts = connectAttemptErrors(err); + const parts = attempts?.length + ? attempts.map((inner) => inner?.message || inner?.code || String(inner)) + : [cause.message || cause.code || String(cause)]; + return `${message} (${parts.join('; ')})`; +} + +/** + * Whether a fetch failed because every address it tried failed to connect. + * Node's network-family auto-selection gives each address a short attempt + * timeout, so a slow IPv4 handshake is abandoned in favour of an unreachable + * IPv6 address and the whole request fails (#68). + * @param {unknown} err + * @returns {boolean} + */ +export function isAddressSelectionFailure(err) { + return err instanceof TypeError && connectAttemptErrors(err)?.length > 0; +} + +/** + * GET over IPv4 only, with one connection attempt bounded by the caller's + * signal rather than by the auto-selection attempt timeout. + * @param {string} url + * @param {{signal?: AbortSignal}} [options] + * @returns {Promise} + */ +export function fetchIpv4(url, { signal } = {}) { + return new Promise((resolve, reject) => { + const request = https.request( + url, + { + method: 'GET', + signal, + family: 4, + autoSelectFamily: false, + agent: false, + }, + (response) => { + // Nothing thrown here may escape: this callback runs outside the + // promise, so an exception would be uncaught and crash the server. + try { + const status = response.statusCode || 500; + // A `Response` only represents statuses up to 599; Node never hands + // this callback a 1xx. + if (status > 599) + throw new Error(`FIRMS returned unsupported status ${status}`); + const headers = new Headers(); + for (const [name, value] of Object.entries(response.headers)) { + if (Array.isArray(value)) + value.forEach((item) => headers.append(name, item)); + else if (value !== undefined) headers.set(name, String(value)); + } + const nullBody = NULL_BODY_STATUSES.has(status); + const upstream = new Response( + nullBody ? null : Readable.toWeb(response), + { status, statusText: response.statusMessage || '', headers }, + ); + if (nullBody) response.resume(); + resolve(upstream); + } catch (error) { + response.destroy(); + reject(error); + } + }, + ); + request.on('error', reject); + request.end(); + }); +} + +/** + * Fetch a FIRMS URL, retrying once over IPv4 when address selection fails. + * Other failures (HTTP status, timeout, DNS) are returned or thrown as is. + * @param {string} url + * @param {{signal?: AbortSignal}} options + * @param {{fetchImpl?: typeof fetch, fetchIpv4Impl?: typeof fetchIpv4, warn?: Function}} [deps] + * @returns {Promise} + */ +export async function fetchFirms( + url, + options, + { fetchImpl = fetch, fetchIpv4Impl = fetchIpv4, warn = console.warn } = {}, +) { + try { + return await fetchImpl(url, options); + } catch (err) { + if (!isAddressSelectionFailure(err) || options?.signal?.aborted) throw err; + warn(`[firms-proxy] ${describeFetchError(err)} — retrying over IPv4`); + return fetchIpv4Impl(url, options); + } +} + /** * NASA FIRMS live active-fire proxy with a memory + disk cache. * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2 @@ -91,8 +214,12 @@ export function firmsProxy() { async function fetchSource(key, source) { const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; const signal = AbortSignal.timeout(60_000); - const res = await fetch(url, { signal }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); + const res = await fetchFirms(url, { signal }); + if (!res.ok) { + // Release the connection now rather than when the deadline fires. + await res.body?.cancel().catch(() => {}); + throw new Error(`HTTP ${res.status}`); + } const records = parseFirmsCsv( await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal), ); @@ -121,7 +248,7 @@ export function firmsProxy() { } catch (err) { console.warn( `[firms-proxy] ${source} fetch failed:`, - err?.message || err, + describeFetchError(err), ); sources.push({ source, count: 0, ok: false }); } @@ -156,8 +283,13 @@ export function firmsProxy() { statusInflight = (async () => { try { const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`; - const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); + const res = await fetchFirms(url, { + signal: AbortSignal.timeout(10_000), + }); + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + throw new Error(`HTTP ${res.status}`); + } const body = await res.json(); const used = Number(body?.current_transactions); const limit = Number(body?.transaction_limit); @@ -167,7 +299,7 @@ export function firmsProxy() { } catch (err) { console.warn( '[firms-proxy] mapkey status failed:', - err?.message || err, + describeFetchError(err), ); return null; } diff --git a/src/data/firmsProxy.test.mjs b/src/data/firmsProxy.test.mjs index a54bd42fe..9ed35ba24 100644 --- a/src/data/firmsProxy.test.mjs +++ b/src/data/firmsProxy.test.mjs @@ -1,7 +1,18 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import test from 'node:test'; +import { EventEmitter } from 'node:events'; +import dns from 'node:dns/promises'; +import https from 'node:https'; +import net from 'node:net'; +import { Readable } from 'node:stream'; import { filterTrailing24h, parseFirmsCsv } from './firmsCsv.js'; +import { + describeFetchError, + fetchFirms, + fetchIpv4, + isAddressSelectionFailure, +} from '../../server/providers/firms.js'; const config = fs.readFileSync(new URL('../../server/providers/firms.js', import.meta.url), 'utf8'); const start = config.indexOf(' async function refreshUpstream(key) {'); @@ -17,8 +28,8 @@ const recent = { acqDate: '2026-09-11', acqTime: '1100' }; // Inject only its upstream, clock and filter dependencies; keep its aggregation // and source-status code intact, including failures while consuming records. function createRefresh(fetchSource, filter = filterTrailing24h) { - return new Function('SOURCES', 'fetchSource', 'filterTrailing24h', 'Date', 'console', - `return (${refreshSource});`)(SOURCES, fetchSource, filter, { now: () => NOW }, { warn() {} }); + return new Function('SOURCES', 'fetchSource', 'filterTrailing24h', 'describeFetchError', 'Date', 'console', + `return (${refreshSource});`)(SOURCES, fetchSource, filter, describeFetchError, { now: () => NOW }, { warn() {} }); } test('FIRMS retains large sources in order and filters expired rows', async () => { @@ -78,3 +89,216 @@ test('FIRMS distinguishes all-source failure from successful empty sources', asy assert.deepEqual(result.fires, []); assert.deepEqual(result.sources, SOURCES.map(source => ({ source, count: 0, ok: true }))); }); + +// #68: Node's network-family auto-selection gives each address a short attempt +// timeout, abandons a slow IPv4 handshake for an unreachable IPv6 address, and +// reports only `fetch failed`. This is the shape Node 24 throws for it. +function addressSelectionFailure() { + const v4 = Object.assign(new Error('connect ETIMEDOUT 198.118.194.34:443'), { code: 'ETIMEDOUT' }); + const v6 = Object.assign(new Error('connect ENETUNREACH 2001:4d0:241a:40c0::34:443'), { code: 'ENETUNREACH' }); + const cause = Object.assign(new AggregateError([v4, v6]), { code: 'ETIMEDOUT' }); + return new TypeError('fetch failed', { cause }); +} +// Node 26's fetch wraps the same AggregateError in a ConnectTimeoutError. +function wrappedAddressSelectionFailure() { + const inner = addressSelectionFailure().cause; + const cause = Object.assign(new Error('Connect Timeout Error (attempted addresses: 198.118.194.34:443, 2001:4d0:241a:40c0::34:443, timeout: 10000ms)'), + { name: 'ConnectTimeoutError', code: 'UND_ERR_CONNECT_TIMEOUT', cause: inner }); + return new TypeError('fetch failed', { cause }); +} +const URL_WITH_KEY = 'https://firms.modaps.eosdis.nasa.gov/api/area/csv/SECRET/VIIRS_SNPP_NRT/world/2'; + +test('FIRMS failures log the socket cause, never the keyed URL', () => { + const described = describeFetchError(addressSelectionFailure()); + assert.equal(described, + 'fetch failed (connect ETIMEDOUT 198.118.194.34:443; connect ENETUNREACH 2001:4d0:241a:40c0::34:443)'); + assert.ok(!described.includes('SECRET')); + assert.equal(describeFetchError(wrappedAddressSelectionFailure()), described, 'Node 26 reports the same attempts'); + const single = new TypeError('fetch failed', { cause: Object.assign(new Error('getaddrinfo ENOTFOUND x'), { code: 'ENOTFOUND' }) }); + assert.equal(describeFetchError(single), 'fetch failed (getaddrinfo ENOTFOUND x)'); + assert.equal(describeFetchError(new Error('HTTP 503')), 'HTTP 503'); +}); + +test('FIRMS retries once over IPv4 when every address fails to connect', async () => { + for (const shape of [addressSelectionFailure, wrappedAddressSelectionFailure]) { + const retried = []; + await fetchFirms(URL_WITH_KEY, {}, { + fetchImpl: async () => { throw shape(); }, + fetchIpv4Impl: async (url) => { retried.push(url); return new Response('csv'); }, + warn() {}, + }); + assert.deepEqual(retried, [URL_WITH_KEY], `${shape.name} is retried`); + } + const signal = new AbortController().signal; + const ok = new Response('csv'); + const retried = []; + const warnings = []; + const response = await fetchFirms(URL_WITH_KEY, { signal }, { + fetchImpl: async () => { throw addressSelectionFailure(); }, + fetchIpv4Impl: async (url, options) => { retried.push([url, options]); return ok; }, + warn: (message) => warnings.push(message), + }); + assert.equal(response, ok); + assert.deepEqual(retried, [[URL_WITH_KEY, { signal }]], 'one retry, same URL and deadline'); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /ETIMEDOUT 198\.118\.194\.34:443; .*ENETUNREACH .* retrying over IPv4/); + assert.ok(!warnings[0].includes('SECRET')); + + const failure = new Error('connect ETIMEDOUT 198.118.194.34:443'); + await assert.rejects(fetchFirms(URL_WITH_KEY, {}, { + fetchImpl: async () => { throw addressSelectionFailure(); }, + fetchIpv4Impl: async () => { throw failure; }, + warn() {}, + }), (err) => err === failure, 'a failed retry reports its own error'); +}); + +test('FIRMS does not retry anything but an address-selection failure', async () => { + const deps = (fetchImpl) => ({ + fetchImpl, + fetchIpv4Impl: async () => assert.fail('must not retry'), + warn: () => assert.fail('must not warn'), + }); + const serverError = new Response('down', { status: 503 }); + assert.equal(await fetchFirms(URL_WITH_KEY, {}, deps(async () => serverError)), serverError); + for (const thrown of [ + new TypeError('fetch failed', { cause: Object.assign(new Error('getaddrinfo ENOTFOUND x'), { code: 'ENOTFOUND' }) }), + new TypeError('fetch failed', { cause: new AggregateError([]) }), + new TypeError('fetch failed'), + new DOMException('The operation was aborted due to timeout', 'TimeoutError'), + ]) { + await assert.rejects(fetchFirms(URL_WITH_KEY, {}, deps(async () => { throw thrown; })), (err) => err === thrown); + } + const controller = new AbortController(); + controller.abort(); + const aborted = addressSelectionFailure(); + await assert.rejects(fetchFirms(URL_WITH_KEY, { signal: controller.signal }, + deps(async () => { throw aborted; })), (err) => err === aborted, 'an expired deadline is not retried'); +}); + +test('the IPv4 retry connects to an IPv4 address only', async (t) => { + // `localhost` resolves to both ::1 and 127.0.0.1. Listen on both with the + // same port and record which one the retry reaches; the TLS handshake then + // fails against a plain socket, which is fine — only the connection matters. + const reached = []; + const listen = (host, port) => new Promise((resolve, reject) => { + const server = net.createServer((socket) => { reached.push(host); socket.destroy(); }); + server.once('error', reject).listen(port, host, () => resolve(server)); + }); + const v4 = await listen('127.0.0.1', 0); + t.after(() => v4.close()); + const { port } = v4.address(); + let v6; + try { + v6 = await listen('::1', port); + } catch { + t.skip('no IPv6 loopback on this host, so IPv4-only cannot be told apart'); + return; + } + t.after(() => v6.close()); + const order = await dns.lookup('localhost', { all: true }); + if (order[0]?.family !== 6) { + t.skip('localhost resolves IPv4 first here, so IPv4-only cannot be told apart'); + return; + } + await assert.rejects(fetchIpv4(`https://localhost:${port}/`, { signal: AbortSignal.timeout(5_000) })); + assert.deepEqual(reached, ['127.0.0.1']); +}); + +// Answer the IPv4 retry with `status`, invoking the response callback +// asynchronously as `https.request` does, outside the caller's promise. +// The returned promise rejects on an uncaught exception, so a regression fails +// the race at once instead of crashing the test runner. +function mockIpv4Status(t, status, statusMessage = '') { + let crash; + const crashed = new Promise((_resolve, reject) => (crash = reject)); + const onUncaught = (error) => + crash(new Error(`uncaught exception: ${error.message}`)); + process.on('uncaughtException', onUncaught); + t.after(() => process.off('uncaughtException', onUncaught)); + t.mock.method(https, 'request', (_url, _options, callback) => { + const request = new EventEmitter(); + request.end = () => + setImmediate(() => + callback( + Object.assign(Readable.from([]), { + statusCode: status, + statusMessage, + headers: { 'content-type': 'text/csv' }, + }), + ), + ); + return request; + }); + return crashed; +} + +for (const status of [204, 205, 304]) { + test(`the IPv4 retry answering null-body status ${status} resolves instead of crashing`, async (t) => { + const crashed = mockIpv4Status(t, status); + const response = await Promise.race([fetchIpv4(URL_WITH_KEY), crashed]); + assert.equal(response.status, status); + assert.equal(response.body, null); + }); +} + +for (const status of [600, 999]) { + test(`the IPv4 retry answering out-of-range status ${status} rejects instead of crashing`, async (t) => { + const crashed = mockIpv4Status(t, status); + await assert.rejects( + Promise.race([fetchIpv4(URL_WITH_KEY), crashed]), + new RegExp(`unsupported status ${status}`), + ); + }); +} + +test('the IPv4 retry answering an invalid reason phrase rejects instead of crashing', async (t) => { + const crashed = mockIpv4Status(t, 200, 'OK\x7f'); + await assert.rejects(Promise.race([fetchIpv4(URL_WITH_KEY), crashed]), TypeError); +}); + +// The fixtures above copy Node's error shape; check it against a real fetch so +// a Node or undici change that stops matching fails here instead of silently +// disabling the retry. A closed port on `localhost` refuses both addresses. +test('a real all-addresses connect failure is recognised as address selection', async (t) => { + const probe = net.createServer(); + try { + await new Promise((resolve, reject) => probe.once('error', reject).listen(0, '::1', resolve)); + } catch { + t.skip('no IPv6 loopback on this host'); + return; + } + const { port } = probe.address(); + await new Promise((resolve) => probe.close(resolve)); + const v4 = net.createServer(); + await new Promise((resolve, reject) => v4.once('error', reject).listen(port, '127.0.0.1', resolve)); + await new Promise((resolve) => v4.close(resolve)); + const err = await fetch(`http://localhost:${port}/`).then(() => null, (error) => error); + assert.ok(err, 'a closed port refuses the connection'); + assert.equal(isAddressSelectionFailure(err), true, describeFetchError(err)); + assert.match(describeFetchError(err), /ECONNREFUSED .*; .*ECONNREFUSED/); +}); + +// #68 only stays fixed while both upstream calls go through fetchFirms; the +// refresh harness injects fetchSource, so guard the call sites directly. +test('every FIRMS upstream request goes through the IPv4 fallback', async () => { + assert.doesNotMatch(config, /await fetch\(/, 'no FIRMS request bypasses fetchFirms'); + const fetchStart = config.indexOf(' async function fetchSource(key, source) {'); + assert.notEqual(fetchStart, -1, 'FIRMS fetchSource exists'); + const fetchSourceText = config.slice(fetchStart, config.indexOf('\n }', fetchStart) + 4); + const calls = []; + const response = new Response('latitude,longitude\n'); + const fetchSource = new Function('fetchFirms', 'readResponseTextCapped', 'parseFirmsCsv', + 'FIRMS_MAX_RESPONSE_BYTES', 'AbortSignal', 'fetch', `return (${fetchSourceText});`)( + async (url, options) => { calls.push(['fetchFirms', url, options.signal]); return response; }, + async (res, cap, signal) => { calls.push(['read', res, signal]); return 'csv'; }, + () => [], + 1024, + AbortSignal, + () => assert.fail('fetchSource must call fetchFirms, not fetch'), + ); + assert.deepEqual(await fetchSource('SECRET', 'MODIS_NRT'), []); + assert.equal(calls.length, 2); + assert.equal(calls[0][0], 'fetchFirms'); + assert.ok(calls[0][2] instanceof AbortSignal); + assert.deepEqual(calls[1], ['read', response, calls[0][2]], 'the body read shares the request deadline'); +});