diff --git a/src/library/parquet.js b/src/library/parquet.js index 29bb7ce99..eba0c047b 100644 --- a/src/library/parquet.js +++ b/src/library/parquet.js @@ -88,11 +88,20 @@ export function rangeBuffer(url, byteLength, { fetchImpl, signal }) { // else must be exactly the bytes asked for. const whole = response.status === 200; const expected = whole ? byteLength : end - start; - const bytes = await readResponseBytesCapped( - response, - whole ? Math.min(byteLength, MAX_UNRANGED_BYTES) : expected, - signal, - ); + let bytes; + try { + bytes = await readResponseBytesCapped( + response, + whole ? Math.min(byteLength, MAX_UNRANGED_BYTES) : expected, + signal, + ); + } catch (error) { + // A body cut off mid-stream (a reset, the proxy's timeout) is a + // fetch failure worth retrying; an oversized one is not. + if (signal?.aborted || error?.code === 'RESPONSE_TOO_LARGE') + throw error; + throw transient(error); + } if (bytes.byteLength !== expected) throw transient( new Error( diff --git a/src/library/parquet.test.mjs b/src/library/parquet.test.mjs index 65cf382ce..8044b4f1e 100644 --- a/src/library/parquet.test.mjs +++ b/src/library/parquet.test.mjs @@ -220,6 +220,50 @@ test('a footer that keeps failing leaves the index incomplete, and a retry recov assert.equal(broken.failed, 1); }); +test('a footer whose body fails mid-stream is retried, not left out', async () => { + const { buildPartitionIndex } = await import('./parquet.js'); + const entry = { + ...epc, + source: { + ...epc.source, + partitions: { + listUrl: 'https://s3.us-west-2.amazonaws.com/bucket?list-type=2', + keyBase: 'https://s3.us-west-2.amazonaws.com/bucket/', + }, + }, + }; + const listing = `a.parquet100`; + let cutOff = 0; + const fetchImpl = async (url) => { + if (url.includes('list-type')) + return new Response(listing, { status: 200 }); + if (cutOff > 0) { + cutOff -= 1; + // Headers arrive, then the connection drops. + const body = new ReadableStream({ + pull(controller) { + controller.error(new Error('connection reset')); + }, + }); + return new Response(body, { status: 206 }); + } + return new Response(new Uint8Array(100), { status: 200 }); + }; + cutOff = 100; + const broken = await buildPartitionIndex(entry, { + fetchImpl, + retryDelayMs: 0, + }); + assert.equal(broken.complete, false, 'never reported whole'); + assert.equal(broken.failed, 1); + cutOff = 1; + const recovered = await buildPartitionIndex(entry, { + fetchImpl, + retryDelayMs: 0, + }); + assert.equal(recovered.complete, true); +}); + test('the browser keeps a complete index over an incomplete rebuild and retries later', async () => { let clock = 30 * 24 * 3600_000; const complete = [{ url: 'https://h/A.parquet', bbox: [-2, 53, -1, 54] }];