mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-13 14:52:19 +00:00
Retry a GeoParquet footer whose body fails mid-stream
Only a failed fetch() marked a footer read as worth retrying. A body cut off after its headers arrived (a reset, the proxy's 20-second timeout) was taken for an unparseable file: the partition was left out and the index still read as complete, so the server cached it without that partition for the entry's cacheSeconds (24 hours for EPC). Such a read is now retried like any fetch failure; an oversized body is still final.
This commit is contained in:
1 parent
7d91a1875b
commit
e05a9566e4
2 files changed
+58
-5
No files matched your search
+14
-5
@@ -88,11 +88,20 @@ export function rangeBuffer(url, byteLength, { fetchImpl, signal }) {
|
||||
// else must be exactly the bytes asked for.
|
||||
const whole = response.status === 200;
|
||||
const expected = whole ? byteLength : end - start;
|
||||
const bytes = await readResponseBytesCapped(
|
||||
response,
|
||||
whole ? Math.min(byteLength, MAX_UNRANGED_BYTES) : expected,
|
||||
signal,
|
||||
);
|
||||
let bytes;
|
||||
try {
|
||||
bytes = await readResponseBytesCapped(
|
||||
response,
|
||||
whole ? Math.min(byteLength, MAX_UNRANGED_BYTES) : expected,
|
||||
signal,
|
||||
);
|
||||
} catch (error) {
|
||||
// A body cut off mid-stream (a reset, the proxy's timeout) is a
|
||||
// fetch failure worth retrying; an oversized one is not.
|
||||
if (signal?.aborted || error?.code === 'RESPONSE_TOO_LARGE')
|
||||
throw error;
|
||||
throw transient(error);
|
||||
}
|
||||
if (bytes.byteLength !== expected)
|
||||
throw transient(
|
||||
new Error(
|
||||
|
||||
@@ -220,6 +220,50 @@ test('a footer that keeps failing leaves the index incomplete, and a retry recov
|
||||
assert.equal(broken.failed, 1);
|
||||
});
|
||||
|
||||
test('a footer whose body fails mid-stream is retried, not left out', async () => {
|
||||
const { buildPartitionIndex } = await import('./parquet.js');
|
||||
const entry = {
|
||||
...epc,
|
||||
source: {
|
||||
...epc.source,
|
||||
partitions: {
|
||||
listUrl: 'https://s3.us-west-2.amazonaws.com/bucket?list-type=2',
|
||||
keyBase: 'https://s3.us-west-2.amazonaws.com/bucket/',
|
||||
},
|
||||
},
|
||||
};
|
||||
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
|
||||
let cutOff = 0;
|
||||
const fetchImpl = async (url) => {
|
||||
if (url.includes('list-type'))
|
||||
return new Response(listing, { status: 200 });
|
||||
if (cutOff > 0) {
|
||||
cutOff -= 1;
|
||||
// Headers arrive, then the connection drops.
|
||||
const body = new ReadableStream({
|
||||
pull(controller) {
|
||||
controller.error(new Error('connection reset'));
|
||||
},
|
||||
});
|
||||
return new Response(body, { status: 206 });
|
||||
}
|
||||
return new Response(new Uint8Array(100), { status: 200 });
|
||||
};
|
||||
cutOff = 100;
|
||||
const broken = await buildPartitionIndex(entry, {
|
||||
fetchImpl,
|
||||
retryDelayMs: 0,
|
||||
});
|
||||
assert.equal(broken.complete, false, 'never reported whole');
|
||||
assert.equal(broken.failed, 1);
|
||||
cutOff = 1;
|
||||
const recovered = await buildPartitionIndex(entry, {
|
||||
fetchImpl,
|
||||
retryDelayMs: 0,
|
||||
});
|
||||
assert.equal(recovered.complete, true);
|
||||
});
|
||||
|
||||
test('the browser keeps a complete index over an incomplete rebuild and retries later', async () => {
|
||||
let clock = 30 * 24 * 3600_000;
|
||||
const complete = [{ url: 'https://h/A.parquet', bbox: [-2, 53, -1, 54] }];
|
||||
|
||||
Reference in new issue
Block a user