Read a view over an incomplete GeoParquet index as PARTIAL

The browser's partition index kept whether it was complete only to
schedule a rebuild and handed the partitions on without it, so a view
counted unread partitions from an index that might be missing some and
could read as complete, even empty, over an area it had never indexed.
A view over an incomplete index now reads PARTIAL until a rebuild
completes.
This commit is contained in:
Sameh Khamis committed 2026-11-12 15:51:52 -08:00
1 parent e05a9566e4
commit 134239fa5a
2 files changed
+40 -4

No files matched your search

+30
View File
@@ -181,6 +181,36 @@ test('a revisit still finds homes at the edge of a partition', async () => {
assert.equal(outside.features.length, 0);
});
test('a view over an incomplete index reads PARTIAL', async () => {
const home = {
type: 'Feature',
geometry: { type: 'Point', coordinates: [-1.2, 53.3] },
properties: { CURRENT_ENERGY_RATING: 'C', UPRN: '1' },
};
const visit = (complete) =>
createLibrarySource({
entries: [epc],
fetchImpl: async () => ({ ok: false, status: 404 }),
storage: null,
loadParquet: async () => ({
...(await import('./parquet.js')),
buildPartitionIndex: async () => ({
partitions: [
{ url: 'https://h/A.parquet', bbox: [-1.5, 53.2, -1.1, 53.5] },
],
complete,
}),
readGeoParquetFeatures: async () => [home],
}),
});
const box = [-1.21, 53.29, -1.19, 53.31];
const whole = await visit(true).loadDataset(epc, { bbox: box });
assert.equal(whole.truncated, false);
const partial = await visit(false).loadDataset(epc, { bbox: box });
assert.equal(partial.features.length, 1);
assert.equal(partial.truncated, true);
});
test('a footer that keeps failing leaves the index incomplete, and a retry recovers', async () => {
const { buildPartitionIndex } = await import('./parquet.js');
const entry = {
+10 -4
View File
@@ -236,7 +236,7 @@ export function createLibrarySource({
current.retryAt = result.complete
? null
: now() + PARTIAL_INDEX_RETRY_MS;
return result.partitions;
return result;
},
(error) => {
if (indexes.get(entry.id)?.promise === promise)
@@ -283,7 +283,7 @@ export function createLibrarySource({
* batches until one batch finds features in view.
*/
async function loadPartitioned(entry, box, { signal }) {
const [{ partitionsForBox, featureInBox }, index] = await Promise.all([
const [{ partitionsForBox, featureInBox }, built] = await Promise.all([
loadParquet(),
partitionIndex(entry, { signal }),
]);
@@ -294,7 +294,12 @@ export function createLibrarySource({
}
const learned =
storedJson(storage, `${EXTENT_STORAGE_PREFIX}${entry.id}`) || {};
const ranked = rankPartitions(index, box, learned, partitionsForBox);
const ranked = rankPartitions(
built.partitions,
box,
learned,
partitionsForBox,
);
const candidates = ranked.slice(0, MAX_PARTITION_READS_PER_VIEW);
// Every partition the box touches but this view leaves unread (the read
// budget, or the early stop below) makes the result partial.
@@ -327,7 +332,8 @@ export function createLibrarySource({
});
return {
features: bounded.features,
truncated: bounded.truncated || unread > 0,
// An incomplete index may be missing partitions this box needs.
truncated: bounded.truncated || unread > 0 || !built.complete,
};
}