mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-13 06:42:23 +00:00
Read a view over an incomplete GeoParquet index as PARTIAL
The browser's partition index kept whether it was complete only to schedule a rebuild and handed the partitions on without it, so a view counted unread partitions from an index that might be missing some and could read as complete, even empty, over an area it had never indexed. A view over an incomplete index now reads PARTIAL until a rebuild completes.
This commit is contained in:
1 parent
e05a9566e4
commit
134239fa5a
2 files changed
+40
-4
No files matched your search
@@ -181,6 +181,36 @@ test('a revisit still finds homes at the edge of a partition', async () => {
|
||||
assert.equal(outside.features.length, 0);
|
||||
});
|
||||
|
||||
test('a view over an incomplete index reads PARTIAL', async () => {
|
||||
const home = {
|
||||
type: 'Feature',
|
||||
geometry: { type: 'Point', coordinates: [-1.2, 53.3] },
|
||||
properties: { CURRENT_ENERGY_RATING: 'C', UPRN: '1' },
|
||||
};
|
||||
const visit = (complete) =>
|
||||
createLibrarySource({
|
||||
entries: [epc],
|
||||
fetchImpl: async () => ({ ok: false, status: 404 }),
|
||||
storage: null,
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => ({
|
||||
partitions: [
|
||||
{ url: 'https://h/A.parquet', bbox: [-1.5, 53.2, -1.1, 53.5] },
|
||||
],
|
||||
complete,
|
||||
}),
|
||||
readGeoParquetFeatures: async () => [home],
|
||||
}),
|
||||
});
|
||||
const box = [-1.21, 53.29, -1.19, 53.31];
|
||||
const whole = await visit(true).loadDataset(epc, { bbox: box });
|
||||
assert.equal(whole.truncated, false);
|
||||
const partial = await visit(false).loadDataset(epc, { bbox: box });
|
||||
assert.equal(partial.features.length, 1);
|
||||
assert.equal(partial.truncated, true);
|
||||
});
|
||||
|
||||
test('a footer that keeps failing leaves the index incomplete, and a retry recovers', async () => {
|
||||
const { buildPartitionIndex } = await import('./parquet.js');
|
||||
const entry = {
|
||||
|
||||
+10
-4
@@ -236,7 +236,7 @@ export function createLibrarySource({
|
||||
current.retryAt = result.complete
|
||||
? null
|
||||
: now() + PARTIAL_INDEX_RETRY_MS;
|
||||
return result.partitions;
|
||||
return result;
|
||||
},
|
||||
(error) => {
|
||||
if (indexes.get(entry.id)?.promise === promise)
|
||||
@@ -283,7 +283,7 @@ export function createLibrarySource({
|
||||
* batches until one batch finds features in view.
|
||||
*/
|
||||
async function loadPartitioned(entry, box, { signal }) {
|
||||
const [{ partitionsForBox, featureInBox }, index] = await Promise.all([
|
||||
const [{ partitionsForBox, featureInBox }, built] = await Promise.all([
|
||||
loadParquet(),
|
||||
partitionIndex(entry, { signal }),
|
||||
]);
|
||||
@@ -294,7 +294,12 @@ export function createLibrarySource({
|
||||
}
|
||||
const learned =
|
||||
storedJson(storage, `${EXTENT_STORAGE_PREFIX}${entry.id}`) || {};
|
||||
const ranked = rankPartitions(index, box, learned, partitionsForBox);
|
||||
const ranked = rankPartitions(
|
||||
built.partitions,
|
||||
box,
|
||||
learned,
|
||||
partitionsForBox,
|
||||
);
|
||||
const candidates = ranked.slice(0, MAX_PARTITION_READS_PER_VIEW);
|
||||
// Every partition the box touches but this view leaves unread (the read
|
||||
// budget, or the early stop below) makes the result partial.
|
||||
@@ -327,7 +332,8 @@ export function createLibrarySource({
|
||||
});
|
||||
return {
|
||||
features: bounded.features,
|
||||
truncated: bounded.truncated || unread > 0,
|
||||
// An incomplete index may be missing partitions this box needs.
|
||||
truncated: bounded.truncated || unread > 0 || !built.complete,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user