mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-15 15:52:21 +00:00
Read a view over an incomplete GeoParquet index as PARTIAL
The browser's partition index kept whether it was complete only to schedule a rebuild and handed the partitions on without it, so a view counted unread partitions from an index that might be missing some and could read as complete, even empty, over an area it had never indexed. A view over an incomplete index now reads PARTIAL until a rebuild completes.
This commit is contained in:
1 parent
e05a9566e4
commit
134239fa5a
2 files changed
+40
-4
No files matched your search
@@ -181,6 +181,36 @@ test('a revisit still finds homes at the edge of a partition', async () => {
|
|||||||
assert.equal(outside.features.length, 0);
|
assert.equal(outside.features.length, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('a view over an incomplete index reads PARTIAL', async () => {
|
||||||
|
const home = {
|
||||||
|
type: 'Feature',
|
||||||
|
geometry: { type: 'Point', coordinates: [-1.2, 53.3] },
|
||||||
|
properties: { CURRENT_ENERGY_RATING: 'C', UPRN: '1' },
|
||||||
|
};
|
||||||
|
const visit = (complete) =>
|
||||||
|
createLibrarySource({
|
||||||
|
entries: [epc],
|
||||||
|
fetchImpl: async () => ({ ok: false, status: 404 }),
|
||||||
|
storage: null,
|
||||||
|
loadParquet: async () => ({
|
||||||
|
...(await import('./parquet.js')),
|
||||||
|
buildPartitionIndex: async () => ({
|
||||||
|
partitions: [
|
||||||
|
{ url: 'https://h/A.parquet', bbox: [-1.5, 53.2, -1.1, 53.5] },
|
||||||
|
],
|
||||||
|
complete,
|
||||||
|
}),
|
||||||
|
readGeoParquetFeatures: async () => [home],
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const box = [-1.21, 53.29, -1.19, 53.31];
|
||||||
|
const whole = await visit(true).loadDataset(epc, { bbox: box });
|
||||||
|
assert.equal(whole.truncated, false);
|
||||||
|
const partial = await visit(false).loadDataset(epc, { bbox: box });
|
||||||
|
assert.equal(partial.features.length, 1);
|
||||||
|
assert.equal(partial.truncated, true);
|
||||||
|
});
|
||||||
|
|
||||||
test('a footer that keeps failing leaves the index incomplete, and a retry recovers', async () => {
|
test('a footer that keeps failing leaves the index incomplete, and a retry recovers', async () => {
|
||||||
const { buildPartitionIndex } = await import('./parquet.js');
|
const { buildPartitionIndex } = await import('./parquet.js');
|
||||||
const entry = {
|
const entry = {
|
||||||
|
|||||||
+10
-4
@@ -236,7 +236,7 @@ export function createLibrarySource({
|
|||||||
current.retryAt = result.complete
|
current.retryAt = result.complete
|
||||||
? null
|
? null
|
||||||
: now() + PARTIAL_INDEX_RETRY_MS;
|
: now() + PARTIAL_INDEX_RETRY_MS;
|
||||||
return result.partitions;
|
return result;
|
||||||
},
|
},
|
||||||
(error) => {
|
(error) => {
|
||||||
if (indexes.get(entry.id)?.promise === promise)
|
if (indexes.get(entry.id)?.promise === promise)
|
||||||
@@ -283,7 +283,7 @@ export function createLibrarySource({
|
|||||||
* batches until one batch finds features in view.
|
* batches until one batch finds features in view.
|
||||||
*/
|
*/
|
||||||
async function loadPartitioned(entry, box, { signal }) {
|
async function loadPartitioned(entry, box, { signal }) {
|
||||||
const [{ partitionsForBox, featureInBox }, index] = await Promise.all([
|
const [{ partitionsForBox, featureInBox }, built] = await Promise.all([
|
||||||
loadParquet(),
|
loadParquet(),
|
||||||
partitionIndex(entry, { signal }),
|
partitionIndex(entry, { signal }),
|
||||||
]);
|
]);
|
||||||
@@ -294,7 +294,12 @@ export function createLibrarySource({
|
|||||||
}
|
}
|
||||||
const learned =
|
const learned =
|
||||||
storedJson(storage, `${EXTENT_STORAGE_PREFIX}${entry.id}`) || {};
|
storedJson(storage, `${EXTENT_STORAGE_PREFIX}${entry.id}`) || {};
|
||||||
const ranked = rankPartitions(index, box, learned, partitionsForBox);
|
const ranked = rankPartitions(
|
||||||
|
built.partitions,
|
||||||
|
box,
|
||||||
|
learned,
|
||||||
|
partitionsForBox,
|
||||||
|
);
|
||||||
const candidates = ranked.slice(0, MAX_PARTITION_READS_PER_VIEW);
|
const candidates = ranked.slice(0, MAX_PARTITION_READS_PER_VIEW);
|
||||||
// Every partition the box touches but this view leaves unread (the read
|
// Every partition the box touches but this view leaves unread (the read
|
||||||
// budget, or the early stop below) makes the result partial.
|
// budget, or the early stop below) makes the result partial.
|
||||||
@@ -327,7 +332,8 @@ export function createLibrarySource({
|
|||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
features: bounded.features,
|
features: bounded.features,
|
||||||
truncated: bounded.truncated || unread > 0,
|
// An incomplete index may be missing partitions this box needs.
|
||||||
|
truncated: bounded.truncated || unread > 0 || !built.complete,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user