mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-15 15:52:21 +00:00
Read an ArcGIS load stopped at maxFeatures as truncated
Paging stopped once maxFeatures were read before it noted that the upstream had more, so a page holding exactly the cap with exceededTransferLimit set read as the whole dataset. FEMA USA Structures (2,000 features, one page) never showed PARTIAL in a dense view. Stopping at the cap or the last page with more upstream now marks the load truncated.
This commit is contained in:
1 parent
134239fa5a
commit
e25cce874d
2 files changed
+30
-2
No files matched your search
@@ -429,8 +429,15 @@ export async function loadLibraryDataset(
|
|||||||
const parsed = parseLibraryBody('arcgis', await readPage(url, budget));
|
const parsed = parseLibraryBody('arcgis', await readPage(url, budget));
|
||||||
candidates.push(...parsed.candidates);
|
candidates.push(...parsed.candidates);
|
||||||
if (!parsed.exceededTransferLimit || !parsed.candidates.length) break;
|
if (!parsed.exceededTransferLimit || !parsed.candidates.length) break;
|
||||||
if (candidates.length >= source.maxFeatures) break;
|
// The upstream holds more than was read: stopping here, at the
|
||||||
if (page === source.maxPages - 1) truncated = true;
|
// feature cap or the last page, leaves the dataset partial.
|
||||||
|
if (
|
||||||
|
candidates.length >= source.maxFeatures ||
|
||||||
|
page === source.maxPages - 1
|
||||||
|
) {
|
||||||
|
truncated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const parsed = parseLibraryBody(
|
const parsed = parseLibraryBody(
|
||||||
|
|||||||
@@ -182,6 +182,27 @@ test('ArcGIS paging follows exceededTransferLimit up to maxPages', async () => {
|
|||||||
assert.equal(collection.truncated, true);
|
assert.equal(collection.truncated, true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('ArcGIS stopping at maxFeatures with more upstream reads as truncated', async () => {
|
||||||
|
const value = entry({
|
||||||
|
format: 'arcgis',
|
||||||
|
url: 'https://s.example.org/FeatureServer/0',
|
||||||
|
maxFeatures: 3,
|
||||||
|
maxPages: 5,
|
||||||
|
});
|
||||||
|
let pages = 0;
|
||||||
|
const collection = await loadLibraryDataset(value, async () => {
|
||||||
|
pages += 1;
|
||||||
|
return JSON.stringify({
|
||||||
|
type: 'FeatureCollection',
|
||||||
|
properties: { exceededTransferLimit: true },
|
||||||
|
features: [point(1, 0), point(2, 0), point(3, 0)],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
assert.equal(pages, 1);
|
||||||
|
assert.equal(collection.features.length, 3);
|
||||||
|
assert.equal(collection.truncated, true);
|
||||||
|
});
|
||||||
|
|
||||||
test('a plain dataset request carries the manifest query once', async () => {
|
test('a plain dataset request carries the manifest query once', async () => {
|
||||||
const value = entry({
|
const value = entry({
|
||||||
format: 'geojson',
|
format: 'geojson',
|
||||||
|
|||||||
Reference in new issue
Block a user