Read an ArcGIS load stopped at maxFeatures as truncated

Paging stopped once maxFeatures were read before it noted that the
upstream had more, so a page holding exactly the cap with
exceededTransferLimit set read as the whole dataset. FEMA USA
Structures (2,000 features, one page) never showed PARTIAL in a dense
view. Stopping at the cap or the last page with more upstream now marks
the load truncated.
This commit is contained in:
Sameh Khamis committed 2026-11-12 15:52:31 -08:00
1 parent 134239fa5a
commit e25cce874d
2 files changed
+30 -2

No files matched your search

+9 -2
View File
@@ -429,8 +429,15 @@ export async function loadLibraryDataset(
const parsed = parseLibraryBody('arcgis', await readPage(url, budget));
candidates.push(...parsed.candidates);
if (!parsed.exceededTransferLimit || !parsed.candidates.length) break;
if (candidates.length >= source.maxFeatures) break;
if (page === source.maxPages - 1) truncated = true;
// The upstream holds more than was read: stopping here, at the
// feature cap or the last page, leaves the dataset partial.
if (
candidates.length >= source.maxFeatures ||
page === source.maxPages - 1
) {
truncated = true;
break;
}
}
} else {
const parsed = parseLibraryBody(
+21
View File
@@ -182,6 +182,27 @@ test('ArcGIS paging follows exceededTransferLimit up to maxPages', async () => {
assert.equal(collection.truncated, true);
});
test('ArcGIS stopping at maxFeatures with more upstream reads as truncated', async () => {
const value = entry({
format: 'arcgis',
url: 'https://s.example.org/FeatureServer/0',
maxFeatures: 3,
maxPages: 5,
});
let pages = 0;
const collection = await loadLibraryDataset(value, async () => {
pages += 1;
return JSON.stringify({
type: 'FeatureCollection',
properties: { exceededTransferLimit: true },
features: [point(1, 0), point(2, 0), point(3, 0)],
});
});
assert.equal(pages, 1);
assert.equal(collection.features.length, 3);
assert.equal(collection.truncated, true);
});
test('a plain dataset request carries the manifest query once', async () => {
const value = entry({
format: 'geojson',