mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
fix(imagery): paginate raw CMR pages within the record budget
Rebased onto current main with review follow-ups folded in. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
This commit is contained in:
1 parent
0265ad4c19
commit
360aa74f50
4 files changed
+72
-6
No files matched your search
@@ -40,6 +40,7 @@ export async function fetchCmrPages({
|
||||
}) {
|
||||
const granules = [];
|
||||
let hits = 0;
|
||||
let readCount = 0;
|
||||
let cursor = null;
|
||||
for (;;) {
|
||||
const init = { signal };
|
||||
@@ -47,19 +48,24 @@ export async function fetchCmrPages({
|
||||
const response = await fetchImpl(url, init);
|
||||
if (!response?.ok)
|
||||
throw new Error(`HTTP ${response?.status ?? 'error'} from CMR`);
|
||||
const page = parseCmrUmm(await response.json(), product);
|
||||
granules.push(...page.granules);
|
||||
const json = await response.json();
|
||||
const rawCount = Array.isArray(json?.items) ? json.items.length : 0;
|
||||
const page = parseCmrUmm(json, product);
|
||||
readCount += rawCount;
|
||||
granules.push(
|
||||
...page.granules.slice(0, Math.max(0, maxRecords - granules.length)),
|
||||
);
|
||||
hits = Math.max(hits, page.hits);
|
||||
cursor = response.headers?.get?.(CMR_SEARCH_AFTER) ?? null;
|
||||
if (
|
||||
!cursor ||
|
||||
page.granules.length < CMR_PAGE_SIZE ||
|
||||
granules.length >= maxRecords ||
|
||||
granules.length >= hits
|
||||
rawCount < CMR_PAGE_SIZE ||
|
||||
readCount >= maxRecords ||
|
||||
readCount >= hits
|
||||
)
|
||||
break;
|
||||
}
|
||||
return { granules, truncated: hits > granules.length };
|
||||
return { granules, truncated: hits > readCount };
|
||||
}
|
||||
|
||||
function finiteOrNull(value) {
|
||||
|
||||
Reference in new issue
Block a user