From 360aa74f50c72cc485734978977bf0884de06282 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Wed, 14 Oct 2026 20:13:52 -0400 Subject: [PATCH 1/2] fix(imagery): paginate raw CMR pages within the record budget Rebased onto current main with review follow-ups folded in. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 + docs/CURRENT-STATE.md | 2 + src/layers/recentImagery/catalog.js | 18 ++++-- .../cmr-raw-page-budget.test.mjs | 56 +++++++++++++++++++ 4 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 src/layers/recentImagery/cmr-raw-page-budget.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index a770fbd35..9b4d82063 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ - Director timeline seeks cross the dateline along the short longitude arc. +- Recent Imagery continues past malformed records in full catalog pages and respects the requested record limit. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1f794a5b5..b65f90ca0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4449,6 +4449,8 @@ are omitted rather than framing the wrong part of the globe. ### Recent Imagery (September 2026) +CMR pagination counts raw rows when checking page completion and the record budget; catalog truncation means upstream hits exceed rows examined, so discarded malformed rows do not imply unseen imagery. + - The layer sits in the Data Layers panel's Cameras group beside Cameras (CCTV); Mapped ALPR Cameras moved to the Infrastructure group to make room. - DATA ▸ **Recent Imagery** (`recent-imagery`, `src/layers/recentImagery/`) lists the last 30 UTC days of imagery for a selected box and drapes the days you pick: HLS S30 (Sentinel-2) and L30 (Landsat 8/9) at 30 m, and the VIIRS NOAA-21 daily overview at 250 m. Keyless and browser-direct: NASA CMR for which days have HLS granules in the box (`catalog.js`, both collections concurrently, paged with `CMR-Search-After` up to 2,000 records each, `truncated` against CMR `hits`), GIBS WMTS for tiles, Worldview Snapshots for thumbnails and exports. Worldview's `Data-Present` header is the only availability signal: GIBS answers an empty tile with HTTP 200, and the JPEG overview would drape black, so a day confirmed empty never drapes. - **Box.** SELECT BOX (`src/ui/imageryBoxTool.js`) claims the pointer like Draw, borrows the stock click actions and freezes the camera only mid-drag; USE VIEW takes the camera rectangle; `boxFromPinAt` grows a 10 km box. Boxes must be ≤ 1,000 km a side, on one side of the dateline and inside the Web-Mercator limit (`validateBox`). A refusal (`boxError`, including the tool's own via `reportBoxRefusal`) stays at the top of the panel until a box succeeds; a view over the cap is refused with its size, never shrunk. An oversized box or view (`Box is … km wide · limit 1,000 km`, `View is … wide · …`) sets the hint to `Zoom in or draw a smaller box` and enables ZOOM IN: `zoomToFit()` flies top-down (1.2 s) to the refused box's centre, or the ground under the canvas centre for a view, at the height whose view is about 400 km wide for the canvas aspect and the camera's vertical FOV (`fitViewHeightM`, clamped 5–400 km), and returns that height. diff --git a/src/layers/recentImagery/catalog.js b/src/layers/recentImagery/catalog.js index 96b314690..fe9a7a021 100644 --- a/src/layers/recentImagery/catalog.js +++ b/src/layers/recentImagery/catalog.js @@ -40,6 +40,7 @@ export async function fetchCmrPages({ }) { const granules = []; let hits = 0; + let readCount = 0; let cursor = null; for (;;) { const init = { signal }; @@ -47,19 +48,24 @@ export async function fetchCmrPages({ const response = await fetchImpl(url, init); if (!response?.ok) throw new Error(`HTTP ${response?.status ?? 'error'} from CMR`); - const page = parseCmrUmm(await response.json(), product); - granules.push(...page.granules); + const json = await response.json(); + const rawCount = Array.isArray(json?.items) ? json.items.length : 0; + const page = parseCmrUmm(json, product); + readCount += rawCount; + granules.push( + ...page.granules.slice(0, Math.max(0, maxRecords - granules.length)), + ); hits = Math.max(hits, page.hits); cursor = response.headers?.get?.(CMR_SEARCH_AFTER) ?? null; if ( !cursor || - page.granules.length < CMR_PAGE_SIZE || - granules.length >= maxRecords || - granules.length >= hits + rawCount < CMR_PAGE_SIZE || + readCount >= maxRecords || + readCount >= hits ) break; } - return { granules, truncated: hits > granules.length }; + return { granules, truncated: hits > readCount }; } function finiteOrNull(value) { diff --git a/src/layers/recentImagery/cmr-raw-page-budget.test.mjs b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs new file mode 100644 index 000000000..4925e7bec --- /dev/null +++ b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs @@ -0,0 +1,56 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { fetchCmrPages } from './catalog.js'; +const item = (id) => ({ + meta: { 'concept-id': id }, + umm: { + TemporalExtent: { + RangeDateTime: { BeginningDateTime: '2026-09-18T12:00:00Z' }, + }, + }, +}); +test('discarded rows do not turn a full CMR page into a final page', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + fetchImpl: async () => { + calls++; + const items = + calls === 1 + ? Array.from({ length: 200 }, (_, i) => + i === 0 ? {} : item(String(i)), + ) + : [item('last')]; + return Response.json( + { hits: 201, items }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 200); + assert.equal(result.granules.at(-1).id, 'last'); + assert.equal(result.truncated, false); +}); +test('a non-page-multiple limit never returns more admitted records than its budget', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + return Response.json( + { + hits: 1000, + items: Array.from({ length: 200 }, (_, i) => item(`${calls}:${i}`)), + }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, true); +}); From 64988f3037e33e46c336f6a914e83411821729fb Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 15 Oct 2026 15:43:38 +0100 Subject: [PATCH 2/2] fix(imagery): mark CMR results truncated when the record cap drops valid granules Co-Authored-By: Claude Opus 5.5 --- src/layers/recentImagery/catalog.js | 13 ++++-- .../cmr-raw-page-budget.test.mjs | 42 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/layers/recentImagery/catalog.js b/src/layers/recentImagery/catalog.js index fe9a7a021..14964b1d2 100644 --- a/src/layers/recentImagery/catalog.js +++ b/src/layers/recentImagery/catalog.js @@ -27,7 +27,8 @@ const CMR_SEARCH_AFTER = 'CMR-Search-After'; /** * Read one product's granules across CMR pages, sending back each response's * `CMR-Search-After` cursor until a short page, a missing cursor or the - * record cap ends the walk. `truncated` is measured against CMR's `hits`. + * record cap ends the walk. `truncated` means CMR's `hits` exceed the rows + * read, or the cap dropped valid granules from the last page read. * @param {{ product: string, url: string, fetchImpl: typeof fetch, signal?: AbortSignal, maxRecords?: number }} request * @returns {Promise<{ granules: Array, truncated: boolean }>} */ @@ -41,6 +42,7 @@ export async function fetchCmrPages({ const granules = []; let hits = 0; let readCount = 0; + let capped = 0; let cursor = null; for (;;) { const init = { signal }; @@ -52,9 +54,12 @@ export async function fetchCmrPages({ const rawCount = Array.isArray(json?.items) ? json.items.length : 0; const page = parseCmrUmm(json, product); readCount += rawCount; - granules.push( - ...page.granules.slice(0, Math.max(0, maxRecords - granules.length)), + const admitted = page.granules.slice( + 0, + Math.max(0, maxRecords - granules.length), ); + granules.push(...admitted); + capped += page.granules.length - admitted.length; hits = Math.max(hits, page.hits); cursor = response.headers?.get?.(CMR_SEARCH_AFTER) ?? null; if ( @@ -65,7 +70,7 @@ export async function fetchCmrPages({ ) break; } - return { granules, truncated: hits > readCount }; + return { granules, truncated: hits > readCount || capped > 0 }; } function finiteOrNull(value) { diff --git a/src/layers/recentImagery/cmr-raw-page-budget.test.mjs b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs index 4925e7bec..c1709b9b2 100644 --- a/src/layers/recentImagery/cmr-raw-page-budget.test.mjs +++ b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs @@ -54,3 +54,45 @@ test('a non-page-multiple limit never returns more admitted records than its bud assert.equal(result.granules.length, 250); assert.equal(result.truncated, true); }); +test('valid records dropped by the cap mark the result truncated even when every hit was read', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + return Response.json( + { + hits: 400, + items: Array.from({ length: 200 }, (_, i) => item(`${calls}:${i}`)), + }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, true); +}); +test('malformed rows discarded under the cap leave a fully examined result untruncated', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + const items = Array.from({ length: 200 }, (_, i) => + calls === 2 && i >= 50 ? {} : item(`${calls}:${i}`), + ); + return Response.json( + { hits: 400, items }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, false); +});