Merge pull request #1042 from rudycelekli/contrib/gods-021-cmr-raw-page-budget-20261011-reviewed

fix(imagery): paginate raw CMR pages within the record budget

CHANGELOG.md

@@ -36,7 +36,9 @@
 
 - Director timeline seeks cross the dateline along the short longitude arc.
 
-- Recent Imagery thumbnails require explicit presence evidence before reporting available pixels.
+- Recent Imagery continues past malformed records in full catalog pages, respects the requested record limit, and reports results as truncated when that limit drops valid records.
+
+- Regional articles retain unknown publication times when compact source dates do not identify a real calendar instant.
 
 - Treat explicitly current, complete empty vessel coverage as a successful update,
   clearing obsolete contacts while retaining stale or incomplete snapshots.

docs/CURRENT-STATE.md

@@ -4542,7 +4542,12 @@ are omitted rather than framing the wrong part of the globe.
 
 ### Recent Imagery (September 2026)
 
-Thumbnail decoding requires explicit `Data-Present: true` evidence; false remains empty, while absent or malformed evidence reports an error. Injected `getThumbnail` sources must return an OK Response with readable `Data-Present: true` to admit image bytes, or `false` for confirmed empty coverage. Cross-origin sources must expose that header to browser JavaScript.
+CMR pagination counts raw rows when checking page completion and the record budget. A catalog is truncated when upstream hits exceed the rows examined or the record cap drops valid granules from the last page; discarded malformed rows do not imply unseen imagery.
+
+Programmatic boxes passed to the exported `setBox` and `searchHls` operations
+reject missing, blank or boolean edges instead of interpreting them as zero
+coordinates. Valid finite numeric edges remain accepted; the app’s drag tool,
+pins and share links already supply complete boxes.
 
 - The layer sits in the Data Layers panel's Cameras group beside Cameras (CCTV); Mapped ALPR Cameras moved to the Infrastructure group to make room.
 - DATA ▸ **Recent Imagery** (`recent-imagery`, `src/layers/recentImagery/`) lists the last 30 UTC days of imagery for a selected box and drapes the days you pick: HLS S30 (Sentinel-2) and L30 (Landsat 8/9) at 30 m, and the VIIRS NOAA-21 daily overview at 250 m. Keyless and browser-direct: NASA CMR for which days have HLS granules in the box (`catalog.js`, both collections concurrently, paged with `CMR-Search-After` up to 2,000 records each, `truncated` against CMR `hits`), GIBS WMTS for tiles, Worldview Snapshots for thumbnails and exports. Worldview's `Data-Present` header is the only availability signal: GIBS answers an empty tile with HTTP 200, and the JPEG overview would drape black, so a day confirmed empty never drapes.

src/layers/recentImagery/catalog.js

@@ -27,7 +27,8 @@ const CMR_SEARCH_AFTER = 'CMR-Search-After';
 /**
  * Read one product's granules across CMR pages, sending back each response's
  * `CMR-Search-After` cursor until a short page, a missing cursor or the
- * record cap ends the walk. `truncated` is measured against CMR's `hits`.
+ * record cap ends the walk. `truncated` means CMR's `hits` exceed the rows
+ * read, or the cap dropped valid granules from the last page read.
  * @param {{ product: string, url: string, fetchImpl: typeof fetch, signal?: AbortSignal, maxRecords?: number }} request
  * @returns {Promise<{ granules: Array<object>, truncated: boolean }>}
  */
@@ -40,6 +41,8 @@ export async function fetchCmrPages({
 }) {
   const granules = [];
   let hits = 0;
+  let readCount = 0;
+  let capped = 0;
   let cursor = null;
   for (;;) {
     const init = { signal };
@@ -47,19 +50,27 @@ export async function fetchCmrPages({
     const response = await fetchImpl(url, init);
     if (!response?.ok)
       throw new Error(`HTTP ${response?.status ?? 'error'} from CMR`);
-    const page = parseCmrUmm(await response.json(), product);
-    granules.push(...page.granules);
+    const json = await response.json();
+    const rawCount = Array.isArray(json?.items) ? json.items.length : 0;
+    const page = parseCmrUmm(json, product);
+    readCount += rawCount;
+    const admitted = page.granules.slice(
+      0,
+      Math.max(0, maxRecords - granules.length),
+    );
+    granules.push(...admitted);
+    capped += page.granules.length - admitted.length;
     hits = Math.max(hits, page.hits);
     cursor = response.headers?.get?.(CMR_SEARCH_AFTER) ?? null;
     if (
       !cursor ||
-      page.granules.length < CMR_PAGE_SIZE ||
-      granules.length >= maxRecords ||
-      granules.length >= hits
+      rawCount < CMR_PAGE_SIZE ||
+      readCount >= maxRecords ||
+      readCount >= hits
     )
       break;
   }
-  return { granules, truncated: hits > granules.length };
+  return { granules, truncated: hits > readCount || capped > 0 };
 }
 
 function finiteOrNull(value) {

src/layers/recentImagery/cmr-raw-page-budget.test.mjs

@@ -0,0 +1,98 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { fetchCmrPages } from './catalog.js';
+const item = (id) => ({
+  meta: { 'concept-id': id },
+  umm: {
+    TemporalExtent: {
+      RangeDateTime: { BeginningDateTime: '2026-09-18T12:00:00Z' },
+    },
+  },
+});
+test('discarded rows do not turn a full CMR page into a final page', async () => {
+  let calls = 0;
+  const result = await fetchCmrPages({
+    product: 'S30',
+    url: 'fixture',
+    fetchImpl: async () => {
+      calls++;
+      const items =
+        calls === 1
+          ? Array.from({ length: 200 }, (_, i) =>
+              i === 0 ? {} : item(String(i)),
+            )
+          : [item('last')];
+      return Response.json(
+        { hits: 201, items },
+        { headers: { 'CMR-Search-After': `cursor-${calls}` } },
+      );
+    },
+  });
+  assert.equal(calls, 2);
+  assert.equal(result.granules.length, 200);
+  assert.equal(result.granules.at(-1).id, 'last');
+  assert.equal(result.truncated, false);
+});
+test('a non-page-multiple limit never returns more admitted records than its budget', async () => {
+  let calls = 0;
+  const result = await fetchCmrPages({
+    product: 'S30',
+    url: 'fixture',
+    maxRecords: 250,
+    fetchImpl: async () => {
+      calls++;
+      return Response.json(
+        {
+          hits: 1000,
+          items: Array.from({ length: 200 }, (_, i) => item(`${calls}:${i}`)),
+        },
+        { headers: { 'CMR-Search-After': `cursor-${calls}` } },
+      );
+    },
+  });
+  assert.equal(calls, 2);
+  assert.equal(result.granules.length, 250);
+  assert.equal(result.truncated, true);
+});
+test('valid records dropped by the cap mark the result truncated even when every hit was read', async () => {
+  let calls = 0;
+  const result = await fetchCmrPages({
+    product: 'S30',
+    url: 'fixture',
+    maxRecords: 250,
+    fetchImpl: async () => {
+      calls++;
+      return Response.json(
+        {
+          hits: 400,
+          items: Array.from({ length: 200 }, (_, i) => item(`${calls}:${i}`)),
+        },
+        { headers: { 'CMR-Search-After': `cursor-${calls}` } },
+      );
+    },
+  });
+  assert.equal(calls, 2);
+  assert.equal(result.granules.length, 250);
+  assert.equal(result.truncated, true);
+});
+test('malformed rows discarded under the cap leave a fully examined result untruncated', async () => {
+  let calls = 0;
+  const result = await fetchCmrPages({
+    product: 'S30',
+    url: 'fixture',
+    maxRecords: 250,
+    fetchImpl: async () => {
+      calls++;
+      const items = Array.from({ length: 200 }, (_, i) =>
+        calls === 2 && i >= 50 ? {} : item(`${calls}:${i}`),
+      );
+      return Response.json(
+        { hits: 400, items },
+        { headers: { 'CMR-Search-After': `cursor-${calls}` } },
+      );
+    },
+  });
+  assert.equal(calls, 2);
+  assert.equal(result.granules.length, 250);
+  assert.equal(result.truncated, false);
+});