diff --git a/CHANGELOG.md b/CHANGELOG.md index cd6ff006b..05fe4fdb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,9 @@ - Director timeline seeks cross the dateline along the short longitude arc. -- Recent Imagery thumbnails require explicit presence evidence before reporting available pixels. +- Recent Imagery continues past malformed records in full catalog pages, respects the requested record limit, and reports results as truncated when that limit drops valid records. + +- Regional articles retain unknown publication times when compact source dates do not identify a real calendar instant. - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index cf3f7648f..72e0c1ae1 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4542,7 +4542,12 @@ are omitted rather than framing the wrong part of the globe. ### Recent Imagery (September 2026) -Thumbnail decoding requires explicit `Data-Present: true` evidence; false remains empty, while absent or malformed evidence reports an error. Injected `getThumbnail` sources must return an OK Response with readable `Data-Present: true` to admit image bytes, or `false` for confirmed empty coverage. Cross-origin sources must expose that header to browser JavaScript. +CMR pagination counts raw rows when checking page completion and the record budget. A catalog is truncated when upstream hits exceed the rows examined or the record cap drops valid granules from the last page; discarded malformed rows do not imply unseen imagery. + +Programmatic boxes passed to the exported `setBox` and `searchHls` operations +reject missing, blank or boolean edges instead of interpreting them as zero +coordinates. Valid finite numeric edges remain accepted; the app’s drag tool, +pins and share links already supply complete boxes. - The layer sits in the Data Layers panel's Cameras group beside Cameras (CCTV); Mapped ALPR Cameras moved to the Infrastructure group to make room. - DATA ▸ **Recent Imagery** (`recent-imagery`, `src/layers/recentImagery/`) lists the last 30 UTC days of imagery for a selected box and drapes the days you pick: HLS S30 (Sentinel-2) and L30 (Landsat 8/9) at 30 m, and the VIIRS NOAA-21 daily overview at 250 m. Keyless and browser-direct: NASA CMR for which days have HLS granules in the box (`catalog.js`, both collections concurrently, paged with `CMR-Search-After` up to 2,000 records each, `truncated` against CMR `hits`), GIBS WMTS for tiles, Worldview Snapshots for thumbnails and exports. Worldview's `Data-Present` header is the only availability signal: GIBS answers an empty tile with HTTP 200, and the JPEG overview would drape black, so a day confirmed empty never drapes. diff --git a/src/layers/recentImagery/catalog.js b/src/layers/recentImagery/catalog.js index 96b314690..14964b1d2 100644 --- a/src/layers/recentImagery/catalog.js +++ b/src/layers/recentImagery/catalog.js @@ -27,7 +27,8 @@ const CMR_SEARCH_AFTER = 'CMR-Search-After'; /** * Read one product's granules across CMR pages, sending back each response's * `CMR-Search-After` cursor until a short page, a missing cursor or the - * record cap ends the walk. `truncated` is measured against CMR's `hits`. + * record cap ends the walk. `truncated` means CMR's `hits` exceed the rows + * read, or the cap dropped valid granules from the last page read. * @param {{ product: string, url: string, fetchImpl: typeof fetch, signal?: AbortSignal, maxRecords?: number }} request * @returns {Promise<{ granules: Array, truncated: boolean }>} */ @@ -40,6 +41,8 @@ export async function fetchCmrPages({ }) { const granules = []; let hits = 0; + let readCount = 0; + let capped = 0; let cursor = null; for (;;) { const init = { signal }; @@ -47,19 +50,27 @@ export async function fetchCmrPages({ const response = await fetchImpl(url, init); if (!response?.ok) throw new Error(`HTTP ${response?.status ?? 'error'} from CMR`); - const page = parseCmrUmm(await response.json(), product); - granules.push(...page.granules); + const json = await response.json(); + const rawCount = Array.isArray(json?.items) ? json.items.length : 0; + const page = parseCmrUmm(json, product); + readCount += rawCount; + const admitted = page.granules.slice( + 0, + Math.max(0, maxRecords - granules.length), + ); + granules.push(...admitted); + capped += page.granules.length - admitted.length; hits = Math.max(hits, page.hits); cursor = response.headers?.get?.(CMR_SEARCH_AFTER) ?? null; if ( !cursor || - page.granules.length < CMR_PAGE_SIZE || - granules.length >= maxRecords || - granules.length >= hits + rawCount < CMR_PAGE_SIZE || + readCount >= maxRecords || + readCount >= hits ) break; } - return { granules, truncated: hits > granules.length }; + return { granules, truncated: hits > readCount || capped > 0 }; } function finiteOrNull(value) { diff --git a/src/layers/recentImagery/cmr-raw-page-budget.test.mjs b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs new file mode 100644 index 000000000..c1709b9b2 --- /dev/null +++ b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs @@ -0,0 +1,98 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { fetchCmrPages } from './catalog.js'; +const item = (id) => ({ + meta: { 'concept-id': id }, + umm: { + TemporalExtent: { + RangeDateTime: { BeginningDateTime: '2026-09-18T12:00:00Z' }, + }, + }, +}); +test('discarded rows do not turn a full CMR page into a final page', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + fetchImpl: async () => { + calls++; + const items = + calls === 1 + ? Array.from({ length: 200 }, (_, i) => + i === 0 ? {} : item(String(i)), + ) + : [item('last')]; + return Response.json( + { hits: 201, items }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 200); + assert.equal(result.granules.at(-1).id, 'last'); + assert.equal(result.truncated, false); +}); +test('a non-page-multiple limit never returns more admitted records than its budget', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + return Response.json( + { + hits: 1000, + items: Array.from({ length: 200 }, (_, i) => item(`${calls}:${i}`)), + }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, true); +}); +test('valid records dropped by the cap mark the result truncated even when every hit was read', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + return Response.json( + { + hits: 400, + items: Array.from({ length: 200 }, (_, i) => item(`${calls}:${i}`)), + }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, true); +}); +test('malformed rows discarded under the cap leave a fully examined result untruncated', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + const items = Array.from({ length: 200 }, (_, i) => + calls === 2 && i >= 50 ? {} : item(`${calls}:${i}`), + ); + return Response.json( + { hits: 400, items }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, false); +});