diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index bfedd6346..41208cd6c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -585,6 +585,24 @@ the ledger (append-only with `--base-ref`, run in CI on pull requests) and the generated DATA_SOURCES.md section; `npm run library:probe` hits every upstream once; `npm run qa:data-library` is the browser gate (fixture-backed). +Shares: the compact `/share/` link carries the switched-on entries in +field 29 (docs/SHARE-LINK-FORMAT.md), so Copy Link keeps them, and a share +opened later in the same tab (pasted address, Back/Forward) restores them too. +Any share decides what is on, as for core layers: one with no library entry +on switches the recipient's stored ones off and leaves them listed. A tile +entry whose coverage needs more than 256 tiles at its minimum zoom (Norway's +cadastre from z14, sea depths from z11) is hidden below that scale through +Cesium's `minimumTerrainLevel`, so zoomed out it requests nothing. EPC +partitions learn two extents (`gev:library:extent:v2:`): every point's box +rules a partition out, the 98 % box only ranks it, so homes at a partition's +edge are found on later visits; the v1 trimmed-only key is dropped. A +partition index whose footers keep failing after two retries is +`complete: false`: neither the server nor the browser keeps it as whole, a +complete earlier index is served instead, and it is rebuilt after five +minutes. A view that leaves partitions touching it unread is PARTIAL. Listing +pages and footer reads are size-capped and ranges are validated. The Browse +dialog rebuilds on a language switch, keeping its filters. + ## Utilities panel — October 29, 2026 UTILITIES is its own left-side panel, between Data Layers and Scenes, with diff --git a/docs/SHARE-LINK-FORMAT.md b/docs/SHARE-LINK-FORMAT.md index 9c61fc7cd..4c3254a99 100644 --- a/docs/SHARE-LINK-FORMAT.md +++ b/docs/SHARE-LINK-FORMAT.md @@ -117,36 +117,37 @@ probability of about 255 in 256 (many of them already fail as base64url). ## Field registry (top level) -| # | State key | Type | Default | Encoding | -| --- | ---------------------------- | ----- | ----------- | ----------------------------------------------------------------- | -| 1 | `view.kind` | enum | see below | 0 place, 1 thing | -| 2 | `lat` | coord | 0 | zigzag of degrees × 10⁴ | -| 3 | `lon` | coord | 0 | zigzag of degrees × 10⁴ | -| 4 | `alt` | sint | 800 | whole metres | -| 5 | `heading` | angle | 0 | whole degrees, see angles | -| 6 | `pitch` | sint | −35 | whole degrees | -| 7 | `roll` | angle | 0 | whole degrees | -| 8 | `sharedAtMs` | epoch | none | zigzag seconds since 2026-01-01T00:00:00Z | -| 9 | booleans | flags | 0 | bit set when the value differs from default | -| 10 | `layerState.enabledLayerIds` | LEN | none | layer bitset, see layers | -| 11 | `layerState.options` | LEN | none | packed option triples, see layers | -| 12 | `panelState` | LEN | none | packed panel entries, see panels | -| 13 | thing | LEN | none | sub-message, see thing views | -| 14 | `style` | enum | `normal` | normal, retro, surveillance, thermal, anime, noir, snow (retired) | -| 15 | `mapStack` | enum | `photoreal` | photoreal, bing-aerial, bing-labels, esri-imagery, osm | -| 16 | `hudVariant` | enum | style | tactical, operator (retired), minimal, cyber | -| 17 | `bloomIntensity` | uint | style | | -| 18 | `bloomVersion` | uint | 2 | | -| 19 | `sharpenIntensity` | uint | style | | -| 20 | `detectionMode` | enum | style | OFF, SPARSE, BALANCED, DENSE | -| 21 | `detectionDensity` | uint | style | one of 0, 25, 50, 75, 100 | -| 22 | `detectionAllocation` | enum | style | ELASTIC, WEIGHTED | -| 23 | `detectionFadePct` | uint | style | 0–40 | -| 24 | `detectionOutsideOpacityPct` | uint | style | 0–100 | -| 25 | `scopeFeatherPct` | uint | 100 | 0–200 | -| 26 | `scopeTerminusPct` | uint | none | 94–100; absent is the adaptive terminus | -| 27 | `styleParams` | LEN | style | packed `(token, value × 100)` pairs | -| 28 | `worldTime` | epoch | none (live) | zigzag seconds since 2026-01-01T00:00:00Z | +| # | State key | Type | Default | Encoding | +| --- | ---------------------------- | ----- | ----------- | --------------------------------------------------------------------- | +| 1 | `view.kind` | enum | see below | 0 place, 1 thing | +| 2 | `lat` | coord | 0 | zigzag of degrees × 10⁴ | +| 3 | `lon` | coord | 0 | zigzag of degrees × 10⁴ | +| 4 | `alt` | sint | 800 | whole metres | +| 5 | `heading` | angle | 0 | whole degrees, see angles | +| 6 | `pitch` | sint | −35 | whole degrees | +| 7 | `roll` | angle | 0 | whole degrees | +| 8 | `sharedAtMs` | epoch | none | zigzag seconds since 2026-01-01T00:00:00Z | +| 9 | booleans | flags | 0 | bit set when the value differs from default | +| 10 | `layerState.enabledLayerIds` | LEN | none | layer bitset, see layers | +| 11 | `layerState.options` | LEN | none | packed option triples, see layers | +| 12 | `panelState` | LEN | none | packed panel entries, see panels | +| 13 | thing | LEN | none | sub-message, see thing views | +| 14 | `style` | enum | `normal` | normal, retro, surveillance, thermal, anime, noir, snow (retired) | +| 15 | `mapStack` | enum | `photoreal` | photoreal, bing-aerial, bing-labels, esri-imagery, osm | +| 16 | `hudVariant` | enum | style | tactical, operator (retired), minimal, cyber | +| 17 | `bloomIntensity` | uint | style | | +| 18 | `bloomVersion` | uint | 2 | | +| 19 | `sharpenIntensity` | uint | style | | +| 20 | `detectionMode` | enum | style | OFF, SPARSE, BALANCED, DENSE | +| 21 | `detectionDensity` | uint | style | one of 0, 25, 50, 75, 100 | +| 22 | `detectionAllocation` | enum | style | ELASTIC, WEIGHTED | +| 23 | `detectionFadePct` | uint | style | 0–40 | +| 24 | `detectionOutsideOpacityPct` | uint | style | 0–100 | +| 25 | `scopeFeatherPct` | uint | 100 | 0–200 | +| 26 | `scopeTerminusPct` | uint | none | 94–100; absent is the adaptive terminus | +| 27 | `styleParams` | LEN | style | packed `(token, value × 100)` pairs | +| 28 | `worldTime` | epoch | none (live) | zigzag seconds since 2026-01-01T00:00:00Z | +| 29 | `library` | LEN | none | Data Library ledger tokens, two base-36 characters each, concatenated | Field 1 absent means `thing` when field 13 is present and `place` otherwise, so v1 never writes it; a later view kind writes its code. @@ -164,6 +165,11 @@ Field 28 is the world time the link was made at, written only when it was away from live. A link without it opens live. Unlike field 8 it may lie ahead of the link's creation (a forecast moment), and it opens paused at that time. +Field 29 lists the Data Library entries switched on, by their permanent +tokens in `src/library/ledger.json` (the long form's `lib=ab.cd`). A share +without it switches no library entry on; the recipient's added entries stay +in their panel, off. Tokens this build does not know are skipped. + Flag bits (field 9): 0 `bloom`, 1 `sharpen`, 2 `hudVisible`, 3 `celestialRing` (each defaulting by style), 4 `scopeEnabled` (default on). @@ -391,7 +397,7 @@ the app that can open it. offers keeps its code: `snow` (6) is retired, so a link that names it opens as Normal and the parameters written for it are dropped. 5. **Add, don't change.** A new setting takes the next free field number - (28 onwards; fields 1–15 have one-byte tags and are all taken). A new + (30 onwards; fields 1–15 have one-byte tags and are all taken). A new boolean takes the next flag bit instead, which costs nothing when it is at its default. 6. **New view kinds and entity kinds** get the next code. A new entity kind's diff --git a/scripts/qa-data-library.mjs b/scripts/qa-data-library.mjs index e8adc49c7..f37f38fbb 100644 --- a/scripts/qa-data-library.mjs +++ b/scripts/qa-data-library.mjs @@ -629,6 +629,107 @@ try { assert.deepEqual(stored, { added: [], on: [] }); }); + const onIds = (target) => + target.evaluate(() => + window.__gevDataLibrary.library + .list() + .filter((row) => row.on) + .map((row) => row.entry.id) + .sort(), + ); + + await step( + 'a copied /share link opens the same library layers in a clean browser', + async () => { + await page.evaluate(async () => { + const { library } = window.__gevDataLibrary; + await library.add('nasa-eonet-events'); + await library.add('nasa-black-marble'); + }); + await page.waitForFunction( + () => /(?:^|&)lib=00\.03(?:&|$)/.test(location.hash.slice(1)), + { timeout: 15_000 }, + ); + const href = await page.evaluate(() => + window.__gevDataLibrary.buildShareLink(), + ); + assert.match(new URL(href).pathname, /\/share\//); + // A separate profile: nothing stored, nothing added. + const context = await browser.createBrowserContext(); + try { + const recipient = await context.newPage(); + await recipient.setRequestInterception(true); + recipient.on('request', (request) => { + const fixture = fixtureFor(new URL(request.url())); + if (fixture) request.respond(fixture); + else request.continue(); + }); + await recipient.evaluateOnNewDocument(() => { + try { + localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); + } catch { + /* the gate still runs */ + } + }); + await recipient.goto(href, { waitUntil: 'domcontentloaded' }); + await recipient.waitForFunction( + () => window.__gevDataLibrary?.library.ready, + { timeout: 180_000 }, + ); + await recipient.waitForFunction( + () => + window.__gevDataLibrary.library.list().filter((row) => row.on) + .length === 2, + { timeout: 60_000 }, + ); + assert.deepEqual(await onIds(recipient), [ + 'nasa-black-marble', + 'nasa-eonet-events', + ]); + } finally { + await context.close(); + } + }, + ); + + await step( + 'a share opened in the same tab switches library layers on and off', + async () => { + const navigate = (lib) => + page.evaluate((value) => { + const params = new URLSearchParams(location.hash.slice(1)); + params.set('lat', String(Number(params.get('lat')) + 1)); + if (value === null) params.delete('lib'); + else params.set('lib', value); + location.hash = params.toString(); + }, lib); + await navigate('00'); + await page.waitForFunction( + () => { + const on = window.__gevDataLibrary.library + .list() + .filter((row) => row.on) + .map((row) => row.entry.id); + return on.length === 1 && on[0] === 'nasa-eonet-events'; + }, + { timeout: 30_000 }, + ); + // A share with every library layer off: the added ones stay listed, off. + await navigate(null); + await page.waitForFunction( + () => window.__gevDataLibrary.library.list().every((row) => !row.on), + { timeout: 30_000 }, + ); + assert.deepEqual( + (await libraryRows(page)).map(({ id, on }) => [id, on]).sort(), + [ + ['nasa-black-marble', false], + ['nasa-eonet-events', false], + ], + ); + }, + ); + assert.deepEqual(errors, [], 'no library errors on the page'); console.log(`# ${count} passed`); } finally { diff --git a/server/providers/library.js b/server/providers/library.js index c01b45aa3..b0c17ebef 100644 --- a/server/providers/library.js +++ b/server/providers/library.js @@ -19,6 +19,8 @@ const DATASET_TIMEOUT_MS = 30_000; const TILE_TIMEOUT_MS = 10_000; /** Each listing page or footer read while building a partition index. */ const INDEX_TIMEOUT_MS = 20_000; +/** How long an incomplete partition index is served before a rebuild. */ +const PARTIAL_INDEX_MS = 5 * 60_000; const TILE_MAX_BYTES = 2 * 1024 * 1024; const TILE_CACHE_LIMIT = 512; const IN_FLIGHT_LIMIT = 64; @@ -107,6 +109,7 @@ export function libraryProxy({ const datasets = new Map(); const tiles = new Map(); const indexes = new Map(); + const partialIndexes = new Map(); const inFlight = new Map(); const allowData = makeRateLimiter({ windowMs: 60_000, @@ -231,25 +234,51 @@ export function libraryProxy({ if (!available) return json(res, 403, { error: reason }); if (!allowData(clientKey(req))) return json(res, 429, { error: 'rate_limited' }); + // Only a complete index is cached for cacheSeconds. An incomplete one + // (footers that kept failing) is served as `complete: false` and kept + // for PARTIAL_INDEX_MS only, so the next request soon rebuilds it; a + // complete index from before keeps being served while rebuilds fail. + const key = `index:${id}`; + const partial = partialIndexes.get(key); + if (partial && now() - partial.savedAt < PARTIAL_INDEX_MS) { + const previous = indexes.get(key); + return json(res, 200, previous?.value ?? partial.value); + } try { const { value } = await cached( indexes, - `index:${id}`, + key, entry.source.cacheSeconds * 1000, - async () => ({ - partitions: await buildPartitionIndex(entry, { + async () => { + const built = await buildPartitionIndex(entry, { fetchImpl: (url, options = {}) => fetchImpl(url, { ...options, signal: options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS), }), - }), - builtAt: now(), - }), + }); + const value = { + partitions: built.partitions, + complete: built.complete, + builtAt: now(), + }; + if (!built.complete) { + // Recorded here too: with an earlier complete index, cached() + // serves that one and swallows this error, and the rebuild + // still waits PARTIAL_INDEX_MS. + partialIndexes.set(key, { value, savedAt: now() }); + throw Object.assign(new Error('index_incomplete'), { + partialIndex: value, + }); + } + partialIndexes.delete(key); + return value; + }, ); return json(res, 200, value); } catch (error) { + if (error?.partialIndex) return json(res, 200, error.partialIndex); const status = error?.status === 429 ? 429 : 502; return json(res, status, { error: 'library_upstream_unavailable' }); } diff --git a/server/providers/library.test.mjs b/server/providers/library.test.mjs index eca9f7060..2e009b33c 100644 --- a/server/providers/library.test.mjs +++ b/server/providers/library.test.mjs @@ -6,6 +6,7 @@ import { parseLibraryTilePath, } from './library.js'; import { validateLibraryCatalog } from '../../src/library/manifest.js'; +import { libraryEntry } from '../../src/library/catalog.js'; const base = { family: 'events', @@ -100,8 +101,8 @@ const point = (title) => ({ properties: { title }, }); -function install({ env = {}, fetchImpl, now } = {}) { - const plugin = libraryProxy({ entries, env, fetchImpl, now }); +function install({ env = {}, fetchImpl, now, catalog = entries } = {}) { + const plugin = libraryProxy({ entries: catalog, env, fetchImpl, now }); let handler; plugin.configurePreviewServer({ middlewares: { @@ -330,3 +331,51 @@ test('only GET is served', async () => { await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res); assert.equal(res.status, 405); }); + +test('a partition index is cached only when complete; a complete one outlives failed rebuilds', async () => { + const epc = libraryEntry('epc-england-wales'); + let clock = 0; + let footers = 'ok'; + let listings = 0; + const listing = `a.parquet100`; + const fetchImpl = async (url) => { + if (url.includes('list-type')) { + listings += 1; + return new Response(listing); + } + return footers === 'ok' + ? new Response(new Uint8Array(100)) + : new Response('', { status: 503 }); + }; + const request = install({ catalog: [epc], now: () => clock, fetchImpl }); + const first = await request('/index/epc-england-wales'); + assert.equal(first.status, 200); + assert.equal(first.body.complete, true); + const builtAt = first.body.builtAt; + + // Past cacheSeconds the rebuild fails: the complete index is still served, + // and no rebuild runs again until the partial pause has passed. + clock += (epc.source.cacheSeconds + 1) * 1000; + footers = 'fail'; + const kept = await request('/index/epc-england-wales'); + assert.equal(kept.body.complete, true); + assert.equal(kept.body.builtAt, builtAt); + const listed = listings; + await request('/index/epc-england-wales'); + assert.equal(listings, listed); + + // A server that never had a complete index serves the partial one as such, + // and rebuilds after the pause. + const fresh = install({ catalog: [epc], now: () => clock, fetchImpl }); + const partial = await fresh('/index/epc-england-wales'); + assert.equal(partial.status, 200); + assert.equal(partial.body.complete, false); + const before = listings; + await fresh('/index/epc-england-wales'); + assert.equal(listings, before); + clock += 6 * 60_000; + footers = 'ok'; + const rebuilt = await fresh('/index/epc-england-wales'); + assert.equal(rebuilt.body.complete, true); + assert.equal(listings, before + 1); +}); diff --git a/src/app/tools.js b/src/app/tools.js index 927e343bb..f22d97b7f 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -142,7 +142,12 @@ export function createApplicationTools({ if (!signal?.aborted) console.warn('[Library] start failed:', error); }); // The live gate's handle (scripts/qa-data-library.mjs). - const libraryHandle = { library: dataLibrary, browser: libraryBrowser }; + const libraryHandle = { + library: dataLibrary, + browser: libraryBrowser, + // What Copy Link would copy. + buildShareLink: () => styleManager.shareLinkManager?.buildLink?.() ?? null, + }; window.__gevDataLibrary = libraryHandle; defer(() => { if (window.__gevDataLibrary === libraryHandle) diff --git a/src/library/controller.js b/src/library/controller.js index 49c02bbae..b90654c7b 100644 --- a/src/library/controller.js +++ b/src/library/controller.js @@ -69,6 +69,9 @@ export function createDataLibrary({ let serverPresent = false; let selection = { added: [], on: [] }; let started = false; + // Set once availability is known; a share navigated to before then waits. + let loaded = false; + let queuedShare = null; let destroyed = false; const listeners = new Set(); const pending = new Map(); @@ -165,6 +168,73 @@ export function createDataLibrary({ persist(); }); + /** Entry ids for share tokens; unknown tokens are skipped. */ + const idsForTokens = (list) => + (list || []).map((token) => tokens.idByToken.get(token)).filter(Boolean); + + /** + * The share this page opened with: null without one, else the entry ids it + * switches on ([] for a share that switches none on). + */ + function openingShare() { + if (shareLink && shareLink.initialLibraryTokens !== undefined) { + const linked = shareLink.initialLibraryTokens; + return linked === null ? null : idsForTokens(linked); + } + const hash = String(initialHash || '').replace(/^#/, ''); + if (!hash) return null; + const params = new URLSearchParams(hash); + const linked = decodeLibraryParam(params, tokens.idByToken); + // Any share (a camera position) decides what is on; no `lib`, nothing. + if (linked === null && params.has('lat') && params.has('lon')) return []; + return linked; + } + + /** + * Apply a selection: register and switch on what `on` names, switch off + * every other added entry that is on. `origin` marks it as a restore so it + * is never remembered as the viewer's own choice. + */ + function applySelection(next, origin) { + selection = normalizeLibrarySelection(next, switchable); + for (const id of selection.added) { + const entry = byId.get(id); + if (!entry || !addable(id)) continue; + const wanted = selection.on.includes(id); + if (wanted && !register(entry)) continue; + if (!isRegistered(id)) continue; + const layerId = libraryLayerId(id); + if (wanted === (dataManager.isEnabled?.(layerId) === true)) continue; + dataManager + .setEnabled(layerId, wanted, { origin }) + .catch((error) => + console.warn(`[Library] ${id} restore failed:`, error), + ); + } + writeStoredLibrarySelection(storage, selection); + shareLink?.onLibraryStateChange?.(); + notify(); + } + + /** + * A share opened in this tab after start-up: its entries join the library + * and it alone decides which are on. + * @param {string[]} shareTokens + */ + function applyShare(shareTokens) { + if (destroyed) return; + if (!loaded) { + queuedShare = [...(shareTokens || [])]; + return; + } + const linked = idsForTokens(shareTokens); + applySelection( + { added: [...selection.added, ...linked], on: linked }, + 'share-restore', + ); + } + shareLink?.setLibraryShareListener?.(applyShare); + shareLink?.setLibraryParamsEncoder?.((params) => encodeLibraryParam( params, @@ -186,29 +256,23 @@ export function createDataLibrary({ availability = answer.availability; serverPresent = answer.server; const stored = readStoredLibrarySelection(storage, switchable); - const hash = String(initialHash || '').replace(/^#/, ''); - const linked = hash - ? decodeLibraryParam(new URLSearchParams(hash), tokens.idByToken) - : null; + // A share opened in this tab while availability loaded wins over the + // opening one. + const linked = queuedShare ? idsForTokens(queuedShare) : openingShare(); + queuedShare = null; + loaded = true; // A link's entries join the viewer's library and open switched on; with a - // link, it alone decides what is on, as it does for core layers. - const added = [...stored.added, ...(linked || [])]; - const on = linked ?? stored.on; - selection = normalizeLibrarySelection({ added, on }, switchable); - const restoreOrigin = linked ? 'share-restore' : 'local-restore'; - for (const id of selection.added) { + // link, it alone decides what is on, as it does for core layers, so a + // share with every library entry off switches the stored ones off. + // Added entries register even while off, so their rows show. + for (const id of [...stored.added, ...(linked || [])]) { const entry = byId.get(id); - if (!entry || !addable(id) || !register(entry)) continue; - if (selection.on.includes(id)) - dataManager - .setEnabled(libraryLayerId(id), true, { origin: restoreOrigin }) - .catch((error) => - console.warn(`[Library] ${id} restore failed:`, error), - ); + if (entry && addable(id)) register(entry); } - writeStoredLibrarySelection(storage, selection); - shareLink?.onLibraryStateChange?.(); - notify(); + applySelection( + { added: [...stored.added, ...(linked || [])], on: linked ?? stored.on }, + linked ? 'share-restore' : 'local-restore', + ); } /** @@ -388,6 +452,7 @@ export function createDataLibrary({ destroyed = true; unsubscribeManager?.(); shareLink?.setLibraryParamsEncoder?.(null); + shareLink?.setLibraryShareListener?.(null); listeners.clear(); }, }; diff --git a/src/library/controller.test.mjs b/src/library/controller.test.mjs index 2e32a120b..53e553233 100644 --- a/src/library/controller.test.mjs +++ b/src/library/controller.test.mjs @@ -207,6 +207,115 @@ test('a link decides what is on and adds its entries to the library', async (t) }); }); +/** A ShareLinkManager stand-in: the opening share and in-tab navigation. */ +function fakeShareLink(initialLibraryTokens) { + const link = { + initialLibraryTokens, + listener: null, + setLibraryShareListener(listener) { + link.listener = listener; + }, + setLibraryParamsEncoder() {}, + onLibraryStateChange() {}, + }; + return link; +} + +test('a share with every library entry off keeps the stored ones off', async (t) => { + const manager = sealedManager(t); + const storage = memoryStorage( + JSON.stringify({ + added: ['night-lights', 'sea-marks'], + on: ['night-lights', 'sea-marks'], + }), + ); + // A compact /share/ link with no library field, and a long-form link + // with no lib=: both are shares that switch no library entry on. + for (const options of [ + { shareLink: fakeShareLink([]) }, + { initialHash: '#lat=1&lon=2&l=' }, + ]) { + const library = createDataLibrary({ + entries, + ledger, + dataManager: manager, + services: {}, + storage, + source: source({}), + ...options, + }); + await library.start(); + assert.equal(manager.isEnabled('lib-night-lights'), false); + assert.equal(manager.isEnabled('lib-sea-marks'), false); + // The recipient's entries stay in their panel, off. + assert.deepEqual( + library + .list() + .filter((row) => row.added) + .map((row) => row.entry.id), + ['night-lights', 'sea-marks'], + ); + library.destroy(); + } +}); + +test('the opening share link restores its library tokens', async (t) => { + const manager = sealedManager(t); + const storage = memoryStorage( + JSON.stringify({ added: ['night-lights'], on: ['night-lights'] }), + ); + const library = createDataLibrary({ + entries, + ledger, + dataManager: manager, + services: {}, + storage, + shareLink: fakeShareLink(['01']), + source: source({}), + }); + await library.start(); + await manager.waitForLayerSettled?.('lib-sea-marks'); + assert.equal(manager.isEnabled('lib-sea-marks'), true); + assert.equal(manager.isEnabled('lib-night-lights'), false); +}); + +test('navigating between shares switches library entries on and off', async (t) => { + const manager = sealedManager(t); + const shareLink = fakeShareLink(['00']); + const library = createDataLibrary({ + entries, + ledger, + dataManager: manager, + services: {}, + storage: memoryStorage(), + shareLink, + source: source({}), + }); + // A share navigated to before availability loads waits for it and wins. + const started = library.start(); + shareLink.listener(['01']); + await started; + const settle = () => + Promise.all( + ['lib-night-lights', 'lib-sea-marks'].map((id) => + manager.waitForLayerSettled?.(id), + ), + ); + await settle(); + assert.equal(manager.isEnabled('lib-sea-marks'), true); + assert.equal(manager.isEnabled('lib-night-lights'), false); + shareLink.listener(['00']); + await settle(); + assert.equal(manager.isEnabled('lib-night-lights'), true); + assert.equal(manager.isEnabled('lib-sea-marks'), false); + shareLink.listener([]); + await settle(); + assert.equal(manager.isEnabled('lib-night-lights'), false); + assert.equal(manager.isEnabled('lib-sea-marks'), false); + library.destroy(); + assert.equal(shareLink.listener, null); +}); + test('zoom flies to an added layer through the caller, reset removes everything', async (t) => { const manager = sealedManager(t); const storage = memoryStorage(); diff --git a/src/library/layers.js b/src/library/layers.js index 68d382dc6..42b6eb237 100644 --- a/src/library/layers.js +++ b/src/library/layers.js @@ -258,6 +258,44 @@ const reliefLegend = () => /** colorToAlpha threshold (max channel distance, 0–1) for `noDataColor`. */ const NO_DATA_THRESHOLD = 0.02; +/** The most tiles an entry's coverage may need at its minimum zoom. */ +const COARSE_TILE_BUDGET = 256; + +function webMercatorTileSpan(bbox, zoom) { + const n = 2 ** zoom; + const clampLat = (lat) => Math.max(-85.0511, Math.min(85.0511, lat)); + const tileX = (lon) => + Math.min(n - 1, Math.max(0, Math.floor(((lon + 180) / 360) * n))); + const tileY = (lat) => { + const rad = (clampLat(lat) * Math.PI) / 180; + const y = (1 - Math.log(Math.tan(rad) + 1 / Math.cos(rad)) / Math.PI) / 2; + return Math.min(n - 1, Math.max(0, Math.floor(y * n))); + }; + const [west, south, east, north] = bbox; + return (tileX(east) - tileX(west) + 1) * (tileY(south) - tileY(north) + 1); +} + +/** + * The globe level below which a tile entry is hidden, or undefined when it + * draws at every scale. Cesium fetches an imagery layer's minimum level for + * every globe tile it covers, however far out the camera is: an entry whose + * coverage needs more than COARSE_TILE_BUDGET tiles at its minimum zoom + * (Norway's cadastre from z14 is about two million) would allocate them all + * when zoomed out. Such an entry is shown only from the globe level that + * matches its minimum zoom (a geographic level spans two Web Mercator ones, + * so level z − 1). + * @param {object} entry + * @returns {number|undefined} + */ +export function minimumTerrainLevelFor(entry) { + const minZoom = entry?.source?.minZoom ?? 0; + if (minZoom <= 0) return undefined; + const bbox = entry.coverage?.bbox ?? [-180, -85.0511, 180, 85.0511]; + return webMercatorTileSpan(bbox, minZoom) > COARSE_TILE_BUDGET + ? minZoom - 1 + : undefined; +} + /** * A tiles entry as a manager layer. * @param {object} entry Validated tiles entry. @@ -406,7 +444,13 @@ export function createLibraryTileLayer( provider.errorEvent?.addEventListener?.(onError); removeErrorListener = () => provider.errorEvent?.removeEventListener?.(onError); - imagery = host.collection.addImageryProvider(provider); + const minimumTerrainLevel = minimumTerrainLevelFor(entry); + imagery = + minimumTerrainLevel !== undefined && cesium.ImageryLayer + ? new cesium.ImageryLayer(provider, { minimumTerrainLevel }) + : null; + if (imagery) host.collection.add(imagery); + else imagery = host.collection.addImageryProvider(provider); imagery.alpha = entry.display.opacity ?? 1; if (entry.display.noDataColor && cesium.Color) { // The provider's "no imagery here" fill, drawn transparent (JPEG diff --git a/src/library/parquet.js b/src/library/parquet.js index 4d5e4f8dc..29bb7ce99 100644 --- a/src/library/parquet.js +++ b/src/library/parquet.js @@ -10,7 +10,10 @@ import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet'; import { decompress as zstdDecompress } from 'fzstd'; import { libraryUpstreamUrl } from './manifest.js'; -import { readResponseBytesCapped } from '../sources/httpBody.js'; +import { + readResponseBytesCapped, + readResponseTextCapped, +} from '../sources/httpBody.js'; /** Footer bytes read first; small files' metadata fits in one request. */ const FOOTER_FETCH_BYTES = 16 * 1024; @@ -19,6 +22,13 @@ export const MAX_PARTITIONS_PER_VIEW = 4; const MAX_LISTED_KEYS = 5000; const MAX_LIST_PAGES = 20; const INDEX_CONCURRENCY = 8; +/** One listing page: 1,000 keys of XML is well under a megabyte. */ +const MAX_LIST_PAGE_BYTES = 4 * 1024 * 1024; +/** A whole file sent in answer to a range request is read only up to this. */ +const MAX_UNRANGED_BYTES = 64 * 1024 * 1024; +/** Extra passes over footers that failed, before the index is incomplete. */ +const FOOTER_RETRIES = 2; +const FOOTER_RETRY_DELAY_MS = 500; /** Decompressors hyparquet does not carry itself (it has Snappy built in). */ export const PARQUET_COMPRESSORS = Object.freeze({ @@ -33,30 +43,67 @@ function onSourceHost(entry, url) { return target.href; } +/** Mark a failure worth retrying: network, 429, 5xx or a short read. */ +const transient = (error) => Object.assign(error, { transient: true }); + /** * A hyparquet AsyncBuffer over HTTP range requests, redirects refused, * credentials and referrer withheld. */ export function rangeBuffer(url, byteLength, { fetchImpl, signal }) { + if (!Number.isSafeInteger(byteLength) || byteLength <= 0) + throw new RangeError(`invalid object size ${byteLength}`); return { byteLength, async slice(start, end = byteLength) { - const response = await fetchImpl(url, { - signal, - redirect: 'error', - credentials: 'omit', - referrerPolicy: 'no-referrer', - headers: { Range: `bytes=${start}-${end - 1}` }, - }); + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start < 0 || + end > byteLength || + end <= start + ) + throw new RangeError(`invalid range ${start}-${end} of ${byteLength}`); + let response; + try { + response = await fetchImpl(url, { + signal, + redirect: 'error', + credentials: 'omit', + referrerPolicy: 'no-referrer', + headers: { Range: `bytes=${start}-${end - 1}` }, + }); + } catch (error) { + if (signal?.aborted) throw error; + throw transient(error); + } if (response.status !== 206 && response.status !== 200) { await response.body?.cancel?.().catch?.(() => {}); - throw new Error(`HTTP ${response.status}`); + const error = new Error(`HTTP ${response.status}`); + throw response.status === 429 || response.status >= 500 + ? transient(error) + : error; } - const buffer = await response.arrayBuffer(); - // A server that ignores Range answers with the whole file. - return response.status === 200 && buffer.byteLength === byteLength - ? buffer.slice(start, end) - : buffer; + // A server that ignores Range answers with the whole file; anything + // else must be exactly the bytes asked for. + const whole = response.status === 200; + const expected = whole ? byteLength : end - start; + const bytes = await readResponseBytesCapped( + response, + whole ? Math.min(byteLength, MAX_UNRANGED_BYTES) : expected, + signal, + ); + if (bytes.byteLength !== expected) + throw transient( + new Error( + `short read: ${bytes.byteLength} of ${expected} bytes from ${start}`, + ), + ); + const buffer = bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength, + ); + return whole ? buffer.slice(start, end) : buffer; }, }; } @@ -79,12 +126,19 @@ export async function listPartitions(entry, { fetchImpl, signal }) { credentials: 'omit', }); if (!response.ok) throw new Error(`HTTP ${response.status}`); - const xml = await response.text(); + const xml = await readResponseTextCapped( + response, + MAX_LIST_PAGE_BYTES, + signal, + ); for (const match of xml.matchAll( /[\s\S]*?([^<]+)<\/Key>[\s\S]*?(\d+)<\/Size>[\s\S]*?<\/Contents>/g, )) { - if (!match[1].endsWith('.parquet')) continue; - objects.push({ key: match[1], size: Number(match[2]) }); + const size = Number(match[2]); + if (!match[1].endsWith('.parquet') || !Number.isSafeInteger(size)) + continue; + if (size <= 0) continue; + objects.push({ key: match[1], size }); if (objects.length >= MAX_LISTED_KEYS) return objects; } token = /([^<]+)<\/NextContinuationToken>/.exec( @@ -119,39 +173,71 @@ export function geoParquetInfo(metadata) { } } +const pause = (ms, signal) => + new Promise((resolve, reject) => { + const timer = setTimeout(resolve, ms); + signal?.addEventListener?.( + 'abort', + () => { + clearTimeout(timer); + reject(signal.reason); + }, + { once: true }, + ); + }); + /** * Where each partition is: list the bucket, then read each file's footer - * (one small range request) for its GeoParquet bbox. Built once and cached - * by the caller; a partition without a bbox is left out. - * @returns {Promise>} + * (one small range request) for its GeoParquet bbox. A partition whose + * metadata is unreadable or has no bbox is left out; a footer that fails to + * load (network, 429, 5xx, short read) is retried + * (FOOTER_RETRIES more passes) and, if it still fails, the index is + * `complete: false` so callers never keep it as if it were whole. + * @returns {Promise<{ partitions: Array<{ url: string, bbox: number[], size: number }>, complete: boolean, failed: number }>} */ -export async function buildPartitionIndex(entry, { fetchImpl, signal }) { +export async function buildPartitionIndex( + entry, + { fetchImpl, signal, retryDelayMs = FOOTER_RETRY_DELAY_MS }, +) { const { keyBase } = entry.source.partitions; const objects = await listPartitions(entry, { fetchImpl, signal }); - const index = []; - let next = 0; - async function worker() { - while (next < objects.length) { - const { key, size } = objects[next++]; - signal?.throwIfAborted?.(); - const url = onSourceHost(entry, new URL(key, keyBase).href); - try { - const metadata = await parquetMetadataAsync( - rangeBuffer(url, size, { fetchImpl, signal }), - { initialFetchSize: FOOTER_FETCH_BYTES }, - ); - const { bbox } = geoParquetInfo(metadata); - if (bbox) index.push({ url, bbox, size }); - } catch (error) { - if (signal?.aborted) throw error; - // One unreadable partition does not sink the index. + const partitions = []; + let pending = objects; + for (let pass = 0; pass <= FOOTER_RETRIES && pending.length; pass++) { + if (pass) await pause(retryDelayMs * pass, signal); + const failed = []; + let next = 0; + const queue = pending; + async function worker() { + while (next < queue.length) { + const object = queue[next++]; + signal?.throwIfAborted?.(); + const url = onSourceHost(entry, new URL(object.key, keyBase).href); + try { + const metadata = await parquetMetadataAsync( + rangeBuffer(url, object.size, { fetchImpl, signal }), + { initialFetchSize: FOOTER_FETCH_BYTES }, + ); + const { bbox } = geoParquetInfo(metadata); + if (bbox) partitions.push({ url, bbox, size: object.size }); + } catch (error) { + if (signal?.aborted) throw error; + // A footer that arrived but does not parse will not parse next + // time either: that file is left out. Only fetch failures retry. + if (error?.transient) failed.push(object); + } } } + await Promise.all( + Array.from({ length: Math.min(INDEX_CONCURRENCY, queue.length) }, worker), + ); + pending = failed; } - await Promise.all( - Array.from({ length: Math.min(INDEX_CONCURRENCY, objects.length) }, worker), - ); - return index.sort((a, b) => a.url.localeCompare(b.url)); + return { + partitions: partitions.sort((a, b) => a.url.localeCompare(b.url)), + complete: pending.length === 0, + failed: pending.length, + }; } const overlapArea = (a, b) => diff --git a/src/library/parquet.test.mjs b/src/library/parquet.test.mjs index 5608194e6..65cf382ce 100644 --- a/src/library/parquet.test.mjs +++ b/src/library/parquet.test.mjs @@ -90,7 +90,7 @@ test('a partitioned view reads candidates in batches until one finds features', }, loadParquet: async () => ({ ...(await import('./parquet.js')), - buildPartitionIndex: async () => index, + buildPartitionIndex: async () => ({ partitions: index, complete: true }), readGeoParquetFeatures: async (entry, url) => { reads.push(url); return url.endsWith('A.parquet') @@ -108,8 +108,11 @@ test('a partitioned view reads candidates in batches until one finds features', ]); // The index is kept in this browser; extents are learned per partition. assert.ok(store.get('gev:library:index:epc-england-wales')); - const learned = JSON.parse(store.get('gev:library:extent:epc-england-wales')); - assert.ok(learned['https://h/A.parquet']); + const learned = JSON.parse( + store.get('gev:library:extent:v2:epc-england-wales'), + ); + assert.ok(learned['https://h/A.parquet'].full); + assert.ok(learned['https://h/A.parquet'].trimmed); // A second view of the same area is served from the decoded cache. await source.loadDataset(epc, { bbox: box }); assert.equal(reads.length, 2); @@ -119,3 +122,185 @@ test('a partitioned view reads candidates in batches until one finds features', truncated: false, }); }); + +test('a revisit still finds homes at the edge of a partition', async () => { + // 200 homes in a cluster and one real home at the partition's edge: the + // trimmed (98 %) extent leaves the edge home out, the full one keeps it. + const point = (lon, lat) => ({ + type: 'Feature', + geometry: { type: 'Point', coordinates: [lon, lat] }, + properties: { CURRENT_ENERGY_RATING: 'C', UPRN: String(lon) }, + }); + const homes = Array.from({ length: 200 }, (_, index) => + point(-1.4 - index / 1e4, 53.4), + ); + const edge = point(-1.2, 53.3); + homes.push(edge); + const index = [ + { url: 'https://h/Edge.parquet', bbox: [-1.5, 53.2, -1.1, 53.5] }, + ]; + const store = new Map([ + // A v1 extent (trimmed only) from an older build: never read again. + [ + 'gev:library:extent:epc-england-wales', + JSON.stringify({ 'https://h/Edge.parquet': [-1.42, 53.4, -1.4, 53.4] }), + ], + ]); + const storage = { + getItem: (key) => store.get(key) ?? null, + setItem: (key, value) => store.set(key, value), + removeItem: (key) => store.delete(key), + }; + const visit = () => + createLibrarySource({ + entries: [epc], + fetchImpl: async () => ({ ok: false, status: 404 }), + storage, + loadParquet: async () => ({ + ...(await import('./parquet.js')), + buildPartitionIndex: async () => ({ + partitions: index, + complete: true, + }), + readGeoParquetFeatures: async () => homes, + }), + }); + const edgeBox = [-1.21, 53.29, -1.19, 53.31]; + // First visit: the old v1 extent would have skipped the partition. + const first = await visit().loadDataset(epc, { bbox: edgeBox }); + assert.equal(first.features.length, 1); + assert.equal(store.has('gev:library:extent:epc-england-wales'), false); + // A later visit (new page, nothing decoded) reads the learned v2 extents. + const later = await visit().loadDataset(epc, { bbox: edgeBox }); + assert.equal(later.features.length, 1); + assert.deepEqual(later.features[0].geometry.coordinates, [-1.2, 53.3]); + // A box outside every point of the partition is still ruled out. + const outside = await visit().loadDataset(epc, { + bbox: [-1.12, 53.45, -1.11, 53.46], + }); + assert.equal(outside.features.length, 0); +}); + +test('a footer that keeps failing leaves the index incomplete, and a retry recovers', async () => { + const { buildPartitionIndex } = await import('./parquet.js'); + const entry = { + ...epc, + source: { + ...epc.source, + partitions: { + listUrl: 'https://s3.us-west-2.amazonaws.com/bucket?list-type=2', + keyBase: 'https://s3.us-west-2.amazonaws.com/bucket/', + }, + }, + }; + const listing = `a.parquet100b.parquet100`; + let footerFailures = 0; + const fetchImpl = async (url) => { + if (url.includes('list-type')) + return new Response(listing, { status: 200 }); + if (url.endsWith('b.parquet') && footerFailures > 0) { + footerFailures -= 1; + return new Response('', { status: 503 }); + } + // Not a Parquet footer: readable, but with no bbox, so left out. + return new Response(new Uint8Array(100), { status: 200 }); + }; + footerFailures = 2; // fails once, then succeeds on the first retry + const recovered = await buildPartitionIndex(entry, { + fetchImpl, + retryDelayMs: 0, + }); + assert.equal(recovered.complete, true); + footerFailures = 100; // fails on every pass + const broken = await buildPartitionIndex(entry, { + fetchImpl, + retryDelayMs: 0, + }); + assert.equal(broken.complete, false); + assert.equal(broken.failed, 1); +}); + +test('the browser keeps a complete index over an incomplete rebuild and retries later', async () => { + let clock = 30 * 24 * 3600_000; + const complete = [{ url: 'https://h/A.parquet', bbox: [-2, 53, -1, 54] }]; + const store = new Map([ + [ + 'gev:library:index:epc-england-wales', + // Past cacheSeconds: due for a rebuild. + JSON.stringify({ savedAt: 0, complete: true, partitions: complete }), + ], + ]); + let builds = 0; + let nextComplete = false; + const source = createLibrarySource({ + entries: [epc], + now: () => clock, + fetchImpl: async () => ({ ok: false, status: 404 }), + storage: { + getItem: (key) => store.get(key) ?? null, + setItem: (key, value) => store.set(key, value), + }, + loadParquet: async () => ({ + ...(await import('./parquet.js')), + buildPartitionIndex: async () => { + builds += 1; + return nextComplete + ? { partitions: complete, complete: true, failed: 0 } + : { partitions: [], complete: false, failed: 3 }; + }, + readGeoParquetFeatures: async () => [], + }), + }); + const box = [-1.5, 53.4, -1.4, 53.5]; + await source.loadDataset(epc, { bbox: box }); + assert.equal(builds, 1); + // The old complete index stays stored; the incomplete one is not saved. + assert.equal( + JSON.parse(store.get('gev:library:index:epc-england-wales')).savedAt, + 0, + ); + // Within the retry pause the incomplete result is reused, not rebuilt. + await source.loadDataset(epc, { bbox: box }); + assert.equal(builds, 1); + // After it, the index is rebuilt; a complete build is kept. + clock += 6 * 60_000; + nextComplete = true; + await source.loadDataset(epc, { bbox: box }); + assert.equal(builds, 2); + assert.equal( + JSON.parse(store.get('gev:library:index:epc-england-wales')).savedAt, + clock, + ); +}); + +test('a view that leaves touching partitions unread is partial', async () => { + const index = Array.from({ length: 6 }, (_, n) => ({ + url: `https://h/P${n}.parquet`, + bbox: [-1.5 - n / 100, 53.3, -1.4, 53.45], + })); + const point = (lon, lat) => ({ + type: 'Feature', + geometry: { type: 'Point', coordinates: [lon, lat] }, + properties: { CURRENT_ENERGY_RATING: 'C', UPRN: '1' }, + }); + const reads = []; + const source = createLibrarySource({ + entries: [epc], + fetchImpl: async () => ({ ok: false, status: 404 }), + storage: null, + loadParquet: async () => ({ + ...(await import('./parquet.js')), + buildPartitionIndex: async () => ({ partitions: index, complete: true }), + readGeoParquetFeatures: async (entry, url) => { + reads.push(url); + return [point(-1.45, 53.4)]; + }, + }), + }); + const result = await source.loadDataset(epc, { + bbox: [-1.46, 53.39, -1.44, 53.41], + }); + // The first batch of four found homes; two touching partitions stay unread. + assert.equal(reads.length, 4); + assert.equal(result.truncated, true); +}); diff --git a/src/library/source.js b/src/library/source.js index 8e15ec0d0..85f37c5d0 100644 --- a/src/library/source.js +++ b/src/library/source.js @@ -17,7 +17,13 @@ const PARTITION_BATCH = 4; /** Decoded partitions kept in memory, most recent first. */ const PARTITION_CACHE_LIMIT = 4; const INDEX_STORAGE_PREFIX = 'gev:library:index:'; -const EXTENT_STORAGE_PREFIX = 'gev:library:extent:'; +/** How long an incomplete partition index is used before a rebuild. */ +const PARTIAL_INDEX_RETRY_MS = 5 * 60_000; +// v2 stores `{ full, trimmed }` per partition. The v1 key held only the +// trimmed box, which wrongly excluded real points at a partition's edge; it +// is no longer read and is removed on first use. +const EXTENT_STORAGE_PREFIX = 'gev:library:extent:v2:'; +const RETIRED_EXTENT_STORAGE_PREFIX = 'gev:library:extent:'; const overlaps = (a, b) => a[0] < b[2] && a[2] > b[0] && a[1] < b[3] && a[3] > b[1]; @@ -47,6 +53,58 @@ export function robustExtent(features) { return [at(lons, 0.01), at(lats, 0.01), at(lons, 0.99), at(lats, 0.99)]; } +/** The box holding every point of a partition, or null. */ +export function fullExtent(features) { + let extent = null; + for (const feature of features) { + let position = feature?.geometry?.coordinates; + while (Array.isArray(position?.[0])) position = position[0]; + if (!Array.isArray(position)) continue; + const [lon, lat] = position; + if (!Number.isFinite(lon) || !Number.isFinite(lat)) continue; + if (!extent) extent = [lon, lat, lon, lat]; + else { + extent[0] = Math.min(extent[0], lon); + extent[1] = Math.min(extent[1], lat); + extent[2] = Math.max(extent[2], lon); + extent[3] = Math.max(extent[3], lat); + } + } + return extent; +} + +const isBox = (value) => + Array.isArray(value) && value.length === 4 && value.every(Number.isFinite); + +/** + * Candidate partitions for a box. A learned full extent that misses the box + * rules a partition out (no point of it can be in view); the learned 98 % + * extent only ranks, so stray geocodes neither stretch the ranking nor hide + * the real points at a partition's edge. + */ +export function rankPartitions(index, box, learned, partitionsForBox) { + const known = (url) => { + const value = learned?.[url]; + return value && isBox(value.full) && isBox(value.trimmed) ? value : null; + }; + const ranked = partitionsForBox( + index + .filter(({ url }) => !known(url) || overlaps(known(url).full, box)) + .map((partition) => { + const extent = known(partition.url); + // Rank by where most points are, unless that misses the view while + // the full extent does not: then the partition's own box ranks it. + return extent && overlaps(extent.trimmed, box) + ? { ...partition, bbox: extent.trimmed } + : partition; + }), + box, + Infinity, + ); + const byUrl = new Map(index.map((partition) => [partition.url, partition])); + return ranked.map(({ url }) => byUrl.get(url)); +} + function storedJson(storage, key) { try { const text = storage?.getItem?.(key); @@ -118,43 +176,76 @@ export function createLibrarySource({ * shared), else built here and kept in this browser for `cacheSeconds`. */ function partitionIndex(entry, { signal } = {}) { - if (!indexes.has(entry.id)) - indexes.set( - entry.id, - (async () => { - try { - const response = await fetchImpl( - `${base}/index/${encodeURIComponent(entry.id)}`, - { signal, headers: { Accept: 'application/json' } }, - ); - if (response.ok) { - const body = await response.json(); - if (Array.isArray(body?.partitions)) return body.partitions; - } - } catch (error) { - if (signal?.aborted) throw error; + const memo = indexes.get(entry.id); + if (memo && (memo.retryAt === null || now() < memo.retryAt)) + return memo.promise; + const promise = (async () => { + const key = `${INDEX_STORAGE_PREFIX}${entry.id}`; + const stored = storedJson(storage, key); + const storedComplete = + stored && Array.isArray(stored.partitions) && stored.complete !== false + ? stored + : null; + let partial = null; + try { + const response = await fetchImpl( + `${base}/index/${encodeURIComponent(entry.id)}`, + { signal, headers: { Accept: 'application/json' } }, + ); + if (response.ok) { + const body = await response.json(); + if (Array.isArray(body?.partitions)) { + if (body.complete !== false) + return { partitions: body.partitions, complete: true }; + partial = body.partitions; } - const key = `${INDEX_STORAGE_PREFIX}${entry.id}`; - const stored = storedJson(storage, key); - if ( - stored && - Array.isArray(stored.partitions) && - now() - stored.savedAt < entry.source.cacheSeconds * 1000 - ) - return stored.partitions; - const { buildPartitionIndex } = await loadParquet(); - const partitions = await buildPartitionIndex(entry, { - fetchImpl, - signal, - }); - storeJson(storage, key, { savedAt: now(), partitions }); - return partitions; - })().catch((error) => { + } + } catch (error) { + if (signal?.aborted) throw error; + } + if ( + storedComplete && + now() - storedComplete.savedAt < entry.source.cacheSeconds * 1000 + ) + return { partitions: storedComplete.partitions, complete: true }; + // The server's index is incomplete, or there is no server: build here. + const { buildPartitionIndex } = await loadParquet(); + const built = await buildPartitionIndex(entry, { fetchImpl, signal }); + if (built.complete) { + storeJson(storage, key, { + savedAt: now(), + complete: true, + partitions: built.partitions, + }); + return { partitions: built.partitions, complete: true }; + } + // Still incomplete: an older complete index beats a partial one, even + // past its age; otherwise use the larger partial set for now. + if (storedComplete) + return { partitions: storedComplete.partitions, complete: false }; + const partitions = + partial && partial.length > built.partitions.length + ? partial + : built.partitions; + return { partitions, complete: false }; + })().then( + (result) => { + // An incomplete index is used for now and rebuilt after a pause. + const current = indexes.get(entry.id); + if (current?.promise === promise) + current.retryAt = result.complete + ? null + : now() + PARTIAL_INDEX_RETRY_MS; + return result.partitions; + }, + (error) => { + if (indexes.get(entry.id)?.promise === promise) indexes.delete(entry.id); - throw error; - }), - ); - return indexes.get(entry.id); + throw error; + }, + ); + indexes.set(entry.id, { promise, retryAt: null }); + return promise; } /** One partition's features, decoded once and kept for a few views. */ @@ -174,11 +265,12 @@ export function createLibrarySource({ while (partitionCache.size > PARTITION_CACHE_LIMIT) partitionCache.delete(partitionCache.keys().next().value); // Learn where this partition really is, for every later view. - const extent = robustExtent(features); - if (extent) { + const full = fullExtent(features); + const trimmed = robustExtent(features); + if (full && trimmed) { const key = `${EXTENT_STORAGE_PREFIX}${entry.id}`; const learned = storedJson(storage, key) || {}; - learned[partition.url] = extent; + learned[partition.url] = { full, trimmed }; storeJson(storage, key, learned); } return features; @@ -195,11 +287,18 @@ export function createLibrarySource({ loadParquet(), partitionIndex(entry, { signal }), ]); + try { + storage?.removeItem?.(`${RETIRED_EXTENT_STORAGE_PREFIX}${entry.id}`); + } catch { + /* a blocked store keeps a key nobody reads */ + } const learned = storedJson(storage, `${EXTENT_STORAGE_PREFIX}${entry.id}`) || {}; - const candidates = partitionsForBox(index, box, Infinity) - .filter(({ url }) => !learned[url] || overlaps(learned[url], box)) - .slice(0, MAX_PARTITION_READS_PER_VIEW); + const ranked = rankPartitions(index, box, learned, partitionsForBox); + const candidates = ranked.slice(0, MAX_PARTITION_READS_PER_VIEW); + // Every partition the box touches but this view leaves unread (the read + // budget, or the early stop below) makes the result partial. + let unread = ranked.length - candidates.length; const inView = []; for (let start = 0; start < candidates.length; start += PARTITION_BATCH) { const batch = candidates.slice(start, start + PARTITION_BATCH); @@ -217,13 +316,19 @@ export function createLibrarySource({ inView.push(...hits); } } - if (found) break; + if (found) { + unread += candidates.length - (start + batch.length); + break; + } } const bounded = boundLibraryFeatures(inView, { maxFeatures: entry.source.maxFeatures, display: entry.display, }); - return { features: bounded.features, truncated: bounded.truncated }; + return { + features: bounded.features, + truncated: bounded.truncated || unread > 0, + }; } /** diff --git a/src/library/tileLayer.test.mjs b/src/library/tileLayer.test.mjs index d26ce18d4..94fe4a999 100644 --- a/src/library/tileLayer.test.mjs +++ b/src/library/tileLayer.test.mjs @@ -1,7 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import * as Cesium from 'cesium'; -import { createLibraryTileLayer } from './layers.js'; +import { createLibraryTileLayer, minimumTerrainLevelFor } from './layers.js'; import { LIBRARY_ENTRIES, libraryEntry } from './catalog.js'; import { libraryTileUrl } from './manifest.js'; import { createLibrarySource } from './source.js'; @@ -133,3 +133,35 @@ test('bbox and TMS templates become the URLs Cesium and the proxy request', () = 'https://t.example.org/3/2/6.png', ); }); + +test('an entry too dense to draw zoomed out is hidden below its scale', async () => { + const cadastre = entryWithoutOutline('kartverket-cadastre'); + assert.equal(cadastre.source.minZoom, 14); + assert.equal(minimumTerrainLevelFor(cadastre), 13); + // Small coverage, or a low minimum zoom, draws at every scale. + assert.equal( + minimumTerrainLevelFor(libraryEntry('lee-county-fl')), + undefined, + ); + assert.equal( + minimumTerrainLevelFor(libraryEntry('ign-orthophotos-france')), + undefined, + ); + + // Switched on zoomed out, then zoomed further out: no globe tile coarser + // than level 13 takes imagery from it, so no z14 tile is ever requested. + const viewer = globeViewer(); + const layer = createLibraryTileLayer(cadastre, { source, cesium: Cesium }); + await layer.init(viewer); + await layer.enable(viewer); + const imagery = viewer.imageryLayers.get(viewer.imageryLayers.length - 1); + const terrain = new Cesium.EllipsoidTerrainProvider(); + for (const level of [0, 4, 12]) + assert.equal( + imagery._createTileImagerySkeletons({ level, data: {} }, terrain), + false, + `globe level ${level} takes no cadastre imagery`, + ); + await layer.disable(); + assert.equal(viewer.imageryLayers.length, 0); +}); diff --git a/src/share/legacyShareState.js b/src/share/legacyShareState.js index 1f54445e4..391293afe 100644 --- a/src/share/legacyShareState.js +++ b/src/share/legacyShareState.js @@ -48,6 +48,7 @@ export const LEGACY_SHARE_STATE_KEYS = Object.freeze([ 'layerState', 'layerStateInvalid', 'panelState', + 'library', 'sharedAtMs', ]); diff --git a/src/share/viewCodec.js b/src/share/viewCodec.js index 4031c9b31..fd8059799 100644 --- a/src/share/viewCodec.js +++ b/src/share/viewCodec.js @@ -390,6 +390,9 @@ export const SHARE_VIEW_FIELDS = freezeAll([ { number: 26, key: 'scopeTerminusPct', type: 'uint', default: null }, { number: 27, key: 'styleParams', type: 'bytes', default: BY_STYLE }, { number: 28, key: 'worldTime', type: 'epoch', default: null }, + // Data Library entries switched on, as their two-character ledger tokens + // concatenated (src/library/ledger.json). Absent means none. + { number: 29, key: 'library', type: 'bytes', default: null }, ]); const FIELD_BY_KEY = new Map(SHARE_VIEW_FIELDS.map((f) => [f.key, f])); const fieldNumber = (key) => FIELD_BY_KEY.get(key).number; @@ -450,6 +453,33 @@ export function createDefaultShareViewState(requested = 'normal') { /* Normalization: the one canonical state both directions agree on. */ /* ------------------------------------------------------------------ */ +/** The most Data Library tokens a share carries (the library's own cap). */ +export const SHARE_LIBRARY_MAX_TOKENS = 64; +const LIBRARY_TOKEN = /^[0-9a-z]{2}$/; + +/** + * Data Library share tokens from a `lib=` value (`ab.cd`) or a token list: + * unique, two base-36 characters each, at most SHARE_LIBRARY_MAX_TOKENS. + * Malformed tokens are skipped. Null or undefined stays null (no field). + * @param {string|string[]|null|undefined} value + * @returns {string[]|null} + */ +export function normalizeLibraryTokens(value) { + if (value === null || value === undefined) return null; + const list = Array.isArray(value) + ? value + : typeof value === 'string' && value.length <= 3 * SHARE_LIBRARY_MAX_TOKENS + ? value.split('.') + : []; + const tokens = []; + for (const token of list) { + if (typeof token !== 'string' || !LIBRARY_TOKEN.test(token)) continue; + if (!tokens.includes(token)) tokens.push(token); + if (tokens.length === SHARE_LIBRARY_MAX_TOKENS) break; + } + return tokens; +} + const finite = (value) => typeof value === 'number' ? Number.isFinite(value) @@ -761,6 +791,7 @@ export function normalizeShareViewState(input = {}) { ? normalizeLayerState(source.layerState) : null; state.panelState = normalizePanelState(source.panelState); + state.library = normalizeLibraryTokens(source.library); state.sharedAtMs = epochMsOr(source.sharedAtMs); state.worldTime = isWorldTime(source.worldTime) ? Math.floor(source.worldTime / 1000) * 1000 @@ -1030,6 +1061,10 @@ export function encodeShareView(input) { case 'panelState': if (state.panelState) encodePanels(writer, state.panelState); break; + case 'library': + if (state.library?.length) + writer.textField(field.number, state.library.join('')); + break; case 'thing': if (thing) encodeThing(writer, thing, state); break; @@ -1438,6 +1473,13 @@ export function decodeShareView(code, { nowMs = Date.now() } = {}) { unknown.code('layer-option', 'without-layers'); if (get('panelState')) raw.panelState = decodePanels(get('panelState').len, unknown); + if (get('library')) { + const text = asciiText(get('library').len.rest()); + const tokens = text.match(/.{1,2}/g) || []; + for (const token of tokens) + if (!LIBRARY_TOKEN.test(token)) unknown.code('library', token); + raw.library = normalizeLibraryTokens(tokens); + } if (get('styleParams') && !retiredStyle) raw.styleParams = decodeStyleParams( get('styleParams').len, diff --git a/src/share/viewCodec.test.mjs b/src/share/viewCodec.test.mjs index 8e663fce9..42dd6b547 100644 --- a/src/share/viewCodec.test.mjs +++ b/src/share/viewCodec.test.mjs @@ -58,6 +58,7 @@ import { followOffsetToEnu, layerTokenFromNumber, layerTokenNumber, + normalizeLibraryTokens, normalizeShareViewState, } from './viewCodec.js'; @@ -507,6 +508,7 @@ test('the field registry is well formed and permanent', () => { [26, 'scopeTerminusPct', null], [27, 'styleParams', BY_STYLE], [28, 'worldTime', null], + [29, 'library', null], ], ); assert.deepEqual( @@ -1571,3 +1573,16 @@ test('the follow offset for a camera is the inverse of followCameraFor', () => { null, ); }); + +test('Data Library tokens round-trip and malformed ones are skipped', () => { + const state = { lat: 1, lon: 2, library: ['00', '1z', '0a'] }; + const decoded = decodeShareView(encodeShareView(state)); + assert.equal(decoded.ok, true); + assert.deepEqual(decoded.state.library, ['00', '1z', '0a']); + // None on: no field, and the decoder reads no tokens. + const none = decodeShareView(encodeShareView({ lat: 1, lon: 2 })); + assert.equal(none.state.library, null); + assert.deepEqual(normalizeLibraryTokens('01.!!.01.zz'), ['01', 'zz']); + assert.equal(normalizeLibraryTokens(null), null); + assert.deepEqual(normalizeLibraryTokens('x'.repeat(500)), []); +}); diff --git a/src/sharelink.js b/src/sharelink.js index 74ad5b05a..6577afeae 100644 --- a/src/sharelink.js +++ b/src/sharelink.js @@ -17,6 +17,7 @@ import { SHARE_VIEW_STYLE_PARAMS, decodeShareView, encodeShareView, + normalizeLibraryTokens, } from './share/viewCodec.js'; import { fromLegacyShareState, @@ -109,6 +110,10 @@ export class ShareLinkManager { this._mapStack = 'photoreal'; this._layerStateProvider = null; this._libraryParamsEncoder = null; + this._libraryShareListener = null; + // The opening share's Data Library tokens: null when the page opened + // without a share, [] for a share that switches no library entry on. + this._initialLibraryTokens = null; this._panelStateProvider = null; this._styleParamStateProvider = null; this._initialRestorePending = false; @@ -161,10 +166,32 @@ export class ShareLinkManager { const state = code === null ? this._parseShareHash(hash) : this._parseShareCode(code); if (!state) return null; + // A share decides which library entries are on, as it does for core + // layers: one without `lib` switches none on. + this._initialLibraryTokens = state.library ?? []; this._beginIncomingRestore(state); return state; } + /** + * The Data Library tokens of the share this page opened with, or null when + * it opened without one. Read by the library once it knows its entries. + * @returns {string[]|null} + */ + get initialLibraryTokens() { + return this._initialLibraryTokens ? [...this._initialLibraryTokens] : null; + } + + /** + * Install the Data Library's listener for shares opened in this tab after + * start-up. It receives the share's tokens ([] when it switches none on). + * @param {((tokens: string[]) => void) | null} listener + */ + setLibraryShareListener(listener) { + this._libraryShareListener = + typeof listener === 'function' ? listener : null; + } + /** * The restore state for a share code, in the long form's parsed shape plus * `shareView` (what the view follows, with its offset and last-known fix) @@ -273,6 +300,7 @@ export class ShareLinkManager { params.has('l') && decodedLayerState === null, panelState: decodePanelStateParams(params), + library: normalizeLibraryTokens(params.get('lib')), sharedAtMs: decodeShareCreatedAtMs(params), }; return state; @@ -324,6 +352,11 @@ export class ShareLinkManager { if (!state) return; this._beginIncomingRestore(state); onNavigate(state); + try { + this._libraryShareListener?.(state.library ?? []); + } catch (error) { + console.warn('[ShareLink] library restore failed:', error); + } } /** @@ -751,6 +784,7 @@ export class ShareLinkManager { this._unwatchNavigation?.(); this._layerStateProvider = null; this._libraryParamsEncoder = null; + this._libraryShareListener = null; this._panelStateProvider = null; this._styleParamStateProvider = null; this._followSubjectProvider = null; diff --git a/src/sharelink.navigation.test.mjs b/src/sharelink.navigation.test.mjs index e486ba58d..10e167bc4 100644 --- a/src/sharelink.navigation.test.mjs +++ b/src/sharelink.navigation.test.mjs @@ -448,3 +448,47 @@ test('a link made away from live carries its world time to the page that opens i assert.equal(liveOpener.parseInitialHash().worldTime, null); liveOpener.destroy(); }); + +test('a copied link carries the Data Library selection to a clean recipient', (t) => { + installTab(t, SHARE_A); + const sharer = new ShareLinkManager(fakeViewer()); + sharer.setLibraryParamsEncoder((params) => params.set('lib', '03.0a')); + const code = new URL(sharer.buildLink()).pathname.split('/').at(-1); + sharer.destroy(); + const opener = new ShareLinkManager(fakeViewer(), { initialShareCode: code }); + assert.deepEqual(opener.parseInitialHash().library, ['03', '0a']); + assert.deepEqual(opener.initialLibraryTokens, ['03', '0a']); + opener.destroy(); + + // A link with every library entry off is a share that switches none on. + const none = new ShareLinkManager(fakeViewer()); + const noneCode = new URL(none.buildLink()).pathname.split('/').at(-1); + none.destroy(); + const noneOpener = new ShareLinkManager(fakeViewer(), { + initialShareCode: noneCode, + }); + noneOpener.parseInitialHash(); + assert.deepEqual(noneOpener.initialLibraryTokens, []); + noneOpener.destroy(); + + // A page opened without a share leaves the library its stored selection. + installTab(t, ''); + const plain = new ShareLinkManager(fakeViewer()); + plain.parseInitialHash(); + assert.equal(plain.initialLibraryTokens, null); + plain.destroy(); +}); + +test('a share navigated to in this tab hands its library tokens over', (t) => { + const tab = installTab(t, SHARE_A); + const manager = new ShareLinkManager(fakeViewer()); + manager.parseInitialHash(); + const received = []; + manager.setLibraryShareListener((tokens) => received.push(tokens)); + manager.watchNavigation(() => {}); + tab.navigate(`${SHARE_B}&lib=01`); + tab.navigate(SHARE_A); + tab.back(); + assert.deepEqual(received, [['01'], [], ['01']]); + manager.destroy(); +}); diff --git a/src/ui/libraryBrowser.js b/src/ui/libraryBrowser.js index 857b0f04d..84989b60b 100644 --- a/src/ui/libraryBrowser.js +++ b/src/ui/libraryBrowser.js @@ -7,7 +7,7 @@ * width, height, labels, initial and return focus, Escape and dismissal, and * closes itself when clean view, recording or Cockpit hides the panels. */ -import { formatNumber, msg, t, tn } from '../i18n/index.js'; +import { formatNumber, msg, onLocaleChange, t, tn } from '../i18n/index.js'; const PANEL_HIDING_MODES = ['ui-clean-view', 'recording-mode', 'cockpit-mode']; /** How long Remove all stays armed after its first press. */ @@ -219,6 +219,12 @@ export function mountLibraryBrowser({ library, trigger }) { node.addEventListener(type, listener); removers.push(() => node.removeEventListener(type, listener)); }; + // Listeners on the dialog's own nodes, released when it is rebuilt. + const dialogRemovers = []; + const listenDialog = (node, type, listener) => { + node.addEventListener(type, listener); + dialogRemovers.push(() => node.removeEventListener(type, listener)); + }; const filter = { query: '', family: '', availableOnly: false }; let dialog = null; let list = null; @@ -259,7 +265,7 @@ export function mountLibraryBrowser({ library, trigger }) { ); close.type = 'button'; close.setAttribute('aria-label', t('Close data library')); - listen(close, 'click', () => api.close()); + listenDialog(close, 'click', () => api.close()); header.append(title, close); const body = element('div', 'data-library-body'); @@ -282,7 +288,8 @@ export function mountLibraryBrowser({ library, trigger }) { search.type = 'search'; search.autocomplete = 'off'; search.placeholder = t('Name, provider or country'); - listen(search, 'input', () => { + search.value = filter.query; + listenDialog(search, 'input', () => { filter.query = search.value; render(); }); @@ -291,10 +298,16 @@ export function mountLibraryBrowser({ library, trigger }) { const familyLabel = element('label', 'data-library-field'); familyLabel.append(element('span', '', t('Theme'))); const family = element('select', 'panel-surface-control'); - family.append(new Option(t('All themes'), '')); + const option = (text, value) => { + const node = element('option', '', text); + node.value = value; + return node; + }; + family.append(option(t('All themes'), '')); for (const [value, label] of Object.entries(FAMILY_LABELS)) - family.append(new Option(t(label), value)); - listen(family, 'change', () => { + family.append(option(t(label), value)); + family.value = filter.family; + listenDialog(family, 'change', () => { filter.family = family.value; render(); }); @@ -303,7 +316,8 @@ export function mountLibraryBrowser({ library, trigger }) { const availableLabel = element('label', 'data-library-check'); const available = element('input'); available.type = 'checkbox'; - listen(available, 'change', () => { + available.checked = filter.availableOnly; + listenDialog(available, 'change', () => { filter.availableOnly = available.checked; render(); }); @@ -319,7 +333,7 @@ export function mountLibraryBrowser({ library, trigger }) { status.setAttribute('aria-live', 'polite'); resetButton = element('button', 'panel-surface-control data-library-reset'); resetButton.type = 'button'; - listen(resetButton, 'click', () => { + listenDialog(resetButton, 'click', () => { if (resetting) return; if (!resetArmed) { resetArmed = true; @@ -344,7 +358,7 @@ export function mountLibraryBrowser({ library, trigger }) { toolbar.append(status, resetButton); list = element('ul', 'data-library-list'); - listen(list, 'click', (event) => { + listenDialog(list, 'click', (event) => { const button = event.target?.closest?.('button[data-library-action]'); if (!button || button.disabled) return; const id = button.dataset.libraryId; @@ -366,13 +380,13 @@ export function mountLibraryBrowser({ library, trigger }) { body.append(stats, intro, filters, toolbar, list); dialog.append(header, body); - listen(dialog, 'cancel', (event) => { + listenDialog(dialog, 'cancel', (event) => { event.preventDefault(); api.close(); }); // A click on the backdrop (the dialog box itself, outside its content) // dismisses, like Escape. - listen(dialog, 'click', (event) => { + listenDialog(dialog, 'click', (event) => { if (event.target === dialog) api.close(); }); document.body.append(dialog); @@ -576,13 +590,33 @@ export function mountLibraryBrowser({ library, trigger }) { destroy() { api.close(); unsubscribe(); + offLocale(); observer?.disconnect(); for (const remove of removers.splice(0)) remove(); + for (const remove of dialogRemovers.splice(0)) remove(); dialog?.remove(); dialog = null; }, }; + /** + * Every label is translated when the dialog is built, so a language switch + * rebuilds it, keeping its filters and, if it was open, opening it again. + */ + function relabel() { + if (!dialog) return; + const wasOpen = dialog.open; + disarmReset(); + clearInterval(statsTimer); + statsTimer = null; + if (wasOpen) dialog.close(); + for (const remove of dialogRemovers.splice(0)) remove(); + dialog.remove(); + dialog = null; + if (wasOpen) api.open(); + } + const offLocale = onLocaleChange(relabel); + const unsubscribe = library.subscribe(render); if (trigger) listen(trigger, 'click', () => api.open()); const observer = diff --git a/src/ui/libraryBrowser.locale.test.mjs b/src/ui/libraryBrowser.locale.test.mjs new file mode 100644 index 000000000..045647b74 --- /dev/null +++ b/src/ui/libraryBrowser.locale.test.mjs @@ -0,0 +1,54 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { installVueDom } from '../testSupport/vueDom.mjs'; +import { setLocale, t } from '../i18n/index.js'; + +test('a language switch relabels the open dialog and keeps its filters', async (context) => { + context.after(installVueDom()); + context.after(() => setLocale('en')); + const { mountLibraryBrowser } = await import('./libraryBrowser.js'); + const library = { + ready: true, + subscribe: () => () => {}, + list: () => [], + summary: () => ({ + entries: 0, + kinds: { dataset: 0, tiles: 0, pack: 0 }, + themes: [], + available: 0, + unavailable: 0, + added: 0, + on: 0, + features: 0, + featureLayers: 0, + }), + reset: async () => 0, + }; + const browser = mountLibraryBrowser({ library, trigger: null }); + context.after(() => browser.destroy()); + browser.open(); + const search = () => + document.querySelector('#data-library input[type="search"]'); + search().value = 'kartverket'; + search().dispatchEvent(new Event('input')); + assert.equal( + document.querySelector('#data-library-title').textContent, + 'DATA LIBRARY', + ); + + setLocale('de'); + const title = document.querySelector('#data-library-title'); + assert.equal(title.textContent, t('DATA LIBRARY')); + assert.notEqual(title.textContent, 'DATA LIBRARY'); + assert.equal(document.querySelectorAll('#data-library').length, 1); + assert.equal(document.querySelector('#data-library').open, true); + assert.equal(search().value, 'kartverket'); + + // Closed and reopened later, it is still in the new language. + browser.close(); + browser.open(); + assert.equal( + document.querySelector('#data-library-title').textContent, + t('DATA LIBRARY'), + ); +});