mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-13 06:42:23 +00:00
Fix Data Library share restore, dense tile layers and EPC partition reads
- Share links: the compact /share/<code> format carries the switched-on library entries (field 29), so Copy Link keeps them; a share opened in the same tab restores them; any share decides what is on, so one with every library entry off no longer switches on the recipient's stored entries. - A tile entry too dense at its minimum zoom (Norway cadastre z14, depth z11) is hidden below that scale via minimumTerrainLevel instead of allocating millions of tiles when zoomed out. - EPC partitions keep the full extent for exclusion and the 98 % extent for ranking only; the old trimmed-only extents are retired. - Partition index footers that fail transiently are retried; an incomplete index is never cached as complete, a complete prior index is kept, and it is rebuilt after five minutes. - Views that leave touching partitions unread are partial; listing and footer bodies are capped and ranges validated. - The Browse dialog relabels after a language switch.
This commit is contained in:
Said Turksever committed
1 parent
871b7f0fcc
commit
24ccdd4dc3
20 files changed
+1220
-162
No files matched your search
@@ -585,6 +585,24 @@ the ledger (append-only with `--base-ref`, run in CI on pull requests) and the
|
||||
generated DATA_SOURCES.md section; `npm run library:probe` hits every upstream
|
||||
once; `npm run qa:data-library` is the browser gate (fixture-backed).
|
||||
|
||||
Shares: the compact `/share/<code>` link carries the switched-on entries in
|
||||
field 29 (docs/SHARE-LINK-FORMAT.md), so Copy Link keeps them, and a share
|
||||
opened later in the same tab (pasted address, Back/Forward) restores them too.
|
||||
Any share decides what is on, as for core layers: one with no library entry
|
||||
on switches the recipient's stored ones off and leaves them listed. A tile
|
||||
entry whose coverage needs more than 256 tiles at its minimum zoom (Norway's
|
||||
cadastre from z14, sea depths from z11) is hidden below that scale through
|
||||
Cesium's `minimumTerrainLevel`, so zoomed out it requests nothing. EPC
|
||||
partitions learn two extents (`gev:library:extent:v2:`): every point's box
|
||||
rules a partition out, the 98 % box only ranks it, so homes at a partition's
|
||||
edge are found on later visits; the v1 trimmed-only key is dropped. A
|
||||
partition index whose footers keep failing after two retries is
|
||||
`complete: false`: neither the server nor the browser keeps it as whole, a
|
||||
complete earlier index is served instead, and it is rebuilt after five
|
||||
minutes. A view that leaves partitions touching it unread is PARTIAL. Listing
|
||||
pages and footer reads are size-capped and ranges are validated. The Browse
|
||||
dialog rebuilds on a language switch, keeping its filters.
|
||||
|
||||
## Utilities panel — October 29, 2026
|
||||
|
||||
UTILITIES is its own left-side panel, between Data Layers and Scenes, with
|
||||
|
||||
+37
-31
@@ -117,36 +117,37 @@ probability of about 255 in 256 (many of them already fail as base64url).
|
||||
|
||||
## Field registry (top level)
|
||||
|
||||
| # | State key | Type | Default | Encoding |
|
||||
| --- | ---------------------------- | ----- | ----------- | ----------------------------------------------------------------- |
|
||||
| 1 | `view.kind` | enum | see below | 0 place, 1 thing |
|
||||
| 2 | `lat` | coord | 0 | zigzag of degrees × 10⁴ |
|
||||
| 3 | `lon` | coord | 0 | zigzag of degrees × 10⁴ |
|
||||
| 4 | `alt` | sint | 800 | whole metres |
|
||||
| 5 | `heading` | angle | 0 | whole degrees, see angles |
|
||||
| 6 | `pitch` | sint | −35 | whole degrees |
|
||||
| 7 | `roll` | angle | 0 | whole degrees |
|
||||
| 8 | `sharedAtMs` | epoch | none | zigzag seconds since 2026-01-01T00:00:00Z |
|
||||
| 9 | booleans | flags | 0 | bit set when the value differs from default |
|
||||
| 10 | `layerState.enabledLayerIds` | LEN | none | layer bitset, see layers |
|
||||
| 11 | `layerState.options` | LEN | none | packed option triples, see layers |
|
||||
| 12 | `panelState` | LEN | none | packed panel entries, see panels |
|
||||
| 13 | thing | LEN | none | sub-message, see thing views |
|
||||
| 14 | `style` | enum | `normal` | normal, retro, surveillance, thermal, anime, noir, snow (retired) |
|
||||
| 15 | `mapStack` | enum | `photoreal` | photoreal, bing-aerial, bing-labels, esri-imagery, osm |
|
||||
| 16 | `hudVariant` | enum | style | tactical, operator (retired), minimal, cyber |
|
||||
| 17 | `bloomIntensity` | uint | style | |
|
||||
| 18 | `bloomVersion` | uint | 2 | |
|
||||
| 19 | `sharpenIntensity` | uint | style | |
|
||||
| 20 | `detectionMode` | enum | style | OFF, SPARSE, BALANCED, DENSE |
|
||||
| 21 | `detectionDensity` | uint | style | one of 0, 25, 50, 75, 100 |
|
||||
| 22 | `detectionAllocation` | enum | style | ELASTIC, WEIGHTED |
|
||||
| 23 | `detectionFadePct` | uint | style | 0–40 |
|
||||
| 24 | `detectionOutsideOpacityPct` | uint | style | 0–100 |
|
||||
| 25 | `scopeFeatherPct` | uint | 100 | 0–200 |
|
||||
| 26 | `scopeTerminusPct` | uint | none | 94–100; absent is the adaptive terminus |
|
||||
| 27 | `styleParams` | LEN | style | packed `(token, value × 100)` pairs |
|
||||
| 28 | `worldTime` | epoch | none (live) | zigzag seconds since 2026-01-01T00:00:00Z |
|
||||
| # | State key | Type | Default | Encoding |
|
||||
| --- | ---------------------------- | ----- | ----------- | --------------------------------------------------------------------- |
|
||||
| 1 | `view.kind` | enum | see below | 0 place, 1 thing |
|
||||
| 2 | `lat` | coord | 0 | zigzag of degrees × 10⁴ |
|
||||
| 3 | `lon` | coord | 0 | zigzag of degrees × 10⁴ |
|
||||
| 4 | `alt` | sint | 800 | whole metres |
|
||||
| 5 | `heading` | angle | 0 | whole degrees, see angles |
|
||||
| 6 | `pitch` | sint | −35 | whole degrees |
|
||||
| 7 | `roll` | angle | 0 | whole degrees |
|
||||
| 8 | `sharedAtMs` | epoch | none | zigzag seconds since 2026-01-01T00:00:00Z |
|
||||
| 9 | booleans | flags | 0 | bit set when the value differs from default |
|
||||
| 10 | `layerState.enabledLayerIds` | LEN | none | layer bitset, see layers |
|
||||
| 11 | `layerState.options` | LEN | none | packed option triples, see layers |
|
||||
| 12 | `panelState` | LEN | none | packed panel entries, see panels |
|
||||
| 13 | thing | LEN | none | sub-message, see thing views |
|
||||
| 14 | `style` | enum | `normal` | normal, retro, surveillance, thermal, anime, noir, snow (retired) |
|
||||
| 15 | `mapStack` | enum | `photoreal` | photoreal, bing-aerial, bing-labels, esri-imagery, osm |
|
||||
| 16 | `hudVariant` | enum | style | tactical, operator (retired), minimal, cyber |
|
||||
| 17 | `bloomIntensity` | uint | style | |
|
||||
| 18 | `bloomVersion` | uint | 2 | |
|
||||
| 19 | `sharpenIntensity` | uint | style | |
|
||||
| 20 | `detectionMode` | enum | style | OFF, SPARSE, BALANCED, DENSE |
|
||||
| 21 | `detectionDensity` | uint | style | one of 0, 25, 50, 75, 100 |
|
||||
| 22 | `detectionAllocation` | enum | style | ELASTIC, WEIGHTED |
|
||||
| 23 | `detectionFadePct` | uint | style | 0–40 |
|
||||
| 24 | `detectionOutsideOpacityPct` | uint | style | 0–100 |
|
||||
| 25 | `scopeFeatherPct` | uint | 100 | 0–200 |
|
||||
| 26 | `scopeTerminusPct` | uint | none | 94–100; absent is the adaptive terminus |
|
||||
| 27 | `styleParams` | LEN | style | packed `(token, value × 100)` pairs |
|
||||
| 28 | `worldTime` | epoch | none (live) | zigzag seconds since 2026-01-01T00:00:00Z |
|
||||
| 29 | `library` | LEN | none | Data Library ledger tokens, two base-36 characters each, concatenated |
|
||||
|
||||
Field 1 absent means `thing` when field 13 is present and `place` otherwise,
|
||||
so v1 never writes it; a later view kind writes its code.
|
||||
@@ -164,6 +165,11 @@ Field 28 is the world time the link was made at, written only when it was away
|
||||
from live. A link without it opens live. Unlike field 8 it may lie ahead of the
|
||||
link's creation (a forecast moment), and it opens paused at that time.
|
||||
|
||||
Field 29 lists the Data Library entries switched on, by their permanent
|
||||
tokens in `src/library/ledger.json` (the long form's `lib=ab.cd`). A share
|
||||
without it switches no library entry on; the recipient's added entries stay
|
||||
in their panel, off. Tokens this build does not know are skipped.
|
||||
|
||||
Flag bits (field 9): 0 `bloom`, 1 `sharpen`, 2 `hudVisible`, 3
|
||||
`celestialRing` (each defaulting by style), 4 `scopeEnabled` (default on).
|
||||
|
||||
@@ -391,7 +397,7 @@ the app that can open it.
|
||||
offers keeps its code: `snow` (6) is retired, so a link that names it
|
||||
opens as Normal and the parameters written for it are dropped.
|
||||
5. **Add, don't change.** A new setting takes the next free field number
|
||||
(28 onwards; fields 1–15 have one-byte tags and are all taken). A new
|
||||
(30 onwards; fields 1–15 have one-byte tags and are all taken). A new
|
||||
boolean takes the next flag bit instead, which costs nothing when it is at
|
||||
its default.
|
||||
6. **New view kinds and entity kinds** get the next code. A new entity kind's
|
||||
|
||||
@@ -629,6 +629,107 @@ try {
|
||||
assert.deepEqual(stored, { added: [], on: [] });
|
||||
});
|
||||
|
||||
const onIds = (target) =>
|
||||
target.evaluate(() =>
|
||||
window.__gevDataLibrary.library
|
||||
.list()
|
||||
.filter((row) => row.on)
|
||||
.map((row) => row.entry.id)
|
||||
.sort(),
|
||||
);
|
||||
|
||||
await step(
|
||||
'a copied /share link opens the same library layers in a clean browser',
|
||||
async () => {
|
||||
await page.evaluate(async () => {
|
||||
const { library } = window.__gevDataLibrary;
|
||||
await library.add('nasa-eonet-events');
|
||||
await library.add('nasa-black-marble');
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() => /(?:^|&)lib=00\.03(?:&|$)/.test(location.hash.slice(1)),
|
||||
{ timeout: 15_000 },
|
||||
);
|
||||
const href = await page.evaluate(() =>
|
||||
window.__gevDataLibrary.buildShareLink(),
|
||||
);
|
||||
assert.match(new URL(href).pathname, /\/share\//);
|
||||
// A separate profile: nothing stored, nothing added.
|
||||
const context = await browser.createBrowserContext();
|
||||
try {
|
||||
const recipient = await context.newPage();
|
||||
await recipient.setRequestInterception(true);
|
||||
recipient.on('request', (request) => {
|
||||
const fixture = fixtureFor(new URL(request.url()));
|
||||
if (fixture) request.respond(fixture);
|
||||
else request.continue();
|
||||
});
|
||||
await recipient.evaluateOnNewDocument(() => {
|
||||
try {
|
||||
localStorage.setItem('gev:first-run-mission:v1', 'suppressed');
|
||||
} catch {
|
||||
/* the gate still runs */
|
||||
}
|
||||
});
|
||||
await recipient.goto(href, { waitUntil: 'domcontentloaded' });
|
||||
await recipient.waitForFunction(
|
||||
() => window.__gevDataLibrary?.library.ready,
|
||||
{ timeout: 180_000 },
|
||||
);
|
||||
await recipient.waitForFunction(
|
||||
() =>
|
||||
window.__gevDataLibrary.library.list().filter((row) => row.on)
|
||||
.length === 2,
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
assert.deepEqual(await onIds(recipient), [
|
||||
'nasa-black-marble',
|
||||
'nasa-eonet-events',
|
||||
]);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
await step(
|
||||
'a share opened in the same tab switches library layers on and off',
|
||||
async () => {
|
||||
const navigate = (lib) =>
|
||||
page.evaluate((value) => {
|
||||
const params = new URLSearchParams(location.hash.slice(1));
|
||||
params.set('lat', String(Number(params.get('lat')) + 1));
|
||||
if (value === null) params.delete('lib');
|
||||
else params.set('lib', value);
|
||||
location.hash = params.toString();
|
||||
}, lib);
|
||||
await navigate('00');
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const on = window.__gevDataLibrary.library
|
||||
.list()
|
||||
.filter((row) => row.on)
|
||||
.map((row) => row.entry.id);
|
||||
return on.length === 1 && on[0] === 'nasa-eonet-events';
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
// A share with every library layer off: the added ones stay listed, off.
|
||||
await navigate(null);
|
||||
await page.waitForFunction(
|
||||
() => window.__gevDataLibrary.library.list().every((row) => !row.on),
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await libraryRows(page)).map(({ id, on }) => [id, on]).sort(),
|
||||
[
|
||||
['nasa-black-marble', false],
|
||||
['nasa-eonet-events', false],
|
||||
],
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(errors, [], 'no library errors on the page');
|
||||
console.log(`# ${count} passed`);
|
||||
} finally {
|
||||
|
||||
@@ -19,6 +19,8 @@ const DATASET_TIMEOUT_MS = 30_000;
|
||||
const TILE_TIMEOUT_MS = 10_000;
|
||||
/** Each listing page or footer read while building a partition index. */
|
||||
const INDEX_TIMEOUT_MS = 20_000;
|
||||
/** How long an incomplete partition index is served before a rebuild. */
|
||||
const PARTIAL_INDEX_MS = 5 * 60_000;
|
||||
const TILE_MAX_BYTES = 2 * 1024 * 1024;
|
||||
const TILE_CACHE_LIMIT = 512;
|
||||
const IN_FLIGHT_LIMIT = 64;
|
||||
@@ -107,6 +109,7 @@ export function libraryProxy({
|
||||
const datasets = new Map();
|
||||
const tiles = new Map();
|
||||
const indexes = new Map();
|
||||
const partialIndexes = new Map();
|
||||
const inFlight = new Map();
|
||||
const allowData = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
@@ -231,25 +234,51 @@ export function libraryProxy({
|
||||
if (!available) return json(res, 403, { error: reason });
|
||||
if (!allowData(clientKey(req)))
|
||||
return json(res, 429, { error: 'rate_limited' });
|
||||
// Only a complete index is cached for cacheSeconds. An incomplete one
|
||||
// (footers that kept failing) is served as `complete: false` and kept
|
||||
// for PARTIAL_INDEX_MS only, so the next request soon rebuilds it; a
|
||||
// complete index from before keeps being served while rebuilds fail.
|
||||
const key = `index:${id}`;
|
||||
const partial = partialIndexes.get(key);
|
||||
if (partial && now() - partial.savedAt < PARTIAL_INDEX_MS) {
|
||||
const previous = indexes.get(key);
|
||||
return json(res, 200, previous?.value ?? partial.value);
|
||||
}
|
||||
try {
|
||||
const { value } = await cached(
|
||||
indexes,
|
||||
`index:${id}`,
|
||||
key,
|
||||
entry.source.cacheSeconds * 1000,
|
||||
async () => ({
|
||||
partitions: await buildPartitionIndex(entry, {
|
||||
async () => {
|
||||
const built = await buildPartitionIndex(entry, {
|
||||
fetchImpl: (url, options = {}) =>
|
||||
fetchImpl(url, {
|
||||
...options,
|
||||
signal:
|
||||
options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS),
|
||||
}),
|
||||
}),
|
||||
builtAt: now(),
|
||||
}),
|
||||
});
|
||||
const value = {
|
||||
partitions: built.partitions,
|
||||
complete: built.complete,
|
||||
builtAt: now(),
|
||||
};
|
||||
if (!built.complete) {
|
||||
// Recorded here too: with an earlier complete index, cached()
|
||||
// serves that one and swallows this error, and the rebuild
|
||||
// still waits PARTIAL_INDEX_MS.
|
||||
partialIndexes.set(key, { value, savedAt: now() });
|
||||
throw Object.assign(new Error('index_incomplete'), {
|
||||
partialIndex: value,
|
||||
});
|
||||
}
|
||||
partialIndexes.delete(key);
|
||||
return value;
|
||||
},
|
||||
);
|
||||
return json(res, 200, value);
|
||||
} catch (error) {
|
||||
if (error?.partialIndex) return json(res, 200, error.partialIndex);
|
||||
const status = error?.status === 429 ? 429 : 502;
|
||||
return json(res, status, { error: 'library_upstream_unavailable' });
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
parseLibraryTilePath,
|
||||
} from './library.js';
|
||||
import { validateLibraryCatalog } from '../../src/library/manifest.js';
|
||||
import { libraryEntry } from '../../src/library/catalog.js';
|
||||
|
||||
const base = {
|
||||
family: 'events',
|
||||
@@ -100,8 +101,8 @@ const point = (title) => ({
|
||||
properties: { title },
|
||||
});
|
||||
|
||||
function install({ env = {}, fetchImpl, now } = {}) {
|
||||
const plugin = libraryProxy({ entries, env, fetchImpl, now });
|
||||
function install({ env = {}, fetchImpl, now, catalog = entries } = {}) {
|
||||
const plugin = libraryProxy({ entries: catalog, env, fetchImpl, now });
|
||||
let handler;
|
||||
plugin.configurePreviewServer({
|
||||
middlewares: {
|
||||
@@ -330,3 +331,51 @@ test('only GET is served', async () => {
|
||||
await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res);
|
||||
assert.equal(res.status, 405);
|
||||
});
|
||||
|
||||
test('a partition index is cached only when complete; a complete one outlives failed rebuilds', async () => {
|
||||
const epc = libraryEntry('epc-england-wales');
|
||||
let clock = 0;
|
||||
let footers = 'ok';
|
||||
let listings = 0;
|
||||
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
|
||||
const fetchImpl = async (url) => {
|
||||
if (url.includes('list-type')) {
|
||||
listings += 1;
|
||||
return new Response(listing);
|
||||
}
|
||||
return footers === 'ok'
|
||||
? new Response(new Uint8Array(100))
|
||||
: new Response('', { status: 503 });
|
||||
};
|
||||
const request = install({ catalog: [epc], now: () => clock, fetchImpl });
|
||||
const first = await request('/index/epc-england-wales');
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(first.body.complete, true);
|
||||
const builtAt = first.body.builtAt;
|
||||
|
||||
// Past cacheSeconds the rebuild fails: the complete index is still served,
|
||||
// and no rebuild runs again until the partial pause has passed.
|
||||
clock += (epc.source.cacheSeconds + 1) * 1000;
|
||||
footers = 'fail';
|
||||
const kept = await request('/index/epc-england-wales');
|
||||
assert.equal(kept.body.complete, true);
|
||||
assert.equal(kept.body.builtAt, builtAt);
|
||||
const listed = listings;
|
||||
await request('/index/epc-england-wales');
|
||||
assert.equal(listings, listed);
|
||||
|
||||
// A server that never had a complete index serves the partial one as such,
|
||||
// and rebuilds after the pause.
|
||||
const fresh = install({ catalog: [epc], now: () => clock, fetchImpl });
|
||||
const partial = await fresh('/index/epc-england-wales');
|
||||
assert.equal(partial.status, 200);
|
||||
assert.equal(partial.body.complete, false);
|
||||
const before = listings;
|
||||
await fresh('/index/epc-england-wales');
|
||||
assert.equal(listings, before);
|
||||
clock += 6 * 60_000;
|
||||
footers = 'ok';
|
||||
const rebuilt = await fresh('/index/epc-england-wales');
|
||||
assert.equal(rebuilt.body.complete, true);
|
||||
assert.equal(listings, before + 1);
|
||||
});
|
||||
+6
-1
@@ -142,7 +142,12 @@ export function createApplicationTools({
|
||||
if (!signal?.aborted) console.warn('[Library] start failed:', error);
|
||||
});
|
||||
// The live gate's handle (scripts/qa-data-library.mjs).
|
||||
const libraryHandle = { library: dataLibrary, browser: libraryBrowser };
|
||||
const libraryHandle = {
|
||||
library: dataLibrary,
|
||||
browser: libraryBrowser,
|
||||
// What Copy Link would copy.
|
||||
buildShareLink: () => styleManager.shareLinkManager?.buildLink?.() ?? null,
|
||||
};
|
||||
window.__gevDataLibrary = libraryHandle;
|
||||
defer(() => {
|
||||
if (window.__gevDataLibrary === libraryHandle)
|
||||
|
||||
+85
-20
@@ -69,6 +69,9 @@ export function createDataLibrary({
|
||||
let serverPresent = false;
|
||||
let selection = { added: [], on: [] };
|
||||
let started = false;
|
||||
// Set once availability is known; a share navigated to before then waits.
|
||||
let loaded = false;
|
||||
let queuedShare = null;
|
||||
let destroyed = false;
|
||||
const listeners = new Set();
|
||||
const pending = new Map();
|
||||
@@ -165,6 +168,73 @@ export function createDataLibrary({
|
||||
persist();
|
||||
});
|
||||
|
||||
/** Entry ids for share tokens; unknown tokens are skipped. */
|
||||
const idsForTokens = (list) =>
|
||||
(list || []).map((token) => tokens.idByToken.get(token)).filter(Boolean);
|
||||
|
||||
/**
|
||||
* The share this page opened with: null without one, else the entry ids it
|
||||
* switches on ([] for a share that switches none on).
|
||||
*/
|
||||
function openingShare() {
|
||||
if (shareLink && shareLink.initialLibraryTokens !== undefined) {
|
||||
const linked = shareLink.initialLibraryTokens;
|
||||
return linked === null ? null : idsForTokens(linked);
|
||||
}
|
||||
const hash = String(initialHash || '').replace(/^#/, '');
|
||||
if (!hash) return null;
|
||||
const params = new URLSearchParams(hash);
|
||||
const linked = decodeLibraryParam(params, tokens.idByToken);
|
||||
// Any share (a camera position) decides what is on; no `lib`, nothing.
|
||||
if (linked === null && params.has('lat') && params.has('lon')) return [];
|
||||
return linked;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a selection: register and switch on what `on` names, switch off
|
||||
* every other added entry that is on. `origin` marks it as a restore so it
|
||||
* is never remembered as the viewer's own choice.
|
||||
*/
|
||||
function applySelection(next, origin) {
|
||||
selection = normalizeLibrarySelection(next, switchable);
|
||||
for (const id of selection.added) {
|
||||
const entry = byId.get(id);
|
||||
if (!entry || !addable(id)) continue;
|
||||
const wanted = selection.on.includes(id);
|
||||
if (wanted && !register(entry)) continue;
|
||||
if (!isRegistered(id)) continue;
|
||||
const layerId = libraryLayerId(id);
|
||||
if (wanted === (dataManager.isEnabled?.(layerId) === true)) continue;
|
||||
dataManager
|
||||
.setEnabled(layerId, wanted, { origin })
|
||||
.catch((error) =>
|
||||
console.warn(`[Library] ${id} restore failed:`, error),
|
||||
);
|
||||
}
|
||||
writeStoredLibrarySelection(storage, selection);
|
||||
shareLink?.onLibraryStateChange?.();
|
||||
notify();
|
||||
}
|
||||
|
||||
/**
|
||||
* A share opened in this tab after start-up: its entries join the library
|
||||
* and it alone decides which are on.
|
||||
* @param {string[]} shareTokens
|
||||
*/
|
||||
function applyShare(shareTokens) {
|
||||
if (destroyed) return;
|
||||
if (!loaded) {
|
||||
queuedShare = [...(shareTokens || [])];
|
||||
return;
|
||||
}
|
||||
const linked = idsForTokens(shareTokens);
|
||||
applySelection(
|
||||
{ added: [...selection.added, ...linked], on: linked },
|
||||
'share-restore',
|
||||
);
|
||||
}
|
||||
shareLink?.setLibraryShareListener?.(applyShare);
|
||||
|
||||
shareLink?.setLibraryParamsEncoder?.((params) =>
|
||||
encodeLibraryParam(
|
||||
params,
|
||||
@@ -186,29 +256,23 @@ export function createDataLibrary({
|
||||
availability = answer.availability;
|
||||
serverPresent = answer.server;
|
||||
const stored = readStoredLibrarySelection(storage, switchable);
|
||||
const hash = String(initialHash || '').replace(/^#/, '');
|
||||
const linked = hash
|
||||
? decodeLibraryParam(new URLSearchParams(hash), tokens.idByToken)
|
||||
: null;
|
||||
// A share opened in this tab while availability loaded wins over the
|
||||
// opening one.
|
||||
const linked = queuedShare ? idsForTokens(queuedShare) : openingShare();
|
||||
queuedShare = null;
|
||||
loaded = true;
|
||||
// A link's entries join the viewer's library and open switched on; with a
|
||||
// link, it alone decides what is on, as it does for core layers.
|
||||
const added = [...stored.added, ...(linked || [])];
|
||||
const on = linked ?? stored.on;
|
||||
selection = normalizeLibrarySelection({ added, on }, switchable);
|
||||
const restoreOrigin = linked ? 'share-restore' : 'local-restore';
|
||||
for (const id of selection.added) {
|
||||
// link, it alone decides what is on, as it does for core layers, so a
|
||||
// share with every library entry off switches the stored ones off.
|
||||
// Added entries register even while off, so their rows show.
|
||||
for (const id of [...stored.added, ...(linked || [])]) {
|
||||
const entry = byId.get(id);
|
||||
if (!entry || !addable(id) || !register(entry)) continue;
|
||||
if (selection.on.includes(id))
|
||||
dataManager
|
||||
.setEnabled(libraryLayerId(id), true, { origin: restoreOrigin })
|
||||
.catch((error) =>
|
||||
console.warn(`[Library] ${id} restore failed:`, error),
|
||||
);
|
||||
if (entry && addable(id)) register(entry);
|
||||
}
|
||||
writeStoredLibrarySelection(storage, selection);
|
||||
shareLink?.onLibraryStateChange?.();
|
||||
notify();
|
||||
applySelection(
|
||||
{ added: [...stored.added, ...(linked || [])], on: linked ?? stored.on },
|
||||
linked ? 'share-restore' : 'local-restore',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -388,6 +452,7 @@ export function createDataLibrary({
|
||||
destroyed = true;
|
||||
unsubscribeManager?.();
|
||||
shareLink?.setLibraryParamsEncoder?.(null);
|
||||
shareLink?.setLibraryShareListener?.(null);
|
||||
listeners.clear();
|
||||
},
|
||||
};
|
||||
|
||||
@@ -207,6 +207,115 @@ test('a link decides what is on and adds its entries to the library', async (t)
|
||||
});
|
||||
});
|
||||
|
||||
/** A ShareLinkManager stand-in: the opening share and in-tab navigation. */
|
||||
function fakeShareLink(initialLibraryTokens) {
|
||||
const link = {
|
||||
initialLibraryTokens,
|
||||
listener: null,
|
||||
setLibraryShareListener(listener) {
|
||||
link.listener = listener;
|
||||
},
|
||||
setLibraryParamsEncoder() {},
|
||||
onLibraryStateChange() {},
|
||||
};
|
||||
return link;
|
||||
}
|
||||
|
||||
test('a share with every library entry off keeps the stored ones off', async (t) => {
|
||||
const manager = sealedManager(t);
|
||||
const storage = memoryStorage(
|
||||
JSON.stringify({
|
||||
added: ['night-lights', 'sea-marks'],
|
||||
on: ['night-lights', 'sea-marks'],
|
||||
}),
|
||||
);
|
||||
// A compact /share/<code> link with no library field, and a long-form link
|
||||
// with no lib=: both are shares that switch no library entry on.
|
||||
for (const options of [
|
||||
{ shareLink: fakeShareLink([]) },
|
||||
{ initialHash: '#lat=1&lon=2&l=' },
|
||||
]) {
|
||||
const library = createDataLibrary({
|
||||
entries,
|
||||
ledger,
|
||||
dataManager: manager,
|
||||
services: {},
|
||||
storage,
|
||||
source: source({}),
|
||||
...options,
|
||||
});
|
||||
await library.start();
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), false);
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), false);
|
||||
// The recipient's entries stay in their panel, off.
|
||||
assert.deepEqual(
|
||||
library
|
||||
.list()
|
||||
.filter((row) => row.added)
|
||||
.map((row) => row.entry.id),
|
||||
['night-lights', 'sea-marks'],
|
||||
);
|
||||
library.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('the opening share link restores its library tokens', async (t) => {
|
||||
const manager = sealedManager(t);
|
||||
const storage = memoryStorage(
|
||||
JSON.stringify({ added: ['night-lights'], on: ['night-lights'] }),
|
||||
);
|
||||
const library = createDataLibrary({
|
||||
entries,
|
||||
ledger,
|
||||
dataManager: manager,
|
||||
services: {},
|
||||
storage,
|
||||
shareLink: fakeShareLink(['01']),
|
||||
source: source({}),
|
||||
});
|
||||
await library.start();
|
||||
await manager.waitForLayerSettled?.('lib-sea-marks');
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), true);
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), false);
|
||||
});
|
||||
|
||||
test('navigating between shares switches library entries on and off', async (t) => {
|
||||
const manager = sealedManager(t);
|
||||
const shareLink = fakeShareLink(['00']);
|
||||
const library = createDataLibrary({
|
||||
entries,
|
||||
ledger,
|
||||
dataManager: manager,
|
||||
services: {},
|
||||
storage: memoryStorage(),
|
||||
shareLink,
|
||||
source: source({}),
|
||||
});
|
||||
// A share navigated to before availability loads waits for it and wins.
|
||||
const started = library.start();
|
||||
shareLink.listener(['01']);
|
||||
await started;
|
||||
const settle = () =>
|
||||
Promise.all(
|
||||
['lib-night-lights', 'lib-sea-marks'].map((id) =>
|
||||
manager.waitForLayerSettled?.(id),
|
||||
),
|
||||
);
|
||||
await settle();
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), true);
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), false);
|
||||
shareLink.listener(['00']);
|
||||
await settle();
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), true);
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), false);
|
||||
shareLink.listener([]);
|
||||
await settle();
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), false);
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), false);
|
||||
library.destroy();
|
||||
assert.equal(shareLink.listener, null);
|
||||
});
|
||||
|
||||
test('zoom flies to an added layer through the caller, reset removes everything', async (t) => {
|
||||
const manager = sealedManager(t);
|
||||
const storage = memoryStorage();
|
||||
|
||||
+45
-1
@@ -258,6 +258,44 @@ const reliefLegend = () =>
|
||||
/** colorToAlpha threshold (max channel distance, 0–1) for `noDataColor`. */
|
||||
const NO_DATA_THRESHOLD = 0.02;
|
||||
|
||||
/** The most tiles an entry's coverage may need at its minimum zoom. */
|
||||
const COARSE_TILE_BUDGET = 256;
|
||||
|
||||
function webMercatorTileSpan(bbox, zoom) {
|
||||
const n = 2 ** zoom;
|
||||
const clampLat = (lat) => Math.max(-85.0511, Math.min(85.0511, lat));
|
||||
const tileX = (lon) =>
|
||||
Math.min(n - 1, Math.max(0, Math.floor(((lon + 180) / 360) * n)));
|
||||
const tileY = (lat) => {
|
||||
const rad = (clampLat(lat) * Math.PI) / 180;
|
||||
const y = (1 - Math.log(Math.tan(rad) + 1 / Math.cos(rad)) / Math.PI) / 2;
|
||||
return Math.min(n - 1, Math.max(0, Math.floor(y * n)));
|
||||
};
|
||||
const [west, south, east, north] = bbox;
|
||||
return (tileX(east) - tileX(west) + 1) * (tileY(south) - tileY(north) + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* The globe level below which a tile entry is hidden, or undefined when it
|
||||
* draws at every scale. Cesium fetches an imagery layer's minimum level for
|
||||
* every globe tile it covers, however far out the camera is: an entry whose
|
||||
* coverage needs more than COARSE_TILE_BUDGET tiles at its minimum zoom
|
||||
* (Norway's cadastre from z14 is about two million) would allocate them all
|
||||
* when zoomed out. Such an entry is shown only from the globe level that
|
||||
* matches its minimum zoom (a geographic level spans two Web Mercator ones,
|
||||
* so level z − 1).
|
||||
* @param {object} entry
|
||||
* @returns {number|undefined}
|
||||
*/
|
||||
export function minimumTerrainLevelFor(entry) {
|
||||
const minZoom = entry?.source?.minZoom ?? 0;
|
||||
if (minZoom <= 0) return undefined;
|
||||
const bbox = entry.coverage?.bbox ?? [-180, -85.0511, 180, 85.0511];
|
||||
return webMercatorTileSpan(bbox, minZoom) > COARSE_TILE_BUDGET
|
||||
? minZoom - 1
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* A tiles entry as a manager layer.
|
||||
* @param {object} entry Validated tiles entry.
|
||||
@@ -406,7 +444,13 @@ export function createLibraryTileLayer(
|
||||
provider.errorEvent?.addEventListener?.(onError);
|
||||
removeErrorListener = () =>
|
||||
provider.errorEvent?.removeEventListener?.(onError);
|
||||
imagery = host.collection.addImageryProvider(provider);
|
||||
const minimumTerrainLevel = minimumTerrainLevelFor(entry);
|
||||
imagery =
|
||||
minimumTerrainLevel !== undefined && cesium.ImageryLayer
|
||||
? new cesium.ImageryLayer(provider, { minimumTerrainLevel })
|
||||
: null;
|
||||
if (imagery) host.collection.add(imagery);
|
||||
else imagery = host.collection.addImageryProvider(provider);
|
||||
imagery.alpha = entry.display.opacity ?? 1;
|
||||
if (entry.display.noDataColor && cesium.Color) {
|
||||
// The provider's "no imagery here" fill, drawn transparent (JPEG
|
||||
|
||||
+128
-42
@@ -10,7 +10,10 @@
|
||||
import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet';
|
||||
import { decompress as zstdDecompress } from 'fzstd';
|
||||
import { libraryUpstreamUrl } from './manifest.js';
|
||||
import { readResponseBytesCapped } from '../sources/httpBody.js';
|
||||
import {
|
||||
readResponseBytesCapped,
|
||||
readResponseTextCapped,
|
||||
} from '../sources/httpBody.js';
|
||||
|
||||
/** Footer bytes read first; small files' metadata fits in one request. */
|
||||
const FOOTER_FETCH_BYTES = 16 * 1024;
|
||||
@@ -19,6 +22,13 @@ export const MAX_PARTITIONS_PER_VIEW = 4;
|
||||
const MAX_LISTED_KEYS = 5000;
|
||||
const MAX_LIST_PAGES = 20;
|
||||
const INDEX_CONCURRENCY = 8;
|
||||
/** One listing page: 1,000 keys of XML is well under a megabyte. */
|
||||
const MAX_LIST_PAGE_BYTES = 4 * 1024 * 1024;
|
||||
/** A whole file sent in answer to a range request is read only up to this. */
|
||||
const MAX_UNRANGED_BYTES = 64 * 1024 * 1024;
|
||||
/** Extra passes over footers that failed, before the index is incomplete. */
|
||||
const FOOTER_RETRIES = 2;
|
||||
const FOOTER_RETRY_DELAY_MS = 500;
|
||||
|
||||
/** Decompressors hyparquet does not carry itself (it has Snappy built in). */
|
||||
export const PARQUET_COMPRESSORS = Object.freeze({
|
||||
@@ -33,30 +43,67 @@ function onSourceHost(entry, url) {
|
||||
return target.href;
|
||||
}
|
||||
|
||||
/** Mark a failure worth retrying: network, 429, 5xx or a short read. */
|
||||
const transient = (error) => Object.assign(error, { transient: true });
|
||||
|
||||
/**
|
||||
* A hyparquet AsyncBuffer over HTTP range requests, redirects refused,
|
||||
* credentials and referrer withheld.
|
||||
*/
|
||||
export function rangeBuffer(url, byteLength, { fetchImpl, signal }) {
|
||||
if (!Number.isSafeInteger(byteLength) || byteLength <= 0)
|
||||
throw new RangeError(`invalid object size ${byteLength}`);
|
||||
return {
|
||||
byteLength,
|
||||
async slice(start, end = byteLength) {
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
credentials: 'omit',
|
||||
referrerPolicy: 'no-referrer',
|
||||
headers: { Range: `bytes=${start}-${end - 1}` },
|
||||
});
|
||||
if (
|
||||
!Number.isSafeInteger(start) ||
|
||||
!Number.isSafeInteger(end) ||
|
||||
start < 0 ||
|
||||
end > byteLength ||
|
||||
end <= start
|
||||
)
|
||||
throw new RangeError(`invalid range ${start}-${end} of ${byteLength}`);
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
credentials: 'omit',
|
||||
referrerPolicy: 'no-referrer',
|
||||
headers: { Range: `bytes=${start}-${end - 1}` },
|
||||
});
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
throw transient(error);
|
||||
}
|
||||
if (response.status !== 206 && response.status !== 200) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
const error = new Error(`HTTP ${response.status}`);
|
||||
throw response.status === 429 || response.status >= 500
|
||||
? transient(error)
|
||||
: error;
|
||||
}
|
||||
const buffer = await response.arrayBuffer();
|
||||
// A server that ignores Range answers with the whole file.
|
||||
return response.status === 200 && buffer.byteLength === byteLength
|
||||
? buffer.slice(start, end)
|
||||
: buffer;
|
||||
// A server that ignores Range answers with the whole file; anything
|
||||
// else must be exactly the bytes asked for.
|
||||
const whole = response.status === 200;
|
||||
const expected = whole ? byteLength : end - start;
|
||||
const bytes = await readResponseBytesCapped(
|
||||
response,
|
||||
whole ? Math.min(byteLength, MAX_UNRANGED_BYTES) : expected,
|
||||
signal,
|
||||
);
|
||||
if (bytes.byteLength !== expected)
|
||||
throw transient(
|
||||
new Error(
|
||||
`short read: ${bytes.byteLength} of ${expected} bytes from ${start}`,
|
||||
),
|
||||
);
|
||||
const buffer = bytes.buffer.slice(
|
||||
bytes.byteOffset,
|
||||
bytes.byteOffset + bytes.byteLength,
|
||||
);
|
||||
return whole ? buffer.slice(start, end) : buffer;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -79,12 +126,19 @@ export async function listPartitions(entry, { fetchImpl, signal }) {
|
||||
credentials: 'omit',
|
||||
});
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
const xml = await response.text();
|
||||
const xml = await readResponseTextCapped(
|
||||
response,
|
||||
MAX_LIST_PAGE_BYTES,
|
||||
signal,
|
||||
);
|
||||
for (const match of xml.matchAll(
|
||||
/<Contents>[\s\S]*?<Key>([^<]+)<\/Key>[\s\S]*?<Size>(\d+)<\/Size>[\s\S]*?<\/Contents>/g,
|
||||
)) {
|
||||
if (!match[1].endsWith('.parquet')) continue;
|
||||
objects.push({ key: match[1], size: Number(match[2]) });
|
||||
const size = Number(match[2]);
|
||||
if (!match[1].endsWith('.parquet') || !Number.isSafeInteger(size))
|
||||
continue;
|
||||
if (size <= 0) continue;
|
||||
objects.push({ key: match[1], size });
|
||||
if (objects.length >= MAX_LISTED_KEYS) return objects;
|
||||
}
|
||||
token = /<NextContinuationToken>([^<]+)<\/NextContinuationToken>/.exec(
|
||||
@@ -119,39 +173,71 @@ export function geoParquetInfo(metadata) {
|
||||
}
|
||||
}
|
||||
|
||||
const pause = (ms, signal) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, ms);
|
||||
signal?.addEventListener?.(
|
||||
'abort',
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
reject(signal.reason);
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* Where each partition is: list the bucket, then read each file's footer
|
||||
* (one small range request) for its GeoParquet bbox. Built once and cached
|
||||
* by the caller; a partition without a bbox is left out.
|
||||
* @returns {Promise<Array<{ url: string, bbox: number[], size: number }>>}
|
||||
* (one small range request) for its GeoParquet bbox. A partition whose
|
||||
* metadata is unreadable or has no bbox is left out; a footer that fails to
|
||||
* load (network, 429, 5xx, short read) is retried
|
||||
* (FOOTER_RETRIES more passes) and, if it still fails, the index is
|
||||
* `complete: false` so callers never keep it as if it were whole.
|
||||
* @returns {Promise<{ partitions: Array<{ url: string, bbox: number[], size: number }>, complete: boolean, failed: number }>}
|
||||
*/
|
||||
export async function buildPartitionIndex(entry, { fetchImpl, signal }) {
|
||||
export async function buildPartitionIndex(
|
||||
entry,
|
||||
{ fetchImpl, signal, retryDelayMs = FOOTER_RETRY_DELAY_MS },
|
||||
) {
|
||||
const { keyBase } = entry.source.partitions;
|
||||
const objects = await listPartitions(entry, { fetchImpl, signal });
|
||||
const index = [];
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (next < objects.length) {
|
||||
const { key, size } = objects[next++];
|
||||
signal?.throwIfAborted?.();
|
||||
const url = onSourceHost(entry, new URL(key, keyBase).href);
|
||||
try {
|
||||
const metadata = await parquetMetadataAsync(
|
||||
rangeBuffer(url, size, { fetchImpl, signal }),
|
||||
{ initialFetchSize: FOOTER_FETCH_BYTES },
|
||||
);
|
||||
const { bbox } = geoParquetInfo(metadata);
|
||||
if (bbox) index.push({ url, bbox, size });
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
// One unreadable partition does not sink the index.
|
||||
const partitions = [];
|
||||
let pending = objects;
|
||||
for (let pass = 0; pass <= FOOTER_RETRIES && pending.length; pass++) {
|
||||
if (pass) await pause(retryDelayMs * pass, signal);
|
||||
const failed = [];
|
||||
let next = 0;
|
||||
const queue = pending;
|
||||
async function worker() {
|
||||
while (next < queue.length) {
|
||||
const object = queue[next++];
|
||||
signal?.throwIfAborted?.();
|
||||
const url = onSourceHost(entry, new URL(object.key, keyBase).href);
|
||||
try {
|
||||
const metadata = await parquetMetadataAsync(
|
||||
rangeBuffer(url, object.size, { fetchImpl, signal }),
|
||||
{ initialFetchSize: FOOTER_FETCH_BYTES },
|
||||
);
|
||||
const { bbox } = geoParquetInfo(metadata);
|
||||
if (bbox) partitions.push({ url, bbox, size: object.size });
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
// A footer that arrived but does not parse will not parse next
|
||||
// time either: that file is left out. Only fetch failures retry.
|
||||
if (error?.transient) failed.push(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(INDEX_CONCURRENCY, queue.length) }, worker),
|
||||
);
|
||||
pending = failed;
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(INDEX_CONCURRENCY, objects.length) }, worker),
|
||||
);
|
||||
return index.sort((a, b) => a.url.localeCompare(b.url));
|
||||
return {
|
||||
partitions: partitions.sort((a, b) => a.url.localeCompare(b.url)),
|
||||
complete: pending.length === 0,
|
||||
failed: pending.length,
|
||||
};
|
||||
}
|
||||
|
||||
const overlapArea = (a, b) =>
|
||||
|
||||
@@ -90,7 +90,7 @@ test('a partitioned view reads candidates in batches until one finds features',
|
||||
},
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => index,
|
||||
buildPartitionIndex: async () => ({ partitions: index, complete: true }),
|
||||
readGeoParquetFeatures: async (entry, url) => {
|
||||
reads.push(url);
|
||||
return url.endsWith('A.parquet')
|
||||
@@ -108,8 +108,11 @@ test('a partitioned view reads candidates in batches until one finds features',
|
||||
]);
|
||||
// The index is kept in this browser; extents are learned per partition.
|
||||
assert.ok(store.get('gev:library:index:epc-england-wales'));
|
||||
const learned = JSON.parse(store.get('gev:library:extent:epc-england-wales'));
|
||||
assert.ok(learned['https://h/A.parquet']);
|
||||
const learned = JSON.parse(
|
||||
store.get('gev:library:extent:v2:epc-england-wales'),
|
||||
);
|
||||
assert.ok(learned['https://h/A.parquet'].full);
|
||||
assert.ok(learned['https://h/A.parquet'].trimmed);
|
||||
// A second view of the same area is served from the decoded cache.
|
||||
await source.loadDataset(epc, { bbox: box });
|
||||
assert.equal(reads.length, 2);
|
||||
@@ -119,3 +122,185 @@ test('a partitioned view reads candidates in batches until one finds features',
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('a revisit still finds homes at the edge of a partition', async () => {
|
||||
// 200 homes in a cluster and one real home at the partition's edge: the
|
||||
// trimmed (98 %) extent leaves the edge home out, the full one keeps it.
|
||||
const point = (lon, lat) => ({
|
||||
type: 'Feature',
|
||||
geometry: { type: 'Point', coordinates: [lon, lat] },
|
||||
properties: { CURRENT_ENERGY_RATING: 'C', UPRN: String(lon) },
|
||||
});
|
||||
const homes = Array.from({ length: 200 }, (_, index) =>
|
||||
point(-1.4 - index / 1e4, 53.4),
|
||||
);
|
||||
const edge = point(-1.2, 53.3);
|
||||
homes.push(edge);
|
||||
const index = [
|
||||
{ url: 'https://h/Edge.parquet', bbox: [-1.5, 53.2, -1.1, 53.5] },
|
||||
];
|
||||
const store = new Map([
|
||||
// A v1 extent (trimmed only) from an older build: never read again.
|
||||
[
|
||||
'gev:library:extent:epc-england-wales',
|
||||
JSON.stringify({ 'https://h/Edge.parquet': [-1.42, 53.4, -1.4, 53.4] }),
|
||||
],
|
||||
]);
|
||||
const storage = {
|
||||
getItem: (key) => store.get(key) ?? null,
|
||||
setItem: (key, value) => store.set(key, value),
|
||||
removeItem: (key) => store.delete(key),
|
||||
};
|
||||
const visit = () =>
|
||||
createLibrarySource({
|
||||
entries: [epc],
|
||||
fetchImpl: async () => ({ ok: false, status: 404 }),
|
||||
storage,
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => ({
|
||||
partitions: index,
|
||||
complete: true,
|
||||
}),
|
||||
readGeoParquetFeatures: async () => homes,
|
||||
}),
|
||||
});
|
||||
const edgeBox = [-1.21, 53.29, -1.19, 53.31];
|
||||
// First visit: the old v1 extent would have skipped the partition.
|
||||
const first = await visit().loadDataset(epc, { bbox: edgeBox });
|
||||
assert.equal(first.features.length, 1);
|
||||
assert.equal(store.has('gev:library:extent:epc-england-wales'), false);
|
||||
// A later visit (new page, nothing decoded) reads the learned v2 extents.
|
||||
const later = await visit().loadDataset(epc, { bbox: edgeBox });
|
||||
assert.equal(later.features.length, 1);
|
||||
assert.deepEqual(later.features[0].geometry.coordinates, [-1.2, 53.3]);
|
||||
// A box outside every point of the partition is still ruled out.
|
||||
const outside = await visit().loadDataset(epc, {
|
||||
bbox: [-1.12, 53.45, -1.11, 53.46],
|
||||
});
|
||||
assert.equal(outside.features.length, 0);
|
||||
});
|
||||
|
||||
test('a footer that keeps failing leaves the index incomplete, and a retry recovers', async () => {
|
||||
const { buildPartitionIndex } = await import('./parquet.js');
|
||||
const entry = {
|
||||
...epc,
|
||||
source: {
|
||||
...epc.source,
|
||||
partitions: {
|
||||
listUrl: 'https://s3.us-west-2.amazonaws.com/bucket?list-type=2',
|
||||
keyBase: 'https://s3.us-west-2.amazonaws.com/bucket/',
|
||||
},
|
||||
},
|
||||
};
|
||||
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents><Contents><Key>b.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
|
||||
let footerFailures = 0;
|
||||
const fetchImpl = async (url) => {
|
||||
if (url.includes('list-type'))
|
||||
return new Response(listing, { status: 200 });
|
||||
if (url.endsWith('b.parquet') && footerFailures > 0) {
|
||||
footerFailures -= 1;
|
||||
return new Response('', { status: 503 });
|
||||
}
|
||||
// Not a Parquet footer: readable, but with no bbox, so left out.
|
||||
return new Response(new Uint8Array(100), { status: 200 });
|
||||
};
|
||||
footerFailures = 2; // fails once, then succeeds on the first retry
|
||||
const recovered = await buildPartitionIndex(entry, {
|
||||
fetchImpl,
|
||||
retryDelayMs: 0,
|
||||
});
|
||||
assert.equal(recovered.complete, true);
|
||||
footerFailures = 100; // fails on every pass
|
||||
const broken = await buildPartitionIndex(entry, {
|
||||
fetchImpl,
|
||||
retryDelayMs: 0,
|
||||
});
|
||||
assert.equal(broken.complete, false);
|
||||
assert.equal(broken.failed, 1);
|
||||
});
|
||||
|
||||
test('the browser keeps a complete index over an incomplete rebuild and retries later', async () => {
|
||||
let clock = 30 * 24 * 3600_000;
|
||||
const complete = [{ url: 'https://h/A.parquet', bbox: [-2, 53, -1, 54] }];
|
||||
const store = new Map([
|
||||
[
|
||||
'gev:library:index:epc-england-wales',
|
||||
// Past cacheSeconds: due for a rebuild.
|
||||
JSON.stringify({ savedAt: 0, complete: true, partitions: complete }),
|
||||
],
|
||||
]);
|
||||
let builds = 0;
|
||||
let nextComplete = false;
|
||||
const source = createLibrarySource({
|
||||
entries: [epc],
|
||||
now: () => clock,
|
||||
fetchImpl: async () => ({ ok: false, status: 404 }),
|
||||
storage: {
|
||||
getItem: (key) => store.get(key) ?? null,
|
||||
setItem: (key, value) => store.set(key, value),
|
||||
},
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => {
|
||||
builds += 1;
|
||||
return nextComplete
|
||||
? { partitions: complete, complete: true, failed: 0 }
|
||||
: { partitions: [], complete: false, failed: 3 };
|
||||
},
|
||||
readGeoParquetFeatures: async () => [],
|
||||
}),
|
||||
});
|
||||
const box = [-1.5, 53.4, -1.4, 53.5];
|
||||
await source.loadDataset(epc, { bbox: box });
|
||||
assert.equal(builds, 1);
|
||||
// The old complete index stays stored; the incomplete one is not saved.
|
||||
assert.equal(
|
||||
JSON.parse(store.get('gev:library:index:epc-england-wales')).savedAt,
|
||||
0,
|
||||
);
|
||||
// Within the retry pause the incomplete result is reused, not rebuilt.
|
||||
await source.loadDataset(epc, { bbox: box });
|
||||
assert.equal(builds, 1);
|
||||
// After it, the index is rebuilt; a complete build is kept.
|
||||
clock += 6 * 60_000;
|
||||
nextComplete = true;
|
||||
await source.loadDataset(epc, { bbox: box });
|
||||
assert.equal(builds, 2);
|
||||
assert.equal(
|
||||
JSON.parse(store.get('gev:library:index:epc-england-wales')).savedAt,
|
||||
clock,
|
||||
);
|
||||
});
|
||||
|
||||
test('a view that leaves touching partitions unread is partial', async () => {
|
||||
const index = Array.from({ length: 6 }, (_, n) => ({
|
||||
url: `https://h/P${n}.parquet`,
|
||||
bbox: [-1.5 - n / 100, 53.3, -1.4, 53.45],
|
||||
}));
|
||||
const point = (lon, lat) => ({
|
||||
type: 'Feature',
|
||||
geometry: { type: 'Point', coordinates: [lon, lat] },
|
||||
properties: { CURRENT_ENERGY_RATING: 'C', UPRN: '1' },
|
||||
});
|
||||
const reads = [];
|
||||
const source = createLibrarySource({
|
||||
entries: [epc],
|
||||
fetchImpl: async () => ({ ok: false, status: 404 }),
|
||||
storage: null,
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => ({ partitions: index, complete: true }),
|
||||
readGeoParquetFeatures: async (entry, url) => {
|
||||
reads.push(url);
|
||||
return [point(-1.45, 53.4)];
|
||||
},
|
||||
}),
|
||||
});
|
||||
const result = await source.loadDataset(epc, {
|
||||
bbox: [-1.46, 53.39, -1.44, 53.41],
|
||||
});
|
||||
// The first batch of four found homes; two touching partitions stay unread.
|
||||
assert.equal(reads.length, 4);
|
||||
assert.equal(result.truncated, true);
|
||||
});
|
||||
+149
-44
@@ -17,7 +17,13 @@ const PARTITION_BATCH = 4;
|
||||
/** Decoded partitions kept in memory, most recent first. */
|
||||
const PARTITION_CACHE_LIMIT = 4;
|
||||
const INDEX_STORAGE_PREFIX = 'gev:library:index:';
|
||||
const EXTENT_STORAGE_PREFIX = 'gev:library:extent:';
|
||||
/** How long an incomplete partition index is used before a rebuild. */
|
||||
const PARTIAL_INDEX_RETRY_MS = 5 * 60_000;
|
||||
// v2 stores `{ full, trimmed }` per partition. The v1 key held only the
|
||||
// trimmed box, which wrongly excluded real points at a partition's edge; it
|
||||
// is no longer read and is removed on first use.
|
||||
const EXTENT_STORAGE_PREFIX = 'gev:library:extent:v2:';
|
||||
const RETIRED_EXTENT_STORAGE_PREFIX = 'gev:library:extent:';
|
||||
|
||||
const overlaps = (a, b) =>
|
||||
a[0] < b[2] && a[2] > b[0] && a[1] < b[3] && a[3] > b[1];
|
||||
@@ -47,6 +53,58 @@ export function robustExtent(features) {
|
||||
return [at(lons, 0.01), at(lats, 0.01), at(lons, 0.99), at(lats, 0.99)];
|
||||
}
|
||||
|
||||
/** The box holding every point of a partition, or null. */
|
||||
export function fullExtent(features) {
|
||||
let extent = null;
|
||||
for (const feature of features) {
|
||||
let position = feature?.geometry?.coordinates;
|
||||
while (Array.isArray(position?.[0])) position = position[0];
|
||||
if (!Array.isArray(position)) continue;
|
||||
const [lon, lat] = position;
|
||||
if (!Number.isFinite(lon) || !Number.isFinite(lat)) continue;
|
||||
if (!extent) extent = [lon, lat, lon, lat];
|
||||
else {
|
||||
extent[0] = Math.min(extent[0], lon);
|
||||
extent[1] = Math.min(extent[1], lat);
|
||||
extent[2] = Math.max(extent[2], lon);
|
||||
extent[3] = Math.max(extent[3], lat);
|
||||
}
|
||||
}
|
||||
return extent;
|
||||
}
|
||||
|
||||
const isBox = (value) =>
|
||||
Array.isArray(value) && value.length === 4 && value.every(Number.isFinite);
|
||||
|
||||
/**
|
||||
* Candidate partitions for a box. A learned full extent that misses the box
|
||||
* rules a partition out (no point of it can be in view); the learned 98 %
|
||||
* extent only ranks, so stray geocodes neither stretch the ranking nor hide
|
||||
* the real points at a partition's edge.
|
||||
*/
|
||||
export function rankPartitions(index, box, learned, partitionsForBox) {
|
||||
const known = (url) => {
|
||||
const value = learned?.[url];
|
||||
return value && isBox(value.full) && isBox(value.trimmed) ? value : null;
|
||||
};
|
||||
const ranked = partitionsForBox(
|
||||
index
|
||||
.filter(({ url }) => !known(url) || overlaps(known(url).full, box))
|
||||
.map((partition) => {
|
||||
const extent = known(partition.url);
|
||||
// Rank by where most points are, unless that misses the view while
|
||||
// the full extent does not: then the partition's own box ranks it.
|
||||
return extent && overlaps(extent.trimmed, box)
|
||||
? { ...partition, bbox: extent.trimmed }
|
||||
: partition;
|
||||
}),
|
||||
box,
|
||||
Infinity,
|
||||
);
|
||||
const byUrl = new Map(index.map((partition) => [partition.url, partition]));
|
||||
return ranked.map(({ url }) => byUrl.get(url));
|
||||
}
|
||||
|
||||
function storedJson(storage, key) {
|
||||
try {
|
||||
const text = storage?.getItem?.(key);
|
||||
@@ -118,43 +176,76 @@ export function createLibrarySource({
|
||||
* shared), else built here and kept in this browser for `cacheSeconds`.
|
||||
*/
|
||||
function partitionIndex(entry, { signal } = {}) {
|
||||
if (!indexes.has(entry.id))
|
||||
indexes.set(
|
||||
entry.id,
|
||||
(async () => {
|
||||
try {
|
||||
const response = await fetchImpl(
|
||||
`${base}/index/${encodeURIComponent(entry.id)}`,
|
||||
{ signal, headers: { Accept: 'application/json' } },
|
||||
);
|
||||
if (response.ok) {
|
||||
const body = await response.json();
|
||||
if (Array.isArray(body?.partitions)) return body.partitions;
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
const memo = indexes.get(entry.id);
|
||||
if (memo && (memo.retryAt === null || now() < memo.retryAt))
|
||||
return memo.promise;
|
||||
const promise = (async () => {
|
||||
const key = `${INDEX_STORAGE_PREFIX}${entry.id}`;
|
||||
const stored = storedJson(storage, key);
|
||||
const storedComplete =
|
||||
stored && Array.isArray(stored.partitions) && stored.complete !== false
|
||||
? stored
|
||||
: null;
|
||||
let partial = null;
|
||||
try {
|
||||
const response = await fetchImpl(
|
||||
`${base}/index/${encodeURIComponent(entry.id)}`,
|
||||
{ signal, headers: { Accept: 'application/json' } },
|
||||
);
|
||||
if (response.ok) {
|
||||
const body = await response.json();
|
||||
if (Array.isArray(body?.partitions)) {
|
||||
if (body.complete !== false)
|
||||
return { partitions: body.partitions, complete: true };
|
||||
partial = body.partitions;
|
||||
}
|
||||
const key = `${INDEX_STORAGE_PREFIX}${entry.id}`;
|
||||
const stored = storedJson(storage, key);
|
||||
if (
|
||||
stored &&
|
||||
Array.isArray(stored.partitions) &&
|
||||
now() - stored.savedAt < entry.source.cacheSeconds * 1000
|
||||
)
|
||||
return stored.partitions;
|
||||
const { buildPartitionIndex } = await loadParquet();
|
||||
const partitions = await buildPartitionIndex(entry, {
|
||||
fetchImpl,
|
||||
signal,
|
||||
});
|
||||
storeJson(storage, key, { savedAt: now(), partitions });
|
||||
return partitions;
|
||||
})().catch((error) => {
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
}
|
||||
if (
|
||||
storedComplete &&
|
||||
now() - storedComplete.savedAt < entry.source.cacheSeconds * 1000
|
||||
)
|
||||
return { partitions: storedComplete.partitions, complete: true };
|
||||
// The server's index is incomplete, or there is no server: build here.
|
||||
const { buildPartitionIndex } = await loadParquet();
|
||||
const built = await buildPartitionIndex(entry, { fetchImpl, signal });
|
||||
if (built.complete) {
|
||||
storeJson(storage, key, {
|
||||
savedAt: now(),
|
||||
complete: true,
|
||||
partitions: built.partitions,
|
||||
});
|
||||
return { partitions: built.partitions, complete: true };
|
||||
}
|
||||
// Still incomplete: an older complete index beats a partial one, even
|
||||
// past its age; otherwise use the larger partial set for now.
|
||||
if (storedComplete)
|
||||
return { partitions: storedComplete.partitions, complete: false };
|
||||
const partitions =
|
||||
partial && partial.length > built.partitions.length
|
||||
? partial
|
||||
: built.partitions;
|
||||
return { partitions, complete: false };
|
||||
})().then(
|
||||
(result) => {
|
||||
// An incomplete index is used for now and rebuilt after a pause.
|
||||
const current = indexes.get(entry.id);
|
||||
if (current?.promise === promise)
|
||||
current.retryAt = result.complete
|
||||
? null
|
||||
: now() + PARTIAL_INDEX_RETRY_MS;
|
||||
return result.partitions;
|
||||
},
|
||||
(error) => {
|
||||
if (indexes.get(entry.id)?.promise === promise)
|
||||
indexes.delete(entry.id);
|
||||
throw error;
|
||||
}),
|
||||
);
|
||||
return indexes.get(entry.id);
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
indexes.set(entry.id, { promise, retryAt: null });
|
||||
return promise;
|
||||
}
|
||||
|
||||
/** One partition's features, decoded once and kept for a few views. */
|
||||
@@ -174,11 +265,12 @@ export function createLibrarySource({
|
||||
while (partitionCache.size > PARTITION_CACHE_LIMIT)
|
||||
partitionCache.delete(partitionCache.keys().next().value);
|
||||
// Learn where this partition really is, for every later view.
|
||||
const extent = robustExtent(features);
|
||||
if (extent) {
|
||||
const full = fullExtent(features);
|
||||
const trimmed = robustExtent(features);
|
||||
if (full && trimmed) {
|
||||
const key = `${EXTENT_STORAGE_PREFIX}${entry.id}`;
|
||||
const learned = storedJson(storage, key) || {};
|
||||
learned[partition.url] = extent;
|
||||
learned[partition.url] = { full, trimmed };
|
||||
storeJson(storage, key, learned);
|
||||
}
|
||||
return features;
|
||||
@@ -195,11 +287,18 @@ export function createLibrarySource({
|
||||
loadParquet(),
|
||||
partitionIndex(entry, { signal }),
|
||||
]);
|
||||
try {
|
||||
storage?.removeItem?.(`${RETIRED_EXTENT_STORAGE_PREFIX}${entry.id}`);
|
||||
} catch {
|
||||
/* a blocked store keeps a key nobody reads */
|
||||
}
|
||||
const learned =
|
||||
storedJson(storage, `${EXTENT_STORAGE_PREFIX}${entry.id}`) || {};
|
||||
const candidates = partitionsForBox(index, box, Infinity)
|
||||
.filter(({ url }) => !learned[url] || overlaps(learned[url], box))
|
||||
.slice(0, MAX_PARTITION_READS_PER_VIEW);
|
||||
const ranked = rankPartitions(index, box, learned, partitionsForBox);
|
||||
const candidates = ranked.slice(0, MAX_PARTITION_READS_PER_VIEW);
|
||||
// Every partition the box touches but this view leaves unread (the read
|
||||
// budget, or the early stop below) makes the result partial.
|
||||
let unread = ranked.length - candidates.length;
|
||||
const inView = [];
|
||||
for (let start = 0; start < candidates.length; start += PARTITION_BATCH) {
|
||||
const batch = candidates.slice(start, start + PARTITION_BATCH);
|
||||
@@ -217,13 +316,19 @@ export function createLibrarySource({
|
||||
inView.push(...hits);
|
||||
}
|
||||
}
|
||||
if (found) break;
|
||||
if (found) {
|
||||
unread += candidates.length - (start + batch.length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
const bounded = boundLibraryFeatures(inView, {
|
||||
maxFeatures: entry.source.maxFeatures,
|
||||
display: entry.display,
|
||||
});
|
||||
return { features: bounded.features, truncated: bounded.truncated };
|
||||
return {
|
||||
features: bounded.features,
|
||||
truncated: bounded.truncated || unread > 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import * as Cesium from 'cesium';
|
||||
import { createLibraryTileLayer } from './layers.js';
|
||||
import { createLibraryTileLayer, minimumTerrainLevelFor } from './layers.js';
|
||||
import { LIBRARY_ENTRIES, libraryEntry } from './catalog.js';
|
||||
import { libraryTileUrl } from './manifest.js';
|
||||
import { createLibrarySource } from './source.js';
|
||||
@@ -133,3 +133,35 @@ test('bbox and TMS templates become the URLs Cesium and the proxy request', () =
|
||||
'https://t.example.org/3/2/6.png',
|
||||
);
|
||||
});
|
||||
|
||||
test('an entry too dense to draw zoomed out is hidden below its scale', async () => {
|
||||
const cadastre = entryWithoutOutline('kartverket-cadastre');
|
||||
assert.equal(cadastre.source.minZoom, 14);
|
||||
assert.equal(minimumTerrainLevelFor(cadastre), 13);
|
||||
// Small coverage, or a low minimum zoom, draws at every scale.
|
||||
assert.equal(
|
||||
minimumTerrainLevelFor(libraryEntry('lee-county-fl')),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
minimumTerrainLevelFor(libraryEntry('ign-orthophotos-france')),
|
||||
undefined,
|
||||
);
|
||||
|
||||
// Switched on zoomed out, then zoomed further out: no globe tile coarser
|
||||
// than level 13 takes imagery from it, so no z14 tile is ever requested.
|
||||
const viewer = globeViewer();
|
||||
const layer = createLibraryTileLayer(cadastre, { source, cesium: Cesium });
|
||||
await layer.init(viewer);
|
||||
await layer.enable(viewer);
|
||||
const imagery = viewer.imageryLayers.get(viewer.imageryLayers.length - 1);
|
||||
const terrain = new Cesium.EllipsoidTerrainProvider();
|
||||
for (const level of [0, 4, 12])
|
||||
assert.equal(
|
||||
imagery._createTileImagerySkeletons({ level, data: {} }, terrain),
|
||||
false,
|
||||
`globe level ${level} takes no cadastre imagery`,
|
||||
);
|
||||
await layer.disable();
|
||||
assert.equal(viewer.imageryLayers.length, 0);
|
||||
});
|
||||
@@ -48,6 +48,7 @@ export const LEGACY_SHARE_STATE_KEYS = Object.freeze([
|
||||
'layerState',
|
||||
'layerStateInvalid',
|
||||
'panelState',
|
||||
'library',
|
||||
'sharedAtMs',
|
||||
]);
|
||||
|
||||
|
||||
@@ -390,6 +390,9 @@ export const SHARE_VIEW_FIELDS = freezeAll([
|
||||
{ number: 26, key: 'scopeTerminusPct', type: 'uint', default: null },
|
||||
{ number: 27, key: 'styleParams', type: 'bytes', default: BY_STYLE },
|
||||
{ number: 28, key: 'worldTime', type: 'epoch', default: null },
|
||||
// Data Library entries switched on, as their two-character ledger tokens
|
||||
// concatenated (src/library/ledger.json). Absent means none.
|
||||
{ number: 29, key: 'library', type: 'bytes', default: null },
|
||||
]);
|
||||
const FIELD_BY_KEY = new Map(SHARE_VIEW_FIELDS.map((f) => [f.key, f]));
|
||||
const fieldNumber = (key) => FIELD_BY_KEY.get(key).number;
|
||||
@@ -450,6 +453,33 @@ export function createDefaultShareViewState(requested = 'normal') {
|
||||
/* Normalization: the one canonical state both directions agree on. */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/** The most Data Library tokens a share carries (the library's own cap). */
|
||||
export const SHARE_LIBRARY_MAX_TOKENS = 64;
|
||||
const LIBRARY_TOKEN = /^[0-9a-z]{2}$/;
|
||||
|
||||
/**
|
||||
* Data Library share tokens from a `lib=` value (`ab.cd`) or a token list:
|
||||
* unique, two base-36 characters each, at most SHARE_LIBRARY_MAX_TOKENS.
|
||||
* Malformed tokens are skipped. Null or undefined stays null (no field).
|
||||
* @param {string|string[]|null|undefined} value
|
||||
* @returns {string[]|null}
|
||||
*/
|
||||
export function normalizeLibraryTokens(value) {
|
||||
if (value === null || value === undefined) return null;
|
||||
const list = Array.isArray(value)
|
||||
? value
|
||||
: typeof value === 'string' && value.length <= 3 * SHARE_LIBRARY_MAX_TOKENS
|
||||
? value.split('.')
|
||||
: [];
|
||||
const tokens = [];
|
||||
for (const token of list) {
|
||||
if (typeof token !== 'string' || !LIBRARY_TOKEN.test(token)) continue;
|
||||
if (!tokens.includes(token)) tokens.push(token);
|
||||
if (tokens.length === SHARE_LIBRARY_MAX_TOKENS) break;
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
const finite = (value) =>
|
||||
typeof value === 'number'
|
||||
? Number.isFinite(value)
|
||||
@@ -761,6 +791,7 @@ export function normalizeShareViewState(input = {}) {
|
||||
? normalizeLayerState(source.layerState)
|
||||
: null;
|
||||
state.panelState = normalizePanelState(source.panelState);
|
||||
state.library = normalizeLibraryTokens(source.library);
|
||||
state.sharedAtMs = epochMsOr(source.sharedAtMs);
|
||||
state.worldTime = isWorldTime(source.worldTime)
|
||||
? Math.floor(source.worldTime / 1000) * 1000
|
||||
@@ -1030,6 +1061,10 @@ export function encodeShareView(input) {
|
||||
case 'panelState':
|
||||
if (state.panelState) encodePanels(writer, state.panelState);
|
||||
break;
|
||||
case 'library':
|
||||
if (state.library?.length)
|
||||
writer.textField(field.number, state.library.join(''));
|
||||
break;
|
||||
case 'thing':
|
||||
if (thing) encodeThing(writer, thing, state);
|
||||
break;
|
||||
@@ -1438,6 +1473,13 @@ export function decodeShareView(code, { nowMs = Date.now() } = {}) {
|
||||
unknown.code('layer-option', 'without-layers');
|
||||
if (get('panelState'))
|
||||
raw.panelState = decodePanels(get('panelState').len, unknown);
|
||||
if (get('library')) {
|
||||
const text = asciiText(get('library').len.rest());
|
||||
const tokens = text.match(/.{1,2}/g) || [];
|
||||
for (const token of tokens)
|
||||
if (!LIBRARY_TOKEN.test(token)) unknown.code('library', token);
|
||||
raw.library = normalizeLibraryTokens(tokens);
|
||||
}
|
||||
if (get('styleParams') && !retiredStyle)
|
||||
raw.styleParams = decodeStyleParams(
|
||||
get('styleParams').len,
|
||||
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
followOffsetToEnu,
|
||||
layerTokenFromNumber,
|
||||
layerTokenNumber,
|
||||
normalizeLibraryTokens,
|
||||
normalizeShareViewState,
|
||||
} from './viewCodec.js';
|
||||
|
||||
@@ -507,6 +508,7 @@ test('the field registry is well formed and permanent', () => {
|
||||
[26, 'scopeTerminusPct', null],
|
||||
[27, 'styleParams', BY_STYLE],
|
||||
[28, 'worldTime', null],
|
||||
[29, 'library', null],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
@@ -1571,3 +1573,16 @@ test('the follow offset for a camera is the inverse of followCameraFor', () => {
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('Data Library tokens round-trip and malformed ones are skipped', () => {
|
||||
const state = { lat: 1, lon: 2, library: ['00', '1z', '0a'] };
|
||||
const decoded = decodeShareView(encodeShareView(state));
|
||||
assert.equal(decoded.ok, true);
|
||||
assert.deepEqual(decoded.state.library, ['00', '1z', '0a']);
|
||||
// None on: no field, and the decoder reads no tokens.
|
||||
const none = decodeShareView(encodeShareView({ lat: 1, lon: 2 }));
|
||||
assert.equal(none.state.library, null);
|
||||
assert.deepEqual(normalizeLibraryTokens('01.!!.01.zz'), ['01', 'zz']);
|
||||
assert.equal(normalizeLibraryTokens(null), null);
|
||||
assert.deepEqual(normalizeLibraryTokens('x'.repeat(500)), []);
|
||||
});
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
SHARE_VIEW_STYLE_PARAMS,
|
||||
decodeShareView,
|
||||
encodeShareView,
|
||||
normalizeLibraryTokens,
|
||||
} from './share/viewCodec.js';
|
||||
import {
|
||||
fromLegacyShareState,
|
||||
@@ -109,6 +110,10 @@ export class ShareLinkManager {
|
||||
this._mapStack = 'photoreal';
|
||||
this._layerStateProvider = null;
|
||||
this._libraryParamsEncoder = null;
|
||||
this._libraryShareListener = null;
|
||||
// The opening share's Data Library tokens: null when the page opened
|
||||
// without a share, [] for a share that switches no library entry on.
|
||||
this._initialLibraryTokens = null;
|
||||
this._panelStateProvider = null;
|
||||
this._styleParamStateProvider = null;
|
||||
this._initialRestorePending = false;
|
||||
@@ -161,10 +166,32 @@ export class ShareLinkManager {
|
||||
const state =
|
||||
code === null ? this._parseShareHash(hash) : this._parseShareCode(code);
|
||||
if (!state) return null;
|
||||
// A share decides which library entries are on, as it does for core
|
||||
// layers: one without `lib` switches none on.
|
||||
this._initialLibraryTokens = state.library ?? [];
|
||||
this._beginIncomingRestore(state);
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Data Library tokens of the share this page opened with, or null when
|
||||
* it opened without one. Read by the library once it knows its entries.
|
||||
* @returns {string[]|null}
|
||||
*/
|
||||
get initialLibraryTokens() {
|
||||
return this._initialLibraryTokens ? [...this._initialLibraryTokens] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Install the Data Library's listener for shares opened in this tab after
|
||||
* start-up. It receives the share's tokens ([] when it switches none on).
|
||||
* @param {((tokens: string[]) => void) | null} listener
|
||||
*/
|
||||
setLibraryShareListener(listener) {
|
||||
this._libraryShareListener =
|
||||
typeof listener === 'function' ? listener : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The restore state for a share code, in the long form's parsed shape plus
|
||||
* `shareView` (what the view follows, with its offset and last-known fix)
|
||||
@@ -273,6 +300,7 @@ export class ShareLinkManager {
|
||||
params.has('l') &&
|
||||
decodedLayerState === null,
|
||||
panelState: decodePanelStateParams(params),
|
||||
library: normalizeLibraryTokens(params.get('lib')),
|
||||
sharedAtMs: decodeShareCreatedAtMs(params),
|
||||
};
|
||||
return state;
|
||||
@@ -324,6 +352,11 @@ export class ShareLinkManager {
|
||||
if (!state) return;
|
||||
this._beginIncomingRestore(state);
|
||||
onNavigate(state);
|
||||
try {
|
||||
this._libraryShareListener?.(state.library ?? []);
|
||||
} catch (error) {
|
||||
console.warn('[ShareLink] library restore failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -751,6 +784,7 @@ export class ShareLinkManager {
|
||||
this._unwatchNavigation?.();
|
||||
this._layerStateProvider = null;
|
||||
this._libraryParamsEncoder = null;
|
||||
this._libraryShareListener = null;
|
||||
this._panelStateProvider = null;
|
||||
this._styleParamStateProvider = null;
|
||||
this._followSubjectProvider = null;
|
||||
|
||||
@@ -448,3 +448,47 @@ test('a link made away from live carries its world time to the page that opens i
|
||||
assert.equal(liveOpener.parseInitialHash().worldTime, null);
|
||||
liveOpener.destroy();
|
||||
});
|
||||
|
||||
test('a copied link carries the Data Library selection to a clean recipient', (t) => {
|
||||
installTab(t, SHARE_A);
|
||||
const sharer = new ShareLinkManager(fakeViewer());
|
||||
sharer.setLibraryParamsEncoder((params) => params.set('lib', '03.0a'));
|
||||
const code = new URL(sharer.buildLink()).pathname.split('/').at(-1);
|
||||
sharer.destroy();
|
||||
const opener = new ShareLinkManager(fakeViewer(), { initialShareCode: code });
|
||||
assert.deepEqual(opener.parseInitialHash().library, ['03', '0a']);
|
||||
assert.deepEqual(opener.initialLibraryTokens, ['03', '0a']);
|
||||
opener.destroy();
|
||||
|
||||
// A link with every library entry off is a share that switches none on.
|
||||
const none = new ShareLinkManager(fakeViewer());
|
||||
const noneCode = new URL(none.buildLink()).pathname.split('/').at(-1);
|
||||
none.destroy();
|
||||
const noneOpener = new ShareLinkManager(fakeViewer(), {
|
||||
initialShareCode: noneCode,
|
||||
});
|
||||
noneOpener.parseInitialHash();
|
||||
assert.deepEqual(noneOpener.initialLibraryTokens, []);
|
||||
noneOpener.destroy();
|
||||
|
||||
// A page opened without a share leaves the library its stored selection.
|
||||
installTab(t, '');
|
||||
const plain = new ShareLinkManager(fakeViewer());
|
||||
plain.parseInitialHash();
|
||||
assert.equal(plain.initialLibraryTokens, null);
|
||||
plain.destroy();
|
||||
});
|
||||
|
||||
test('a share navigated to in this tab hands its library tokens over', (t) => {
|
||||
const tab = installTab(t, SHARE_A);
|
||||
const manager = new ShareLinkManager(fakeViewer());
|
||||
manager.parseInitialHash();
|
||||
const received = [];
|
||||
manager.setLibraryShareListener((tokens) => received.push(tokens));
|
||||
manager.watchNavigation(() => {});
|
||||
tab.navigate(`${SHARE_B}&lib=01`);
|
||||
tab.navigate(SHARE_A);
|
||||
tab.back();
|
||||
assert.deepEqual(received, [['01'], [], ['01']]);
|
||||
manager.destroy();
|
||||
});
|
||||
+45
-11
@@ -7,7 +7,7 @@
|
||||
* width, height, labels, initial and return focus, Escape and dismissal, and
|
||||
* closes itself when clean view, recording or Cockpit hides the panels.
|
||||
*/
|
||||
import { formatNumber, msg, t, tn } from '../i18n/index.js';
|
||||
import { formatNumber, msg, onLocaleChange, t, tn } from '../i18n/index.js';
|
||||
|
||||
const PANEL_HIDING_MODES = ['ui-clean-view', 'recording-mode', 'cockpit-mode'];
|
||||
/** How long Remove all stays armed after its first press. */
|
||||
@@ -219,6 +219,12 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
node.addEventListener(type, listener);
|
||||
removers.push(() => node.removeEventListener(type, listener));
|
||||
};
|
||||
// Listeners on the dialog's own nodes, released when it is rebuilt.
|
||||
const dialogRemovers = [];
|
||||
const listenDialog = (node, type, listener) => {
|
||||
node.addEventListener(type, listener);
|
||||
dialogRemovers.push(() => node.removeEventListener(type, listener));
|
||||
};
|
||||
const filter = { query: '', family: '', availableOnly: false };
|
||||
let dialog = null;
|
||||
let list = null;
|
||||
@@ -259,7 +265,7 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
);
|
||||
close.type = 'button';
|
||||
close.setAttribute('aria-label', t('Close data library'));
|
||||
listen(close, 'click', () => api.close());
|
||||
listenDialog(close, 'click', () => api.close());
|
||||
header.append(title, close);
|
||||
|
||||
const body = element('div', 'data-library-body');
|
||||
@@ -282,7 +288,8 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
search.type = 'search';
|
||||
search.autocomplete = 'off';
|
||||
search.placeholder = t('Name, provider or country');
|
||||
listen(search, 'input', () => {
|
||||
search.value = filter.query;
|
||||
listenDialog(search, 'input', () => {
|
||||
filter.query = search.value;
|
||||
render();
|
||||
});
|
||||
@@ -291,10 +298,16 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
const familyLabel = element('label', 'data-library-field');
|
||||
familyLabel.append(element('span', '', t('Theme')));
|
||||
const family = element('select', 'panel-surface-control');
|
||||
family.append(new Option(t('All themes'), ''));
|
||||
const option = (text, value) => {
|
||||
const node = element('option', '', text);
|
||||
node.value = value;
|
||||
return node;
|
||||
};
|
||||
family.append(option(t('All themes'), ''));
|
||||
for (const [value, label] of Object.entries(FAMILY_LABELS))
|
||||
family.append(new Option(t(label), value));
|
||||
listen(family, 'change', () => {
|
||||
family.append(option(t(label), value));
|
||||
family.value = filter.family;
|
||||
listenDialog(family, 'change', () => {
|
||||
filter.family = family.value;
|
||||
render();
|
||||
});
|
||||
@@ -303,7 +316,8 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
const availableLabel = element('label', 'data-library-check');
|
||||
const available = element('input');
|
||||
available.type = 'checkbox';
|
||||
listen(available, 'change', () => {
|
||||
available.checked = filter.availableOnly;
|
||||
listenDialog(available, 'change', () => {
|
||||
filter.availableOnly = available.checked;
|
||||
render();
|
||||
});
|
||||
@@ -319,7 +333,7 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
status.setAttribute('aria-live', 'polite');
|
||||
resetButton = element('button', 'panel-surface-control data-library-reset');
|
||||
resetButton.type = 'button';
|
||||
listen(resetButton, 'click', () => {
|
||||
listenDialog(resetButton, 'click', () => {
|
||||
if (resetting) return;
|
||||
if (!resetArmed) {
|
||||
resetArmed = true;
|
||||
@@ -344,7 +358,7 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
toolbar.append(status, resetButton);
|
||||
|
||||
list = element('ul', 'data-library-list');
|
||||
listen(list, 'click', (event) => {
|
||||
listenDialog(list, 'click', (event) => {
|
||||
const button = event.target?.closest?.('button[data-library-action]');
|
||||
if (!button || button.disabled) return;
|
||||
const id = button.dataset.libraryId;
|
||||
@@ -366,13 +380,13 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
|
||||
body.append(stats, intro, filters, toolbar, list);
|
||||
dialog.append(header, body);
|
||||
listen(dialog, 'cancel', (event) => {
|
||||
listenDialog(dialog, 'cancel', (event) => {
|
||||
event.preventDefault();
|
||||
api.close();
|
||||
});
|
||||
// A click on the backdrop (the dialog box itself, outside its content)
|
||||
// dismisses, like Escape.
|
||||
listen(dialog, 'click', (event) => {
|
||||
listenDialog(dialog, 'click', (event) => {
|
||||
if (event.target === dialog) api.close();
|
||||
});
|
||||
document.body.append(dialog);
|
||||
@@ -576,13 +590,33 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
destroy() {
|
||||
api.close();
|
||||
unsubscribe();
|
||||
offLocale();
|
||||
observer?.disconnect();
|
||||
for (const remove of removers.splice(0)) remove();
|
||||
for (const remove of dialogRemovers.splice(0)) remove();
|
||||
dialog?.remove();
|
||||
dialog = null;
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Every label is translated when the dialog is built, so a language switch
|
||||
* rebuilds it, keeping its filters and, if it was open, opening it again.
|
||||
*/
|
||||
function relabel() {
|
||||
if (!dialog) return;
|
||||
const wasOpen = dialog.open;
|
||||
disarmReset();
|
||||
clearInterval(statsTimer);
|
||||
statsTimer = null;
|
||||
if (wasOpen) dialog.close();
|
||||
for (const remove of dialogRemovers.splice(0)) remove();
|
||||
dialog.remove();
|
||||
dialog = null;
|
||||
if (wasOpen) api.open();
|
||||
}
|
||||
const offLocale = onLocaleChange(relabel);
|
||||
|
||||
const unsubscribe = library.subscribe(render);
|
||||
if (trigger) listen(trigger, 'click', () => api.open());
|
||||
const observer =
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { installVueDom } from '../testSupport/vueDom.mjs';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
|
||||
test('a language switch relabels the open dialog and keeps its filters', async (context) => {
|
||||
context.after(installVueDom());
|
||||
context.after(() => setLocale('en'));
|
||||
const { mountLibraryBrowser } = await import('./libraryBrowser.js');
|
||||
const library = {
|
||||
ready: true,
|
||||
subscribe: () => () => {},
|
||||
list: () => [],
|
||||
summary: () => ({
|
||||
entries: 0,
|
||||
kinds: { dataset: 0, tiles: 0, pack: 0 },
|
||||
themes: [],
|
||||
available: 0,
|
||||
unavailable: 0,
|
||||
added: 0,
|
||||
on: 0,
|
||||
features: 0,
|
||||
featureLayers: 0,
|
||||
}),
|
||||
reset: async () => 0,
|
||||
};
|
||||
const browser = mountLibraryBrowser({ library, trigger: null });
|
||||
context.after(() => browser.destroy());
|
||||
browser.open();
|
||||
const search = () =>
|
||||
document.querySelector('#data-library input[type="search"]');
|
||||
search().value = 'kartverket';
|
||||
search().dispatchEvent(new Event('input'));
|
||||
assert.equal(
|
||||
document.querySelector('#data-library-title').textContent,
|
||||
'DATA LIBRARY',
|
||||
);
|
||||
|
||||
setLocale('de');
|
||||
const title = document.querySelector('#data-library-title');
|
||||
assert.equal(title.textContent, t('DATA LIBRARY'));
|
||||
assert.notEqual(title.textContent, 'DATA LIBRARY');
|
||||
assert.equal(document.querySelectorAll('#data-library').length, 1);
|
||||
assert.equal(document.querySelector('#data-library').open, true);
|
||||
assert.equal(search().value, 'kartverket');
|
||||
|
||||
// Closed and reopened later, it is still in the new language.
|
||||
browser.close();
|
||||
browser.open();
|
||||
assert.equal(
|
||||
document.querySelector('#data-library-title').textContent,
|
||||
t('DATA LIBRARY'),
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user