mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-13 14:52:19 +00:00
Fix Data Library share restore, dense tile layers and EPC partition reads
- Share links: the compact /share/<code> format carries the switched-on library entries (field 29), so Copy Link keeps them; a share opened in the same tab restores them; any share decides what is on, so one with every library entry off no longer switches on the recipient's stored entries. - A tile entry too dense at its minimum zoom (Norway cadastre z14, depth z11) is hidden below that scale via minimumTerrainLevel instead of allocating millions of tiles when zoomed out. - EPC partitions keep the full extent for exclusion and the 98 % extent for ranking only; the old trimmed-only extents are retired. - Partition index footers that fail transiently are retried; an incomplete index is never cached as complete, a complete prior index is kept, and it is rebuilt after five minutes. - Views that leave touching partitions unread are partial; listing and footer bodies are capped and ranges validated. - The Browse dialog relabels after a language switch.
This commit is contained in:
1 parent
871b7f0fcc
commit
24ccdd4dc3
20 files changed
+1220
-162
No files matched your search
+6
-1
@@ -142,7 +142,12 @@ export function createApplicationTools({
|
||||
if (!signal?.aborted) console.warn('[Library] start failed:', error);
|
||||
});
|
||||
// The live gate's handle (scripts/qa-data-library.mjs).
|
||||
const libraryHandle = { library: dataLibrary, browser: libraryBrowser };
|
||||
const libraryHandle = {
|
||||
library: dataLibrary,
|
||||
browser: libraryBrowser,
|
||||
// What Copy Link would copy.
|
||||
buildShareLink: () => styleManager.shareLinkManager?.buildLink?.() ?? null,
|
||||
};
|
||||
window.__gevDataLibrary = libraryHandle;
|
||||
defer(() => {
|
||||
if (window.__gevDataLibrary === libraryHandle)
|
||||
|
||||
+85
-20
@@ -69,6 +69,9 @@ export function createDataLibrary({
|
||||
let serverPresent = false;
|
||||
let selection = { added: [], on: [] };
|
||||
let started = false;
|
||||
// Set once availability is known; a share navigated to before then waits.
|
||||
let loaded = false;
|
||||
let queuedShare = null;
|
||||
let destroyed = false;
|
||||
const listeners = new Set();
|
||||
const pending = new Map();
|
||||
@@ -165,6 +168,73 @@ export function createDataLibrary({
|
||||
persist();
|
||||
});
|
||||
|
||||
/** Entry ids for share tokens; unknown tokens are skipped. */
|
||||
const idsForTokens = (list) =>
|
||||
(list || []).map((token) => tokens.idByToken.get(token)).filter(Boolean);
|
||||
|
||||
/**
|
||||
* The share this page opened with: null without one, else the entry ids it
|
||||
* switches on ([] for a share that switches none on).
|
||||
*/
|
||||
function openingShare() {
|
||||
if (shareLink && shareLink.initialLibraryTokens !== undefined) {
|
||||
const linked = shareLink.initialLibraryTokens;
|
||||
return linked === null ? null : idsForTokens(linked);
|
||||
}
|
||||
const hash = String(initialHash || '').replace(/^#/, '');
|
||||
if (!hash) return null;
|
||||
const params = new URLSearchParams(hash);
|
||||
const linked = decodeLibraryParam(params, tokens.idByToken);
|
||||
// Any share (a camera position) decides what is on; no `lib`, nothing.
|
||||
if (linked === null && params.has('lat') && params.has('lon')) return [];
|
||||
return linked;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a selection: register and switch on what `on` names, switch off
|
||||
* every other added entry that is on. `origin` marks it as a restore so it
|
||||
* is never remembered as the viewer's own choice.
|
||||
*/
|
||||
function applySelection(next, origin) {
|
||||
selection = normalizeLibrarySelection(next, switchable);
|
||||
for (const id of selection.added) {
|
||||
const entry = byId.get(id);
|
||||
if (!entry || !addable(id)) continue;
|
||||
const wanted = selection.on.includes(id);
|
||||
if (wanted && !register(entry)) continue;
|
||||
if (!isRegistered(id)) continue;
|
||||
const layerId = libraryLayerId(id);
|
||||
if (wanted === (dataManager.isEnabled?.(layerId) === true)) continue;
|
||||
dataManager
|
||||
.setEnabled(layerId, wanted, { origin })
|
||||
.catch((error) =>
|
||||
console.warn(`[Library] ${id} restore failed:`, error),
|
||||
);
|
||||
}
|
||||
writeStoredLibrarySelection(storage, selection);
|
||||
shareLink?.onLibraryStateChange?.();
|
||||
notify();
|
||||
}
|
||||
|
||||
/**
|
||||
* A share opened in this tab after start-up: its entries join the library
|
||||
* and it alone decides which are on.
|
||||
* @param {string[]} shareTokens
|
||||
*/
|
||||
function applyShare(shareTokens) {
|
||||
if (destroyed) return;
|
||||
if (!loaded) {
|
||||
queuedShare = [...(shareTokens || [])];
|
||||
return;
|
||||
}
|
||||
const linked = idsForTokens(shareTokens);
|
||||
applySelection(
|
||||
{ added: [...selection.added, ...linked], on: linked },
|
||||
'share-restore',
|
||||
);
|
||||
}
|
||||
shareLink?.setLibraryShareListener?.(applyShare);
|
||||
|
||||
shareLink?.setLibraryParamsEncoder?.((params) =>
|
||||
encodeLibraryParam(
|
||||
params,
|
||||
@@ -186,29 +256,23 @@ export function createDataLibrary({
|
||||
availability = answer.availability;
|
||||
serverPresent = answer.server;
|
||||
const stored = readStoredLibrarySelection(storage, switchable);
|
||||
const hash = String(initialHash || '').replace(/^#/, '');
|
||||
const linked = hash
|
||||
? decodeLibraryParam(new URLSearchParams(hash), tokens.idByToken)
|
||||
: null;
|
||||
// A share opened in this tab while availability loaded wins over the
|
||||
// opening one.
|
||||
const linked = queuedShare ? idsForTokens(queuedShare) : openingShare();
|
||||
queuedShare = null;
|
||||
loaded = true;
|
||||
// A link's entries join the viewer's library and open switched on; with a
|
||||
// link, it alone decides what is on, as it does for core layers.
|
||||
const added = [...stored.added, ...(linked || [])];
|
||||
const on = linked ?? stored.on;
|
||||
selection = normalizeLibrarySelection({ added, on }, switchable);
|
||||
const restoreOrigin = linked ? 'share-restore' : 'local-restore';
|
||||
for (const id of selection.added) {
|
||||
// link, it alone decides what is on, as it does for core layers, so a
|
||||
// share with every library entry off switches the stored ones off.
|
||||
// Added entries register even while off, so their rows show.
|
||||
for (const id of [...stored.added, ...(linked || [])]) {
|
||||
const entry = byId.get(id);
|
||||
if (!entry || !addable(id) || !register(entry)) continue;
|
||||
if (selection.on.includes(id))
|
||||
dataManager
|
||||
.setEnabled(libraryLayerId(id), true, { origin: restoreOrigin })
|
||||
.catch((error) =>
|
||||
console.warn(`[Library] ${id} restore failed:`, error),
|
||||
);
|
||||
if (entry && addable(id)) register(entry);
|
||||
}
|
||||
writeStoredLibrarySelection(storage, selection);
|
||||
shareLink?.onLibraryStateChange?.();
|
||||
notify();
|
||||
applySelection(
|
||||
{ added: [...stored.added, ...(linked || [])], on: linked ?? stored.on },
|
||||
linked ? 'share-restore' : 'local-restore',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -388,6 +452,7 @@ export function createDataLibrary({
|
||||
destroyed = true;
|
||||
unsubscribeManager?.();
|
||||
shareLink?.setLibraryParamsEncoder?.(null);
|
||||
shareLink?.setLibraryShareListener?.(null);
|
||||
listeners.clear();
|
||||
},
|
||||
};
|
||||
|
||||
@@ -207,6 +207,115 @@ test('a link decides what is on and adds its entries to the library', async (t)
|
||||
});
|
||||
});
|
||||
|
||||
/** A ShareLinkManager stand-in: the opening share and in-tab navigation. */
|
||||
function fakeShareLink(initialLibraryTokens) {
|
||||
const link = {
|
||||
initialLibraryTokens,
|
||||
listener: null,
|
||||
setLibraryShareListener(listener) {
|
||||
link.listener = listener;
|
||||
},
|
||||
setLibraryParamsEncoder() {},
|
||||
onLibraryStateChange() {},
|
||||
};
|
||||
return link;
|
||||
}
|
||||
|
||||
test('a share with every library entry off keeps the stored ones off', async (t) => {
|
||||
const manager = sealedManager(t);
|
||||
const storage = memoryStorage(
|
||||
JSON.stringify({
|
||||
added: ['night-lights', 'sea-marks'],
|
||||
on: ['night-lights', 'sea-marks'],
|
||||
}),
|
||||
);
|
||||
// A compact /share/<code> link with no library field, and a long-form link
|
||||
// with no lib=: both are shares that switch no library entry on.
|
||||
for (const options of [
|
||||
{ shareLink: fakeShareLink([]) },
|
||||
{ initialHash: '#lat=1&lon=2&l=' },
|
||||
]) {
|
||||
const library = createDataLibrary({
|
||||
entries,
|
||||
ledger,
|
||||
dataManager: manager,
|
||||
services: {},
|
||||
storage,
|
||||
source: source({}),
|
||||
...options,
|
||||
});
|
||||
await library.start();
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), false);
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), false);
|
||||
// The recipient's entries stay in their panel, off.
|
||||
assert.deepEqual(
|
||||
library
|
||||
.list()
|
||||
.filter((row) => row.added)
|
||||
.map((row) => row.entry.id),
|
||||
['night-lights', 'sea-marks'],
|
||||
);
|
||||
library.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('the opening share link restores its library tokens', async (t) => {
|
||||
const manager = sealedManager(t);
|
||||
const storage = memoryStorage(
|
||||
JSON.stringify({ added: ['night-lights'], on: ['night-lights'] }),
|
||||
);
|
||||
const library = createDataLibrary({
|
||||
entries,
|
||||
ledger,
|
||||
dataManager: manager,
|
||||
services: {},
|
||||
storage,
|
||||
shareLink: fakeShareLink(['01']),
|
||||
source: source({}),
|
||||
});
|
||||
await library.start();
|
||||
await manager.waitForLayerSettled?.('lib-sea-marks');
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), true);
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), false);
|
||||
});
|
||||
|
||||
test('navigating between shares switches library entries on and off', async (t) => {
|
||||
const manager = sealedManager(t);
|
||||
const shareLink = fakeShareLink(['00']);
|
||||
const library = createDataLibrary({
|
||||
entries,
|
||||
ledger,
|
||||
dataManager: manager,
|
||||
services: {},
|
||||
storage: memoryStorage(),
|
||||
shareLink,
|
||||
source: source({}),
|
||||
});
|
||||
// A share navigated to before availability loads waits for it and wins.
|
||||
const started = library.start();
|
||||
shareLink.listener(['01']);
|
||||
await started;
|
||||
const settle = () =>
|
||||
Promise.all(
|
||||
['lib-night-lights', 'lib-sea-marks'].map((id) =>
|
||||
manager.waitForLayerSettled?.(id),
|
||||
),
|
||||
);
|
||||
await settle();
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), true);
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), false);
|
||||
shareLink.listener(['00']);
|
||||
await settle();
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), true);
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), false);
|
||||
shareLink.listener([]);
|
||||
await settle();
|
||||
assert.equal(manager.isEnabled('lib-night-lights'), false);
|
||||
assert.equal(manager.isEnabled('lib-sea-marks'), false);
|
||||
library.destroy();
|
||||
assert.equal(shareLink.listener, null);
|
||||
});
|
||||
|
||||
test('zoom flies to an added layer through the caller, reset removes everything', async (t) => {
|
||||
const manager = sealedManager(t);
|
||||
const storage = memoryStorage();
|
||||
|
||||
+45
-1
@@ -258,6 +258,44 @@ const reliefLegend = () =>
|
||||
/** colorToAlpha threshold (max channel distance, 0–1) for `noDataColor`. */
|
||||
const NO_DATA_THRESHOLD = 0.02;
|
||||
|
||||
/** The most tiles an entry's coverage may need at its minimum zoom. */
|
||||
const COARSE_TILE_BUDGET = 256;
|
||||
|
||||
function webMercatorTileSpan(bbox, zoom) {
|
||||
const n = 2 ** zoom;
|
||||
const clampLat = (lat) => Math.max(-85.0511, Math.min(85.0511, lat));
|
||||
const tileX = (lon) =>
|
||||
Math.min(n - 1, Math.max(0, Math.floor(((lon + 180) / 360) * n)));
|
||||
const tileY = (lat) => {
|
||||
const rad = (clampLat(lat) * Math.PI) / 180;
|
||||
const y = (1 - Math.log(Math.tan(rad) + 1 / Math.cos(rad)) / Math.PI) / 2;
|
||||
return Math.min(n - 1, Math.max(0, Math.floor(y * n)));
|
||||
};
|
||||
const [west, south, east, north] = bbox;
|
||||
return (tileX(east) - tileX(west) + 1) * (tileY(south) - tileY(north) + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* The globe level below which a tile entry is hidden, or undefined when it
|
||||
* draws at every scale. Cesium fetches an imagery layer's minimum level for
|
||||
* every globe tile it covers, however far out the camera is: an entry whose
|
||||
* coverage needs more than COARSE_TILE_BUDGET tiles at its minimum zoom
|
||||
* (Norway's cadastre from z14 is about two million) would allocate them all
|
||||
* when zoomed out. Such an entry is shown only from the globe level that
|
||||
* matches its minimum zoom (a geographic level spans two Web Mercator ones,
|
||||
* so level z − 1).
|
||||
* @param {object} entry
|
||||
* @returns {number|undefined}
|
||||
*/
|
||||
export function minimumTerrainLevelFor(entry) {
|
||||
const minZoom = entry?.source?.minZoom ?? 0;
|
||||
if (minZoom <= 0) return undefined;
|
||||
const bbox = entry.coverage?.bbox ?? [-180, -85.0511, 180, 85.0511];
|
||||
return webMercatorTileSpan(bbox, minZoom) > COARSE_TILE_BUDGET
|
||||
? minZoom - 1
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* A tiles entry as a manager layer.
|
||||
* @param {object} entry Validated tiles entry.
|
||||
@@ -406,7 +444,13 @@ export function createLibraryTileLayer(
|
||||
provider.errorEvent?.addEventListener?.(onError);
|
||||
removeErrorListener = () =>
|
||||
provider.errorEvent?.removeEventListener?.(onError);
|
||||
imagery = host.collection.addImageryProvider(provider);
|
||||
const minimumTerrainLevel = minimumTerrainLevelFor(entry);
|
||||
imagery =
|
||||
minimumTerrainLevel !== undefined && cesium.ImageryLayer
|
||||
? new cesium.ImageryLayer(provider, { minimumTerrainLevel })
|
||||
: null;
|
||||
if (imagery) host.collection.add(imagery);
|
||||
else imagery = host.collection.addImageryProvider(provider);
|
||||
imagery.alpha = entry.display.opacity ?? 1;
|
||||
if (entry.display.noDataColor && cesium.Color) {
|
||||
// The provider's "no imagery here" fill, drawn transparent (JPEG
|
||||
|
||||
+128
-42
@@ -10,7 +10,10 @@
|
||||
import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet';
|
||||
import { decompress as zstdDecompress } from 'fzstd';
|
||||
import { libraryUpstreamUrl } from './manifest.js';
|
||||
import { readResponseBytesCapped } from '../sources/httpBody.js';
|
||||
import {
|
||||
readResponseBytesCapped,
|
||||
readResponseTextCapped,
|
||||
} from '../sources/httpBody.js';
|
||||
|
||||
/** Footer bytes read first; small files' metadata fits in one request. */
|
||||
const FOOTER_FETCH_BYTES = 16 * 1024;
|
||||
@@ -19,6 +22,13 @@ export const MAX_PARTITIONS_PER_VIEW = 4;
|
||||
const MAX_LISTED_KEYS = 5000;
|
||||
const MAX_LIST_PAGES = 20;
|
||||
const INDEX_CONCURRENCY = 8;
|
||||
/** One listing page: 1,000 keys of XML is well under a megabyte. */
|
||||
const MAX_LIST_PAGE_BYTES = 4 * 1024 * 1024;
|
||||
/** A whole file sent in answer to a range request is read only up to this. */
|
||||
const MAX_UNRANGED_BYTES = 64 * 1024 * 1024;
|
||||
/** Extra passes over footers that failed, before the index is incomplete. */
|
||||
const FOOTER_RETRIES = 2;
|
||||
const FOOTER_RETRY_DELAY_MS = 500;
|
||||
|
||||
/** Decompressors hyparquet does not carry itself (it has Snappy built in). */
|
||||
export const PARQUET_COMPRESSORS = Object.freeze({
|
||||
@@ -33,30 +43,67 @@ function onSourceHost(entry, url) {
|
||||
return target.href;
|
||||
}
|
||||
|
||||
/** Mark a failure worth retrying: network, 429, 5xx or a short read. */
|
||||
const transient = (error) => Object.assign(error, { transient: true });
|
||||
|
||||
/**
|
||||
* A hyparquet AsyncBuffer over HTTP range requests, redirects refused,
|
||||
* credentials and referrer withheld.
|
||||
*/
|
||||
export function rangeBuffer(url, byteLength, { fetchImpl, signal }) {
|
||||
if (!Number.isSafeInteger(byteLength) || byteLength <= 0)
|
||||
throw new RangeError(`invalid object size ${byteLength}`);
|
||||
return {
|
||||
byteLength,
|
||||
async slice(start, end = byteLength) {
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
credentials: 'omit',
|
||||
referrerPolicy: 'no-referrer',
|
||||
headers: { Range: `bytes=${start}-${end - 1}` },
|
||||
});
|
||||
if (
|
||||
!Number.isSafeInteger(start) ||
|
||||
!Number.isSafeInteger(end) ||
|
||||
start < 0 ||
|
||||
end > byteLength ||
|
||||
end <= start
|
||||
)
|
||||
throw new RangeError(`invalid range ${start}-${end} of ${byteLength}`);
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
credentials: 'omit',
|
||||
referrerPolicy: 'no-referrer',
|
||||
headers: { Range: `bytes=${start}-${end - 1}` },
|
||||
});
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
throw transient(error);
|
||||
}
|
||||
if (response.status !== 206 && response.status !== 200) {
|
||||
await response.body?.cancel?.().catch?.(() => {});
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
const error = new Error(`HTTP ${response.status}`);
|
||||
throw response.status === 429 || response.status >= 500
|
||||
? transient(error)
|
||||
: error;
|
||||
}
|
||||
const buffer = await response.arrayBuffer();
|
||||
// A server that ignores Range answers with the whole file.
|
||||
return response.status === 200 && buffer.byteLength === byteLength
|
||||
? buffer.slice(start, end)
|
||||
: buffer;
|
||||
// A server that ignores Range answers with the whole file; anything
|
||||
// else must be exactly the bytes asked for.
|
||||
const whole = response.status === 200;
|
||||
const expected = whole ? byteLength : end - start;
|
||||
const bytes = await readResponseBytesCapped(
|
||||
response,
|
||||
whole ? Math.min(byteLength, MAX_UNRANGED_BYTES) : expected,
|
||||
signal,
|
||||
);
|
||||
if (bytes.byteLength !== expected)
|
||||
throw transient(
|
||||
new Error(
|
||||
`short read: ${bytes.byteLength} of ${expected} bytes from ${start}`,
|
||||
),
|
||||
);
|
||||
const buffer = bytes.buffer.slice(
|
||||
bytes.byteOffset,
|
||||
bytes.byteOffset + bytes.byteLength,
|
||||
);
|
||||
return whole ? buffer.slice(start, end) : buffer;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -79,12 +126,19 @@ export async function listPartitions(entry, { fetchImpl, signal }) {
|
||||
credentials: 'omit',
|
||||
});
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
const xml = await response.text();
|
||||
const xml = await readResponseTextCapped(
|
||||
response,
|
||||
MAX_LIST_PAGE_BYTES,
|
||||
signal,
|
||||
);
|
||||
for (const match of xml.matchAll(
|
||||
/<Contents>[\s\S]*?<Key>([^<]+)<\/Key>[\s\S]*?<Size>(\d+)<\/Size>[\s\S]*?<\/Contents>/g,
|
||||
)) {
|
||||
if (!match[1].endsWith('.parquet')) continue;
|
||||
objects.push({ key: match[1], size: Number(match[2]) });
|
||||
const size = Number(match[2]);
|
||||
if (!match[1].endsWith('.parquet') || !Number.isSafeInteger(size))
|
||||
continue;
|
||||
if (size <= 0) continue;
|
||||
objects.push({ key: match[1], size });
|
||||
if (objects.length >= MAX_LISTED_KEYS) return objects;
|
||||
}
|
||||
token = /<NextContinuationToken>([^<]+)<\/NextContinuationToken>/.exec(
|
||||
@@ -119,39 +173,71 @@ export function geoParquetInfo(metadata) {
|
||||
}
|
||||
}
|
||||
|
||||
const pause = (ms, signal) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, ms);
|
||||
signal?.addEventListener?.(
|
||||
'abort',
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
reject(signal.reason);
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* Where each partition is: list the bucket, then read each file's footer
|
||||
* (one small range request) for its GeoParquet bbox. Built once and cached
|
||||
* by the caller; a partition without a bbox is left out.
|
||||
* @returns {Promise<Array<{ url: string, bbox: number[], size: number }>>}
|
||||
* (one small range request) for its GeoParquet bbox. A partition whose
|
||||
* metadata is unreadable or has no bbox is left out; a footer that fails to
|
||||
* load (network, 429, 5xx, short read) is retried
|
||||
* (FOOTER_RETRIES more passes) and, if it still fails, the index is
|
||||
* `complete: false` so callers never keep it as if it were whole.
|
||||
* @returns {Promise<{ partitions: Array<{ url: string, bbox: number[], size: number }>, complete: boolean, failed: number }>}
|
||||
*/
|
||||
export async function buildPartitionIndex(entry, { fetchImpl, signal }) {
|
||||
export async function buildPartitionIndex(
|
||||
entry,
|
||||
{ fetchImpl, signal, retryDelayMs = FOOTER_RETRY_DELAY_MS },
|
||||
) {
|
||||
const { keyBase } = entry.source.partitions;
|
||||
const objects = await listPartitions(entry, { fetchImpl, signal });
|
||||
const index = [];
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (next < objects.length) {
|
||||
const { key, size } = objects[next++];
|
||||
signal?.throwIfAborted?.();
|
||||
const url = onSourceHost(entry, new URL(key, keyBase).href);
|
||||
try {
|
||||
const metadata = await parquetMetadataAsync(
|
||||
rangeBuffer(url, size, { fetchImpl, signal }),
|
||||
{ initialFetchSize: FOOTER_FETCH_BYTES },
|
||||
);
|
||||
const { bbox } = geoParquetInfo(metadata);
|
||||
if (bbox) index.push({ url, bbox, size });
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
// One unreadable partition does not sink the index.
|
||||
const partitions = [];
|
||||
let pending = objects;
|
||||
for (let pass = 0; pass <= FOOTER_RETRIES && pending.length; pass++) {
|
||||
if (pass) await pause(retryDelayMs * pass, signal);
|
||||
const failed = [];
|
||||
let next = 0;
|
||||
const queue = pending;
|
||||
async function worker() {
|
||||
while (next < queue.length) {
|
||||
const object = queue[next++];
|
||||
signal?.throwIfAborted?.();
|
||||
const url = onSourceHost(entry, new URL(object.key, keyBase).href);
|
||||
try {
|
||||
const metadata = await parquetMetadataAsync(
|
||||
rangeBuffer(url, object.size, { fetchImpl, signal }),
|
||||
{ initialFetchSize: FOOTER_FETCH_BYTES },
|
||||
);
|
||||
const { bbox } = geoParquetInfo(metadata);
|
||||
if (bbox) partitions.push({ url, bbox, size: object.size });
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
// A footer that arrived but does not parse will not parse next
|
||||
// time either: that file is left out. Only fetch failures retry.
|
||||
if (error?.transient) failed.push(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(INDEX_CONCURRENCY, queue.length) }, worker),
|
||||
);
|
||||
pending = failed;
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(INDEX_CONCURRENCY, objects.length) }, worker),
|
||||
);
|
||||
return index.sort((a, b) => a.url.localeCompare(b.url));
|
||||
return {
|
||||
partitions: partitions.sort((a, b) => a.url.localeCompare(b.url)),
|
||||
complete: pending.length === 0,
|
||||
failed: pending.length,
|
||||
};
|
||||
}
|
||||
|
||||
const overlapArea = (a, b) =>
|
||||
|
||||
@@ -90,7 +90,7 @@ test('a partitioned view reads candidates in batches until one finds features',
|
||||
},
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => index,
|
||||
buildPartitionIndex: async () => ({ partitions: index, complete: true }),
|
||||
readGeoParquetFeatures: async (entry, url) => {
|
||||
reads.push(url);
|
||||
return url.endsWith('A.parquet')
|
||||
@@ -108,8 +108,11 @@ test('a partitioned view reads candidates in batches until one finds features',
|
||||
]);
|
||||
// The index is kept in this browser; extents are learned per partition.
|
||||
assert.ok(store.get('gev:library:index:epc-england-wales'));
|
||||
const learned = JSON.parse(store.get('gev:library:extent:epc-england-wales'));
|
||||
assert.ok(learned['https://h/A.parquet']);
|
||||
const learned = JSON.parse(
|
||||
store.get('gev:library:extent:v2:epc-england-wales'),
|
||||
);
|
||||
assert.ok(learned['https://h/A.parquet'].full);
|
||||
assert.ok(learned['https://h/A.parquet'].trimmed);
|
||||
// A second view of the same area is served from the decoded cache.
|
||||
await source.loadDataset(epc, { bbox: box });
|
||||
assert.equal(reads.length, 2);
|
||||
@@ -119,3 +122,185 @@ test('a partitioned view reads candidates in batches until one finds features',
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('a revisit still finds homes at the edge of a partition', async () => {
|
||||
// 200 homes in a cluster and one real home at the partition's edge: the
|
||||
// trimmed (98 %) extent leaves the edge home out, the full one keeps it.
|
||||
const point = (lon, lat) => ({
|
||||
type: 'Feature',
|
||||
geometry: { type: 'Point', coordinates: [lon, lat] },
|
||||
properties: { CURRENT_ENERGY_RATING: 'C', UPRN: String(lon) },
|
||||
});
|
||||
const homes = Array.from({ length: 200 }, (_, index) =>
|
||||
point(-1.4 - index / 1e4, 53.4),
|
||||
);
|
||||
const edge = point(-1.2, 53.3);
|
||||
homes.push(edge);
|
||||
const index = [
|
||||
{ url: 'https://h/Edge.parquet', bbox: [-1.5, 53.2, -1.1, 53.5] },
|
||||
];
|
||||
const store = new Map([
|
||||
// A v1 extent (trimmed only) from an older build: never read again.
|
||||
[
|
||||
'gev:library:extent:epc-england-wales',
|
||||
JSON.stringify({ 'https://h/Edge.parquet': [-1.42, 53.4, -1.4, 53.4] }),
|
||||
],
|
||||
]);
|
||||
const storage = {
|
||||
getItem: (key) => store.get(key) ?? null,
|
||||
setItem: (key, value) => store.set(key, value),
|
||||
removeItem: (key) => store.delete(key),
|
||||
};
|
||||
const visit = () =>
|
||||
createLibrarySource({
|
||||
entries: [epc],
|
||||
fetchImpl: async () => ({ ok: false, status: 404 }),
|
||||
storage,
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => ({
|
||||
partitions: index,
|
||||
complete: true,
|
||||
}),
|
||||
readGeoParquetFeatures: async () => homes,
|
||||
}),
|
||||
});
|
||||
const edgeBox = [-1.21, 53.29, -1.19, 53.31];
|
||||
// First visit: the old v1 extent would have skipped the partition.
|
||||
const first = await visit().loadDataset(epc, { bbox: edgeBox });
|
||||
assert.equal(first.features.length, 1);
|
||||
assert.equal(store.has('gev:library:extent:epc-england-wales'), false);
|
||||
// A later visit (new page, nothing decoded) reads the learned v2 extents.
|
||||
const later = await visit().loadDataset(epc, { bbox: edgeBox });
|
||||
assert.equal(later.features.length, 1);
|
||||
assert.deepEqual(later.features[0].geometry.coordinates, [-1.2, 53.3]);
|
||||
// A box outside every point of the partition is still ruled out.
|
||||
const outside = await visit().loadDataset(epc, {
|
||||
bbox: [-1.12, 53.45, -1.11, 53.46],
|
||||
});
|
||||
assert.equal(outside.features.length, 0);
|
||||
});
|
||||
|
||||
test('a footer that keeps failing leaves the index incomplete, and a retry recovers', async () => {
|
||||
const { buildPartitionIndex } = await import('./parquet.js');
|
||||
const entry = {
|
||||
...epc,
|
||||
source: {
|
||||
...epc.source,
|
||||
partitions: {
|
||||
listUrl: 'https://s3.us-west-2.amazonaws.com/bucket?list-type=2',
|
||||
keyBase: 'https://s3.us-west-2.amazonaws.com/bucket/',
|
||||
},
|
||||
},
|
||||
};
|
||||
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents><Contents><Key>b.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
|
||||
let footerFailures = 0;
|
||||
const fetchImpl = async (url) => {
|
||||
if (url.includes('list-type'))
|
||||
return new Response(listing, { status: 200 });
|
||||
if (url.endsWith('b.parquet') && footerFailures > 0) {
|
||||
footerFailures -= 1;
|
||||
return new Response('', { status: 503 });
|
||||
}
|
||||
// Not a Parquet footer: readable, but with no bbox, so left out.
|
||||
return new Response(new Uint8Array(100), { status: 200 });
|
||||
};
|
||||
footerFailures = 2; // fails once, then succeeds on the first retry
|
||||
const recovered = await buildPartitionIndex(entry, {
|
||||
fetchImpl,
|
||||
retryDelayMs: 0,
|
||||
});
|
||||
assert.equal(recovered.complete, true);
|
||||
footerFailures = 100; // fails on every pass
|
||||
const broken = await buildPartitionIndex(entry, {
|
||||
fetchImpl,
|
||||
retryDelayMs: 0,
|
||||
});
|
||||
assert.equal(broken.complete, false);
|
||||
assert.equal(broken.failed, 1);
|
||||
});
|
||||
|
||||
test('the browser keeps a complete index over an incomplete rebuild and retries later', async () => {
|
||||
let clock = 30 * 24 * 3600_000;
|
||||
const complete = [{ url: 'https://h/A.parquet', bbox: [-2, 53, -1, 54] }];
|
||||
const store = new Map([
|
||||
[
|
||||
'gev:library:index:epc-england-wales',
|
||||
// Past cacheSeconds: due for a rebuild.
|
||||
JSON.stringify({ savedAt: 0, complete: true, partitions: complete }),
|
||||
],
|
||||
]);
|
||||
let builds = 0;
|
||||
let nextComplete = false;
|
||||
const source = createLibrarySource({
|
||||
entries: [epc],
|
||||
now: () => clock,
|
||||
fetchImpl: async () => ({ ok: false, status: 404 }),
|
||||
storage: {
|
||||
getItem: (key) => store.get(key) ?? null,
|
||||
setItem: (key, value) => store.set(key, value),
|
||||
},
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => {
|
||||
builds += 1;
|
||||
return nextComplete
|
||||
? { partitions: complete, complete: true, failed: 0 }
|
||||
: { partitions: [], complete: false, failed: 3 };
|
||||
},
|
||||
readGeoParquetFeatures: async () => [],
|
||||
}),
|
||||
});
|
||||
const box = [-1.5, 53.4, -1.4, 53.5];
|
||||
await source.loadDataset(epc, { bbox: box });
|
||||
assert.equal(builds, 1);
|
||||
// The old complete index stays stored; the incomplete one is not saved.
|
||||
assert.equal(
|
||||
JSON.parse(store.get('gev:library:index:epc-england-wales')).savedAt,
|
||||
0,
|
||||
);
|
||||
// Within the retry pause the incomplete result is reused, not rebuilt.
|
||||
await source.loadDataset(epc, { bbox: box });
|
||||
assert.equal(builds, 1);
|
||||
// After it, the index is rebuilt; a complete build is kept.
|
||||
clock += 6 * 60_000;
|
||||
nextComplete = true;
|
||||
await source.loadDataset(epc, { bbox: box });
|
||||
assert.equal(builds, 2);
|
||||
assert.equal(
|
||||
JSON.parse(store.get('gev:library:index:epc-england-wales')).savedAt,
|
||||
clock,
|
||||
);
|
||||
});
|
||||
|
||||
test('a view that leaves touching partitions unread is partial', async () => {
|
||||
const index = Array.from({ length: 6 }, (_, n) => ({
|
||||
url: `https://h/P${n}.parquet`,
|
||||
bbox: [-1.5 - n / 100, 53.3, -1.4, 53.45],
|
||||
}));
|
||||
const point = (lon, lat) => ({
|
||||
type: 'Feature',
|
||||
geometry: { type: 'Point', coordinates: [lon, lat] },
|
||||
properties: { CURRENT_ENERGY_RATING: 'C', UPRN: '1' },
|
||||
});
|
||||
const reads = [];
|
||||
const source = createLibrarySource({
|
||||
entries: [epc],
|
||||
fetchImpl: async () => ({ ok: false, status: 404 }),
|
||||
storage: null,
|
||||
loadParquet: async () => ({
|
||||
...(await import('./parquet.js')),
|
||||
buildPartitionIndex: async () => ({ partitions: index, complete: true }),
|
||||
readGeoParquetFeatures: async (entry, url) => {
|
||||
reads.push(url);
|
||||
return [point(-1.45, 53.4)];
|
||||
},
|
||||
}),
|
||||
});
|
||||
const result = await source.loadDataset(epc, {
|
||||
bbox: [-1.46, 53.39, -1.44, 53.41],
|
||||
});
|
||||
// The first batch of four found homes; two touching partitions stay unread.
|
||||
assert.equal(reads.length, 4);
|
||||
assert.equal(result.truncated, true);
|
||||
});
|
||||
+149
-44
@@ -17,7 +17,13 @@ const PARTITION_BATCH = 4;
|
||||
/** Decoded partitions kept in memory, most recent first. */
|
||||
const PARTITION_CACHE_LIMIT = 4;
|
||||
const INDEX_STORAGE_PREFIX = 'gev:library:index:';
|
||||
const EXTENT_STORAGE_PREFIX = 'gev:library:extent:';
|
||||
/** How long an incomplete partition index is used before a rebuild. */
|
||||
const PARTIAL_INDEX_RETRY_MS = 5 * 60_000;
|
||||
// v2 stores `{ full, trimmed }` per partition. The v1 key held only the
|
||||
// trimmed box, which wrongly excluded real points at a partition's edge; it
|
||||
// is no longer read and is removed on first use.
|
||||
const EXTENT_STORAGE_PREFIX = 'gev:library:extent:v2:';
|
||||
const RETIRED_EXTENT_STORAGE_PREFIX = 'gev:library:extent:';
|
||||
|
||||
const overlaps = (a, b) =>
|
||||
a[0] < b[2] && a[2] > b[0] && a[1] < b[3] && a[3] > b[1];
|
||||
@@ -47,6 +53,58 @@ export function robustExtent(features) {
|
||||
return [at(lons, 0.01), at(lats, 0.01), at(lons, 0.99), at(lats, 0.99)];
|
||||
}
|
||||
|
||||
/** The box holding every point of a partition, or null. */
|
||||
export function fullExtent(features) {
|
||||
let extent = null;
|
||||
for (const feature of features) {
|
||||
let position = feature?.geometry?.coordinates;
|
||||
while (Array.isArray(position?.[0])) position = position[0];
|
||||
if (!Array.isArray(position)) continue;
|
||||
const [lon, lat] = position;
|
||||
if (!Number.isFinite(lon) || !Number.isFinite(lat)) continue;
|
||||
if (!extent) extent = [lon, lat, lon, lat];
|
||||
else {
|
||||
extent[0] = Math.min(extent[0], lon);
|
||||
extent[1] = Math.min(extent[1], lat);
|
||||
extent[2] = Math.max(extent[2], lon);
|
||||
extent[3] = Math.max(extent[3], lat);
|
||||
}
|
||||
}
|
||||
return extent;
|
||||
}
|
||||
|
||||
const isBox = (value) =>
|
||||
Array.isArray(value) && value.length === 4 && value.every(Number.isFinite);
|
||||
|
||||
/**
|
||||
* Candidate partitions for a box. A learned full extent that misses the box
|
||||
* rules a partition out (no point of it can be in view); the learned 98 %
|
||||
* extent only ranks, so stray geocodes neither stretch the ranking nor hide
|
||||
* the real points at a partition's edge.
|
||||
*/
|
||||
export function rankPartitions(index, box, learned, partitionsForBox) {
|
||||
const known = (url) => {
|
||||
const value = learned?.[url];
|
||||
return value && isBox(value.full) && isBox(value.trimmed) ? value : null;
|
||||
};
|
||||
const ranked = partitionsForBox(
|
||||
index
|
||||
.filter(({ url }) => !known(url) || overlaps(known(url).full, box))
|
||||
.map((partition) => {
|
||||
const extent = known(partition.url);
|
||||
// Rank by where most points are, unless that misses the view while
|
||||
// the full extent does not: then the partition's own box ranks it.
|
||||
return extent && overlaps(extent.trimmed, box)
|
||||
? { ...partition, bbox: extent.trimmed }
|
||||
: partition;
|
||||
}),
|
||||
box,
|
||||
Infinity,
|
||||
);
|
||||
const byUrl = new Map(index.map((partition) => [partition.url, partition]));
|
||||
return ranked.map(({ url }) => byUrl.get(url));
|
||||
}
|
||||
|
||||
function storedJson(storage, key) {
|
||||
try {
|
||||
const text = storage?.getItem?.(key);
|
||||
@@ -118,43 +176,76 @@ export function createLibrarySource({
|
||||
* shared), else built here and kept in this browser for `cacheSeconds`.
|
||||
*/
|
||||
function partitionIndex(entry, { signal } = {}) {
|
||||
if (!indexes.has(entry.id))
|
||||
indexes.set(
|
||||
entry.id,
|
||||
(async () => {
|
||||
try {
|
||||
const response = await fetchImpl(
|
||||
`${base}/index/${encodeURIComponent(entry.id)}`,
|
||||
{ signal, headers: { Accept: 'application/json' } },
|
||||
);
|
||||
if (response.ok) {
|
||||
const body = await response.json();
|
||||
if (Array.isArray(body?.partitions)) return body.partitions;
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
const memo = indexes.get(entry.id);
|
||||
if (memo && (memo.retryAt === null || now() < memo.retryAt))
|
||||
return memo.promise;
|
||||
const promise = (async () => {
|
||||
const key = `${INDEX_STORAGE_PREFIX}${entry.id}`;
|
||||
const stored = storedJson(storage, key);
|
||||
const storedComplete =
|
||||
stored && Array.isArray(stored.partitions) && stored.complete !== false
|
||||
? stored
|
||||
: null;
|
||||
let partial = null;
|
||||
try {
|
||||
const response = await fetchImpl(
|
||||
`${base}/index/${encodeURIComponent(entry.id)}`,
|
||||
{ signal, headers: { Accept: 'application/json' } },
|
||||
);
|
||||
if (response.ok) {
|
||||
const body = await response.json();
|
||||
if (Array.isArray(body?.partitions)) {
|
||||
if (body.complete !== false)
|
||||
return { partitions: body.partitions, complete: true };
|
||||
partial = body.partitions;
|
||||
}
|
||||
const key = `${INDEX_STORAGE_PREFIX}${entry.id}`;
|
||||
const stored = storedJson(storage, key);
|
||||
if (
|
||||
stored &&
|
||||
Array.isArray(stored.partitions) &&
|
||||
now() - stored.savedAt < entry.source.cacheSeconds * 1000
|
||||
)
|
||||
return stored.partitions;
|
||||
const { buildPartitionIndex } = await loadParquet();
|
||||
const partitions = await buildPartitionIndex(entry, {
|
||||
fetchImpl,
|
||||
signal,
|
||||
});
|
||||
storeJson(storage, key, { savedAt: now(), partitions });
|
||||
return partitions;
|
||||
})().catch((error) => {
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
}
|
||||
if (
|
||||
storedComplete &&
|
||||
now() - storedComplete.savedAt < entry.source.cacheSeconds * 1000
|
||||
)
|
||||
return { partitions: storedComplete.partitions, complete: true };
|
||||
// The server's index is incomplete, or there is no server: build here.
|
||||
const { buildPartitionIndex } = await loadParquet();
|
||||
const built = await buildPartitionIndex(entry, { fetchImpl, signal });
|
||||
if (built.complete) {
|
||||
storeJson(storage, key, {
|
||||
savedAt: now(),
|
||||
complete: true,
|
||||
partitions: built.partitions,
|
||||
});
|
||||
return { partitions: built.partitions, complete: true };
|
||||
}
|
||||
// Still incomplete: an older complete index beats a partial one, even
|
||||
// past its age; otherwise use the larger partial set for now.
|
||||
if (storedComplete)
|
||||
return { partitions: storedComplete.partitions, complete: false };
|
||||
const partitions =
|
||||
partial && partial.length > built.partitions.length
|
||||
? partial
|
||||
: built.partitions;
|
||||
return { partitions, complete: false };
|
||||
})().then(
|
||||
(result) => {
|
||||
// An incomplete index is used for now and rebuilt after a pause.
|
||||
const current = indexes.get(entry.id);
|
||||
if (current?.promise === promise)
|
||||
current.retryAt = result.complete
|
||||
? null
|
||||
: now() + PARTIAL_INDEX_RETRY_MS;
|
||||
return result.partitions;
|
||||
},
|
||||
(error) => {
|
||||
if (indexes.get(entry.id)?.promise === promise)
|
||||
indexes.delete(entry.id);
|
||||
throw error;
|
||||
}),
|
||||
);
|
||||
return indexes.get(entry.id);
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
indexes.set(entry.id, { promise, retryAt: null });
|
||||
return promise;
|
||||
}
|
||||
|
||||
/** One partition's features, decoded once and kept for a few views. */
|
||||
@@ -174,11 +265,12 @@ export function createLibrarySource({
|
||||
while (partitionCache.size > PARTITION_CACHE_LIMIT)
|
||||
partitionCache.delete(partitionCache.keys().next().value);
|
||||
// Learn where this partition really is, for every later view.
|
||||
const extent = robustExtent(features);
|
||||
if (extent) {
|
||||
const full = fullExtent(features);
|
||||
const trimmed = robustExtent(features);
|
||||
if (full && trimmed) {
|
||||
const key = `${EXTENT_STORAGE_PREFIX}${entry.id}`;
|
||||
const learned = storedJson(storage, key) || {};
|
||||
learned[partition.url] = extent;
|
||||
learned[partition.url] = { full, trimmed };
|
||||
storeJson(storage, key, learned);
|
||||
}
|
||||
return features;
|
||||
@@ -195,11 +287,18 @@ export function createLibrarySource({
|
||||
loadParquet(),
|
||||
partitionIndex(entry, { signal }),
|
||||
]);
|
||||
try {
|
||||
storage?.removeItem?.(`${RETIRED_EXTENT_STORAGE_PREFIX}${entry.id}`);
|
||||
} catch {
|
||||
/* a blocked store keeps a key nobody reads */
|
||||
}
|
||||
const learned =
|
||||
storedJson(storage, `${EXTENT_STORAGE_PREFIX}${entry.id}`) || {};
|
||||
const candidates = partitionsForBox(index, box, Infinity)
|
||||
.filter(({ url }) => !learned[url] || overlaps(learned[url], box))
|
||||
.slice(0, MAX_PARTITION_READS_PER_VIEW);
|
||||
const ranked = rankPartitions(index, box, learned, partitionsForBox);
|
||||
const candidates = ranked.slice(0, MAX_PARTITION_READS_PER_VIEW);
|
||||
// Every partition the box touches but this view leaves unread (the read
|
||||
// budget, or the early stop below) makes the result partial.
|
||||
let unread = ranked.length - candidates.length;
|
||||
const inView = [];
|
||||
for (let start = 0; start < candidates.length; start += PARTITION_BATCH) {
|
||||
const batch = candidates.slice(start, start + PARTITION_BATCH);
|
||||
@@ -217,13 +316,19 @@ export function createLibrarySource({
|
||||
inView.push(...hits);
|
||||
}
|
||||
}
|
||||
if (found) break;
|
||||
if (found) {
|
||||
unread += candidates.length - (start + batch.length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
const bounded = boundLibraryFeatures(inView, {
|
||||
maxFeatures: entry.source.maxFeatures,
|
||||
display: entry.display,
|
||||
});
|
||||
return { features: bounded.features, truncated: bounded.truncated };
|
||||
return {
|
||||
features: bounded.features,
|
||||
truncated: bounded.truncated || unread > 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import * as Cesium from 'cesium';
|
||||
import { createLibraryTileLayer } from './layers.js';
|
||||
import { createLibraryTileLayer, minimumTerrainLevelFor } from './layers.js';
|
||||
import { LIBRARY_ENTRIES, libraryEntry } from './catalog.js';
|
||||
import { libraryTileUrl } from './manifest.js';
|
||||
import { createLibrarySource } from './source.js';
|
||||
@@ -133,3 +133,35 @@ test('bbox and TMS templates become the URLs Cesium and the proxy request', () =
|
||||
'https://t.example.org/3/2/6.png',
|
||||
);
|
||||
});
|
||||
|
||||
test('an entry too dense to draw zoomed out is hidden below its scale', async () => {
|
||||
const cadastre = entryWithoutOutline('kartverket-cadastre');
|
||||
assert.equal(cadastre.source.minZoom, 14);
|
||||
assert.equal(minimumTerrainLevelFor(cadastre), 13);
|
||||
// Small coverage, or a low minimum zoom, draws at every scale.
|
||||
assert.equal(
|
||||
minimumTerrainLevelFor(libraryEntry('lee-county-fl')),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
minimumTerrainLevelFor(libraryEntry('ign-orthophotos-france')),
|
||||
undefined,
|
||||
);
|
||||
|
||||
// Switched on zoomed out, then zoomed further out: no globe tile coarser
|
||||
// than level 13 takes imagery from it, so no z14 tile is ever requested.
|
||||
const viewer = globeViewer();
|
||||
const layer = createLibraryTileLayer(cadastre, { source, cesium: Cesium });
|
||||
await layer.init(viewer);
|
||||
await layer.enable(viewer);
|
||||
const imagery = viewer.imageryLayers.get(viewer.imageryLayers.length - 1);
|
||||
const terrain = new Cesium.EllipsoidTerrainProvider();
|
||||
for (const level of [0, 4, 12])
|
||||
assert.equal(
|
||||
imagery._createTileImagerySkeletons({ level, data: {} }, terrain),
|
||||
false,
|
||||
`globe level ${level} takes no cadastre imagery`,
|
||||
);
|
||||
await layer.disable();
|
||||
assert.equal(viewer.imageryLayers.length, 0);
|
||||
});
|
||||
@@ -48,6 +48,7 @@ export const LEGACY_SHARE_STATE_KEYS = Object.freeze([
|
||||
'layerState',
|
||||
'layerStateInvalid',
|
||||
'panelState',
|
||||
'library',
|
||||
'sharedAtMs',
|
||||
]);
|
||||
|
||||
|
||||
@@ -390,6 +390,9 @@ export const SHARE_VIEW_FIELDS = freezeAll([
|
||||
{ number: 26, key: 'scopeTerminusPct', type: 'uint', default: null },
|
||||
{ number: 27, key: 'styleParams', type: 'bytes', default: BY_STYLE },
|
||||
{ number: 28, key: 'worldTime', type: 'epoch', default: null },
|
||||
// Data Library entries switched on, as their two-character ledger tokens
|
||||
// concatenated (src/library/ledger.json). Absent means none.
|
||||
{ number: 29, key: 'library', type: 'bytes', default: null },
|
||||
]);
|
||||
const FIELD_BY_KEY = new Map(SHARE_VIEW_FIELDS.map((f) => [f.key, f]));
|
||||
const fieldNumber = (key) => FIELD_BY_KEY.get(key).number;
|
||||
@@ -450,6 +453,33 @@ export function createDefaultShareViewState(requested = 'normal') {
|
||||
/* Normalization: the one canonical state both directions agree on. */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/** The most Data Library tokens a share carries (the library's own cap). */
|
||||
export const SHARE_LIBRARY_MAX_TOKENS = 64;
|
||||
const LIBRARY_TOKEN = /^[0-9a-z]{2}$/;
|
||||
|
||||
/**
|
||||
* Data Library share tokens from a `lib=` value (`ab.cd`) or a token list:
|
||||
* unique, two base-36 characters each, at most SHARE_LIBRARY_MAX_TOKENS.
|
||||
* Malformed tokens are skipped. Null or undefined stays null (no field).
|
||||
* @param {string|string[]|null|undefined} value
|
||||
* @returns {string[]|null}
|
||||
*/
|
||||
export function normalizeLibraryTokens(value) {
|
||||
if (value === null || value === undefined) return null;
|
||||
const list = Array.isArray(value)
|
||||
? value
|
||||
: typeof value === 'string' && value.length <= 3 * SHARE_LIBRARY_MAX_TOKENS
|
||||
? value.split('.')
|
||||
: [];
|
||||
const tokens = [];
|
||||
for (const token of list) {
|
||||
if (typeof token !== 'string' || !LIBRARY_TOKEN.test(token)) continue;
|
||||
if (!tokens.includes(token)) tokens.push(token);
|
||||
if (tokens.length === SHARE_LIBRARY_MAX_TOKENS) break;
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
const finite = (value) =>
|
||||
typeof value === 'number'
|
||||
? Number.isFinite(value)
|
||||
@@ -761,6 +791,7 @@ export function normalizeShareViewState(input = {}) {
|
||||
? normalizeLayerState(source.layerState)
|
||||
: null;
|
||||
state.panelState = normalizePanelState(source.panelState);
|
||||
state.library = normalizeLibraryTokens(source.library);
|
||||
state.sharedAtMs = epochMsOr(source.sharedAtMs);
|
||||
state.worldTime = isWorldTime(source.worldTime)
|
||||
? Math.floor(source.worldTime / 1000) * 1000
|
||||
@@ -1030,6 +1061,10 @@ export function encodeShareView(input) {
|
||||
case 'panelState':
|
||||
if (state.panelState) encodePanels(writer, state.panelState);
|
||||
break;
|
||||
case 'library':
|
||||
if (state.library?.length)
|
||||
writer.textField(field.number, state.library.join(''));
|
||||
break;
|
||||
case 'thing':
|
||||
if (thing) encodeThing(writer, thing, state);
|
||||
break;
|
||||
@@ -1438,6 +1473,13 @@ export function decodeShareView(code, { nowMs = Date.now() } = {}) {
|
||||
unknown.code('layer-option', 'without-layers');
|
||||
if (get('panelState'))
|
||||
raw.panelState = decodePanels(get('panelState').len, unknown);
|
||||
if (get('library')) {
|
||||
const text = asciiText(get('library').len.rest());
|
||||
const tokens = text.match(/.{1,2}/g) || [];
|
||||
for (const token of tokens)
|
||||
if (!LIBRARY_TOKEN.test(token)) unknown.code('library', token);
|
||||
raw.library = normalizeLibraryTokens(tokens);
|
||||
}
|
||||
if (get('styleParams') && !retiredStyle)
|
||||
raw.styleParams = decodeStyleParams(
|
||||
get('styleParams').len,
|
||||
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
followOffsetToEnu,
|
||||
layerTokenFromNumber,
|
||||
layerTokenNumber,
|
||||
normalizeLibraryTokens,
|
||||
normalizeShareViewState,
|
||||
} from './viewCodec.js';
|
||||
|
||||
@@ -507,6 +508,7 @@ test('the field registry is well formed and permanent', () => {
|
||||
[26, 'scopeTerminusPct', null],
|
||||
[27, 'styleParams', BY_STYLE],
|
||||
[28, 'worldTime', null],
|
||||
[29, 'library', null],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
@@ -1571,3 +1573,16 @@ test('the follow offset for a camera is the inverse of followCameraFor', () => {
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('Data Library tokens round-trip and malformed ones are skipped', () => {
|
||||
const state = { lat: 1, lon: 2, library: ['00', '1z', '0a'] };
|
||||
const decoded = decodeShareView(encodeShareView(state));
|
||||
assert.equal(decoded.ok, true);
|
||||
assert.deepEqual(decoded.state.library, ['00', '1z', '0a']);
|
||||
// None on: no field, and the decoder reads no tokens.
|
||||
const none = decodeShareView(encodeShareView({ lat: 1, lon: 2 }));
|
||||
assert.equal(none.state.library, null);
|
||||
assert.deepEqual(normalizeLibraryTokens('01.!!.01.zz'), ['01', 'zz']);
|
||||
assert.equal(normalizeLibraryTokens(null), null);
|
||||
assert.deepEqual(normalizeLibraryTokens('x'.repeat(500)), []);
|
||||
});
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
SHARE_VIEW_STYLE_PARAMS,
|
||||
decodeShareView,
|
||||
encodeShareView,
|
||||
normalizeLibraryTokens,
|
||||
} from './share/viewCodec.js';
|
||||
import {
|
||||
fromLegacyShareState,
|
||||
@@ -109,6 +110,10 @@ export class ShareLinkManager {
|
||||
this._mapStack = 'photoreal';
|
||||
this._layerStateProvider = null;
|
||||
this._libraryParamsEncoder = null;
|
||||
this._libraryShareListener = null;
|
||||
// The opening share's Data Library tokens: null when the page opened
|
||||
// without a share, [] for a share that switches no library entry on.
|
||||
this._initialLibraryTokens = null;
|
||||
this._panelStateProvider = null;
|
||||
this._styleParamStateProvider = null;
|
||||
this._initialRestorePending = false;
|
||||
@@ -161,10 +166,32 @@ export class ShareLinkManager {
|
||||
const state =
|
||||
code === null ? this._parseShareHash(hash) : this._parseShareCode(code);
|
||||
if (!state) return null;
|
||||
// A share decides which library entries are on, as it does for core
|
||||
// layers: one without `lib` switches none on.
|
||||
this._initialLibraryTokens = state.library ?? [];
|
||||
this._beginIncomingRestore(state);
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Data Library tokens of the share this page opened with, or null when
|
||||
* it opened without one. Read by the library once it knows its entries.
|
||||
* @returns {string[]|null}
|
||||
*/
|
||||
get initialLibraryTokens() {
|
||||
return this._initialLibraryTokens ? [...this._initialLibraryTokens] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Install the Data Library's listener for shares opened in this tab after
|
||||
* start-up. It receives the share's tokens ([] when it switches none on).
|
||||
* @param {((tokens: string[]) => void) | null} listener
|
||||
*/
|
||||
setLibraryShareListener(listener) {
|
||||
this._libraryShareListener =
|
||||
typeof listener === 'function' ? listener : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The restore state for a share code, in the long form's parsed shape plus
|
||||
* `shareView` (what the view follows, with its offset and last-known fix)
|
||||
@@ -273,6 +300,7 @@ export class ShareLinkManager {
|
||||
params.has('l') &&
|
||||
decodedLayerState === null,
|
||||
panelState: decodePanelStateParams(params),
|
||||
library: normalizeLibraryTokens(params.get('lib')),
|
||||
sharedAtMs: decodeShareCreatedAtMs(params),
|
||||
};
|
||||
return state;
|
||||
@@ -324,6 +352,11 @@ export class ShareLinkManager {
|
||||
if (!state) return;
|
||||
this._beginIncomingRestore(state);
|
||||
onNavigate(state);
|
||||
try {
|
||||
this._libraryShareListener?.(state.library ?? []);
|
||||
} catch (error) {
|
||||
console.warn('[ShareLink] library restore failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -751,6 +784,7 @@ export class ShareLinkManager {
|
||||
this._unwatchNavigation?.();
|
||||
this._layerStateProvider = null;
|
||||
this._libraryParamsEncoder = null;
|
||||
this._libraryShareListener = null;
|
||||
this._panelStateProvider = null;
|
||||
this._styleParamStateProvider = null;
|
||||
this._followSubjectProvider = null;
|
||||
|
||||
@@ -448,3 +448,47 @@ test('a link made away from live carries its world time to the page that opens i
|
||||
assert.equal(liveOpener.parseInitialHash().worldTime, null);
|
||||
liveOpener.destroy();
|
||||
});
|
||||
|
||||
test('a copied link carries the Data Library selection to a clean recipient', (t) => {
|
||||
installTab(t, SHARE_A);
|
||||
const sharer = new ShareLinkManager(fakeViewer());
|
||||
sharer.setLibraryParamsEncoder((params) => params.set('lib', '03.0a'));
|
||||
const code = new URL(sharer.buildLink()).pathname.split('/').at(-1);
|
||||
sharer.destroy();
|
||||
const opener = new ShareLinkManager(fakeViewer(), { initialShareCode: code });
|
||||
assert.deepEqual(opener.parseInitialHash().library, ['03', '0a']);
|
||||
assert.deepEqual(opener.initialLibraryTokens, ['03', '0a']);
|
||||
opener.destroy();
|
||||
|
||||
// A link with every library entry off is a share that switches none on.
|
||||
const none = new ShareLinkManager(fakeViewer());
|
||||
const noneCode = new URL(none.buildLink()).pathname.split('/').at(-1);
|
||||
none.destroy();
|
||||
const noneOpener = new ShareLinkManager(fakeViewer(), {
|
||||
initialShareCode: noneCode,
|
||||
});
|
||||
noneOpener.parseInitialHash();
|
||||
assert.deepEqual(noneOpener.initialLibraryTokens, []);
|
||||
noneOpener.destroy();
|
||||
|
||||
// A page opened without a share leaves the library its stored selection.
|
||||
installTab(t, '');
|
||||
const plain = new ShareLinkManager(fakeViewer());
|
||||
plain.parseInitialHash();
|
||||
assert.equal(plain.initialLibraryTokens, null);
|
||||
plain.destroy();
|
||||
});
|
||||
|
||||
test('a share navigated to in this tab hands its library tokens over', (t) => {
|
||||
const tab = installTab(t, SHARE_A);
|
||||
const manager = new ShareLinkManager(fakeViewer());
|
||||
manager.parseInitialHash();
|
||||
const received = [];
|
||||
manager.setLibraryShareListener((tokens) => received.push(tokens));
|
||||
manager.watchNavigation(() => {});
|
||||
tab.navigate(`${SHARE_B}&lib=01`);
|
||||
tab.navigate(SHARE_A);
|
||||
tab.back();
|
||||
assert.deepEqual(received, [['01'], [], ['01']]);
|
||||
manager.destroy();
|
||||
});
|
||||
+45
-11
@@ -7,7 +7,7 @@
|
||||
* width, height, labels, initial and return focus, Escape and dismissal, and
|
||||
* closes itself when clean view, recording or Cockpit hides the panels.
|
||||
*/
|
||||
import { formatNumber, msg, t, tn } from '../i18n/index.js';
|
||||
import { formatNumber, msg, onLocaleChange, t, tn } from '../i18n/index.js';
|
||||
|
||||
const PANEL_HIDING_MODES = ['ui-clean-view', 'recording-mode', 'cockpit-mode'];
|
||||
/** How long Remove all stays armed after its first press. */
|
||||
@@ -219,6 +219,12 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
node.addEventListener(type, listener);
|
||||
removers.push(() => node.removeEventListener(type, listener));
|
||||
};
|
||||
// Listeners on the dialog's own nodes, released when it is rebuilt.
|
||||
const dialogRemovers = [];
|
||||
const listenDialog = (node, type, listener) => {
|
||||
node.addEventListener(type, listener);
|
||||
dialogRemovers.push(() => node.removeEventListener(type, listener));
|
||||
};
|
||||
const filter = { query: '', family: '', availableOnly: false };
|
||||
let dialog = null;
|
||||
let list = null;
|
||||
@@ -259,7 +265,7 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
);
|
||||
close.type = 'button';
|
||||
close.setAttribute('aria-label', t('Close data library'));
|
||||
listen(close, 'click', () => api.close());
|
||||
listenDialog(close, 'click', () => api.close());
|
||||
header.append(title, close);
|
||||
|
||||
const body = element('div', 'data-library-body');
|
||||
@@ -282,7 +288,8 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
search.type = 'search';
|
||||
search.autocomplete = 'off';
|
||||
search.placeholder = t('Name, provider or country');
|
||||
listen(search, 'input', () => {
|
||||
search.value = filter.query;
|
||||
listenDialog(search, 'input', () => {
|
||||
filter.query = search.value;
|
||||
render();
|
||||
});
|
||||
@@ -291,10 +298,16 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
const familyLabel = element('label', 'data-library-field');
|
||||
familyLabel.append(element('span', '', t('Theme')));
|
||||
const family = element('select', 'panel-surface-control');
|
||||
family.append(new Option(t('All themes'), ''));
|
||||
const option = (text, value) => {
|
||||
const node = element('option', '', text);
|
||||
node.value = value;
|
||||
return node;
|
||||
};
|
||||
family.append(option(t('All themes'), ''));
|
||||
for (const [value, label] of Object.entries(FAMILY_LABELS))
|
||||
family.append(new Option(t(label), value));
|
||||
listen(family, 'change', () => {
|
||||
family.append(option(t(label), value));
|
||||
family.value = filter.family;
|
||||
listenDialog(family, 'change', () => {
|
||||
filter.family = family.value;
|
||||
render();
|
||||
});
|
||||
@@ -303,7 +316,8 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
const availableLabel = element('label', 'data-library-check');
|
||||
const available = element('input');
|
||||
available.type = 'checkbox';
|
||||
listen(available, 'change', () => {
|
||||
available.checked = filter.availableOnly;
|
||||
listenDialog(available, 'change', () => {
|
||||
filter.availableOnly = available.checked;
|
||||
render();
|
||||
});
|
||||
@@ -319,7 +333,7 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
status.setAttribute('aria-live', 'polite');
|
||||
resetButton = element('button', 'panel-surface-control data-library-reset');
|
||||
resetButton.type = 'button';
|
||||
listen(resetButton, 'click', () => {
|
||||
listenDialog(resetButton, 'click', () => {
|
||||
if (resetting) return;
|
||||
if (!resetArmed) {
|
||||
resetArmed = true;
|
||||
@@ -344,7 +358,7 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
toolbar.append(status, resetButton);
|
||||
|
||||
list = element('ul', 'data-library-list');
|
||||
listen(list, 'click', (event) => {
|
||||
listenDialog(list, 'click', (event) => {
|
||||
const button = event.target?.closest?.('button[data-library-action]');
|
||||
if (!button || button.disabled) return;
|
||||
const id = button.dataset.libraryId;
|
||||
@@ -366,13 +380,13 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
|
||||
body.append(stats, intro, filters, toolbar, list);
|
||||
dialog.append(header, body);
|
||||
listen(dialog, 'cancel', (event) => {
|
||||
listenDialog(dialog, 'cancel', (event) => {
|
||||
event.preventDefault();
|
||||
api.close();
|
||||
});
|
||||
// A click on the backdrop (the dialog box itself, outside its content)
|
||||
// dismisses, like Escape.
|
||||
listen(dialog, 'click', (event) => {
|
||||
listenDialog(dialog, 'click', (event) => {
|
||||
if (event.target === dialog) api.close();
|
||||
});
|
||||
document.body.append(dialog);
|
||||
@@ -576,13 +590,33 @@ export function mountLibraryBrowser({ library, trigger }) {
|
||||
destroy() {
|
||||
api.close();
|
||||
unsubscribe();
|
||||
offLocale();
|
||||
observer?.disconnect();
|
||||
for (const remove of removers.splice(0)) remove();
|
||||
for (const remove of dialogRemovers.splice(0)) remove();
|
||||
dialog?.remove();
|
||||
dialog = null;
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Every label is translated when the dialog is built, so a language switch
|
||||
* rebuilds it, keeping its filters and, if it was open, opening it again.
|
||||
*/
|
||||
function relabel() {
|
||||
if (!dialog) return;
|
||||
const wasOpen = dialog.open;
|
||||
disarmReset();
|
||||
clearInterval(statsTimer);
|
||||
statsTimer = null;
|
||||
if (wasOpen) dialog.close();
|
||||
for (const remove of dialogRemovers.splice(0)) remove();
|
||||
dialog.remove();
|
||||
dialog = null;
|
||||
if (wasOpen) api.open();
|
||||
}
|
||||
const offLocale = onLocaleChange(relabel);
|
||||
|
||||
const unsubscribe = library.subscribe(render);
|
||||
if (trigger) listen(trigger, 'click', () => api.open());
|
||||
const observer =
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { installVueDom } from '../testSupport/vueDom.mjs';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
|
||||
test('a language switch relabels the open dialog and keeps its filters', async (context) => {
|
||||
context.after(installVueDom());
|
||||
context.after(() => setLocale('en'));
|
||||
const { mountLibraryBrowser } = await import('./libraryBrowser.js');
|
||||
const library = {
|
||||
ready: true,
|
||||
subscribe: () => () => {},
|
||||
list: () => [],
|
||||
summary: () => ({
|
||||
entries: 0,
|
||||
kinds: { dataset: 0, tiles: 0, pack: 0 },
|
||||
themes: [],
|
||||
available: 0,
|
||||
unavailable: 0,
|
||||
added: 0,
|
||||
on: 0,
|
||||
features: 0,
|
||||
featureLayers: 0,
|
||||
}),
|
||||
reset: async () => 0,
|
||||
};
|
||||
const browser = mountLibraryBrowser({ library, trigger: null });
|
||||
context.after(() => browser.destroy());
|
||||
browser.open();
|
||||
const search = () =>
|
||||
document.querySelector('#data-library input[type="search"]');
|
||||
search().value = 'kartverket';
|
||||
search().dispatchEvent(new Event('input'));
|
||||
assert.equal(
|
||||
document.querySelector('#data-library-title').textContent,
|
||||
'DATA LIBRARY',
|
||||
);
|
||||
|
||||
setLocale('de');
|
||||
const title = document.querySelector('#data-library-title');
|
||||
assert.equal(title.textContent, t('DATA LIBRARY'));
|
||||
assert.notEqual(title.textContent, 'DATA LIBRARY');
|
||||
assert.equal(document.querySelectorAll('#data-library').length, 1);
|
||||
assert.equal(document.querySelector('#data-library').open, true);
|
||||
assert.equal(search().value, 'kartverket');
|
||||
|
||||
// Closed and reopened later, it is still in the new language.
|
||||
browser.close();
|
||||
browser.open();
|
||||
assert.equal(
|
||||
document.querySelector('#data-library-title').textContent,
|
||||
t('DATA LIBRARY'),
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user