mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-13 06:42:23 +00:00
Fix Data Library share restore, dense tile layers and EPC partition reads
- Share links: the compact /share/<code> format carries the switched-on library entries (field 29), so Copy Link keeps them; a share opened in the same tab restores them; any share decides what is on, so one with every library entry off no longer switches on the recipient's stored entries. - A tile entry too dense at its minimum zoom (Norway cadastre z14, depth z11) is hidden below that scale via minimumTerrainLevel instead of allocating millions of tiles when zoomed out. - EPC partitions keep the full extent for exclusion and the 98 % extent for ranking only; the old trimmed-only extents are retired. - Partition index footers that fail transiently are retried; an incomplete index is never cached as complete, a complete prior index is kept, and it is rebuilt after five minutes. - Views that leave touching partitions unread are partial; listing and footer bodies are capped and ranges validated. - The Browse dialog relabels after a language switch.
This commit is contained in:
1 parent
871b7f0fcc
commit
24ccdd4dc3
20 files changed
+1220
-162
No files matched your search
@@ -19,6 +19,8 @@ const DATASET_TIMEOUT_MS = 30_000;
|
||||
const TILE_TIMEOUT_MS = 10_000;
|
||||
/** Each listing page or footer read while building a partition index. */
|
||||
const INDEX_TIMEOUT_MS = 20_000;
|
||||
/** How long an incomplete partition index is served before a rebuild. */
|
||||
const PARTIAL_INDEX_MS = 5 * 60_000;
|
||||
const TILE_MAX_BYTES = 2 * 1024 * 1024;
|
||||
const TILE_CACHE_LIMIT = 512;
|
||||
const IN_FLIGHT_LIMIT = 64;
|
||||
@@ -107,6 +109,7 @@ export function libraryProxy({
|
||||
const datasets = new Map();
|
||||
const tiles = new Map();
|
||||
const indexes = new Map();
|
||||
const partialIndexes = new Map();
|
||||
const inFlight = new Map();
|
||||
const allowData = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
@@ -231,25 +234,51 @@ export function libraryProxy({
|
||||
if (!available) return json(res, 403, { error: reason });
|
||||
if (!allowData(clientKey(req)))
|
||||
return json(res, 429, { error: 'rate_limited' });
|
||||
// Only a complete index is cached for cacheSeconds. An incomplete one
|
||||
// (footers that kept failing) is served as `complete: false` and kept
|
||||
// for PARTIAL_INDEX_MS only, so the next request soon rebuilds it; a
|
||||
// complete index from before keeps being served while rebuilds fail.
|
||||
const key = `index:${id}`;
|
||||
const partial = partialIndexes.get(key);
|
||||
if (partial && now() - partial.savedAt < PARTIAL_INDEX_MS) {
|
||||
const previous = indexes.get(key);
|
||||
return json(res, 200, previous?.value ?? partial.value);
|
||||
}
|
||||
try {
|
||||
const { value } = await cached(
|
||||
indexes,
|
||||
`index:${id}`,
|
||||
key,
|
||||
entry.source.cacheSeconds * 1000,
|
||||
async () => ({
|
||||
partitions: await buildPartitionIndex(entry, {
|
||||
async () => {
|
||||
const built = await buildPartitionIndex(entry, {
|
||||
fetchImpl: (url, options = {}) =>
|
||||
fetchImpl(url, {
|
||||
...options,
|
||||
signal:
|
||||
options.signal ?? AbortSignal.timeout(INDEX_TIMEOUT_MS),
|
||||
}),
|
||||
}),
|
||||
builtAt: now(),
|
||||
}),
|
||||
});
|
||||
const value = {
|
||||
partitions: built.partitions,
|
||||
complete: built.complete,
|
||||
builtAt: now(),
|
||||
};
|
||||
if (!built.complete) {
|
||||
// Recorded here too: with an earlier complete index, cached()
|
||||
// serves that one and swallows this error, and the rebuild
|
||||
// still waits PARTIAL_INDEX_MS.
|
||||
partialIndexes.set(key, { value, savedAt: now() });
|
||||
throw Object.assign(new Error('index_incomplete'), {
|
||||
partialIndex: value,
|
||||
});
|
||||
}
|
||||
partialIndexes.delete(key);
|
||||
return value;
|
||||
},
|
||||
);
|
||||
return json(res, 200, value);
|
||||
} catch (error) {
|
||||
if (error?.partialIndex) return json(res, 200, error.partialIndex);
|
||||
const status = error?.status === 429 ? 429 : 502;
|
||||
return json(res, status, { error: 'library_upstream_unavailable' });
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
parseLibraryTilePath,
|
||||
} from './library.js';
|
||||
import { validateLibraryCatalog } from '../../src/library/manifest.js';
|
||||
import { libraryEntry } from '../../src/library/catalog.js';
|
||||
|
||||
const base = {
|
||||
family: 'events',
|
||||
@@ -100,8 +101,8 @@ const point = (title) => ({
|
||||
properties: { title },
|
||||
});
|
||||
|
||||
function install({ env = {}, fetchImpl, now } = {}) {
|
||||
const plugin = libraryProxy({ entries, env, fetchImpl, now });
|
||||
function install({ env = {}, fetchImpl, now, catalog = entries } = {}) {
|
||||
const plugin = libraryProxy({ entries: catalog, env, fetchImpl, now });
|
||||
let handler;
|
||||
plugin.configurePreviewServer({
|
||||
middlewares: {
|
||||
@@ -330,3 +331,51 @@ test('only GET is served', async () => {
|
||||
await plugin.handler({ url: '/catalog', method: 'POST', headers: {} }, res);
|
||||
assert.equal(res.status, 405);
|
||||
});
|
||||
|
||||
test('a partition index is cached only when complete; a complete one outlives failed rebuilds', async () => {
|
||||
const epc = libraryEntry('epc-england-wales');
|
||||
let clock = 0;
|
||||
let footers = 'ok';
|
||||
let listings = 0;
|
||||
const listing = `<ListBucketResult><Contents><Key>a.parquet</Key><Size>100</Size></Contents></ListBucketResult>`;
|
||||
const fetchImpl = async (url) => {
|
||||
if (url.includes('list-type')) {
|
||||
listings += 1;
|
||||
return new Response(listing);
|
||||
}
|
||||
return footers === 'ok'
|
||||
? new Response(new Uint8Array(100))
|
||||
: new Response('', { status: 503 });
|
||||
};
|
||||
const request = install({ catalog: [epc], now: () => clock, fetchImpl });
|
||||
const first = await request('/index/epc-england-wales');
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(first.body.complete, true);
|
||||
const builtAt = first.body.builtAt;
|
||||
|
||||
// Past cacheSeconds the rebuild fails: the complete index is still served,
|
||||
// and no rebuild runs again until the partial pause has passed.
|
||||
clock += (epc.source.cacheSeconds + 1) * 1000;
|
||||
footers = 'fail';
|
||||
const kept = await request('/index/epc-england-wales');
|
||||
assert.equal(kept.body.complete, true);
|
||||
assert.equal(kept.body.builtAt, builtAt);
|
||||
const listed = listings;
|
||||
await request('/index/epc-england-wales');
|
||||
assert.equal(listings, listed);
|
||||
|
||||
// A server that never had a complete index serves the partial one as such,
|
||||
// and rebuilds after the pause.
|
||||
const fresh = install({ catalog: [epc], now: () => clock, fetchImpl });
|
||||
const partial = await fresh('/index/epc-england-wales');
|
||||
assert.equal(partial.status, 200);
|
||||
assert.equal(partial.body.complete, false);
|
||||
const before = listings;
|
||||
await fresh('/index/epc-england-wales');
|
||||
assert.equal(listings, before);
|
||||
clock += 6 * 60_000;
|
||||
footers = 'ok';
|
||||
const rebuilt = await fresh('/index/epc-england-wales');
|
||||
assert.equal(rebuilt.body.complete, true);
|
||||
assert.equal(listings, before + 1);
|
||||
});
|
||||
Reference in new issue
Block a user