diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index cca71070f..0548e3aae 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -2880,7 +2880,8 @@ "src/ui/kit/UiSelect.js", "src/ui/news/NewsCard.vue", "src/ui/news/NewsFeed.vue", - "src/ui/newsPanel.js" + "src/ui/newsPanel.js", + "src/globe/navara/missingTiles.js" ], "external": [ "hls.js", diff --git a/src/globe/navara/WORKAROUNDS.md b/src/globe/navara/WORKAROUNDS.md index ed4780bb2..3c5f748c8 100644 --- a/src/globe/navara/WORKAROUNDS.md +++ b/src/globe/navara/WORKAROUNDS.md @@ -74,6 +74,7 @@ is fixed. Neither excluded check is a signal to remove a workaround. | `src/globe/navara/viewOptions.js:93`, `src/globe/navara/look.js:172` | Cloud bases are ellipsoid-relative and do not clear local terrain. | 0.2.1 | No canary: needs sampled relief and cloud-base inspection at local scale. | Remove terrain-dependent cloud offsets if native cloud altitude can be terrain-relative. | | `src/globe/navara/credits.js:13` | Content-based deduplication has no holder reference count. | 0.2.1 | No canary: current credit tests stub attribution; they do not test the plugin. | Remove local counts when native removal retains a credit until its last holder releases it. | | `src/globe/navara/credits.js:28` | No separate persistent on-screen credit strip. | 0.2.1 | No canary: UI/API gap; opening the attribution card is only a partial substitute. | Replace forced `attribution.show()` with native on-screen credits. | +| `src/globe/navara/missingTiles.js:28`, `src/globe/navara/maps.js:108` | Raster sources have no tile discard policy, so Esri's grey "Map data not yet available" tiles (200 responses) draw over the parent level. | 0.2.1 | No canary: needs Esri's live placeholder; `missingTiles.test.mjs` covers the fetch wrapper only. | Remove the fetch wrapper when a raster source can reject a loaded tile (discard callback or missing-tile image). | | `src/globe/navara/bloom.js:59` | Native selective bloom uses emissive/effect masks before the transparent pass; the contract needs whole-frame bloom after overlays. | 0.2.1 | No canary: capability gap; custom effect tests verify our implementation. | Replace the custom bloom descriptor with a native effect only if ordering and settings match. | Private-member checks run through `navaraHooks(view)` before engine construction diff --git a/src/globe/navara/index.js b/src/globe/navara/index.js index 8f2346078..674d2fb2e 100644 --- a/src/globe/navara/index.js +++ b/src/globe/navara/index.js @@ -43,10 +43,6 @@ import { resolveNavaraLook, } from './viewOptions.js'; -// Room for tiles in Navara's tile cache, over the screen-sized render -// targets that its budget also counts (fixedGpuBytes). -const TILE_CACHE_ROOM = 768 * 1024 * 1024; - // Navara 0.2.1 has no preserveDrawingBuffer option but takes a canvas whose // WebGL2 context is already created; three then uses that context. function preservedCanvas(container) { @@ -225,11 +221,14 @@ export async function createNavaraGlobe({ container, search, hd = false }) { engine.render.request('navara-look'); }), ]; - // The tile cache keeps tiles that left the view until it reaches its - // budget, by default a quarter of device memory up to 2 GiB: a long - // session grew to about 1.4 GiB of GPU memory. The budget counts the - // screen-sized targets too, which Navara reports after it draws them, so - // it follows them (a fixed total coarsened the view on a 2x screen). + // Navara's tile budget counts the screen-sized render targets too, which + // it reports only after drawing them, so the budget follows them (a fixed + // total coarsened the view on a 2x screen). The room for tiles over them + // is Navara's device default: a quarter of device memory, at most 2 GiB + // (less on phones). An oblique view needs about 0.9 GiB of tiles at + // 960x640 and 1.2 GiB at 1440x900; over budget Navara stops refining, so + // a smaller room left the imagery about two levels coarse. + const tileRoom = view.cacheBytes; let fixedGpuBytes = 0; let budgetedAt = -Infinity; removers.push( @@ -240,7 +239,7 @@ export async function createNavaraGlobe({ container, search, hd = false }) { const fixed = view.memoryStats()?.fixedGpuBytes ?? 0; if (fixed <= 0 || fixed === fixedGpuBytes) return; fixedGpuBytes = fixed; - view.cacheBytes = fixed + TILE_CACHE_ROOM; + view.cacheBytes = fixed + tileRoom; }), ); removeLook = () => removers.forEach((remove) => remove()); diff --git a/src/globe/navara/maps.js b/src/globe/navara/maps.js index 24b45f648..ae0b1317d 100644 --- a/src/globe/navara/maps.js +++ b/src/globe/navara/maps.js @@ -5,7 +5,8 @@ // src/globe/cesium/mapImagery.js and terrain.js, which import Cesium. // // - imagery: OSM and Esri as `raster-tile` layers; Cesium ion imagery (Bing) -// has no Navara path yet and rejects. +// has no Navara path yet and rejects. Esri's grey "no imagery" tiles are +// discarded, as on Cesium (missingTiles.js). // - terrain: the keyless Re:Earth quantized mesh, Cesium World Terrain through // the ion endpoint with an explicit token (Navara sends it as a Bearer // token), or none (the ellipsoid). Navara 0.2.1 panics on a quantized-mesh @@ -23,14 +24,16 @@ import { SENTINEL2_URL, } from '../../maps/sentinel2.js'; import { localTilesetPath } from '../../maps/qaTileset.js'; +import { ESRI_TILE_ROOT, discardMissingEsriTiles } from './missingTiles.js'; const OSM_URL = 'https://tile.openstreetmap.org/{z}/{x}/{y}.png'; +// The deepest level tile.openstreetmap.org serves. +const OSM_MAX_ZOOM = 19; const OSM_CREDIT = { attribution: '© OpenStreetMap contributors', attributionUrl: 'https://www.openstreetmap.org/copyright', }; -const ESRI_URL = - 'https://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}'; +const ESRI_URL = `${ESRI_TILE_ROOT}{z}/{y}/{x}`; const ESRI_CREDIT = { attributionHtml: 'Powered by Esri', @@ -38,6 +41,11 @@ const ESRI_CREDIT = { const KEYLESS_TERRAIN_URL = 'https://terrain.reearth.land/cesium-mesh/ellipsoid/{z}/{x}/{y}.terrain'; const KEYLESS_TERRAIN_MAX_ZOOM = 14; +// The globe drapes imagery on terrain tiles, about one level finer than the +// tile, so terrain tiles that stop at the mesh's level 14 hold imagery at +// level 15: blurry below a few kilometres. Splitting tiles past the mesh's +// data, as Cesium does, lets the imagery refine to street level. +const TERRAIN_OVERSCALED_MAX_ZOOM = 20; const ION_WORLD_TERRAIN_ENDPOINT = 'https://api.cesium.com/v1/assets/1/endpoint?access_token='; const GOOGLE_TILES_ROOT = 'https://tile.googleapis.com/v1/3dtiles/root.json'; @@ -60,7 +68,11 @@ function imageryFor(source) { credit: { attributionHtml: SENTINEL2_CREDIT }, }; case 'osm': - return { url: OSM_URL, credit: credit ?? OSM_CREDIT }; + return { + url: OSM_URL, + maxZoom: OSM_MAX_ZOOM, + credit: credit ?? OSM_CREDIT, + }; case 'esri-imagery': return { url: ESRI_URL, credit: credit ?? ESRI_CREDIT }; case 'ion': @@ -92,6 +104,8 @@ export function createNavaraMaps( now = () => performance.now(), } = {}, ) { + // Esri's "no imagery" tiles fail instead of drawing (missingTiles.js). + const stopDiscardingTiles = discardMissingEsriTiles(); let settleTimer = null; let settleUntil = 0; const stopSettling = () => { @@ -192,6 +206,7 @@ export function createNavaraMaps( type: 'quantized-mesh', url: KEYLESS_TERRAIN_URL, maxZoom: KEYLESS_TERRAIN_MAX_ZOOM, + overscaledMaxZoom: TERRAIN_OVERSCALED_MAX_ZOOM, requestVertexNormals: true, requestWaterMask: true, }; @@ -376,6 +391,7 @@ export function createNavaraMaps( if (destroyed) return; destroyed = true; stopSettling(); + stopDiscardingTiles(); for (const overlay of overlays) { overlay.layer.delete(); overlay.source.delete(); diff --git a/src/globe/navara/maps.test.mjs b/src/globe/navara/maps.test.mjs index 4085b113e..b4d692eab 100644 --- a/src/globe/navara/maps.test.mjs +++ b/src/globe/navara/maps.test.mjs @@ -70,6 +70,7 @@ test('setBaseImagery builds Navara raster layers for OSM and Esri, one at a time view.sources[0].desc.url, 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', ); + assert.equal(view.sources[0].desc.maxZoom, 19); assert.equal(layer.desc.source, view.sources[0]); assert.ok( view.shown.some((item) => /OpenStreetMap/.test(JSON.stringify(item))), @@ -148,6 +149,9 @@ test('setTerrain: keyless Re:Earth mesh, flat ellipsoid, and ion world terrain w 'https://terrain.reearth.land/cesium-mesh/ellipsoid/{z}/{x}/{y}.terrain', ); assert.equal(terrain.desc.source.desc.type, 'quantized-mesh'); + // Tiles split past the mesh's level 14, so imagery can refine beyond 15. + assert.equal(terrain.desc.source.desc.maxZoom, 14); + assert.equal(terrain.desc.source.desc.overscaledMaxZoom, 20); await maps.setTerrain({ kind: 'ellipsoid' }); assert.equal(layersOf(view, 'terrain').length, 0); assert.equal(view.sources.length, 0); diff --git a/src/globe/navara/missingTiles.js b/src/globe/navara/missingTiles.js new file mode 100644 index 000000000..5d04280c4 --- /dev/null +++ b/src/globe/navara/missingTiles.js @@ -0,0 +1,75 @@ +// Where Esri's World Imagery has nothing at a level, it answers with a grey +// "Map data not yet available" JPEG (200) rather than an error: over open +// ocean from level 14, over deserts from about 18. Cesium discards that +// image (DiscardMissingTileImagePolicy) and keeps the parent level; Navara +// 0.2.1 has no discard policy and draws it. While installed, an Esri tile +// whose bytes are that image answers 404 instead, which Navara treats as a +// failed tile, keeping the parent level. Like Cesium, the reference is +// Esri's tile 0/0 at level 23, which never has imagery. + +export const ESRI_TILE_ROOT = + 'https://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/'; +const MISSING_TILE_URL = `${ESRI_TILE_ROOT}23/0/0`; + +const installs = new WeakMap(); + +function sameBytes(a, b) { + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i++) if (a[i] !== b[i]) return false; + return true; +} + +/** + * Wraps `target.fetch` so Esri's missing-imagery tiles answer 404. Returns + * the uninstaller; installs on one target share a single wrapper. + * @param {{ fetch: typeof fetch }} [target] + * @returns {() => void} + */ +export function discardMissingEsriTiles(target = globalThis) { + let install = installs.get(target); + if (!install) { + const realFetch = target.fetch; + // window.fetch throws "Illegal invocation" when called without its window. + const fetchFrom = (input, init) => realFetch.call(target, input, init); + let reference = null; + // The missing-imagery bytes, fetched once; null (tiles pass) on failure, + // tried again on the next tile. + const missingBytes = () => + (reference ??= fetchFrom(MISSING_TILE_URL) + .then((response) => (response.ok ? response.arrayBuffer() : null)) + .then((buffer) => (buffer ? new Uint8Array(buffer) : null)) + .catch(() => null) + .then((bytes) => { + if (!bytes) reference = null; + return bytes; + })); + const wrapped = async (input, init) => { + const response = await fetchFrom(input, init); + const url = + typeof input === 'string' ? input : String(input?.url ?? input); + if (!response.ok || !url.startsWith(ESRI_TILE_ROOT)) return response; + const missing = await missingBytes(); + if ( + !missing || + response.headers.get('content-length') !== String(missing.byteLength) + ) + return response; + const bytes = new Uint8Array(await response.clone().arrayBuffer()); + return sameBytes(bytes, missing) + ? new Response(null, { status: 404, statusText: 'No imagery' }) + : response; + }; + install = { realFetch, wrapped, users: 0 }; + installs.set(target, install); + target.fetch = wrapped; + } + install.users++; + let removed = false; + return () => { + if (removed) return; + removed = true; + if (--install.users > 0) return; + installs.delete(target); + if (target.fetch === install.wrapped) target.fetch = install.realFetch; + }; +} diff --git a/src/globe/navara/missingTiles.test.mjs b/src/globe/navara/missingTiles.test.mjs new file mode 100644 index 000000000..3b3484a3f --- /dev/null +++ b/src/globe/navara/missingTiles.test.mjs @@ -0,0 +1,88 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { ESRI_TILE_ROOT, discardMissingEsriTiles } from './missingTiles.js'; + +const MISSING = new Uint8Array([1, 2, 3, 4, 5]); +const jpeg = (bytes) => + new Response(bytes, { + status: 200, + headers: { + 'content-type': 'image/jpeg', + 'content-length': String(bytes.byteLength), + }, + }); + +// A window-like target whose fetch, like window.fetch, needs its `this`. +function fakeWindow(tiles, { referenceFails = false } = {}) { + const target = { requests: [] }; + target.fetch = function (input) { + assert.equal(this, target, 'fetch called without its window'); + const url = typeof input === 'string' ? input : input.url; + target.requests.push(url); + if (url === `${ESRI_TILE_ROOT}23/0/0`) + return Promise.resolve( + referenceFails ? new Response(null, { status: 503 }) : jpeg(MISSING), + ); + return Promise.resolve(tiles[url] ?? new Response(null, { status: 404 })); + }; + return target; +} + +test("Esri's missing-imagery tile answers 404; real tiles and other hosts pass", async () => { + const missingTile = `${ESRI_TILE_ROOT}18/1/2`; + const sameLength = `${ESRI_TILE_ROOT}18/1/3`; + const real = `${ESRI_TILE_ROOT}18/1/4`; + const other = 'https://tile.openstreetmap.org/18/1/2.png'; + const target = fakeWindow({ + [missingTile]: jpeg(MISSING), + [sameLength]: jpeg(new Uint8Array([1, 2, 3, 4, 6])), + [real]: jpeg(new Uint8Array(40)), + [other]: jpeg(MISSING), + }); + const uninstall = discardMissingEsriTiles(target); + assert.equal((await target.fetch(missingTile)).status, 404); + const kept = await target.fetch(sameLength); + assert.equal(kept.status, 200); + assert.deepEqual( + new Uint8Array(await kept.arrayBuffer()), + new Uint8Array([1, 2, 3, 4, 6]), + ); + assert.equal((await target.fetch(new Request(real))).status, 200); + assert.equal((await target.fetch(other)).status, 200); + assert.equal( + target.requests.filter((url) => url.endsWith('23/0/0')).length, + 1, + 'the reference is fetched once', + ); + uninstall(); +}); + +test('without the reference image Esri tiles pass, and it is tried again', async () => { + const missingTile = `${ESRI_TILE_ROOT}18/1/2`; + const target = fakeWindow( + { [missingTile]: jpeg(MISSING) }, + { referenceFails: true }, + ); + const uninstall = discardMissingEsriTiles(target); + assert.equal((await target.fetch(missingTile)).status, 200); + assert.equal((await target.fetch(missingTile)).status, 200); + assert.equal( + target.requests.filter((url) => url.endsWith('23/0/0')).length, + 2, + ); + uninstall(); +}); + +test('installs share one wrapper; the last uninstall restores fetch', () => { + const target = fakeWindow({}); + const original = target.fetch; + const first = discardMissingEsriTiles(target); + const wrapped = target.fetch; + const second = discardMissingEsriTiles(target); + assert.equal(target.fetch, wrapped); + first(); + first(); + assert.equal(target.fetch, wrapped); + second(); + assert.equal(target.fetch, original); +});