From ce671ce500a393be27e3cbb2a08799fbca9b6e28 Mon Sep 17 00:00:00 2001 From: Bilawal Sidhu <106619546+bilawalsidhu@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:52:41 -0500 Subject: [PATCH] fix(data): load bundled region packs in the browser; bound region lookups (#743) The Natural Earth and neighborhood packs were loaded with a JSON-attributed import(). The Vite dev server answers that request with a JavaScript module, which the browser rejects, so every analyst region scope failed and fell through to a geocode lookup that could take 12-28 s. - src/data/bundledJson.js fetches the pack from a new URL(..., import.meta.url) asset in the browser; under Node the caller's attributed import runs. - resolveRegionRingForQuery caps the geocode + admin-boundary rung at 3 s and returns { error: 'region-timeout' }; the lookup keeps running and fills the cache. The analyst query reports it as code 'region-timeout'. The Nepal locator opts out of the budget. - Unit tests for the loader, the timeout and the analyst error; the behavior harness checks the Gulf of Mexico scope and retries Texas on region-timeout. Co-authored-by: Bilawal Sidhu --- CHANGELOG.md | 7 +++ scripts/format-scope.json | 1 + scripts/package-boundaries.json | 4 ++ scripts/qa-voice-routing.mjs | 17 +++++- src/annotations/regionRing.test.mjs | 86 +++++++++++++++++++++++++++++ src/annotations/resolver.js | 30 +++++++++- src/browserModuleBoundary.test.mjs | 2 +- src/data/analystEngine.js | 10 +++- src/data/analystEngine.test.mjs | 13 +++++ src/data/bundledJson.js | 29 ++++++++++ src/data/bundledJson.test.mjs | 36 ++++++++++++ src/data/naturalEarthRegions.js | 41 +++++++------- src/data/neighborhoodPolygons.js | 12 ++-- src/standalone/application.js | 2 + src/voice/gevActions.js | 1 + 15 files changed, 263 insertions(+), 28 deletions(-) create mode 100644 src/annotations/regionRing.test.mjs create mode 100644 src/data/bundledJson.js create mode 100644 src/data/bundledJson.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index dfe29059d..79d393162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +- Region scopes in voice analyst queries ("in the Gulf of Mexico", "over + the Alps") work again in the dev server: the bundled Natural Earth and + neighborhood packs are fetched as JSON in the browser + (`src/data/bundledJson.js`). When a region is not in the bundled packs, the + geocode and admin-boundary fallback answers `region-timeout` after 3 s + instead of holding the reply. + - Transit and Directions rows repaint as soon as their data lands again: `refreshLayerStats()` now lives on the layer lifecycle, not only on the compatibility facade. `scripts/qa-radio.mjs` uses it instead of a private diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 0d629d0f3..fb425c09b 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -562,6 +562,7 @@ "src/data/telegeographySubmarineCables.test.mjs", "src/data/naturalEarthRegions.js", "src/data/neighborhoodPolygons.js", + "src/data/bundledJson.js", "src/layers/submarineCables/bundledSource.test.mjs", "src/maps/availability.js", "src/maps/catalog.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 0184b4684..13b758c6e 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1174,6 +1174,7 @@ "modules": [ "src/data/naturalEarthRegions.js", "src/data/neighborhoodPolygons.js", + "src/data/bundledJson.js", "src/data/retryableLoad.js", "src/data/local_data/neighborhoods/san-francisco.json", "src/data/local_data/natural_earth/regions.json", @@ -1408,6 +1409,7 @@ "src/data/regionalBrief.js", "src/data/regionalModel.js", "src/data/renderAltitude.js", + "src/data/bundledJson.js", "src/data/retryableLoad.js", "src/data/rocketLaunches.js", "src/data/routePlausible.js", @@ -2059,6 +2061,7 @@ "src/data/pickRegistry.js", "src/data/radioCountry.js", "src/data/renderAltitude.js", + "src/data/bundledJson.js", "src/data/retryableLoad.js", "src/data/routePlausible.js", "src/data/satelliteClass.js", @@ -2436,6 +2439,7 @@ "src/data/local_data/neighborhoods/san-francisco.json", "src/data/naturalEarthRegions.js", "src/data/neighborhoodPolygons.js", + "src/data/bundledJson.js", "src/data/retryableLoad.js", "src/data/scenePick.js", "src/keylessGeocoder.js", diff --git a/scripts/qa-voice-routing.mjs b/scripts/qa-voice-routing.mjs index 4c5c9dedc..de5f44fec 100644 --- a/scripts/qa-voice-routing.mjs +++ b/scripts/qa-voice-routing.mjs @@ -625,12 +625,27 @@ async function runBehaviorLayer() { // (4b) analyst engine end-to-end: count flights over Texas (region ring // via NE-pack/admin machinery), then a follow-up over the same set. + // A cold boundary lookup can exceed the resolver's budget; it answers + // region-timeout and keeps loading, so asking again must succeed. + let texasTimeouts = 0; r = await run('analyst_query', { layers: ['flights'], scope: { kind: 'region', name: 'Texas' }, limit: 5 }); + while (r?.code === 'region-timeout' && texasTimeouts < 8) { + texasTimeouts += 1; + await settle(5000); + r = await run('analyst_query', { layers: ['flights'], scope: { kind: 'region', name: 'Texas' }, limit: 5 }); + } report(r?.ok === true && Number.isFinite(r?.count) && r.count > 0 && String(r?.coverage?.scope || '').includes('Texas'), - 'behavior: analyst counts flights over Texas', `count=${r?.count} scope=${r?.coverage?.scope} err=${r?.error || ''}`); + 'behavior: analyst counts flights over Texas', `count=${r?.count} scope=${r?.coverage?.scope} timeouts=${texasTimeouts} err=${r?.error || ''}`); r = await run('analyst_query', { followUp: true, filters: [{ field: 'onGround', op: 'eq', value: false }], sortBy: 'altitudeM', limit: 3 }); report(r?.ok === true && r?.coverage?.followUp === true, 'behavior: analyst follow-up re-filters the remembered set', `count=${r?.count} followUp=${r?.coverage?.followUp}`); + // A marine region resolves from the bundled Natural Earth pack in the page, + // without the slower geocode fallback. + const gulfStarted = Date.now(); + r = await run('analyst_query', { layers: ['flights'], scope: { kind: 'region', name: 'Gulf of Mexico' }, limit: 5 }); + const gulfMs = Date.now() - gulfStarted; + report(r?.ok === true && String(r?.coverage?.scope || '').includes('Gulf of Mexico') && gulfMs < 3000, + 'behavior: analyst resolves the Gulf of Mexico from the bundled pack', `scope=${r?.coverage?.scope} ms=${gulfMs} err=${r?.error || ''}`); // (5) zoom_to_globe is ABSOLUTE full-earth (>12,000 km band) r = await run('zoom_to_globe', {}); diff --git a/src/annotations/regionRing.test.mjs b/src/annotations/regionRing.test.mjs new file mode 100644 index 000000000..b84bd861b --- /dev/null +++ b/src/annotations/regionRing.test.mjs @@ -0,0 +1,86 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { FEATURE_SOURCE_METHODS } from '../sources/featureSource.js'; +import { createAnnotationResolver } from './resolver.js'; + +function featureSource(overrides = {}) { + const source = {}; + for (const method of FEATURE_SOURCE_METHODS) + source[method] = async () => null; + return Object.assign(source, overrides); +} + +const texasGeocode = { + async geocode() { + return { + place: { + lat: 31.0, + lng: -99.9, + label: 'Texas, USA', + name: 'Texas', + types: ['administrative_area_level_1', 'political'], + }, + }; + }, +}; + +test('region ring: Natural Earth names resolve without a geocoder', async () => { + const { resolveRegionRingForQuery } = createAnnotationResolver({ + featureSource: featureSource(), + }); + for (const name of ['Gulf of Mexico', 'the Alps']) { + const region = await resolveRegionRingForQuery(name); + assert.ok(region?.ring?.length >= 3, `${name} resolves to a ring`); + assert.equal(region.error, undefined); + } +}); + +test('region ring: a slow admin-boundary lookup returns region-timeout within the budget', async () => { + let adminSignal; + let release; + const { resolveRegionRingForQuery } = createAnnotationResolver({ + featureSource: featureSource({ + getAdministrativeAreas(_point, { signal }) { + adminSignal = signal; + return new Promise((resolve) => { + release = resolve; + }); + }, + }), + }); + const started = Date.now(); + const region = await resolveRegionRingForQuery( + 'Texas', + undefined, + texasGeocode, + { budgetMs: 50 }, + ); + assert.deepEqual(region, { + name: 'Texas', + ring: null, + error: 'region-timeout', + }); + assert.ok( + Date.now() - started < 1000, + 'returns at the budget, not the lookup', + ); + // The lookup is left running so it can fill the boundary cache. + assert.equal(adminSignal?.aborted ?? false, false); + release(null); +}); + +test('region ring: an unbounded budget waits for the lookup', async () => { + const { resolveRegionRingForQuery } = createAnnotationResolver({ + featureSource: featureSource({ + getAdministrativeAreas: () => + new Promise((resolve) => setTimeout(() => resolve(null), 30)), + }), + }); + const region = await resolveRegionRingForQuery( + 'Texas', + undefined, + texasGeocode, + { budgetMs: Infinity }, + ); + assert.equal(region, null); +}); diff --git a/src/annotations/resolver.js b/src/annotations/resolver.js index 3f757159e..214223ff1 100644 --- a/src/annotations/resolver.js +++ b/src/annotations/resolver.js @@ -25,6 +25,9 @@ import { import { unavailablePlaceSearch } from '../search/placeSearch.js'; import { isPickedWorldPosition } from '../data/scenePick.js'; +/** Ms the analyst region lookup waits on geocode + admin boundary. */ +export const REGION_FALLBACK_BUDGET_MS = 3_000; + /** Own annotation lookup caches and ranking of supplied feature candidates. */ export function createAnnotationResolver({ boundarySource, @@ -1877,14 +1880,22 @@ export function createAnnotationResolver({ * resolve to a region-like boundary — the analyst engine reports that * honestly rather than silently scoping to nothing. * + * The geocode + admin-boundary rung is capped at `budgetMs`. Past it the + * call returns `{name, ring: null, error: 'region-timeout'}` so a voice + * answer is not held for tens of seconds; the lookup keeps running and + * fills the boundary cache, so asking again shortly is fast. + * * @param {string} name e.g. "Texas", "the Alps", "France", "Gulf of Mexico" * @param {AbortSignal} [signal] - * @returns {Promise<{name:string, ring:Array<[number,number]>}|null>} + * @param {object} [placeSearch] + * @param {{budgetMs?: number}} [options] `Infinity` waits for the lookup. + * @returns {Promise<{name:string, ring:Array<[number,number]>}|{name:string, ring:null, error:'region-timeout'}|null>} */ async function resolveRegionRingForQuery( name, signal, placeSearch = unavailablePlaceSearch, + { budgetMs = REGION_FALLBACK_BUDGET_MS } = {}, ) { signal = lifetime ? signal @@ -1901,6 +1912,23 @@ export function createAnnotationResolver({ const ring = [...ne.polygons].sort((a, b) => b.length - a.length)[0]; if (ring?.length >= 3) return { name: ne.name, ring }; } + const lookup = resolveAdminRegionRing(q, signal, placeSearch); + if (!Number.isFinite(budgetMs)) return lookup; + let timer; + const timeout = new Promise((resolve) => { + timer = setTimeout( + () => resolve({ name: q, ring: null, error: 'region-timeout' }), + budgetMs, + ); + }); + try { + return await Promise.race([lookup, timeout]); + } finally { + clearTimeout(timer); + } + } + + async function resolveAdminRegionRing(q, signal, placeSearch) { const geo = await geocodePlace(q, null, signal, placeSearch).catch( () => null, ); diff --git a/src/browserModuleBoundary.test.mjs b/src/browserModuleBoundary.test.mjs index d4c1c3bdb..ff883e46f 100644 --- a/src/browserModuleBoundary.test.mjs +++ b/src/browserModuleBoundary.test.mjs @@ -22,7 +22,7 @@ test('no browser-built module imports a Node core module', () => { // import survives the build and turns into a runtime failure the moment the // guard around it is wrong. src/data/naturalEarthRegions.js and // src/data/neighborhoodPolygons.js both carried one to read their bundled - // JSON packs under node:test; an import attribute serves both runtimes. + // JSON packs under node:test; src/data/bundledJson.js now serves both runtimes. const offenders = []; for (const file of browserModules()) { const source = readFileSync(file, 'utf8'); diff --git a/src/data/analystEngine.js b/src/data/analystEngine.js index b77121c12..6016048d3 100644 --- a/src/data/analystEngine.js +++ b/src/data/analystEngine.js @@ -18,7 +18,7 @@ * * Providers (injected — keeps the engine pure and node-testable): * getRecords(layerKey) → Array (layer accessor snapshot) - * resolveRegionRing(name) → Promise<{ring, name}|null> (NE pack / admin boundary) + * resolveRegionRing(name) → Promise<{ring, name}|{error:'region-timeout'}|null> * getViewContext() → {lat, lon, viewRadiusKm, bounds?} (camera-derived) * * @module data/analystEngine @@ -234,6 +234,14 @@ export function createAnalystEngine(providers) { const scope = spec.scope || { kind: 'view' }; if (scope.kind === 'region' && scope.name) { const region = await providers.resolveRegionRing(scope.name); + if (region?.error === 'region-timeout') { + return { + ok: false, + code: 'region-timeout', + error: `Looking up the boundary for "${scope.name}" is taking too long — ask again in a moment.`, + coverage: { layersQueried, scope: `region:${scope.name}:timeout` }, + }; + } if (!region?.ring) { return { ok: false, diff --git a/src/data/analystEngine.test.mjs b/src/data/analystEngine.test.mjs index 72a0d0f5c..2b2e6dae4 100644 --- a/src/data/analystEngine.test.mjs +++ b/src/data/analystEngine.test.mjs @@ -187,6 +187,19 @@ test('analyst: unresolved region is an honest failure, not empty success', async assert.match(r.error, /Atlantis/); }); +test('analyst: a region lookup timeout is reported as region-timeout', async () => { + const eng = createAnalystEngine({ + getRecords: () => FLIGHTS, + resolveRegionRing: async (name) => ({ name, ring: null, error: 'region-timeout' }), + getViewContext: () => ({ lat: 30.27, lon: -97.74, viewRadiusKm: 150 }), + }); + const r = await eng.query({ layers: ['flights'], scope: { kind: 'region', name: 'Texas' } }); + assert.equal(r.ok, false); + assert.equal(r.code, 'region-timeout'); + assert.match(r.error, /Texas/); + assert.equal(r.coverage.scope, 'region:Texas:timeout'); +}); + test('analyst: route fields queryable from cached enrichment only', async () => { const r = await makeEngine().query({ layers: ['flights'], scope: { kind: 'anywhere' }, diff --git a/src/data/bundledJson.js b/src/data/bundledJson.js new file mode 100644 index 000000000..61e98fc14 --- /dev/null +++ b/src/data/bundledJson.js @@ -0,0 +1,29 @@ +/** + * @file Load a JSON file shipped beside the source modules. + * + * The browser fetches the file from `url` — a `new URL('./….json', + * import.meta.url)` written in the calling module, which Vite serves as-is in + * dev and emits as an asset in the production build. Under Node (node:test) + * that URL is a `file:` URL, which `fetch` cannot read, so the caller's + * JSON-attributed import runs instead. + * + * The attributed import cannot serve the browser: the Vite dev server answers + * it with a JavaScript module, which the browser rejects for a JSON import. + * + * @module data/bundledJson + */ + +/** + * @param {URL} url - Resolved against the calling module's `import.meta.url`. + * @param {() => Promise<{default: any}>} importJson - The caller's literal + * `import('./….json', { with: { type: 'json' } })`, used only under Node. + * @returns {Promise} The parsed JSON. + */ +export async function loadBundledJson(url, importJson) { + if (url.protocol === 'file:') return (await importJson()).default; + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HTTP ${response.status} for ${url.pathname}`); + } + return response.json(); +} diff --git a/src/data/bundledJson.test.mjs b/src/data/bundledJson.test.mjs new file mode 100644 index 000000000..2a7dfe293 --- /dev/null +++ b/src/data/bundledJson.test.mjs @@ -0,0 +1,36 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { loadBundledJson } from './bundledJson.js'; + +const MARINE = new URL('./local_data/natural_earth/marine.json', import.meta.url); +const importMarine = () => + import('./local_data/natural_earth/marine.json', { with: { type: 'json' } }); + +test('bundled JSON: a file URL uses the JSON import under Node', async (t) => { + const fetchMock = t.mock.method(globalThis, 'fetch'); + const pack = await loadBundledJson(MARINE, importMarine); + assert.ok(pack.features.some((ft) => ft.name === 'Gulf of Mexico')); + assert.equal(fetchMock.mock.callCount(), 0); +}); + +test('bundled JSON: a served URL is fetched as plain JSON', async (t) => { + const requests = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requests.push(String(url)); + return new Response('{"features":[]}', { + headers: { 'content-type': 'application/json' }, + }); + }); + const url = new URL('http://localhost/src/data/local_data/natural_earth/marine.json'); + const importJson = () => assert.fail('the browser path must not import JSON'); + assert.deepEqual(await loadBundledJson(url, importJson), { features: [] }); + assert.deepEqual(requests, [url.href]); +}); + +test('bundled JSON: an HTTP error rejects so the retryable loader can retry', async (t) => { + t.mock.method(globalThis, 'fetch', async () => new Response('', { status: 404 })); + await assert.rejects( + loadBundledJson(new URL('http://localhost/missing.json'), importMarine), + /HTTP 404/, + ); +}); diff --git a/src/data/naturalEarthRegions.js b/src/data/naturalEarthRegions.js index 891e7e9a1..5eba85231 100644 --- a/src/data/naturalEarthRegions.js +++ b/src/data/naturalEarthRegions.js @@ -11,11 +11,12 @@ * * PURE data module — no Cesium imports, node-testable. The packs are lazy- * loaded on first lookup and cached in module scope (bbox/area computed once - * at load). In the browser Vite bundles the JSON via dynamic import; under - * node the same files are read from disk. A failed load is retried on the - * next lookup rather than cached (see `createRetryableLoader`). + * at load) through `loadBundledJson`, which works in the browser and under + * node:test. A failed load is retried on the next lookup rather than cached + * (see `createRetryableLoader`). */ +import { loadBundledJson } from './bundledJson.js'; import { createRetryableLoader } from './retryableLoad.js'; const EARTH_RADIUS_KM = 6371; @@ -120,20 +121,22 @@ function suffixVariants(norm) { /** @type {Array|null} flat entry list for listRegions() */ let _entries = null; -async function loadPackFile(base) { - // Vite bundles these JSON files as modules; the import attribute is what Node - // needs to load the same files under node:test (same pattern as - // neighborhoodPolygons.js). One path, so no node: import reaches the browser. - const mod = - base === 'regions' - ? await import('./local_data/natural_earth/regions.json', { - with: { type: 'json' }, - }) - : await import('./local_data/natural_earth/marine.json', { - with: { type: 'json' }, - }); - return mod.default || mod; -} +const PACKS = { + regions: { + url: new URL('./local_data/natural_earth/regions.json', import.meta.url), + importJson: () => + import('./local_data/natural_earth/regions.json', { + with: { type: 'json' }, + }), + }, + marine: { + url: new URL('./local_data/natural_earth/marine.json', import.meta.url), + importJson: () => + import('./local_data/natural_earth/marine.json', { + with: { type: 'json' }, + }), + }, +}; function buildEntries(pack, kind) { const out = []; @@ -175,8 +178,8 @@ function buildEntries(pack, kind) { */ const loadIndex = createRetryableLoader(async () => { const [regions, marine] = await Promise.all([ - loadPackFile('regions'), - loadPackFile('marine'), + loadBundledJson(PACKS.regions.url, PACKS.regions.importJson), + loadBundledJson(PACKS.marine.url, PACKS.marine.importJson), ]); _entries = [ ...buildEntries(regions, 'natural'), diff --git a/src/data/neighborhoodPolygons.js b/src/data/neighborhoodPolygons.js index 2861f9318..2a3c8ca87 100644 --- a/src/data/neighborhoodPolygons.js +++ b/src/data/neighborhoodPolygons.js @@ -10,6 +10,7 @@ * touching this module — see `local_data/neighborhoods/SOURCE.md`. */ +import { loadBundledJson } from './bundledJson.js'; import { createRetryableLoader } from './retryableLoad.js'; // bbox = [west, south, east, north]; only load a city file when the point falls in its box. @@ -17,7 +18,11 @@ const CITY_FILES = [ { id: 'san-francisco', bbox: [-122.55, 37.7, -122.35, 37.84], - loader: () => + url: new URL( + './local_data/neighborhoods/san-francisco.json', + import.meta.url, + ), + importJson: () => import('./local_data/neighborhoods/san-francisco.json', { with: { type: 'json' }, }), @@ -107,10 +112,7 @@ function cityLoader(city) { let loader = _cityLoaders.get(city.id); if (!loader) { loader = createRetryableLoader(async () => { - // One path for both runtimes: Vite bundles the JSON as a module, and the - // import attribute is what Node needs to load the same file under node:test. - const mod = await city.loader(); - const fc = mod.default || mod; + const fc = await loadBundledJson(city.url, city.importJson); return Array.isArray(fc.features) ? fc.features : []; }); _cityLoaders.set(city.id, loader); diff --git a/src/standalone/application.js b/src/standalone/application.js index 3aaa9bde8..cbdf9cb7f 100644 --- a/src/standalone/application.js +++ b/src/standalone/application.js @@ -48,6 +48,8 @@ export function createStandaloneApplication({ 'Nepal', signal, placeSearch, + // The locator draws the border whenever it arrives. + { budgetMs: Infinity }, ), signal: context.signal, surface: scene.operations.surface, diff --git a/src/voice/gevActions.js b/src/voice/gevActions.js index df025e867..cfdb1fb0b 100644 --- a/src/voice/gevActions.js +++ b/src/voice/gevActions.js @@ -4323,6 +4323,7 @@ async function runAnalystQuery( return { ok: false, action: 'analyst_query', + ...(result.code ? { code: result.code } : {}), error: result.error, coverage: result.coverage, };