fix(data): load bundled region packs in the browser; bound region lookups (#743)

The Natural Earth and neighborhood packs were loaded with a JSON-attributed
import(). The Vite dev server answers that request with a JavaScript module,
which the browser rejects, so every analyst region scope failed and fell
through to a geocode lookup that could take 12-28 s.

- src/data/bundledJson.js fetches the pack from a new URL(..., import.meta.url)
  asset in the browser; under Node the caller's attributed import runs.
- resolveRegionRingForQuery caps the geocode + admin-boundary rung at 3 s and
  returns { error: 'region-timeout' }; the lookup keeps running and fills the
  cache. The analyst query reports it as code 'region-timeout'. The Nepal
  locator opts out of the budget.
- Unit tests for the loader, the timeout and the analyst error; the behavior
  harness checks the Gulf of Mexico scope and retries Texas on region-timeout.

Co-authored-by: Bilawal Sidhu <[email protected]>

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+- Region scopes in voice analyst queries ("in the Gulf of Mexico", "over
+  the Alps") work again in the dev server: the bundled Natural Earth and
+  neighborhood packs are fetched as JSON in the browser
+  (`src/data/bundledJson.js`). When a region is not in the bundled packs, the
+  geocode and admin-boundary fallback answers `region-timeout` after 3 s
+  instead of holding the reply.
+
 - Transit and Directions rows repaint as soon as their data lands again:
   `refreshLayerStats()` now lives on the layer lifecycle, not only on the
   compatibility facade. `scripts/qa-radio.mjs` uses it instead of a private

scripts/format-scope.json

@@ -562,6 +562,7 @@
   "src/data/telegeographySubmarineCables.test.mjs",
   "src/data/naturalEarthRegions.js",
   "src/data/neighborhoodPolygons.js",
+  "src/data/bundledJson.js",
   "src/layers/submarineCables/bundledSource.test.mjs",
   "src/maps/availability.js",
   "src/maps/catalog.js",

scripts/package-boundaries.json

@@ -1174,6 +1174,7 @@
     "modules": [
       "src/data/naturalEarthRegions.js",
       "src/data/neighborhoodPolygons.js",
+      "src/data/bundledJson.js",
       "src/data/retryableLoad.js",
       "src/data/local_data/neighborhoods/san-francisco.json",
       "src/data/local_data/natural_earth/regions.json",
@@ -1408,6 +1409,7 @@
       "src/data/regionalBrief.js",
       "src/data/regionalModel.js",
       "src/data/renderAltitude.js",
+      "src/data/bundledJson.js",
       "src/data/retryableLoad.js",
       "src/data/rocketLaunches.js",
       "src/data/routePlausible.js",
@@ -2059,6 +2061,7 @@
       "src/data/pickRegistry.js",
       "src/data/radioCountry.js",
       "src/data/renderAltitude.js",
+      "src/data/bundledJson.js",
       "src/data/retryableLoad.js",
       "src/data/routePlausible.js",
       "src/data/satelliteClass.js",
@@ -2436,6 +2439,7 @@
       "src/data/local_data/neighborhoods/san-francisco.json",
       "src/data/naturalEarthRegions.js",
       "src/data/neighborhoodPolygons.js",
+      "src/data/bundledJson.js",
       "src/data/retryableLoad.js",
       "src/data/scenePick.js",
       "src/keylessGeocoder.js",

scripts/qa-voice-routing.mjs

@@ -625,12 +625,27 @@ async function runBehaviorLayer() {
 
     // (4b) analyst engine end-to-end: count flights over Texas (region ring
     // via NE-pack/admin machinery), then a follow-up over the same set.
+    // A cold boundary lookup can exceed the resolver's budget; it answers
+    // region-timeout and keeps loading, so asking again must succeed.
+    let texasTimeouts = 0;
     r = await run('analyst_query', { layers: ['flights'], scope: { kind: 'region', name: 'Texas' }, limit: 5 });
+    while (r?.code === 'region-timeout' && texasTimeouts < 8) {
+      texasTimeouts += 1;
+      await settle(5000);
+      r = await run('analyst_query', { layers: ['flights'], scope: { kind: 'region', name: 'Texas' }, limit: 5 });
+    }
     report(r?.ok === true && Number.isFinite(r?.count) && r.count > 0 && String(r?.coverage?.scope || '').includes('Texas'),
-      'behavior: analyst counts flights over Texas', `count=${r?.count} scope=${r?.coverage?.scope} err=${r?.error || ''}`);
+      'behavior: analyst counts flights over Texas', `count=${r?.count} scope=${r?.coverage?.scope} timeouts=${texasTimeouts} err=${r?.error || ''}`);
     r = await run('analyst_query', { followUp: true, filters: [{ field: 'onGround', op: 'eq', value: false }], sortBy: 'altitudeM', limit: 3 });
     report(r?.ok === true && r?.coverage?.followUp === true,
       'behavior: analyst follow-up re-filters the remembered set', `count=${r?.count} followUp=${r?.coverage?.followUp}`);
+    // A marine region resolves from the bundled Natural Earth pack in the page,
+    // without the slower geocode fallback.
+    const gulfStarted = Date.now();
+    r = await run('analyst_query', { layers: ['flights'], scope: { kind: 'region', name: 'Gulf of Mexico' }, limit: 5 });
+    const gulfMs = Date.now() - gulfStarted;
+    report(r?.ok === true && String(r?.coverage?.scope || '').includes('Gulf of Mexico') && gulfMs < 3000,
+      'behavior: analyst resolves the Gulf of Mexico from the bundled pack', `scope=${r?.coverage?.scope} ms=${gulfMs} err=${r?.error || ''}`);
 
     // (5) zoom_to_globe is ABSOLUTE full-earth (>12,000 km band)
     r = await run('zoom_to_globe', {});

src/annotations/regionRing.test.mjs

@@ -0,0 +1,86 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { FEATURE_SOURCE_METHODS } from '../sources/featureSource.js';
+import { createAnnotationResolver } from './resolver.js';
+
+function featureSource(overrides = {}) {
+  const source = {};
+  for (const method of FEATURE_SOURCE_METHODS)
+    source[method] = async () => null;
+  return Object.assign(source, overrides);
+}
+
+const texasGeocode = {
+  async geocode() {
+    return {
+      place: {
+        lat: 31.0,
+        lng: -99.9,
+        label: 'Texas, USA',
+        name: 'Texas',
+        types: ['administrative_area_level_1', 'political'],
+      },
+    };
+  },
+};
+
+test('region ring: Natural Earth names resolve without a geocoder', async () => {
+  const { resolveRegionRingForQuery } = createAnnotationResolver({
+    featureSource: featureSource(),
+  });
+  for (const name of ['Gulf of Mexico', 'the Alps']) {
+    const region = await resolveRegionRingForQuery(name);
+    assert.ok(region?.ring?.length >= 3, `${name} resolves to a ring`);
+    assert.equal(region.error, undefined);
+  }
+});
+
+test('region ring: a slow admin-boundary lookup returns region-timeout within the budget', async () => {
+  let adminSignal;
+  let release;
+  const { resolveRegionRingForQuery } = createAnnotationResolver({
+    featureSource: featureSource({
+      getAdministrativeAreas(_point, { signal }) {
+        adminSignal = signal;
+        return new Promise((resolve) => {
+          release = resolve;
+        });
+      },
+    }),
+  });
+  const started = Date.now();
+  const region = await resolveRegionRingForQuery(
+    'Texas',
+    undefined,
+    texasGeocode,
+    { budgetMs: 50 },
+  );
+  assert.deepEqual(region, {
+    name: 'Texas',
+    ring: null,
+    error: 'region-timeout',
+  });
+  assert.ok(
+    Date.now() - started < 1000,
+    'returns at the budget, not the lookup',
+  );
+  // The lookup is left running so it can fill the boundary cache.
+  assert.equal(adminSignal?.aborted ?? false, false);
+  release(null);
+});
+
+test('region ring: an unbounded budget waits for the lookup', async () => {
+  const { resolveRegionRingForQuery } = createAnnotationResolver({
+    featureSource: featureSource({
+      getAdministrativeAreas: () =>
+        new Promise((resolve) => setTimeout(() => resolve(null), 30)),
+    }),
+  });
+  const region = await resolveRegionRingForQuery(
+    'Texas',
+    undefined,
+    texasGeocode,
+    { budgetMs: Infinity },
+  );
+  assert.equal(region, null);
+});

src/annotations/resolver.js

@@ -25,6 +25,9 @@ import {
 import { unavailablePlaceSearch } from '../search/placeSearch.js';
 import { isPickedWorldPosition } from '../data/scenePick.js';
 
+/** Ms the analyst region lookup waits on geocode + admin boundary. */
+export const REGION_FALLBACK_BUDGET_MS = 3_000;
+
 /** Own annotation lookup caches and ranking of supplied feature candidates. */
 export function createAnnotationResolver({
   boundarySource,
@@ -1877,14 +1880,22 @@ export function createAnnotationResolver({
    * resolve to a region-like boundary — the analyst engine reports that
    * honestly rather than silently scoping to nothing.
    *
+   * The geocode + admin-boundary rung is capped at `budgetMs`. Past it the
+   * call returns `{name, ring: null, error: 'region-timeout'}` so a voice
+   * answer is not held for tens of seconds; the lookup keeps running and
+   * fills the boundary cache, so asking again shortly is fast.
+   *
    * @param {string} name  e.g. "Texas", "the Alps", "France", "Gulf of Mexico"
    * @param {AbortSignal} [signal]
-   * @returns {Promise<{name:string, ring:Array<[number,number]>}|null>}
+   * @param {object} [placeSearch]
+   * @param {{budgetMs?: number}} [options]  `Infinity` waits for the lookup.
+   * @returns {Promise<{name:string, ring:Array<[number,number]>}|{name:string, ring:null, error:'region-timeout'}|null>}
    */
   async function resolveRegionRingForQuery(
     name,
     signal,
     placeSearch = unavailablePlaceSearch,
+    { budgetMs = REGION_FALLBACK_BUDGET_MS } = {},
   ) {
     signal = lifetime
       ? signal
@@ -1901,6 +1912,23 @@ export function createAnnotationResolver({
       const ring = [...ne.polygons].sort((a, b) => b.length - a.length)[0];
       if (ring?.length >= 3) return { name: ne.name, ring };
     }
+    const lookup = resolveAdminRegionRing(q, signal, placeSearch);
+    if (!Number.isFinite(budgetMs)) return lookup;
+    let timer;
+    const timeout = new Promise((resolve) => {
+      timer = setTimeout(
+        () => resolve({ name: q, ring: null, error: 'region-timeout' }),
+        budgetMs,
+      );
+    });
+    try {
+      return await Promise.race([lookup, timeout]);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async function resolveAdminRegionRing(q, signal, placeSearch) {
     const geo = await geocodePlace(q, null, signal, placeSearch).catch(
       () => null,
     );

src/browserModuleBoundary.test.mjs

@@ -22,7 +22,7 @@ test('no browser-built module imports a Node core module', () => {
   // import survives the build and turns into a runtime failure the moment the
   // guard around it is wrong. src/data/naturalEarthRegions.js and
   // src/data/neighborhoodPolygons.js both carried one to read their bundled
-  // JSON packs under node:test; an import attribute serves both runtimes.
+  // JSON packs under node:test; src/data/bundledJson.js now serves both runtimes.
   const offenders = [];
   for (const file of browserModules()) {
     const source = readFileSync(file, 'utf8');

src/data/analystEngine.js

@@ -18,7 +18,7 @@
  *
  * Providers (injected — keeps the engine pure and node-testable):
  *   getRecords(layerKey) → Array<record>            (layer accessor snapshot)
- *   resolveRegionRing(name) → Promise<{ring, name}|null>  (NE pack / admin boundary)
+ *   resolveRegionRing(name) → Promise<{ring, name}|{error:'region-timeout'}|null>
  *   getViewContext() → {lat, lon, viewRadiusKm, bounds?}  (camera-derived)
  *
  * @module data/analystEngine
@@ -234,6 +234,14 @@ export function createAnalystEngine(providers) {
     const scope = spec.scope || { kind: 'view' };
     if (scope.kind === 'region' && scope.name) {
       const region = await providers.resolveRegionRing(scope.name);
+      if (region?.error === 'region-timeout') {
+        return {
+          ok: false,
+          code: 'region-timeout',
+          error: `Looking up the boundary for "${scope.name}" is taking too long — ask again in a moment.`,
+          coverage: { layersQueried, scope: `region:${scope.name}:timeout` },
+        };
+      }
       if (!region?.ring) {
         return {
           ok: false,

src/data/analystEngine.test.mjs

@@ -187,6 +187,19 @@ test('analyst: unresolved region is an honest failure, not empty success', async
   assert.match(r.error, /Atlantis/);
 });
 
+test('analyst: a region lookup timeout is reported as region-timeout', async () => {
+  const eng = createAnalystEngine({
+    getRecords: () => FLIGHTS,
+    resolveRegionRing: async (name) => ({ name, ring: null, error: 'region-timeout' }),
+    getViewContext: () => ({ lat: 30.27, lon: -97.74, viewRadiusKm: 150 }),
+  });
+  const r = await eng.query({ layers: ['flights'], scope: { kind: 'region', name: 'Texas' } });
+  assert.equal(r.ok, false);
+  assert.equal(r.code, 'region-timeout');
+  assert.match(r.error, /Texas/);
+  assert.equal(r.coverage.scope, 'region:Texas:timeout');
+});
+
 test('analyst: route fields queryable from cached enrichment only', async () => {
   const r = await makeEngine().query({
     layers: ['flights'], scope: { kind: 'anywhere' },

src/data/bundledJson.js

@@ -0,0 +1,29 @@
+/**
+ * @file Load a JSON file shipped beside the source modules.
+ *
+ * The browser fetches the file from `url` — a `new URL('./….json',
+ * import.meta.url)` written in the calling module, which Vite serves as-is in
+ * dev and emits as an asset in the production build. Under Node (node:test)
+ * that URL is a `file:` URL, which `fetch` cannot read, so the caller's
+ * JSON-attributed import runs instead.
+ *
+ * The attributed import cannot serve the browser: the Vite dev server answers
+ * it with a JavaScript module, which the browser rejects for a JSON import.
+ *
+ * @module data/bundledJson
+ */
+
+/**
+ * @param {URL} url - Resolved against the calling module's `import.meta.url`.
+ * @param {() => Promise<{default: any}>} importJson - The caller's literal
+ *   `import('./….json', { with: { type: 'json' } })`, used only under Node.
+ * @returns {Promise<any>} The parsed JSON.
+ */
+export async function loadBundledJson(url, importJson) {
+  if (url.protocol === 'file:') return (await importJson()).default;
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} for ${url.pathname}`);
+  }
+  return response.json();
+}

src/data/bundledJson.test.mjs

@@ -0,0 +1,36 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { loadBundledJson } from './bundledJson.js';
+
+const MARINE = new URL('./local_data/natural_earth/marine.json', import.meta.url);
+const importMarine = () =>
+  import('./local_data/natural_earth/marine.json', { with: { type: 'json' } });
+
+test('bundled JSON: a file URL uses the JSON import under Node', async (t) => {
+  const fetchMock = t.mock.method(globalThis, 'fetch');
+  const pack = await loadBundledJson(MARINE, importMarine);
+  assert.ok(pack.features.some((ft) => ft.name === 'Gulf of Mexico'));
+  assert.equal(fetchMock.mock.callCount(), 0);
+});
+
+test('bundled JSON: a served URL is fetched as plain JSON', async (t) => {
+  const requests = [];
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    requests.push(String(url));
+    return new Response('{"features":[]}', {
+      headers: { 'content-type': 'application/json' },
+    });
+  });
+  const url = new URL('http://localhost/src/data/local_data/natural_earth/marine.json');
+  const importJson = () => assert.fail('the browser path must not import JSON');
+  assert.deepEqual(await loadBundledJson(url, importJson), { features: [] });
+  assert.deepEqual(requests, [url.href]);
+});
+
+test('bundled JSON: an HTTP error rejects so the retryable loader can retry', async (t) => {
+  t.mock.method(globalThis, 'fetch', async () => new Response('', { status: 404 }));
+  await assert.rejects(
+    loadBundledJson(new URL('http://localhost/missing.json'), importMarine),
+    /HTTP 404/,
+  );
+});

src/data/naturalEarthRegions.js

@@ -11,11 +11,12 @@
  *
  * PURE data module — no Cesium imports, node-testable. The packs are lazy-
  * loaded on first lookup and cached in module scope (bbox/area computed once
- * at load). In the browser Vite bundles the JSON via dynamic import; under
- * node the same files are read from disk. A failed load is retried on the
- * next lookup rather than cached (see `createRetryableLoader`).
+ * at load) through `loadBundledJson`, which works in the browser and under
+ * node:test. A failed load is retried on the next lookup rather than cached
+ * (see `createRetryableLoader`).
  */
 
+import { loadBundledJson } from './bundledJson.js';
 import { createRetryableLoader } from './retryableLoad.js';
 
 const EARTH_RADIUS_KM = 6371;
@@ -120,20 +121,22 @@ function suffixVariants(norm) {
 /** @type {Array|null} flat entry list for listRegions() */
 let _entries = null;
 
-async function loadPackFile(base) {
-  // Vite bundles these JSON files as modules; the import attribute is what Node
-  // needs to load the same files under node:test (same pattern as
-  // neighborhoodPolygons.js). One path, so no node: import reaches the browser.
-  const mod =
-    base === 'regions'
-      ? await import('./local_data/natural_earth/regions.json', {
-          with: { type: 'json' },
-        })
-      : await import('./local_data/natural_earth/marine.json', {
-          with: { type: 'json' },
-        });
-  return mod.default || mod;
-}
+const PACKS = {
+  regions: {
+    url: new URL('./local_data/natural_earth/regions.json', import.meta.url),
+    importJson: () =>
+      import('./local_data/natural_earth/regions.json', {
+        with: { type: 'json' },
+      }),
+  },
+  marine: {
+    url: new URL('./local_data/natural_earth/marine.json', import.meta.url),
+    importJson: () =>
+      import('./local_data/natural_earth/marine.json', {
+        with: { type: 'json' },
+      }),
+  },
+};
 
 function buildEntries(pack, kind) {
   const out = [];
@@ -175,8 +178,8 @@ function buildEntries(pack, kind) {
  */
 const loadIndex = createRetryableLoader(async () => {
   const [regions, marine] = await Promise.all([
-    loadPackFile('regions'),
-    loadPackFile('marine'),
+    loadBundledJson(PACKS.regions.url, PACKS.regions.importJson),
+    loadBundledJson(PACKS.marine.url, PACKS.marine.importJson),
   ]);
   _entries = [
     ...buildEntries(regions, 'natural'),

src/data/neighborhoodPolygons.js

@@ -10,6 +10,7 @@
  * touching this module — see `local_data/neighborhoods/SOURCE.md`.
  */
 
+import { loadBundledJson } from './bundledJson.js';
 import { createRetryableLoader } from './retryableLoad.js';
 
 // bbox = [west, south, east, north]; only load a city file when the point falls in its box.
@@ -17,7 +18,11 @@ const CITY_FILES = [
   {
     id: 'san-francisco',
     bbox: [-122.55, 37.7, -122.35, 37.84],
-    loader: () =>
+    url: new URL(
+      './local_data/neighborhoods/san-francisco.json',
+      import.meta.url,
+    ),
+    importJson: () =>
       import('./local_data/neighborhoods/san-francisco.json', {
         with: { type: 'json' },
       }),
@@ -107,10 +112,7 @@ function cityLoader(city) {
   let loader = _cityLoaders.get(city.id);
   if (!loader) {
     loader = createRetryableLoader(async () => {
-      // One path for both runtimes: Vite bundles the JSON as a module, and the
-      // import attribute is what Node needs to load the same file under node:test.
-      const mod = await city.loader();
-      const fc = mod.default || mod;
+      const fc = await loadBundledJson(city.url, city.importJson);
       return Array.isArray(fc.features) ? fc.features : [];
     });
     _cityLoaders.set(city.id, loader);

src/standalone/application.js

@@ -48,6 +48,8 @@ export function createStandaloneApplication({
             'Nepal',
             signal,
             placeSearch,
+            // The locator draws the border whenever it arrives.
+            { budgetMs: Infinity },
           ),
         signal: context.signal,
         surface: scene.operations.surface,

src/voice/gevActions.js

@@ -4323,6 +4323,7 @@ async function runAnalystQuery(
     return {
       ok: false,
       action: 'analyst_query',
+      ...(result.code ? { code: result.code } : {}),
       error: result.error,
       coverage: result.coverage,
     };