Merge main into feat/voice-core

Keeps the voice-core prompt rules and adds main's world-query and
vessels-vs-visuals rules; the tool catalog wraps the deixis-aware runner.

Also fixes two things the branch had before the merge:
- Live voice could not start: OpenAI rejects track_entity's top-level
  anyOf. Model-facing tools now omit top-level combinators; the runner
  and benchmark still require a query or a referent.
- The Contacts-centred analyst answer dropped the viewport coverage note.
This commit is contained in:
Bilawal Sidhu committed 2026-10-03 23:30:02 -07:00
commit e7cb9b02b5
191 files changed
+20005 -1450

No files matched your search

+25 -1
View File
@@ -1,3 +1,5 @@
import { toFunctionOutput } from '../tools/functions.js';
import { GEV_ACTION_SCHEMAS } from './actionSchemas.js';
import { createGevActionRunner } from './gevActions.js';
import { createVoiceCommands } from './commands.js';
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
@@ -5,6 +7,25 @@ import { createPointerPicker } from './pointerPick.js';
import { createReferentRegistry } from './referents.js';
export * from './realtimeController.js';
const ACTION_NAMES = new Set(GEV_ACTION_SCHEMAS.map((schema) => schema.name));
/**
* Run app actions through `runner` and every other tool the catalog has
* through the catalog. `loadCatalog` resolves the catalog when first needed.
*/
export function withToolCatalog(runner, loadCatalog) {
if (typeof loadCatalog !== 'function') return runner;
return async function runGevTool(name, args, options = {}) {
if (ACTION_NAMES.has(name)) return runner(name, args, options);
const catalog = await loadCatalog();
if (!catalog?.get(name)) return runner(name, args, options);
const result = await catalog.call(name, args ?? {}, {
signal: options.signal,
});
return toFunctionOutput(name, result);
};
}
/** Compose the standalone action runner with the voice controls. */
export function initGevVoiceCommands(options) {
const canvas = options.viewer?.scene?.canvas || null;
@@ -29,7 +50,10 @@ export function initGevVoiceCommands(options) {
...options,
pointer,
referents,
runner: createGevActionRunner({ ...options, deixis }),
runner: withToolCatalog(
createGevActionRunner({ ...options, deixis }),
options.toolCatalog,
),
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();