mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
Merge main into feat/voice-core
Keeps the voice-core prompt rules and adds main's world-query and vessels-vs-visuals rules; the tool catalog wraps the deixis-aware runner. Also fixes two things the branch had before the merge: - Live voice could not start: OpenAI rejects track_entity's top-level anyOf. Model-facing tools now omit top-level combinators; the runner and benchmark still require a query or a referent. - The Contacts-centred analyst answer dropped the viewport coverage note.
This commit is contained in:
commit
e7cb9b02b5
191 files changed
+20005
-1450
No files matched your search
@@ -1,3 +1,5 @@
|
||||
import { toFunctionOutput } from '../tools/functions.js';
|
||||
import { GEV_ACTION_SCHEMAS } from './actionSchemas.js';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
import { createVoiceCommands } from './commands.js';
|
||||
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
|
||||
@@ -5,6 +7,25 @@ import { createPointerPicker } from './pointerPick.js';
|
||||
import { createReferentRegistry } from './referents.js';
|
||||
export * from './realtimeController.js';
|
||||
|
||||
const ACTION_NAMES = new Set(GEV_ACTION_SCHEMAS.map((schema) => schema.name));
|
||||
|
||||
/**
|
||||
* Run app actions through `runner` and every other tool the catalog has
|
||||
* through the catalog. `loadCatalog` resolves the catalog when first needed.
|
||||
*/
|
||||
export function withToolCatalog(runner, loadCatalog) {
|
||||
if (typeof loadCatalog !== 'function') return runner;
|
||||
return async function runGevTool(name, args, options = {}) {
|
||||
if (ACTION_NAMES.has(name)) return runner(name, args, options);
|
||||
const catalog = await loadCatalog();
|
||||
if (!catalog?.get(name)) return runner(name, args, options);
|
||||
const result = await catalog.call(name, args ?? {}, {
|
||||
signal: options.signal,
|
||||
});
|
||||
return toFunctionOutput(name, result);
|
||||
};
|
||||
}
|
||||
|
||||
/** Compose the standalone action runner with the voice controls. */
|
||||
export function initGevVoiceCommands(options) {
|
||||
const canvas = options.viewer?.scene?.canvas || null;
|
||||
@@ -29,7 +50,10 @@ export function initGevVoiceCommands(options) {
|
||||
...options,
|
||||
pointer,
|
||||
referents,
|
||||
runner: createGevActionRunner({ ...options, deixis }),
|
||||
runner: withToolCatalog(
|
||||
createGevActionRunner({ ...options, deixis }),
|
||||
options.toolCatalog,
|
||||
),
|
||||
});
|
||||
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
|
||||
deixis.onPointerUsed = () => commands.announcePointer?.();
|
||||
|
||||
Reference in new issue
Block a user