feat: share the tool catalog with voice

src/standalone/toolCatalog.js

@@ -0,0 +1,22 @@
+/** The standalone tool catalog for voice, loaded the first time voice needs it. */
+
+let pending = null;
+
+export function loadToolCatalog() {
+  pending ??= Promise.all([
+    import('../tools/index.js'),
+    import('../tools/services.js'),
+  ]).then(([{ composeCatalog, coreTools }, { createToolServices }]) =>
+    composeCatalog({
+      tools: coreTools,
+      services: createToolServices({
+        fetchImpl: (...args) => globalThis.fetch(...args),
+        appUrl: globalThis.location.origin,
+      }),
+    }),
+  );
+  pending.catch(() => {
+    pending = null;
+  });
+  return pending;
+}

src/standalone/tools.js

@@ -1,6 +1,7 @@
 import { createAssetDirectorySource } from '../director/packs/source.js';
 import { createApplicationTools } from '../app/tools.js';
 import { startStandaloneChrome } from './startupChrome.js';
+import { loadToolCatalog } from './toolCatalog.js';
 export function createStandaloneTools(options) {
   return createApplicationTools({
     startChrome: startStandaloneChrome,
@@ -12,5 +13,6 @@ export function createStandaloneTools(options) {
       },
     },
     ...options,
+    voice: { toolCatalog: loadToolCatalog, ...options?.voice },
   });
 }

src/tooling/localServices.test.mjs

@@ -21,6 +21,7 @@ import { openAiRealtimeProxy } from 'gods-eye-view/server/providers/openai';
 import { keySetupEndpoint } from 'gods-eye-view/server/standalone/key-setup';
 import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
 import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
+import { standaloneVoiceTools } from '../../server/standalone/voiceTools.js';
 
 function install(plugin, preview = false) {
   const routes = new Map();
@@ -195,6 +196,38 @@ test('Realtime handler preserves tools and default instructions, isolates suppli
   assert.equal(sent[0].session.instructions, sent[2].session.instructions);
 });
 
+test('Realtime sessions carry supplied tools, and the standalone voice adds the catalog queries', async (t) => {
+  env(t, 'OPENAI_API_KEY', 'fixture-upstream-secret');
+  env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined);
+  const sent = [];
+  t.mock.method(globalThis, 'fetch', async (url, options) => {
+    sent.push(JSON.parse(options.body));
+    return Response.json({ value: 'fixture-ephemeral' });
+  });
+  const tools = standaloneVoiceTools();
+  const response = await request(
+    install(openAiRealtimeProxy({ realtime: { tools } })).get(
+      '/api/realtime/token',
+    ),
+  );
+  assert.equal(response.status, 200);
+  assert.deepEqual(sent.at(-1).session.tools, tools);
+  const names = tools.map((tool) => tool.name);
+  assert.equal(new Set(names).size, names.length);
+  assert.deepEqual(
+    tools.slice(0, GEV_REALTIME_TOOLS.length),
+    GEV_REALTIME_TOOLS,
+  );
+  assert.ok(names.includes('get_weather'));
+  assert.ok(names.includes('military_awareness'));
+  assert.ok(!names.includes('open_in_gods_eye_view'));
+  // The action of the same name answers satellite passes.
+  assert.equal(
+    tools.findLast((tool) => tool.name === 'next_satellite_pass'),
+    GEV_REALTIME_TOOLS.find((tool) => tool.name === 'next_satellite_pass'),
+  );
+});
+
 test('debug logging resolves each supplied application directory independently', async (t) => {
   const first = root(t),
     second = root(t);

src/tools/functions.js

@@ -0,0 +1,28 @@
+/** Function-calling adapter for a tool catalog: tool records and results. */
+
+/** Tools as `{ type: 'function', name, description, parameters }` records. */
+export function toFunctionTools(tools, { exclude = [] } = {}) {
+  const skipped = new Set(exclude);
+  return tools
+    .filter((tool) => !skipped.has(tool.name))
+    .map((tool) => ({
+      type: 'function',
+      name: tool.name,
+      description: tool.description,
+      parameters: structuredClone(tool.inputSchema),
+    }));
+}
+
+/**
+ * A tool result as one JSON function output. Images are counted rather than
+ * sent, since function outputs carry text.
+ */
+export function toFunctionOutput(name, result) {
+  return {
+    ok: true,
+    tool: name,
+    summary: result.summary,
+    data: result.data,
+    ...(result.images?.length ? { images_omitted: result.images.length } : {}),
+  };
+}

src/tools/functions.test.mjs

@@ -0,0 +1,50 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { coreTools, toFunctionOutput, toFunctionTools } from './index.js';
+
+test('catalog tools become independent function records', () => {
+  const records = toFunctionTools(coreTools, {
+    exclude: ['open_in_gods_eye_view'],
+  });
+  assert.equal(records.length, coreTools.length - 1);
+  assert.ok(!records.some((tool) => tool.name === 'open_in_gods_eye_view'));
+  const weather = records.find((tool) => tool.name === 'get_weather');
+  const source = coreTools.find((tool) => tool.name === 'get_weather');
+  assert.deepEqual(Object.keys(weather), [
+    'type',
+    'name',
+    'description',
+    'parameters',
+  ]);
+  assert.equal(weather.type, 'function');
+  assert.equal(weather.description, source.description);
+  assert.deepEqual(weather.parameters, source.inputSchema);
+  assert.notEqual(weather.parameters, source.inputSchema);
+  weather.parameters.properties.location.description = 'changed';
+  assert.notEqual(
+    source.inputSchema.properties.location.description,
+    'changed',
+  );
+  assert.ok(records.every((tool) => tool.parameters.type === 'object'));
+});
+
+test('results become function outputs without image data', () => {
+  assert.deepEqual(
+    toFunctionOutput('get_wind', { summary: 'Calm.', data: { calm: true } }),
+    { ok: true, tool: 'get_wind', summary: 'Calm.', data: { calm: true } },
+  );
+  assert.deepEqual(
+    toFunctionOutput('get_weather_map', {
+      summary: 'Radar.',
+      data: {},
+      images: [{ mimeType: 'image/png', data: 'AAAA' }],
+    }),
+    {
+      ok: true,
+      tool: 'get_weather_map',
+      summary: 'Radar.',
+      data: {},
+      images_omitted: 1,
+    },
+  );
+});

src/tools/index.js

@@ -67,6 +67,7 @@ export {
   distanceKm,
 } from './area.js';
 export { LIMIT_SCHEMA, DEFAULT_LIMIT, MAX_LIMIT, capRows } from './results.js';
+export { toFunctionOutput, toFunctionTools } from './functions.js';
 export {
   createGeocodePlaceService,
   createPlaceSearchService,

src/voice/gevRealtime.js

@@ -1,11 +1,35 @@
+import { toFunctionOutput } from '../tools/functions.js';
+import { GEV_ACTION_SCHEMAS } from './actionSchemas.js';
 import { createGevActionRunner } from './gevActions.js';
 import { createVoiceCommands } from './commands.js';
 export * from './realtimeController.js';
 
+const ACTION_NAMES = new Set(GEV_ACTION_SCHEMAS.map((schema) => schema.name));
+
+/**
+ * Run app actions through `runner` and every other tool the catalog has
+ * through the catalog. `loadCatalog` resolves the catalog when first needed.
+ */
+export function withToolCatalog(runner, loadCatalog) {
+  if (typeof loadCatalog !== 'function') return runner;
+  return async function runGevTool(name, args, options = {}) {
+    if (ACTION_NAMES.has(name)) return runner(name, args, options);
+    const catalog = await loadCatalog();
+    if (!catalog?.get(name)) return runner(name, args, options);
+    const result = await catalog.call(name, args ?? {}, {
+      signal: options.signal,
+    });
+    return toFunctionOutput(name, result);
+  };
+}
+
 /** Compose the standalone action runner with the voice controls. */
 export function initGevVoiceCommands(options) {
   return createVoiceCommands({
     ...options,
-    runner: createGevActionRunner(options),
+    runner: withToolCatalog(
+      createGevActionRunner(options),
+      options.toolCatalog,
+    ),
   });
 }

src/voice/gevRealtime.test.mjs

@@ -31,6 +31,7 @@ import {
   readStoredVoiceLimits,
   writeStoredVoiceTier,
   writeStoredVoiceLimits,
+  withToolCatalog,
 } from './gevRealtime.js';
 import { createVoiceCostTracker } from './voiceCost.js';
 
@@ -3681,3 +3682,43 @@ test('a genuinely different refused call still gets its own output', async () =>
 const testPlaceSearch = () => createStandalonePlaceSearch({ resolveApiKey: () => globalThis.window?.__GOOGLE_MAPS_API_KEY__ });
 function createGevActionRunner(options) { return createActionRunner({ placeSearch: testPlaceSearch(), ...options }); }
 function controlRadio(viewer, manager, args, options) { return runControlRadio(viewer, manager, args, { placeSearch: testPlaceSearch(), ...options }); }
+
+test('voice runs app actions itself and other tools through the catalog', async () => {
+  const actions = [];
+  const runner = async (name, args) => {
+    actions.push([name, args]);
+    return { ok: true, action: name };
+  };
+  const calls = [];
+  let loads = 0;
+  const catalog = {
+    get: (name) => (name === 'get_wind' ? { name } : undefined),
+    async call(name, args, options) {
+      calls.push([name, args, options.signal]);
+      return { summary: 'Calm.', data: { calm: true } };
+    },
+  };
+  const run = withToolCatalog(runner, async () => {
+    loads += 1;
+    return catalog;
+  });
+  const signal = new AbortController().signal;
+  assert.deepEqual(await run('zoom_to_globe', {}), {
+    ok: true,
+    action: 'zoom_to_globe',
+  });
+  assert.equal(loads, 0);
+  assert.deepEqual(
+    await run('get_wind', { location: { place: 'Oslo' } }, { signal }),
+    { ok: true, tool: 'get_wind', summary: 'Calm.', data: { calm: true } },
+  );
+  assert.deepEqual(calls, [
+    ['get_wind', { location: { place: 'Oslo' } }, signal],
+  ]);
+  await run('not_a_tool', {});
+  assert.deepEqual(
+    actions.map(([name]) => name),
+    ['zoom_to_globe', 'not_a_tool'],
+  );
+  assert.equal(withToolCatalog(runner, undefined), runner);
+});