Merge pull request #1638 from avaldini/fix/realtime-voice-unicode-patterns

fix(voice): send OpenAI Realtime tool schemas the API accepts

changelog.d/1638-openai-realtime-tool-patterns.md

@@ -0,0 +1,5 @@
+- Voice on OpenAI connects again: OpenAI Realtime refused every session, so
+  the mic failed with "Failed to create Realtime token", because
+  `resolve_area`'s area-id pattern uses Unicode classes it no longer accepts.
+  Voice leaves those patterns out, as typed chat does; area ids are still
+  checked before the tool runs (avaldini, #1638).

server/providers/openai/chat.js

@@ -69,7 +69,7 @@ function validMessages(messages) {
  * 400 for a whole request whose tool uses Unicode property classes (`\p{L}`),
  * as `resolve_area`'s area ids do. Every argument is still checked against
  * the action's own contract before the tool runs, so the model loses a hint,
- * not a rule. Realtime voice accepts these patterns and keeps them.
+ * not a rule. Realtime voice refuses them too now (#1618) and uses this as well.
  */
 export function chatSchema(schema) {
   if (Array.isArray(schema)) return schema.map(chatSchema);

server/providers/openai/realtime.js

@@ -14,6 +14,7 @@ import {
 } from './constants.js';
 import { realtimeInstructions } from './instructions.js';
 import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
+import { chatSchema } from './chat.js';
 import { resolveOverpassUpstreams } from '../overpass/constants.js';
 import {
   preferCodexOAuth,
@@ -128,6 +129,19 @@ function createRealtimeOAuthLoginHandler({
   };
 }
 
+/**
+ * The tools one Realtime session is offered, as OpenAI accepts them: it now
+ * refuses a whole session whose tool schema uses `\p{…}` patterns (#1618), as
+ * Chat Completions did (#1616), so voice drops the same pattern hints chat does.
+ */
+function realtimeSessionTools(tools, { overpass = false } = {}) {
+  return offeredTools(tools, { overpass }).map((tool) =>
+    tool.parameters
+      ? { ...tool, parameters: chatSchema(tool.parameters) }
+      : tool,
+  );
+}
+
 function createRealtimeTokenHandler({
   annotationGuidance,
   endpoint = 'https://api.openai.com/v1/realtime/client_secrets',
@@ -303,7 +317,7 @@ function createRealtimeTokenHandler({
           output: { voice },
         },
         instructions: realtimeInstructions(annotationGuidance, { overpass }),
-        tools: offeredTools(tools, { overpass }),
+        tools: realtimeSessionTools(tools, { overpass }),
         tool_choice: 'auto',
       },
     };
@@ -370,4 +384,5 @@ export {
   createRealtimeOAuthLoginHandler,
   createRealtimeOAuthStatusHandler,
   createRealtimeTokenHandler,
+  realtimeSessionTools,
 };

src/tooling/localServices.test.mjs

@@ -20,12 +20,9 @@ import {
 import { openAiRealtimeProxy } from 'gods-eye-view/server/providers/openai';
 import { keySetupEndpoint } from 'gods-eye-view/server/standalone/key-setup';
 import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
-import {
-  GEV_REALTIME_TOOLS,
-  offeredTools,
-  realtimeTools,
-} from '../../server/providers/openai/tools.js';
+import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
 import { createDebugLogHandler } from '../../server/providers/openai/debug-log.js';
+import { realtimeSessionTools } from '../../server/providers/openai/realtime.js';
 import { attachVoiceResult } from '../voice/speech.js';
 import { sanitizeDebugValue } from '../voice/realtimeDiagnostics.js';
 import { standaloneVoiceTools } from '../../server/standalone/voiceTools.js';
@@ -199,7 +196,10 @@ test('Realtime handler preserves tools and default instructions, isolates suppli
       sent.at(-1).session.instructions,
       realtimeInstructions(guidance),
     );
-    assert.deepEqual(sent.at(-1).session.tools, realtimeTools());
+    assert.deepEqual(
+      sent.at(-1).session.tools,
+      realtimeSessionTools(GEV_REALTIME_TOOLS),
+    );
   }
   assert.notEqual(sent[0].session.instructions, sent[1].session.instructions);
   assert.equal(sent[0].session.instructions, sent[2].session.instructions);
@@ -607,7 +607,10 @@ test('osm_query is offered only when an operator configured Overpass', async (t)
   assert.doesNotMatch(plain.instructions, /osm_query/);
   assert.ok(names(plain).includes('resolve_area'));
   assert.ok(names(plain).includes('find_imagery'));
-  assert.deepEqual(configured.tools, GEV_REALTIME_TOOLS);
+  assert.deepEqual(
+    configured.tools,
+    realtimeSessionTools(GEV_REALTIME_TOOLS, { overpass: true }),
+  );
   assert.match(configured.instructions, /osm_query/);
   assert.equal(
     configured.instructions,
@@ -656,7 +659,7 @@ test('Realtime sessions carry supplied tools, and the standalone voice adds the
   );
   assert.equal(response.status, 200);
   // Without an operator Overpass, osm_query is left out of the session.
-  assert.deepEqual(sent.at(-1).session.tools, offeredTools(tools));
+  assert.deepEqual(sent.at(-1).session.tools, realtimeSessionTools(tools));
   assert.ok(
     !sent.at(-1).session.tools.some(({ name }) => name === 'osm_query'),
   );
@@ -740,6 +743,26 @@ test('key setup writes only the supplied application root, retains request guard
   assert.equal(existsSync(path.join(untouched, '.env')), false);
 });
 
+test('Realtime sessions leave out the Unicode property patterns OpenAI refuses (#1618)', async (t) => {
+  // OpenAI answers 400 invalid_function_parameters for a whole Realtime session
+  // whose tool schema uses `\p{…}`, as resolve_area's area ids do, so voice
+  // never connects. The session drops only those patterns, as typed chat does.
+  env(t, 'OPENAI_API_KEY', 'fixture-upstream-secret');
+  env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined);
+  const sent = [];
+  t.mock.method(globalThis, 'fetch', async (_url, options) => {
+    sent.push(JSON.parse(options.body).session);
+    return Response.json({ value: 'fixture-ephemeral' });
+  });
+  const response = await request(
+    install(openAiRealtimeProxy({})).get('/api/realtime/token'),
+  );
+  assert.equal(response.status, 200);
+  const { tools } = sent.at(-1);
+  assert.ok(tools.some(({ name }) => name === 'resolve_area'));
+  assert.doesNotMatch(JSON.stringify(tools), /\\\\[pP]\{/);
+});
+
 test('Realtime service configuration selects compatible endpoint/model without forwarding request model IDs or keys', async () => {
   const handler = install(
     openAiRealtimeProxy({
@@ -757,7 +780,7 @@ test('Realtime service configuration selects compatible endpoint/model without f
           assert.equal(payload.session.model, 'configured-model');
           assert.deepEqual(
             payload.session.tools,
-            realtimeTools({ overpass: false }),
+            realtimeSessionTools(GEV_REALTIME_TOOLS),
           );
           return Response.json({ value: 'short-lived-fixture' });
         },