From 9f2dfdf8208efdd831b0f0743edb12f76774c809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Valdez?= <762946+avaldini@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:34:33 +0000 Subject: [PATCH 1/2] fix(voice): send OpenAI Realtime tool schemas the API accepts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI Realtime now answers 400 invalid_function_parameters for a whole session whose tool schema uses Unicode property classes, as resolve_area's area-id pattern does, so voice never connects ("Failed to create Realtime token"). The session sends its tools through chatSchema, the filter typed chat adopted in #1616, which leaves out only \p{…} patterns; area ids are still checked against the action's own contract before the tool runs. Fixes #1618 --- server/providers/openai/chat.js | 2 +- server/providers/openai/realtime.js | 17 +++++++++++- src/tooling/localServices.test.mjs | 41 ++++++++++++++++++++++------- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/server/providers/openai/chat.js b/server/providers/openai/chat.js index 6ca4648d6..b0f7fe211 100644 --- a/server/providers/openai/chat.js +++ b/server/providers/openai/chat.js @@ -69,7 +69,7 @@ function validMessages(messages) { * 400 for a whole request whose tool uses Unicode property classes (`\p{L}`), * as `resolve_area`'s area ids do. Every argument is still checked against * the action's own contract before the tool runs, so the model loses a hint, - * not a rule. Realtime voice accepts these patterns and keeps them. + * not a rule. Realtime voice refuses them too now (#1618) and uses this as well. */ export function chatSchema(schema) { if (Array.isArray(schema)) return schema.map(chatSchema); diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index 73ac8d2cf..4412eb331 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -14,6 +14,7 @@ import { } from './constants.js'; import { realtimeInstructions } from './instructions.js'; import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js'; +import { chatSchema } from './chat.js'; import { resolveOverpassUpstreams } from '../overpass/constants.js'; import { preferCodexOAuth, @@ -128,6 +129,19 @@ function createRealtimeOAuthLoginHandler({ }; } +/** + * The tools one Realtime session is offered, as OpenAI accepts them: it now + * refuses a whole session whose tool schema uses `\p{…}` patterns (#1618), as + * Chat Completions did (#1616), so voice drops the same pattern hints chat does. + */ +function realtimeSessionTools(tools, { overpass = false } = {}) { + return offeredTools(tools, { overpass }).map((tool) => + tool.parameters + ? { ...tool, parameters: chatSchema(tool.parameters) } + : tool, + ); +} + function createRealtimeTokenHandler({ annotationGuidance, endpoint = 'https://api.openai.com/v1/realtime/client_secrets', @@ -303,7 +317,7 @@ function createRealtimeTokenHandler({ output: { voice }, }, instructions: realtimeInstructions(annotationGuidance, { overpass }), - tools: offeredTools(tools, { overpass }), + tools: realtimeSessionTools(tools, { overpass }), tool_choice: 'auto', }, }; @@ -370,4 +384,5 @@ export { createRealtimeOAuthLoginHandler, createRealtimeOAuthStatusHandler, createRealtimeTokenHandler, + realtimeSessionTools, }; diff --git a/src/tooling/localServices.test.mjs b/src/tooling/localServices.test.mjs index 84a2f6926..7a69b5e7f 100644 --- a/src/tooling/localServices.test.mjs +++ b/src/tooling/localServices.test.mjs @@ -20,12 +20,9 @@ import { import { openAiRealtimeProxy } from 'gods-eye-view/server/providers/openai'; import { keySetupEndpoint } from 'gods-eye-view/server/standalone/key-setup'; import { realtimeInstructions } from '../../server/providers/openai/instructions.js'; -import { - GEV_REALTIME_TOOLS, - offeredTools, - realtimeTools, -} from '../../server/providers/openai/tools.js'; +import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js'; import { createDebugLogHandler } from '../../server/providers/openai/debug-log.js'; +import { realtimeSessionTools } from '../../server/providers/openai/realtime.js'; import { attachVoiceResult } from '../voice/speech.js'; import { sanitizeDebugValue } from '../voice/realtimeDiagnostics.js'; import { standaloneVoiceTools } from '../../server/standalone/voiceTools.js'; @@ -199,7 +196,10 @@ test('Realtime handler preserves tools and default instructions, isolates suppli sent.at(-1).session.instructions, realtimeInstructions(guidance), ); - assert.deepEqual(sent.at(-1).session.tools, realtimeTools()); + assert.deepEqual( + sent.at(-1).session.tools, + realtimeSessionTools(GEV_REALTIME_TOOLS), + ); } assert.notEqual(sent[0].session.instructions, sent[1].session.instructions); assert.equal(sent[0].session.instructions, sent[2].session.instructions); @@ -607,7 +607,10 @@ test('osm_query is offered only when an operator configured Overpass', async (t) assert.doesNotMatch(plain.instructions, /osm_query/); assert.ok(names(plain).includes('resolve_area')); assert.ok(names(plain).includes('find_imagery')); - assert.deepEqual(configured.tools, GEV_REALTIME_TOOLS); + assert.deepEqual( + configured.tools, + realtimeSessionTools(GEV_REALTIME_TOOLS, { overpass: true }), + ); assert.match(configured.instructions, /osm_query/); assert.equal( configured.instructions, @@ -656,7 +659,7 @@ test('Realtime sessions carry supplied tools, and the standalone voice adds the ); assert.equal(response.status, 200); // Without an operator Overpass, osm_query is left out of the session. - assert.deepEqual(sent.at(-1).session.tools, offeredTools(tools)); + assert.deepEqual(sent.at(-1).session.tools, realtimeSessionTools(tools)); assert.ok( !sent.at(-1).session.tools.some(({ name }) => name === 'osm_query'), ); @@ -740,6 +743,26 @@ test('key setup writes only the supplied application root, retains request guard assert.equal(existsSync(path.join(untouched, '.env')), false); }); +test('Realtime sessions leave out the Unicode property patterns OpenAI refuses (#1618)', async (t) => { + // OpenAI answers 400 invalid_function_parameters for a whole Realtime session + // whose tool schema uses `\p{…}`, as resolve_area's area ids do, so voice + // never connects. The session drops only those patterns, as typed chat does. + env(t, 'OPENAI_API_KEY', 'fixture-upstream-secret'); + env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined); + const sent = []; + t.mock.method(globalThis, 'fetch', async (_url, options) => { + sent.push(JSON.parse(options.body).session); + return Response.json({ value: 'fixture-ephemeral' }); + }); + const response = await request( + install(openAiRealtimeProxy({})).get('/api/realtime/token'), + ); + assert.equal(response.status, 200); + const { tools } = sent.at(-1); + assert.ok(tools.some(({ name }) => name === 'resolve_area')); + assert.doesNotMatch(JSON.stringify(tools), /\\\\[pP]\{/); +}); + test('Realtime service configuration selects compatible endpoint/model without forwarding request model IDs or keys', async () => { const handler = install( openAiRealtimeProxy({ @@ -757,7 +780,7 @@ test('Realtime service configuration selects compatible endpoint/model without f assert.equal(payload.session.model, 'configured-model'); assert.deepEqual( payload.session.tools, - realtimeTools({ overpass: false }), + realtimeSessionTools(GEV_REALTIME_TOOLS), ); return Response.json({ value: 'short-lived-fixture' }); }, From d52c719ef645657aea470b8c6aa3b2f2bd791c2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Valdez?= <762946+avaldini@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:52:24 +0000 Subject: [PATCH 2/2] docs(changelog): OpenAI Realtime voice tool patterns (#1638) --- changelog.d/1638-openai-realtime-tool-patterns.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog.d/1638-openai-realtime-tool-patterns.md diff --git a/changelog.d/1638-openai-realtime-tool-patterns.md b/changelog.d/1638-openai-realtime-tool-patterns.md new file mode 100644 index 000000000..20e167112 --- /dev/null +++ b/changelog.d/1638-openai-realtime-tool-patterns.md @@ -0,0 +1,5 @@ +- Voice on OpenAI connects again: OpenAI Realtime refused every session, so + the mic failed with "Failed to create Realtime token", because + `resolve_area`'s area-id pattern uses Unicode classes it no longer accepts. + Voice leaves those patterns out, as typed chat does; area ids are still + checked before the tool runs (avaldini, #1638).