From ae3612a356b7dc9e8fa7c015d23d33ef0079ca2f Mon Sep 17 00:00:00 2001 From: manjunath22466 <301209272+manjunath22466@users.noreply.github.com> Date: Mon, 5 Oct 2026 05:46:47 +0530 Subject: [PATCH] fix voice debug-log privacy and docs drift omit tool arguments unless content logging is enabled correct the remaining current tool-count references --- .env.example | 5 +- README.md | 4 +- docs/CURRENT-STATE.md | 6 +- server/providers/openai/debug-log.js | 29 +++-- src/tooling/localServices.test.mjs | 187 ++++++++++++++++++++++++--- 5 files changed, 195 insertions(+), 36 deletions(-) diff --git a/.env.example b/.env.example index 5181ff7b3..c1df70a08 100644 --- a/.env.example +++ b/.env.example @@ -71,8 +71,9 @@ OPENAI_REALTIME_CONTEXT_RETENTION=0.5 # counted in the session cost meter and cap; off disables them. OPENAI_REALTIME_TRANSCRIBE_MODEL=gpt-4o-mini-transcribe # The voice debug log (.gev-logs/realtime-conversations.jsonl, rotated at -# 32 MB, never expired) omits transcripts and typed or spoken text by default. -# Set to 1 to keep them while debugging; delete the log when you are done. +# 32 MB, never expired) omits transcripts, typed or spoken text, and tool +# arguments by default. Set to 1 to keep them while debugging; delete the log +# when you are done. GEV_VOICE_LOG_CONTENT= OPENAI_HUD_SUMMARY_MODEL=gpt-5-nano # Per-IP rate limit for the OpenAI cost endpoints diff --git a/README.md b/README.md index 85e66938d..292aa726f 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ Click **GEV MIC**, grant the microphone, and just talk. This is more than a voic - **๐ŸŽฌ Cinematic framing.** _"Show me the planes overhead"_ pulls the camera back, angles it, and frames the live traffic like a director. - **๐Ÿ”’ Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. -Twenty-nine tools, four jobs โ€” the commands below come straight from the product's voice test suite and tool playbook: +Thirty tools, four jobs โ€” the commands below come straight from the product's voice test suite and tool playbook: **๐ŸŽฅ Direct it** โ€” drone-operator camera verbs: @@ -384,7 +384,7 @@ src/ โ”œโ”€โ”€ hud.js # Intelligence HUD + AI scene summary โ”œโ”€โ”€ keySetup.js # POWER UP panel โ€” in-app provider keys (dev server only) โ”œโ”€โ”€ mapStackController.js # Basemap switching โ€” Google 3D / Esri / OSM / ion stacks -โ”œโ”€โ”€ voice/ # OpenAI Realtime session + 29 voice tools +โ”œโ”€โ”€ voice/ # OpenAI Realtime session + 30 voice tools โ”œโ”€โ”€ layers/ # Layer components โ€” weather, wind, cyclones, transit, ALPR, โ€ฆ โ”œโ”€โ”€ data/ # One module per layer + orchestration + context store โ”‚ โ”œโ”€โ”€ iconOrientation.js # Screen-projected headings + horizon cull diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index b07670734..5284a67b8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -383,8 +383,8 @@ the speaker (`mayVoiceClaimSpeaker()`). Progress lines stay outside the reply lifecycle and Radio handoff. `turn.span` debug records carry generation and playback latency, silence, preamble and word-count figures. Caption transcription (`OPENAI_REALTIME_TRANSCRIBE_MODEL`, default on) is metered into -the session cost and cap. The debug log omits transcripts and text unless -`GEV_VOICE_LOG_CONTENT=1`. +the session cost and cap. The debug log omits transcripts, text and tool +arguments unless `GEV_VOICE_LOG_CONTENT=1`. Point and ask: `pointerContext.js` reads what the cursor is on. A Space hold is the pointing gesture: its keydown snapshot (released with the key) is sent @@ -2939,7 +2939,7 @@ test:track` 43 tracking invariants ยท headless QA harnesses under > ISS pass prediction, and per-layer data attribution. Gate at close: unit 98/98, build clean, > track 19/19, + five QA harnesses (heading 16/16, sprites 9/9, cctv 5/5, failstate 5/5, > attribution 18/18). New modules: `src/data/{motionModel,aircraftMeta,aircraftClass,aircraftIcons,issPass,routePlausible,dataCredits}.js`. -> The live runtime now declares 29 voice tools; the 17โ†’20 count above is retained only as milestone history. +> The live runtime now declares 30 voice tools; the 17โ†’20 count above is retained only as milestone history. ## Canonical Docs Order diff --git a/server/providers/openai/debug-log.js b/server/providers/openai/debug-log.js index 1ca86c29f..f411128d4 100644 --- a/server/providers/openai/debug-log.js +++ b/server/providers/openai/debug-log.js @@ -25,30 +25,41 @@ const REALTIME_DEBUG_LOG_MAX_PER_MIN = 120; /** * Fields that carry what people said or heard: transcripts, typed and spoken - * text, streamed deltas and the span's quoted replies. They are dropped from - * the persisted log unless content logging is explicitly enabled + * text, streamed deltas, structured tool arguments and the span's quoted + * replies. They are dropped from the persisted log unless content logging is explicitly enabled * (GEV_VOICE_LOG_CONTENT=1), so timing and tool diagnostics never retain * speech by default. */ -const VOICE_CONTENT_FIELDS = new Set([ +const VOICE_TEXT_FIELDS = new Set([ 'transcript', 'text', 'delta', 'answer_text', 'preamble_text', ]); +const VOICE_CONTENT_MAX_DEPTH = 12; +const VOICE_CONTENT_MAX_DEPTH_MARKER = '[omitted: max depth]'; -/** Replace spoken or typed content with its length. */ +/** Replace spoken or typed content and structured tool arguments. */ function omitVoiceContent(value, depth = 0) { - if (depth > 12 || value === null || typeof value !== 'object') return value; + if (depth > VOICE_CONTENT_MAX_DEPTH) return VOICE_CONTENT_MAX_DEPTH_MARKER; + if (value === null || typeof value !== 'object') return value; if (Array.isArray(value)) return value.map((item) => omitVoiceContent(item, depth + 1)); const output = {}; for (const [key, item] of Object.entries(value)) { - output[key] = - VOICE_CONTENT_FIELDS.has(key) && typeof item === 'string' - ? `[omitted ${item.length} chars]` - : omitVoiceContent(item, depth + 1); + if (key === 'arguments') { + output[key] = + typeof item === 'string' + ? `[omitted ${item.length} chars]` + : '[omitted]'; + continue; + } + if (VOICE_TEXT_FIELDS.has(key) && typeof item === 'string') { + output[key] = `[omitted ${item.length} chars]`; + continue; + } + output[key] = omitVoiceContent(item, depth + 1); } return output; } diff --git a/src/tooling/localServices.test.mjs b/src/tooling/localServices.test.mjs index 73c5d9e50..ec4cf2a56 100644 --- a/src/tooling/localServices.test.mjs +++ b/src/tooling/localServices.test.mjs @@ -399,37 +399,184 @@ test('OpenAI routes answer generically when the upstream or the request fails', assert.equal(token.body.includes('fixture-upstream-secret'), false); }); -test('the debug-log sink omits what people said unless content logging is enabled', async (t) => { - const record = { - event: 'server.event', - payload: { - type: 'conversation.item.input_audio_transcription.completed', - transcript: 'my home address is 12 Elm Street', - usage: { input_tokens: 40 }, - nested: [{ type: 'input_text', text: 'call my sister' }], - }, - span: { answer_text: 'Flying to Tokyo.', answer_words: 3 }, +test('the debug-log sink omits what people said and tool arguments unless content logging is enabled', async (t) => { + const nestObject = (value, levels) => { + let nested = value; + for (let index = 0; index < levels; index += 1) nested = { nested }; + return nested; }; + const nestArray = (value, levels) => { + let nested = value; + for (let index = 0; index < levels; index += 1) nested = [nested]; + return nested; + }; + const records = [ + { + timestamp: '2026-10-05T00:00:00.000Z', + sessionId: 'gev-session-1', + event: 'tool.call', + status: 'active', + payload: { + name: 'analyst_query', + callId: 'call-1', + arguments: { location: '12 Elm Street', note: 'meet Alice there' }, + }, + }, + { + timestamp: '2026-10-05T00:00:01.000Z', + sessionId: 'gev-session-1', + event: 'server.event', + status: 'active', + payload: { + type: 'response.function_call_arguments.done', + eventId: 'event-1', + responseId: 'response-1', + payload: { + type: 'response.function_call_arguments.done', + event_id: 'event-1', + response_id: 'response-1', + item_id: 'item-1', + output_index: 0, + call_id: 'call-1', + name: 'analyst_query', + arguments: '{"query":"Tokyo Station"}', + }, + }, + }, + { + event: 'server.event', + status: 'active', + payload: { + type: 'response.audio_transcript.done', + transcript: 'my home address is 12 Elm Street', + text: 'call my sister', + delta: 'Flying to Tokyo.', + usage: { input_tokens: 40 }, + }, + }, + { + event: 'voice.cost', + status: 'active', + payload: { delta: 17, total: 41 }, + }, + { + event: 'tool.call', + status: 'active', + payload: { name: 'adjust_camera_zoom', callId: 'call-2', arguments: 2 }, + }, + { + event: 'deep.object', + status: 'active', + payload: nestObject( + { + transcript: 'deep-object-transcript-secret', + arguments: { query: 'deep-object-argument-secret' }, + }, + 14, + ), + }, + { + event: 'deep.array', + status: 'active', + payload: nestArray( + { + transcript: 'deep-array-transcript-secret', + arguments: { query: 'deep-array-argument-secret' }, + }, + 14, + ), + }, + ]; for (const includeContent of [false, true]) { const sourceRoot = root(t); const handler = createDebugLogHandler({ sourceRoot, includeContent }); - const response = await request(handler, { - method: 'POST', - body: JSON.stringify(record), - }); - assert.equal(response.status, 204); + for (const record of records) { + const response = await request(handler, { + method: 'POST', + body: JSON.stringify(record), + }); + assert.equal(response.status, 204); + } const logged = readFileSync( path.join(sourceRoot, '.gev-logs/realtime-conversations.jsonl'), 'utf8', ); assert.equal(logged.includes('Elm Street'), includeContent); assert.equal(logged.includes('call my sister'), includeContent); + assert.equal(logged.includes('meet Alice there'), includeContent); + assert.equal(logged.includes('Tokyo Station'), includeContent); assert.equal(logged.includes('Flying to Tokyo'), includeContent); - const parsed = JSON.parse(logged); - assert.equal(parsed.payload.usage.input_tokens, 40, 'numbers stay'); - assert.equal(parsed.span.answer_words, 3); - if (!includeContent) - assert.equal(parsed.payload.transcript, '[omitted 32 chars]'); + assert.equal( + logged.includes('deep-object-transcript-secret'), + includeContent, + ); + assert.equal( + logged.includes('deep-object-argument-secret'), + includeContent, + ); + assert.equal( + logged.includes('deep-array-transcript-secret'), + includeContent, + ); + assert.equal(logged.includes('deep-array-argument-secret'), includeContent); + const [ + toolCall, + serverEvent, + transcriptEvent, + numericDelta, + numericArguments, + deepObject, + deepArray, + ] = logged + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)); + assert.equal(toolCall.timestamp, records[0].timestamp); + assert.equal(toolCall.sessionId, 'gev-session-1'); + assert.equal(toolCall.event, 'tool.call'); + assert.equal(toolCall.status, 'active'); + assert.equal(toolCall.payload.name, 'analyst_query'); + assert.equal(toolCall.payload.callId, 'call-1'); + assert.equal(serverEvent.payload.eventId, 'event-1'); + assert.equal(serverEvent.payload.responseId, 'response-1'); + assert.equal(serverEvent.payload.payload.event_id, 'event-1'); + assert.equal(serverEvent.payload.payload.response_id, 'response-1'); + assert.equal(serverEvent.payload.payload.item_id, 'item-1'); + assert.equal(serverEvent.payload.payload.output_index, 0); + assert.equal(serverEvent.payload.payload.call_id, 'call-1'); + assert.equal(serverEvent.payload.payload.name, 'analyst_query'); + assert.equal(transcriptEvent.payload.usage.input_tokens, 40); + assert.equal(numericDelta.payload.delta, 17); + assert.equal(numericDelta.payload.total, 41); + if (!includeContent) { + assert.equal(toolCall.payload.arguments, '[omitted]'); + assert.match( + serverEvent.payload.payload.arguments, + /^\[omitted \d+ chars\]$/, + ); + assert.equal(transcriptEvent.payload.transcript, '[omitted 32 chars]'); + assert.equal(transcriptEvent.payload.text, '[omitted 14 chars]'); + assert.equal(transcriptEvent.payload.delta, '[omitted 16 chars]'); + assert.equal(numericArguments.payload.arguments, '[omitted]'); + assert.equal( + JSON.stringify(deepObject.payload).includes('[omitted: max depth]'), + true, + ); + assert.equal( + JSON.stringify(deepArray.payload).includes('[omitted: max depth]'), + true, + ); + } else { + assert.deepEqual( + toolCall.payload.arguments, + records[0].payload.arguments, + ); + assert.equal( + serverEvent.payload.payload.arguments, + records[1].payload.payload.arguments, + ); + assert.equal(numericArguments.payload.arguments, 2); + } } });