From da8b0dd8cd966587dc4ef321a0261e2024aaed12 Mon Sep 17 00:00:00 2001 From: manjunath22466 <301209272+manjunath22466@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:01:08 +0530 Subject: [PATCH] fix(voice): address review of typed AI chat Typed chat sessions no longer end on the voice inactivity timeout, and a failed or stopped reply keeps the conversation: Stop cancels only the reply in progress, and map actions that already ran stay in the model's context. Each upstream request and map action gets its own deadline, old tool results shrink instead of the history overflowing, and step limits, a missing or rejected key, the OAuth lock and throttling each get their own message. - Server: OpenAI chat is off under GEV_PREFER_CODEX_OAUTH; chat throttles default to the provider's throttle; replies carry API_KEY_MISSING, API_KEY_REJECTED, CODEX_OAUTH_REQUIRED and RATE_LIMITED codes; upstream Retry-After passes through, and a rejected key (Gemini's invalid-key 400 included) cools down for a minute - Pinokio forwards the chat throttle settings - UI: the reply reveal publishes only the message list; the border orbit runs only while busy; layers attach with a button or Enter; the log is aria-busy while a reply types out; auto-scroll respects a reader who scrolled up; Escape closes the attach menu; the chat gear says Settings; a note says chat bills the OpenAI key when voice uses ChatGPT sign-in - Layout does less work while panel height animates; unused blurs removed - Gemini voice start ends chat; the voice preview clears on every new session - Tests for the route gates, client limits, attachments and the fixes --- .env.example | 20 +- README.md | 6 +- SECURITY.md | 4 +- changelog.d/1507-typed-ai-chat.md | 11 +- docs/CURRENT-STATE.md | 18 +- pinokio/_ENVIRONMENT | 4 + pinokio/install.js | 2 + pinokio/start.js | 2 + pinokio/update.js | 2 + scripts/format-scope.json | 3 + scripts/package-boundaries.json | 1 + scripts/pinokio-environment.mjs | 2 + server/providers/common/textChat.js | 82 +++++- server/providers/gemini/chat.js | 6 + server/providers/openai/chat.js | 17 ++ src/costEndpointThrottle.test.mjs | 90 ++++++ src/i18n/locales/ar.json | 9 +- src/i18n/locales/de.json | 9 +- src/i18n/locales/es.json | 9 +- src/i18n/locales/fr.json | 9 +- src/i18n/locales/it.json | 9 +- src/i18n/locales/ja.json | 9 +- src/i18n/locales/pt.json | 9 +- src/i18n/locales/ru.json | 9 +- src/i18n/locales/uk.json | 9 +- src/i18n/locales/zh-CN.json | 9 +- src/i18n/locales/zh-Hant.json | 9 +- src/pinokioLauncherContract.test.mjs | 2 + src/textChatServer.test.mjs | 395 ++++++++++++++++++++++++--- src/ui/styles/ai-chat.css | 41 ++- src/ui/views/panels/ChatPanel.vue | 52 +++- src/ui/views/panels/VoiceControl.vue | 3 +- src/voice/chatAttachments.js | 18 +- src/voice/chatAttachments.test.mjs | 139 ++++++++++ src/voice/chatControl.js | 79 +++++- src/voice/chatControlView.test.mjs | 331 ++++++++++++++++++++++ src/voice/control.js | 4 + src/voice/geminiSession.js | 3 + src/voice/geminiTextChat.test.mjs | 59 +++- src/voice/panel/host.js | 4 +- src/voice/panel/host.test.mjs | 30 ++ src/voice/panelLayout.js | 65 ++++- src/voice/realtimeSession.js | 2 + src/voice/session.test.mjs | 50 ++++ src/voice/textChat.js | 185 ++++++++++--- src/voice/textChat.test.mjs | 392 ++++++++++++++++++++++++++ src/voice/textProtocol.js | 37 +++ src/voice/voiceMiniPreview.js | 4 +- src/voice/voiceMiniPreview.test.mjs | 17 +- 49 files changed, 2121 insertions(+), 160 deletions(-) create mode 100644 src/voice/chatAttachments.test.mjs create mode 100644 src/voice/chatControlView.test.mjs create mode 100644 src/voice/textChat.test.mjs diff --git a/.env.example b/.env.example index 49f80e99e..9b8b8db83 100644 --- a/.env.example +++ b/.env.example @@ -61,14 +61,15 @@ CESIUM_ION_TOKEN= # ChatGPT/Codex OAuth session; the browser still receives only a short-lived # Realtime client secret. Do not prefix with VITE_. OPENAI_API_KEY= -# Typed chat uses Chat Completions, independently of the voice model. It always -# bills OPENAI_API_KEY; GEV_PREFER_CODEX_OAUTH below applies to voice only. +# Typed chat uses Chat Completions, independently of the voice model. It can +# only bill OPENAI_API_KEY, so GEV_PREFER_CODEX_OAUTH below turns it off. OPENAI_CHAT_MODEL=gpt-4.1-mini -# Typed chat requests (/api/openai/chat) per IP per minute, separate from the -# voice throttle; 0 disables it. Not a billing cap. -# GEV_RATELIMIT_OPENAI_CHAT_PER_MIN=60 # the default -# Optional: forbid API-key voice sessions. Select ChatGPT OAuth in Provider -# Settings; missing or expired Codex sign-in then fails closed. +# Typed chat requests (/api/openai/chat) per IP per minute, counted apart from +# voice. Unset, it takes GEV_RATELIMIT_OPENAI_PER_MIN's cap; 0 disables it. +# Not a billing cap. +# GEV_RATELIMIT_OPENAI_CHAT_PER_MIN=30 +# Optional: forbid API-key voice sessions and OpenAI typed chat. Select ChatGPT +# OAuth in Provider Settings; missing or expired Codex sign-in then fails closed. # GEV_PREFER_CODEX_OAUTH=true OPENAI_REALTIME_MODEL=gpt-realtime-2 # Cheaper voice tier, used when the GEV MIC panel's MINI toggle is on. @@ -304,5 +305,6 @@ OVERPASS_UPSTREAMS= # Gemini typed chat uses generateContent; voice continues to use Gemini Live. GEMINI_CHAT_MODEL=gemini-3.5-flash-lite -# Typed chat requests (/api/gemini/chat) per IP per minute; 0 disables it. -# GEV_RATELIMIT_GEMINI_CHAT_PER_MIN=60 # the default +# Typed chat requests (/api/gemini/chat) per IP per minute. Unset, it takes +# GEV_RATELIMIT_GEMINI_PER_MIN's cap (default 30); 0 disables it. +# GEV_RATELIMIT_GEMINI_CHAT_PER_MIN=30 diff --git a/README.md b/README.md index 3b77bd0b6..a699a3e44 100644 --- a/README.md +++ b/README.md @@ -362,11 +362,11 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain — an ## 🎙️ Talk to It -The standalone app offers **OpenAI** and **Gemini** in the GEV MIC provider chooser. OpenAI is the initial default; your choice is saved in this browser. To use Gemini, add `GEMINI_API_KEY` in Provider Settings or your local `.env` file, then select Gemini. `GEMINI_LIVE_MODEL` defaults to `gemini-3.8-live`. The permanent key stays on the local server, which issues a constrained, single-use session credential. +The standalone app offers **OpenAI** and **Gemini** under **AI Provider** in Settings, one choice shared by voice and chat. OpenAI is the initial default; your choice is saved in this browser. To use Gemini, add `GEMINI_API_KEY` in Provider Settings or your local `.env` file, then select Gemini. `GEMINI_LIVE_MODEL` defaults to `gemini-3.8-live`. The permanent key stays on the local server, which issues a constrained, single-use session credential. -Changing provider ends the current conversation. If voice is on, the new provider starts with the same open-mic or push-to-talk mode; a held Space key is not carried over. Gemini uses the same map actions and captions, but has no OpenAI cost meter or MINI tier. When Gemini reports that its session has expired, start voice again for a new conversation. The provider chooser does not change the OpenAI-backed AI HUD summary. +Changing provider ends the current conversation. If voice is on, the new provider starts with the same open-mic or push-to-talk mode; a held Space key is not carried over. Gemini uses the same map actions and captions, but has no OpenAI cost meter or MINI tier. When Gemini reports that its session has expired, start voice again for a new conversation. The AI Provider setting does not change the OpenAI-backed AI HUD summary. -Next to the mic, **CHAT** opens a typed chat with silent replies. It uses the selected provider and the same map actions, and never asks for the microphone. OpenAI chat needs an API key (`OPENAI_CHAT_MODEL`); Gemini chat uses a separate text model (`GEMINI_CHAT_MODEL`). Chat and voice panels sit above the dock or dock to the right. +Next to the mic, **CHAT** opens a typed chat with silent replies. It uses the selected provider and the same map actions, and never asks for the microphone. OpenAI chat needs an API key (`OPENAI_CHAT_MODEL`), so it is off while `GEV_PREFER_CODEX_OAUTH` pins OpenAI to ChatGPT OAuth; Gemini chat uses a separate text model (`GEMINI_CHAT_MODEL`). Chat and voice panels share the left panels' glass surface and sit above the dock or dock to the right. > Voice can use an **OpenAI API key** or, on the same machine, an opt-in signed-in **ChatGPT/Codex OAuth** session. The API-key path remains the default. Selecting ChatGPT OAuth in Provider Settings reuses an existing local sign-in or starts the Codex browser sign-in flow and selects OAuth when it completes. The OpenAI key still drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. diff --git a/SECURITY.md b/SECURITY.md index b32c5b60f..a3043eaa7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -85,7 +85,7 @@ The data proxies under `server/providers/` are written so the browser cannot tur Allowing `maps.googleapis.com` in `script-src` also trusts its JSONP endpoints, which can bypass the script allowlist if an HTML-injection bug exists; this is an accepted limitation of loading the Maps JavaScript API, alongside the existing `'unsafe-eval'` allowance. - **Full Content-Security-Policy.** The dev and preview servers send a real CSP (`script-src 'self' 'unsafe-eval' blob:` plus the YouTube, Facebook and X scripts the Nepal event's embedded media loads, with a matching `frame-src` for their players and posts: no inline script and no other script origin; `blob:` covers the object URLs Cesium's bundled workers bootstrap through in the built app), alongside the existing `X-Frame-Options: DENY` and `frame-ancestors 'none'`. `'unsafe-eval'` stays because Knockout, bundled inside Cesium's widgets package, resolves the global object with `eval` at load time and the globe does not initialize without it; styles permit `'unsafe-inline'` and Google Fonts because Vite's dev client injects `