mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
feat(voice): add typed AI chat beside voice
Chat sits next to the mic in the command dock. Replies are silent and never request the microphone; the first message connects with the selected provider and runs the same map actions as voice through each provider's text API (OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which never starts Live). Only one mode runs at a time. - Chat panel rendered by the voice control presenter: conversation, recent and quick-command suggestions, Up/Down history, attachments (map-layer references or small UTF-8 text files, sent only with the message) - Chat and voice panels sit above the dock or dock into the right rail's column; minimized bars sit centred above the dock, and height animates on swaps - A centred panel steps down to its bar for the Location and Visual Presets trays without moving focus, rests above an open tray, and docks right while expanded over one; it returns to the centre once the trays close - Minimized voice previews the last sentence from each side; the empty Voice Preview suggests things to say - Settings opened from chat keep chat's frame and show the shared AI provider and quick commands; voice Settings show Back to voice - Closing chat ends a typed session; the voice panel's close turns voice off - The dock widens its voice control for the chat button and restyles its wings, which show icons only at 900px and below; assistant panels share the left panels' glass, and Cyber keeps its own surfaces - Server text-chat handlers with body, time and per-IP limits (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all eleven locales Co-authored-by: Darren Dignam <[email protected]>
This commit is contained in:
1 parent
720260af58
commit
af461febae
57 files changed
+6500
-63
No files matched your search
@@ -0,0 +1,172 @@
|
||||
import { createTextChatHandler } from '../common/textChat.js';
|
||||
import { invalidCalls } from '../common/textTools.js';
|
||||
import { textInstructions } from '../common/textInstructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
|
||||
import { normalizeGeminiModel } from './config.js';
|
||||
|
||||
const object = (value) =>
|
||||
value && typeof value === 'object' && !Array.isArray(value);
|
||||
const keys = (value, allowed) =>
|
||||
Object.keys(value).every((key) => allowed.includes(key));
|
||||
|
||||
// Only text and function exchanges may cross this route, never media or config.
|
||||
export function validChatContents(contents, names) {
|
||||
return (
|
||||
Array.isArray(contents) &&
|
||||
contents.length > 0 &&
|
||||
contents.length <= 96 &&
|
||||
contents.every(
|
||||
(content) =>
|
||||
object(content) &&
|
||||
keys(content, ['role', 'parts']) &&
|
||||
['user', 'model'].includes(content.role) &&
|
||||
Array.isArray(content.parts) &&
|
||||
content.parts.length > 0 &&
|
||||
content.parts.length <= 32 &&
|
||||
content.parts.every((part) => {
|
||||
if (
|
||||
!object(part) ||
|
||||
!keys(part, [
|
||||
'text',
|
||||
'functionCall',
|
||||
'functionResponse',
|
||||
'thoughtSignature',
|
||||
])
|
||||
)
|
||||
return false;
|
||||
if (
|
||||
part.thoughtSignature !== undefined &&
|
||||
(typeof part.thoughtSignature !== 'string' ||
|
||||
part.thoughtSignature.length > 32000 ||
|
||||
content.role !== 'model')
|
||||
)
|
||||
return false;
|
||||
if (Object.hasOwn(part, 'text'))
|
||||
return (
|
||||
keys(part, ['text', 'thoughtSignature']) &&
|
||||
typeof part.text === 'string'
|
||||
);
|
||||
const call = part.functionCall ?? part.functionResponse;
|
||||
if (
|
||||
!object(call) ||
|
||||
typeof call.name !== 'string' ||
|
||||
call.name.length > 128 ||
|
||||
(names && !names.has(call.name)) ||
|
||||
(call.id !== undefined &&
|
||||
(typeof call.id !== 'string' || call.id.length > 1024))
|
||||
)
|
||||
return false;
|
||||
if (part.functionCall)
|
||||
return (
|
||||
content.role === 'model' &&
|
||||
keys(part, ['functionCall', 'thoughtSignature']) &&
|
||||
keys(call, ['name', 'args', 'id']) &&
|
||||
(call.args === null || object(call.args ?? {}))
|
||||
);
|
||||
return (
|
||||
content.role === 'user' &&
|
||||
keys(part, ['functionResponse']) &&
|
||||
keys(call, ['name', 'response', 'id']) &&
|
||||
object(call.response)
|
||||
);
|
||||
}),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function createGeminiChatHandler({
|
||||
annotationGuidance,
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
resolveChatModel = () =>
|
||||
process.env.GEMINI_CHAT_MODEL || 'gemini-3.5-flash-lite',
|
||||
resolveApiKey = () => process.env.GEMINI_API_KEY,
|
||||
...options
|
||||
} = {}) {
|
||||
return createTextChatHandler({
|
||||
...options,
|
||||
provider: 'gemini',
|
||||
resolveApiKey,
|
||||
prepare(input) {
|
||||
const model = normalizeGeminiModel(resolveChatModel());
|
||||
if (!model || /live|audio|tts/i.test(model)) return { errorStatus: 503 };
|
||||
if (
|
||||
!object(input) ||
|
||||
!keys(input, ['contents']) ||
|
||||
!validChatContents(input.contents, null)
|
||||
)
|
||||
return null;
|
||||
if (!pairedContents(input.contents)) return null;
|
||||
return {
|
||||
endpoint: `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
|
||||
headers: (key) => ({ 'x-goog-api-key': key }),
|
||||
body: {
|
||||
contents: input.contents,
|
||||
systemInstruction: {
|
||||
parts: [{ text: textInstructions(annotationGuidance) }],
|
||||
},
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: tools.map(
|
||||
({ name, description, parameters }) => ({
|
||||
name,
|
||||
description,
|
||||
parametersJsonSchema: parameters,
|
||||
}),
|
||||
),
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ['TEXT'],
|
||||
maxOutputTokens: 4096,
|
||||
thinkingConfig: { thinkingLevel: 'LOW' },
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
parse(data) {
|
||||
const candidate = data?.candidates?.[0];
|
||||
const content = candidate?.content;
|
||||
if (
|
||||
!content ||
|
||||
!validChatContents([content], null) ||
|
||||
content.role !== 'model'
|
||||
)
|
||||
return null;
|
||||
const calls = content.parts
|
||||
.filter((part) => part.functionCall)
|
||||
.map((part) => part.functionCall);
|
||||
return {
|
||||
content,
|
||||
finishReason: candidate.finishReason,
|
||||
invalidCalls: invalidCalls(calls, tools),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Never forward an orphan result or an unfinished batch in browser history.
|
||||
function pairedContents(contents) {
|
||||
let pending = [];
|
||||
for (const content of contents) {
|
||||
const calls = content.parts
|
||||
.filter((part) => part.functionCall)
|
||||
.map((part) => part.functionCall);
|
||||
const results = content.parts
|
||||
.filter((part) => part.functionResponse)
|
||||
.map((part) => part.functionResponse);
|
||||
if (pending.length) {
|
||||
if (
|
||||
calls.length ||
|
||||
results.length !== pending.length ||
|
||||
results.some(
|
||||
(result, i) =>
|
||||
result.name !== pending[i].name || result.id !== pending[i].id,
|
||||
)
|
||||
)
|
||||
return false;
|
||||
pending = [];
|
||||
} else if (results.length) return false;
|
||||
if (calls.length) pending = calls;
|
||||
}
|
||||
return !pending.length;
|
||||
}
|
||||
Reference in new issue
Block a user