feat(voice): add typed AI chat beside voice

Chat sits next to the mic in the command dock. Replies are silent and never
request the microphone; the first message connects with the selected provider
and runs the same map actions as voice through each provider's text API
(OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which
never starts Live). Only one mode runs at a time.

- Chat panel rendered by the voice control presenter: conversation, recent and
  quick-command suggestions, Up/Down history, attachments (map-layer
  references or small UTF-8 text files, sent only with the message)
- Chat and voice panels sit above the dock or dock into the right rail's
  column; minimized bars sit centred above the dock, and height animates on
  swaps
- A centred panel steps down to its bar for the Location and Visual Presets
  trays without moving focus, rests above an open tray, and docks right while
  expanded over one; it returns to the centre once the trays close
- Minimized voice previews the last sentence from each side; the empty Voice
  Preview suggests things to say
- Settings opened from chat keep chat's frame and show the shared AI provider
  and quick commands; voice Settings show Back to voice
- Closing chat ends a typed session; the voice panel's close turns voice off
- The dock widens its voice control for the chat button and restyles its
  wings, which show icons only at 900px and below; assistant panels share the
  left panels' glass, and Cyber keeps its own surfaces
- Server text-chat handlers with body, time and per-IP limits
  (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all
  eleven locales

Co-authored-by: Darren Dignam <[email protected]>
This commit is contained in:
manjunath22466andDarren Dignam committed 2026-11-02 20:56:39 +05:30
1 parent 720260af58
commit af461febae
57 files changed
+6500 -63

No files matched your search

+136
View File
@@ -0,0 +1,136 @@
import { readRequestBody } from './request.js';
import { clientKey, makeCostRateLimiter } from './rate-limit.js';
import { waitForSignal } from '../../../src/sources/requestDeadline.js';
import { readResponseJsonCapped } from '../../../src/sources/httpBody.js';
import { isGeminiRequestOriginAllowed } from '../gemini/request.js';
import { parseUsage } from './textTools.js';
const MAX_BYTES = 512 * 1024;
const FAILURE = 'Could not connect. Try again.';
export function createTextChatHandler({
provider,
prepare,
parse,
fetchImpl = (...args) => fetch(...args),
resolveApiKey,
resolveHost = () => process.env.HOST,
resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS,
resolveRateLimit = () =>
process.env[`GEV_RATELIMIT_${provider.toUpperCase()}_CHAT_PER_MIN`] ?? '60',
timeoutMs = 60000,
} = {}) {
let limiter;
return async (req, res) => {
if (req.aborted || res.destroyed || res.writableEnded) return;
res.setHeader('Cache-Control', 'no-store');
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('X-Content-Type-Options', 'nosniff');
const reply = (status, value) => {
if (req.aborted || res.destroyed || res.writableEnded) return;
res.statusCode = status;
if (status >= 400) res.setHeader('Connection', 'close');
res.end(
JSON.stringify(typeof value === 'string' ? { error: value } : value),
);
};
if (req.method !== 'POST') {
res.setHeader('Allow', 'POST');
return reply(405, 'Method not allowed');
}
const lifetime = new AbortController();
const disconnect = () => {
if (!res.writableEnded) lifetime.abort();
};
req.once('aborted', disconnect);
res.once('close', disconnect);
const timer = setTimeout(() => lifetime.abort(), timeoutMs);
timer.unref?.();
let response;
let usage = null;
try {
if (
!isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts())
)
return reply(403, FAILURE);
const apiKey = String(resolveApiKey() || '').trim();
if (!apiKey) return reply(503, 'Add an API key in Provider Settings');
if (limiter === undefined)
limiter = makeCostRateLimiter(resolveRateLimit(), 60);
if (limiter && !limiter(clientKey(req))) {
res.setHeader('Retry-After', '60');
return reply(429, FAILURE);
}
if (
!/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')
)
return reply(415, FAILURE);
if (Number(req.headers['content-length']) > MAX_BYTES)
return reply(413, FAILURE);
const raw = await readRequestBody(req, MAX_BYTES, lifetime.signal);
let input;
try {
input = JSON.parse(raw);
} catch {
return reply(400, FAILURE);
}
const prepared = prepare(input);
if (!prepared) return reply(400, FAILURE);
if (prepared.errorStatus) return reply(prepared.errorStatus, FAILURE);
const { endpoint, body, headers } = prepared;
lifetime.signal.throwIfAborted();
const acquisition = Promise.resolve(
fetchImpl(endpoint, {
method: 'POST',
redirect: 'error',
signal: lifetime.signal,
headers: { 'Content-Type': 'application/json', ...headers(apiKey) },
body: JSON.stringify(body),
}),
);
void acquisition.then(
(late) => {
response = late;
if (lifetime.signal.aborted) void late.body?.cancel().catch(() => {});
},
() => {},
);
response = await waitForSignal(acquisition, lifetime.signal);
lifetime.signal.throwIfAborted();
if (response.redirected || (response.url && response.url !== endpoint))
throw new Error('Unexpected destination');
if (!response.ok) {
void response.body?.cancel().catch(() => {});
return reply(
[429, 503].includes(response.status) ? response.status : 502,
FAILURE,
);
}
const data = await readResponseJsonCapped(
response,
MAX_BYTES,
lifetime.signal,
);
lifetime.signal.throwIfAborted();
usage = parseUsage(provider, data);
const result = parse(data);
if (!result || JSON.stringify(result).includes(apiKey))
return reply(502, { error: FAILURE, usage });
reply(200, { ...result, usage });
} catch (error) {
reply(
error?.code === 'BODY_TOO_LARGE'
? 413
: lifetime.signal.aborted
? 504
: 502,
{ error: FAILURE, usage },
);
} finally {
if (lifetime.signal.aborted)
void response?.body?.cancel().catch(() => {});
clearTimeout(timer);
req.off('aborted', disconnect);
res.off('close', disconnect);
}
};
}
@@ -0,0 +1,31 @@
import { realtimeDirectives } from '../openai/instructions.js';
// Voice wording in the shared directives and its written form. A test checks
// each phrase still occurs in the directives, so rewording one there fails
// rather than leaking spoken wording into chat.
export const TEXT_WORDING = Object.freeze([
['without speaking first', 'without writing a preamble'],
['before speaking', 'before confirming'],
['may first say ONE sentence', 'may first write ONE sentence'],
["speak the result's say line", "write the result's say line"],
['on the voice card: speak a caveat', 'in the tool result: include a caveat'],
['as you talk', 'as you write'],
['you can read aloud', 'you can report'],
]);
// Shared-manual adaptation follows Darren Dignam's GEV COMMAND approach (#976).
// Keep our Radio handoff and annotation outcome policies from the shared manual.
export function textInstructions(annotationGuidance) {
return realtimeDirectives(annotationGuidance)
.map((line, index) => {
if (index === 0)
return "You are GEV Chat, a concise text assistant for a Cesium geospatial app called God's Eye View.";
if (index === 1)
return 'Have a natural written conversation. This is text-only chat; no microphone or generated audio is used.';
return TEXT_WORDING.reduce(
(text, [spoken, written]) => text.replace(spoken, written),
line,
);
})
.join('\n');
}
+54
View File
@@ -0,0 +1,54 @@
import { validateValue } from '../../../src/tools/schema.js';
export function invalidCalls(calls, tools) {
const schemas = new Map(tools.map((tool) => [tool.name, tool.parameters]));
return calls.flatMap((call, index) => {
const schema = schemas.get(call.name);
const errors = !schema
? ['Unknown tool']
: call.parseError
? ['Arguments must be valid JSON']
: validateValue(schema, call.args === undefined ? {} : call.args);
return errors.length
? [{ index, error: errors.slice(0, 4).join('; ').slice(0, 1000) }]
: [];
});
}
// Missing usage stays unknown, never an invented zero-cost request.
export function parseUsage(provider, data) {
const raw = provider === 'gemini' ? data?.usageMetadata : data?.usage;
const number = (value) =>
Number.isSafeInteger(value) && value >= 0 ? value : null;
const inputTokens = number(
provider === 'gemini' ? raw?.promptTokenCount : raw?.prompt_tokens,
);
const candidateTokens = number(
provider === 'gemini' ? raw?.candidatesTokenCount : raw?.completion_tokens,
);
const totalTokens = number(
provider === 'gemini' ? raw?.totalTokenCount : raw?.total_tokens,
);
if (inputTokens === null || candidateTokens === null || totalTokens === null)
return null;
const reasoningTokens =
number(
provider === 'gemini'
? raw?.thoughtsTokenCount
: raw?.completion_tokens_details?.reasoning_tokens,
) ?? 0;
const outputTokens =
candidateTokens + (provider === 'gemini' ? reasoningTokens : 0);
return {
inputTokens,
outputTokens,
totalTokens,
reasoningTokens,
cachedInputTokens:
number(
provider === 'gemini'
? raw?.cachedContentTokenCount
: raw?.prompt_tokens_details?.cached_tokens,
) ?? 0,
};
}