Merge origin/main into german/street-level-slim

Conflicts: the Street Level and Gemini voice strings both appended to the
es/ru catalogs, and both sides' modules to package-boundaries.json — kept
both. scripts/qa-panel-resize.mjs stays deleted (floating panels removed).

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/providers/common/request.js

@@ -14,16 +14,34 @@ async function readRequestBodyCapped(req, maxBytes) {
   return Buffer.concat(chunks);
 }
 
-function readRequestBody(req, maxBytes = 1024 * 1024) {
+/** Optional cancellation discards buffered data while the request drains. */
+function readRequestBody(req, maxBytes = 1024 * 1024, signal) {
   return new Promise((resolve, reject) => {
     let total = 0;
     let bodyTooLarge = false;
+    let settled = false;
     const chunks = [];
-    req.on('data', (chunk) => {
+    const stopReading = () => {
+      if (!signal) return;
+      req.off('data', data);
+      req.off('end', end);
+      signal.removeEventListener('abort', abort);
+    };
+    const fail = (error) => {
+      if (signal) {
+        if (settled) return;
+        settled = true;
+        chunks.length = 0;
+        signal.removeEventListener('abort', abort);
+      }
+      reject(error);
+    };
+    const abort = () => fail(signal.reason);
+    const data = (chunk) => {
       // Keep draining after the cap without retaining more bytes. Destroying
       // the request also destroys its socket, so the handler cannot deliver
       // the fixed error response it constructs for BODY_TOO_LARGE.
-      if (bodyTooLarge) return;
+      if (bodyTooLarge || settled) return;
       total += chunk.length;
       if (total > maxBytes) {
         // Same marker readRequestBodyCapped sets, so a caller can tell an
@@ -32,15 +50,42 @@ function readRequestBody(req, maxBytes = 1024 * 1024) {
         chunks.length = 0;
         const error = new Error(`Request body exceeds ${maxBytes} bytes`);
         error.code = 'BODY_TOO_LARGE';
-        reject(error);
+        fail(error);
         return;
       }
       chunks.push(chunk);
-    });
-    req.on('end', () => {
+    };
+    const end = () => {
+      if (settled) {
+        stopReading();
+        return;
+      }
       if (!bodyTooLarge) resolve(Buffer.concat(chunks).toString('utf8'));
-    });
-    req.on('error', reject);
+      if (signal) {
+        settled = true;
+        chunks.length = 0;
+        stopReading();
+      }
+    };
+    const close = () => {
+      fail(
+        new DOMException('Request closed before its body ended', 'AbortError'),
+      );
+      stopReading();
+      req.off('error', fail);
+      req.off('close', close);
+    };
+    if (signal) {
+      // Abort/overflow release retained bytes but keep draining. Retain an
+      // error handler until close, including a late error after end/abort.
+      req.on('close', close);
+      signal.addEventListener('abort', abort, { once: true });
+      if (signal.aborted) abort();
+    }
+    req.on('data', data);
+    req.on('end', end);
+    req.on('error', fail);
+    if (signal && req.destroyed) close();
   });
 }
 

server/providers/common/request.test.mjs

@@ -0,0 +1,137 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { EventEmitter, getEventListeners } from 'node:events';
+import { readRequestBody } from './request.js';
+
+function fixture() {
+  const req = new EventEmitter();
+  req.pause = () => assert.fail('The reader must keep draining');
+  req.destroy = () =>
+    assert.fail('The reader must preserve the response socket');
+  const owner = new AbortController();
+  return { req, owner };
+}
+
+function finish(req, owner) {
+  assert.doesNotThrow(() => req.emit('error', new Error('late stream error')));
+  req.emit('end');
+  assert.doesNotThrow(() => req.emit('error', new Error('error before close')));
+  req.emit('close');
+  for (const event of ['data', 'end', 'error', 'close'])
+    assert.equal(req.listenerCount(event), 0, event);
+  assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
+}
+
+test('default and cancellable readers preserve byte caps and UTF-8 across chunks', async () => {
+  for (const useSignal of [false, true]) {
+    for (const text of ['', 'hello', 'é']) {
+      const { req, owner } = fixture();
+      const bytes = Buffer.from(text);
+      const body = readRequestBody(
+        req,
+        bytes.length,
+        useSignal ? owner.signal : undefined,
+      );
+      for (const byte of bytes) req.emit('data', Buffer.from([byte]));
+      req.emit('end');
+      assert.equal(await body, text);
+      if (useSignal) finish(req, owner);
+    }
+  }
+});
+
+test('default overflow rejects before end and drains without pausing or destroying', async () => {
+  const { req } = fixture();
+  const rejection = assert.rejects(readRequestBody(req, 2), {
+    code: 'BODY_TOO_LARGE',
+  });
+  req.emit('data', Buffer.from('é'));
+  req.emit('data', Buffer.from('!'));
+  await rejection;
+  req.emit('data', Buffer.alloc(4096));
+  assert.doesNotThrow(() => req.emit('error', new Error('late')));
+  req.emit('end');
+});
+
+test('partial abort settles before end, discards late data and guards errors until close', async () => {
+  const { req, owner } = fixture();
+  const reason = new Error('request deadline');
+  const rejection = assert.rejects(
+    readRequestBody(req, 1024, owner.signal),
+    (error) => error === reason,
+  );
+  req.emit('data', Buffer.from('partial'));
+  owner.abort(reason);
+  await rejection;
+  assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
+  req.emit('data', Buffer.alloc(4096));
+  finish(req, owner);
+});
+
+test('already-aborted and already-closed requests settle and release their listeners', async () => {
+  for (const closed of [false, true]) {
+    const { req, owner } = fixture();
+    if (closed) req.destroyed = true;
+    else owner.abort();
+    await assert.rejects(readRequestBody(req, 1024, owner.signal), {
+      name: 'AbortError',
+    });
+    if (!closed) finish(req, owner);
+    for (const event of ['data', 'end', 'error', 'close'])
+      assert.equal(req.listenerCount(event), 0);
+    assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
+  }
+});
+
+test('overflow keeps its first failure through later abort and stream errors', async () => {
+  const { req, owner } = fixture();
+  const rejection = assert.rejects(readRequestBody(req, 2, owner.signal), {
+    code: 'BODY_TOO_LARGE',
+  });
+  req.emit('data', Buffer.from('é!'));
+  await rejection;
+  owner.abort(new Error('later deadline'));
+  req.emit('data', Buffer.alloc(4096));
+  finish(req, owner);
+});
+
+test('early close and request error reject once; successful end wins over later abort', async () => {
+  for (const event of ['close', 'error', 'end']) {
+    const { req, owner } = fixture();
+    const reason = new Error('request failed');
+    const body = readRequestBody(req, 32, owner.signal);
+    const observed =
+      event === 'end'
+        ? body.then((value) => assert.equal(value, 'ok'))
+        : assert.rejects(
+            body,
+            event === 'close'
+              ? { name: 'AbortError' }
+              : (error) => error === reason,
+          );
+    req.emit('data', Buffer.from('ok'));
+    req.emit(event, reason);
+    owner.abort(new Error('later abort'));
+    await observed;
+    if (event !== 'close') finish(req, owner);
+    assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
+  }
+});
+
+test('a prior data observer can abort synchronously without retaining or resolving that chunk', async () => {
+  const { req, owner } = fixture();
+  const reason = new Error('observer stopped request');
+  const external = () => owner.abort(reason);
+  req.on('data', external);
+  const rejection = assert.rejects(
+    readRequestBody(req, 1024, owner.signal),
+    (error) => error === reason,
+  );
+  req.emit('data', Buffer.from('must be discarded'));
+  await rejection;
+  req.emit('end');
+  req.emit('close');
+  assert.deepEqual(req.listeners('data'), [external]);
+  assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
+  req.off('data', external);
+});

server/providers/gemini.js

@@ -0,0 +1,22 @@
+import { createGeminiTokenHandler } from './gemini/realtime.js';
+
+/** Mint constrained Live API credentials; the permanent key stays on Node. */
+function geminiLiveProxy({ annotationGuidance, realtime = {} } = {}) {
+  function install(middlewares) {
+    middlewares.use(
+      '/api/gemini/token',
+      createGeminiTokenHandler({ ...realtime, annotationGuidance }),
+    );
+  }
+  return {
+    name: 'gemini-live-proxy',
+    configureServer(server) {
+      install(server.middlewares);
+    },
+    configurePreviewServer(server) {
+      install(server.middlewares);
+    },
+  };
+}
+
+export { geminiLiveProxy };

server/providers/gemini/config.js

@@ -0,0 +1,53 @@
+import { realtimeInstructions } from '../openai/instructions.js';
+import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
+
+export const GEMINI_LIVE_MODEL_DEFAULT = 'gemini-3.8-live';
+export const GEMINI_TOKEN_ENDPOINT =
+  'https://generativelanguage.googleapis.com/v1beta/auth_tokens';
+export const GEMINI_LIVE_WEBSOCKET_URL =
+  'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained';
+
+export function normalizeGeminiModel(value) {
+  const model = String(value ?? GEMINI_LIVE_MODEL_DEFAULT)
+    .trim()
+    .replace(/^models\//, '');
+  return /^gemini-[a-z0-9][a-z0-9.-]{0,119}$/.test(model) ? model : null;
+}
+
+/** REST BidiGenerateContentSetup fields, excluding its model resource name. */
+export function createGeminiLiveConfig(
+  annotationGuidance,
+  inputMode = 'open-mic',
+  tools = GEV_REALTIME_TOOLS,
+) {
+  if (!['open-mic', 'push-to-talk'].includes(inputMode))
+    throw new Error('Invalid Gemini input mode');
+  const config = {
+    generationConfig: { responseModalities: ['AUDIO'] },
+    systemInstruction: {
+      parts: [{ text: realtimeInstructions(annotationGuidance) }],
+    },
+    tools: [
+      {
+        functionDeclarations: tools.map(
+          ({ name, description, parameters }) => ({
+            name,
+            description,
+            // GEV's canonical schemas are JSON Schema, not Google's Schema.
+            parametersJsonSchema: structuredClone(parameters),
+            // App actions must finish before the model confirms their result.
+            behavior: 'BLOCKING',
+          }),
+        ),
+      },
+    ],
+    inputAudioTranscription: {},
+    outputAudioTranscription: {},
+    contextWindowCompression: { slidingWindow: {} },
+  };
+  if (inputMode === 'push-to-talk')
+    config.realtimeInputConfig = {
+      automaticActivityDetection: { disabled: true },
+    };
+  return config;
+}

server/providers/gemini/realtime.js

@@ -0,0 +1,208 @@
+import { waitForSignal } from '../../../src/sources/requestDeadline.js';
+import { clientKey, makeCostRateLimiter } from '../common/rate-limit.js';
+import { readResponseJsonCapped } from '../../../src/sources/httpBody.js';
+import {
+  GEMINI_TOKEN_ENDPOINT,
+  GEMINI_LIVE_WEBSOCKET_URL,
+  GEMINI_LIVE_MODEL_DEFAULT,
+  createGeminiLiveConfig,
+  normalizeGeminiModel,
+} from './config.js';
+import {
+  isGeminiRequestOriginAllowed,
+  readGeminiTokenRequest,
+} from './request.js';
+
+const UPSTREAM_RESPONSE_BYTES = 32 * 1024;
+const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
+
+export function createGeminiTokenHandler({
+  annotationGuidance,
+  tools,
+  fetchImpl = (...args) => fetch(...args),
+  resolveApiKey = () => process.env.GEMINI_API_KEY,
+  resolveModel = () =>
+    process.env.GEMINI_LIVE_MODEL || GEMINI_LIVE_MODEL_DEFAULT,
+  resolveHost = () => process.env.HOST,
+  resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS,
+  resolveRateLimit = () => process.env.GEV_RATELIMIT_GEMINI_PER_MIN ?? '30',
+  now = Date.now,
+  timeoutMs = 15_000,
+} = {}) {
+  let limiter;
+  return async (req, res) => {
+    if (req.aborted || res.destroyed || res.writableEnded) return;
+    res.setHeader('Cache-Control', 'no-store');
+    res.setHeader('Content-Type', 'application/json; charset=utf-8');
+    res.setHeader('X-Content-Type-Options', 'nosniff');
+    const reply = (status, errorOrPayload) => {
+      if (res.destroyed || res.writableEnded) return;
+      res.statusCode = status;
+      if (status >= 400) res.setHeader('Connection', 'close');
+      res.end(
+        JSON.stringify(
+          typeof errorOrPayload === 'string'
+            ? { error: errorOrPayload }
+            : errorOrPayload,
+        ),
+      );
+    };
+    if (req.method !== 'POST') {
+      res.setHeader('Allow', 'POST');
+      return reply(405, 'Method not allowed');
+    }
+    let apiKey;
+    let model;
+    try {
+      if (
+        !isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts())
+      )
+        return reply(403, 'Gemini voice requires a same-origin app request');
+      apiKey = String(resolveApiKey() || '').trim();
+      if (!apiKey)
+        return reply(503, 'Add a Gemini API key in Provider Settings');
+      model = normalizeGeminiModel(resolveModel());
+      if (!model)
+        return reply(503, 'The Gemini Live model configuration is invalid');
+      if (limiter === undefined)
+        limiter = makeCostRateLimiter(resolveRateLimit(), 30);
+      if (limiter && !limiter(clientKey(req))) {
+        res.setHeader('Retry-After', '60');
+        return reply(
+          429,
+          'Gemini voice is starting too often. Try again shortly.',
+        );
+      }
+    } catch {
+      // Connect does not observe the handler's returned promise. A synchronous
+      // configuration resolver must fail with a bounded, fixed response.
+      return reply(503, MINT_FAILURE);
+    }
+
+    const lifetime = new AbortController();
+    let disconnected = false;
+    const disconnect = () => {
+      disconnected = true;
+      lifetime.abort(new DOMException('Request closed', 'AbortError'));
+    };
+    const responseClosed = () => {
+      if (!res.writableEnded) disconnect();
+    };
+    req.once('aborted', disconnect);
+    res.once('close', responseClosed);
+    const deadline = setTimeout(() => {
+      lifetime.abort(
+        new DOMException('Token request timed out', 'TimeoutError'),
+      );
+    }, timeoutMs);
+    deadline.unref?.();
+    let acquiredResponse;
+    try {
+      const inputMode = await readGeminiTokenRequest(req, lifetime.signal);
+      lifetime.signal.throwIfAborted();
+      const config = createGeminiLiveConfig(
+        annotationGuidance,
+        inputMode,
+        tools,
+      );
+      const issuedAt = now();
+      // REST uses bidiGenerateContentSetup. liveConnectConstraints is the SDK
+      // input shape, converted before transport. With no fieldMask the complete
+      // setup is locked to this server-owned configuration.
+      const request = {
+        uses: 1,
+        expireTime: new Date(issuedAt + 30 * 60_000).toISOString(),
+        newSessionExpireTime: new Date(issuedAt + 60_000).toISOString(),
+        bidiGenerateContentSetup: { model: `models/${model}`, ...config },
+      };
+      const acquisition = Promise.resolve(
+        fetchImpl(GEMINI_TOKEN_ENDPOINT, {
+          method: 'POST',
+          redirect: 'error',
+          signal: lifetime.signal,
+          headers: {
+            'x-goog-api-key': apiKey,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(request),
+        }),
+      );
+      // A fetch implementation can ignore abort. Bound this caller's wait and
+      // discard a response that arrives after timeout/disconnection.
+      void acquisition.then(
+        (late) => {
+          acquiredResponse = late;
+          if (lifetime.signal.aborted) void late.body?.cancel().catch(() => {});
+        },
+        () => {},
+      );
+      const response = await waitForSignal(acquisition, lifetime.signal);
+      lifetime.signal.throwIfAborted();
+      if (
+        response.redirected ||
+        (response.url && response.url !== GEMINI_TOKEN_ENDPOINT)
+      ) {
+        void response.body?.cancel().catch(() => {});
+        throw new Error('Unexpected token response destination');
+      }
+      if (!response.ok) {
+        // Never forward upstream bodies, headers or exception messages.
+        void response.body?.cancel().catch(() => {});
+        if (response.status === 429) {
+          res.setHeader('Retry-After', '60');
+          return reply(429, 'Gemini quota is unavailable. Try again later.');
+        }
+        if (response.status === 401 || response.status === 403)
+          return reply(
+            503,
+            'Gemini access was refused. Check the API key and model access.',
+          );
+        if (response.status === 400 || response.status === 404)
+          return reply(
+            502,
+            'Gemini Live is unavailable for the configured model.',
+          );
+        return reply(502, MINT_FAILURE);
+      }
+      const data = await readResponseJsonCapped(
+        response,
+        UPSTREAM_RESPONSE_BYTES,
+        lifetime.signal,
+      );
+      lifetime.signal.throwIfAborted();
+      if (
+        !data ||
+        Array.isArray(data) ||
+        typeof data !== 'object' ||
+        typeof data.name !== 'string' ||
+        data.name.length > 8192 ||
+        !/^auth_tokens\/[A-Za-z0-9._~+/=-]+$/.test(data.name) ||
+        data.name.includes(apiKey)
+      )
+        throw new Error('Invalid ephemeral token');
+      reply(200, {
+        token: data.name,
+        model,
+        config,
+        websocketUrl: GEMINI_LIVE_WEBSOCKET_URL,
+      });
+    } catch (error) {
+      if (disconnected || req.aborted) return;
+      if (error?.code === 'BODY_TOO_LARGE')
+        return reply(413, 'Token request is too large');
+      if (error?.code === 'BAD_CONTENT_TYPE')
+        return reply(415, 'Use an empty JSON token request');
+      if (error?.code === 'BAD_BODY')
+        return reply(400, 'Use an empty JSON token request');
+      reply(lifetime.signal.aborted ? 504 : 502, MINT_FAILURE);
+    } finally {
+      // Abort can happen after acquisition resolves but before this handler
+      // resumes and passes ownership to the body reader.
+      if (lifetime.signal.aborted)
+        void acquiredResponse?.body?.cancel().catch(() => {});
+      clearTimeout(deadline);
+      req.off('aborted', disconnect);
+      res.off('close', responseClosed);
+    }
+  };
+}

server/providers/gemini/request.js

@@ -0,0 +1,79 @@
+import { readRequestBody } from '../common/request.js';
+import { admitSameSiteRequest } from '../../../src/localRequestGate.mjs';
+import {
+  isAllowedHost,
+  resolveAllowedHosts,
+} from '../../../build/allowedHosts.js';
+
+/** Use the app's Host and request policies, with a required browser Origin. */
+export function isGeminiRequestOriginAllowed(
+  req,
+  configuredHost = '',
+  configuredAllowedHosts,
+) {
+  const host = req.headers?.host;
+  const origin = req.headers?.origin;
+  if (
+    typeof host !== 'string' ||
+    typeof origin !== 'string' ||
+    !host ||
+    !origin ||
+    /[\s/?#@\\]/.test(host)
+  )
+    return false;
+  const protocol = req.socket?.encrypted ? 'https:' : 'http:';
+  if (
+    !admitSameSiteRequest({
+      hostHeader: host,
+      protocol,
+      origin,
+      secFetchSite: req.headers?.['sec-fetch-site'],
+      proxyHeaders: req.headers || {},
+    }).ok
+  )
+    return false;
+  // The shared gate compares a present Origin exactly with this authority;
+  // this route also requires the Origin and an allowed Host.
+  const boundHost = String(configuredHost || '')
+    .trim()
+    .toLowerCase();
+  return isAllowedHost(host, resolveAllowedHosts(configuredAllowedHosts), [
+    boundHost,
+  ]);
+}
+
+/** Admit only the existing input gesture; callers cannot supply model/setup. */
+export async function readGeminiTokenRequest(req, signal, maxBytes = 1024) {
+  const declared = Number(req.headers?.['content-length']);
+  const fail = (code) =>
+    Object.assign(new Error('Invalid token request'), { code });
+  const body = await readRequestBody(
+    req,
+    maxBytes,
+    Number.isFinite(declared) && declared > maxBytes
+      ? AbortSignal.abort(fail('BODY_TOO_LARGE'))
+      : signal,
+  );
+  if (!body.trim()) return 'open-mic';
+  if (
+    !/^application\/json(?:\s*;|$)/i.test(req.headers?.['content-type'] || '')
+  )
+    throw fail('BAD_CONTENT_TYPE');
+  let parsed;
+  try {
+    parsed = JSON.parse(body);
+  } catch {
+    throw fail('BAD_BODY');
+  }
+  if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object')
+    throw fail('BAD_BODY');
+  const keys = Object.keys(parsed);
+  if (!keys.length) return 'open-mic';
+  if (
+    keys.length === 1 &&
+    keys[0] === 'inputMode' &&
+    parsed.inputMode === 'push-to-talk'
+  )
+    return 'push-to-talk';
+  throw fail('BAD_BODY');
+}

server/providers/local.js

@@ -19,6 +19,7 @@ import { adsbLolProxy } from './aircraft/adsb-lol.js';
 import { aisLiveProxy } from './vessels/ais-live.js';
 import { trackBackfillProxies } from './aircraft/tracks.js';
 import { openAiRealtimeProxy } from './openai.js';
+import { geminiLiveProxy } from './gemini.js';
 import { googlePlacesContextProxy } from './places.js';
 import { mapillaryProxy } from './mapillary.js';
 import { keySetupEndpoint } from '../standalone/key-setup.js';
@@ -30,9 +31,9 @@ import { windProxy } from './wind.js';
 
 /**
  * Construct the local provider plugins in their established order.
- * `realtime` configures the voice session token endpoint.
+ * `realtime` configures OpenAI; `gemini` explicitly opts into Gemini Live.
  */
-function localProviderPlugins({ realtime } = {}) {
+function localProviderPlugins({ realtime, gemini } = {}) {
   return [
     openSkyProxy(),
     celestrakProxy(),
@@ -55,6 +56,7 @@ function localProviderPlugins({ realtime } = {}) {
     aisLiveProxy(),
     trackBackfillProxies(),
     openAiRealtimeProxy({ realtime }),
+    ...(gemini ? [geminiLiveProxy({ realtime: gemini })] : []),
     googlePlacesContextProxy(),
     windProxy(),
     weatherProxy(),

server/standalone/vite.config.js

@@ -16,9 +16,10 @@ export default defineConfig(({ command, mode }) => {
   for (const [key, value] of Object.entries(loaded)) {
     if (process.env[key] === undefined) process.env[key] = value;
   }
+  const tools = standaloneVoiceTools();
   return createBrowserViteConfig({
     plugins: [
-      ...localProviderPlugins({ realtime: { tools: standaloneVoiceTools() } }),
+      ...localProviderPlugins({ realtime: { tools }, gemini: { tools } }),
       localMcpPlugin(),
       apiNotFoundPlugin(),
       unwatchedRuntimeDirsPlugin(),