Merge main into feat/voice-core

Keeps the voice-core prompt rules and adds main's world-query and
vessels-vs-visuals rules; the tool catalog wraps the deixis-aware runner.

Also fixes two things the branch had before the merge:
- Live voice could not start: OpenAI rejects track_entity's top-level
  anyOf. Model-facing tools now omit top-level combinators; the runner
  and benchmark still require a query or a referent.
- The Contacts-centred analyst answer dropped the viewport coverage note.

server/providers/openai/debug-log.js

@@ -114,10 +114,11 @@ function createDebugLogHandler({
       const body = await readRequestBody(req, REALTIME_DEBUG_LOG_MAX_BYTES);
       const parsed = JSON.parse(body || '{}');
       const record = includeContent ? parsed : omitVoiceContent(parsed);
+      // The server's own timestamp comes last, so a record cannot supply one.
       await append(
         `${JSON.stringify({
-          loggedAt: new Date().toISOString(),
           ...record,
+          loggedAt: new Date().toISOString(),
         })}\n`,
       );
       res.statusCode = 204;

server/providers/openai/hud-summary.js

@@ -2,7 +2,7 @@ import {
   HUD_SUMMARY_INSTRUCTIONS,
   keylessHudSummaryResponse,
 } from '../../../src/hudSummaryResponse.js';
-import { enforceOptInRateLimit, openAiRateLimiter } from './rate-limit.js';
+import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
 import { readRequestBody } from '../common/request.js';
 import { OPENAI_HUD_SUMMARY_MODEL_DEFAULT } from './constants.js';
 
@@ -46,10 +46,10 @@ async function handleHudSummary(req, res) {
     return;
   }
 
-  // Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). Keyless HUD
-  // fallback has no provider cost and resolves above without consuming a
-  // paid-endpoint quota slot.
-  if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return;
+  // Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables.
+  // Keyless HUD fallback has no provider cost and resolves above without
+  // consuming a paid-endpoint quota slot.
+  if (!enforceRateLimit(openAiRateLimiter(), req, res)) return;
 
   try {
     const body = await readRequestBody(req, 64 * 1024);

server/providers/openai/instructions.js

@@ -19,6 +19,7 @@ function realtimeInstructions(
     'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
     'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
     'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
+    'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
     'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
     'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
     'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, "near / nearby / how many aircraft" means the Contacts window: answer from contactsWindow in the tool result — the exact numbers on the user\'s panel (set_context_mode, analyst_query and get_current_view_state carry it once Contacts settles). For "Open Contacts and tell me how many aircraft are within 250 km", call set_context_mode{mode:"contacts"} first and answer from contactsWindow.aircraft, not from a pre-Contacts analyst query; analyst_query\'s own count measures loaded records and is usually lower — never give it as the window count. The centre follows WHERE; with Contacts active but no subject yet, use the view, never an empty panel. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
@@ -45,6 +46,7 @@ function realtimeInstructions(
     // string is the whole rollback.
     'NAMED VIEWS are shorthand for tool calls you already have — there is no "mode" tool for them. Treat ONLY these as the shorthand: "infrastructure mode" / "the infrastructure view" / "show me global infrastructure" means three set_layer_visibility calls (local-datacenters, local-dams, telegeography-submarine-cables) plus zoom_to_globe; "environmental mode" / "earth watch" / "active events", said as the name of a view, means set_layer_visibility for local-firms and earthquakes plus zoom_to_globe. Anything vaguer is NOT this shorthand — an open-ended question about the world or the news is an ordinary question: answer it, or use analyst_query over the layers already on. Never switch a whole view on to answer a question nobody asked to see. When you do run one, make every call before speaking, then give one confirmation naming the resulting state; if the fires layer comes back unavailable because no FIRMS key is configured, say so plainly — the earthquakes still loaded. "Live contacts" and "space missions" are NOT this pattern: they stay set_context_mode{mode:"contacts"} and set_context_mode{mode:"space-missions"}.',
     'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
+    'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, snow, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
     'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any other mention of "satellite" or "satellites" means the satellites DATA LAYER via set_layer_visibility, never a basemap — except "recent/latest satellite imagery" (the recent-imagery layer) and "weather satellite"/clouds (weather-satellite). "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
     'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
     'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',

server/providers/openai/rate-limit.js

@@ -1,33 +1,46 @@
-import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js';
+import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js';
+
+/**
+ * Requests/min/IP applied when GEV_RATELIMIT_OPENAI_PER_MIN is unset. Both
+ * places that already name a value agree on it: `.env.example` recommends 30
+ * and the Pinokio build ships 30 (pinokio/_ENVIRONMENT), so the packaged app
+ * is unaffected and only an unconfigured server changes. The HUD asks for a
+ * summary every 15s (HUD_SUMMARY_INTERVAL_MS), so an open tab spends 4/min and
+ * a voice session adds one token mint; the cap clears ordinary use several
+ * times over and bites only a caller working the endpoint far harder than the
+ * app does.
+ */
+const OPENAI_DEFAULT_PER_MIN = 30;
 
 // Built LAZILY on first request, NOT at module load: `.env` values are applied to process.env later
 // (the plugin config hook calls loadEnv → process.env, AFTER this module is imported), so reading
-// process.env here at import time would always see them unset and silently stay unlimited even when
+// process.env here at import time would always see them unset and fall back to the default even when
 // configured via .env. Building on first request (like the OPENAI_API_KEY reads) sees the loaded env;
-// the result is cached so the limiter's per-IP window state persists. `null` = unlimited (default).
+// the result is cached so the limiter's per-IP window state persists. `null` = the explicit 0 opt-out.
 let _openAiRateLimiter;
 
-/** OpenAI cost endpoints (realtime/token + hud-summary). Null = unlimited (default). */
+/** OpenAI cost endpoints (realtime/token + hud-summary). Null only when set to 0. */
 function openAiRateLimiter() {
   if (_openAiRateLimiter === undefined)
-    _openAiRateLimiter = makeOptInRateLimiter(
+    _openAiRateLimiter = makeCostRateLimiter(
       process.env.GEV_RATELIMIT_OPENAI_PER_MIN,
+      OPENAI_DEFAULT_PER_MIN,
     );
   return _openAiRateLimiter;
 }
 
 /**
- * Apply an opt-in limiter to a request, writing a 429 when over the cap.
- * When `limiter` is null (unlimited, the default) this is a no-op returning
- * `true`, so the handler proceeds exactly as before.
+ * Apply the limiter to a request, writing a 429 when over the cap.
+ * When `limiter` is null — the explicit `0` opt-out — this is a no-op
+ * returning `true`, so the handler proceeds unthrottled.
  *
  * @param {((key:string)=>boolean)|null} limiter
  * @param {import('http').IncomingMessage} req
  * @param {import('http').ServerResponse} res
  * @returns {boolean} True if the request may proceed; false if a 429 was sent.
  */
-function enforceOptInRateLimit(limiter, req, res) {
-  if (!limiter) return true; // unlimited (default) — no behavior change
+function enforceRateLimit(limiter, req, res) {
+  if (!limiter) return true; // explicit 0 opt-out — unthrottled
   if (limiter(clientKey(req))) return true;
   res.statusCode = 429;
   res.setHeader('Content-Type', 'application/json');
@@ -36,4 +49,4 @@ function enforceOptInRateLimit(limiter, req, res) {
   return false;
 }
 
-export { enforceOptInRateLimit, openAiRateLimiter };
+export { enforceRateLimit, openAiRateLimiter, OPENAI_DEFAULT_PER_MIN };

server/providers/openai/realtime.js

@@ -1,4 +1,4 @@
-import { enforceOptInRateLimit, openAiRateLimiter } from './rate-limit.js';
+import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
 import {
   resolveVoiceModel,
   isKnownVoiceTier,
@@ -21,6 +21,7 @@ function createRealtimeTokenHandler({
   fetchImpl = (...args) => fetch(...args),
   resolveApiKey = () => process.env.OPENAI_API_KEY,
   models = {},
+  tools = GEV_REALTIME_TOOLS,
 } = {}) {
   return async (req, res) => {
     res.setHeader('Cache-Control', 'no-store');
@@ -31,8 +32,8 @@ function createRealtimeTokenHandler({
       return;
     }
 
-    // Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). No-op when unset.
-    if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return;
+    // Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables.
+    if (!enforceRateLimit(openAiRateLimiter(), req, res)) return;
 
     const apiKey = resolveApiKey();
     if (!apiKey) {
@@ -123,7 +124,7 @@ function createRealtimeTokenHandler({
           output: { voice },
         },
         instructions: realtimeInstructions(annotationGuidance),
-        tools: GEV_REALTIME_TOOLS,
+        tools,
         tool_choice: 'auto',
       },
     };

server/providers/openai/tools.js

@@ -2,3 +2,15 @@ import { createActionTools } from '../../../src/voice/actionSchemas.js';
 import { ACTION_DESCRIPTIONS } from './toolDescriptions.js';
 
 export const GEV_REALTIME_TOOLS = createActionTools(ACTION_DESCRIPTIONS);
+
+/**
+ * The session's tools: the app actions, then additional function tools
+ * whose names no action already uses.
+ */
+export function realtimeSessionTools(additional = []) {
+  const names = new Set(GEV_REALTIME_TOOLS.map((tool) => tool.name));
+  return [
+    ...GEV_REALTIME_TOOLS,
+    ...additional.filter((tool) => !names.has(tool.name)),
+  ];
+}