Merge upstream main into ChatGPT Realtime OAuth

This commit is contained in:
devv-shayan committed 2026-10-04 12:29:54 +05:00
commit 6c88b78140
402 files changed
+48518 -3623

No files matched your search

+2 -1
View File
@@ -80,10 +80,11 @@ function createDebugLogHandler({ sourceRoot = defaultSourceRoot } = {}) {
try {
const body = await readRequestBody(req, REALTIME_DEBUG_LOG_MAX_BYTES);
const record = JSON.parse(body || '{}');
// The server's own timestamp comes last, so a record cannot supply one.
await append(
`${JSON.stringify({
loggedAt: new Date().toISOString(),
...record,
loggedAt: new Date().toISOString(),
})}\n`,
);
res.statusCode = 204;
+5 -5
View File
@@ -2,7 +2,7 @@ import {
HUD_SUMMARY_INSTRUCTIONS,
keylessHudSummaryResponse,
} from '../../../src/hudSummaryResponse.js';
import { enforceOptInRateLimit, openAiRateLimiter } from './rate-limit.js';
import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
import { readRequestBody } from '../common/request.js';
import { OPENAI_HUD_SUMMARY_MODEL_DEFAULT } from './constants.js';
@@ -46,10 +46,10 @@ async function handleHudSummary(req, res) {
return;
}
// Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). Keyless HUD
// fallback has no provider cost and resolves above without consuming a
// paid-endpoint quota slot.
if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return;
// Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables.
// Keyless HUD fallback has no provider cost and resolves above without
// consuming a paid-endpoint quota slot.
if (!enforceRateLimit(openAiRateLimiter(), req, res)) return;
try {
const body = await readRequestBody(req, 64 * 1024);
+2
View File
@@ -19,6 +19,7 @@ function realtimeInstructions(
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data: report the returned feed source briefly, never call it an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
'For ANALYTICAL questions about layer data — how many / which / fastest / highest / biggest / nearest flights, ships, fires, earthquakes, satellites, datacenters, or dams ("how many flights over Texas", "biggest fire near LA", "which ships are headed to Oakland", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Narrate the count plus two or three notable examples by name, and reflect the result\'s coverage note honestly: the answer covers data loaded by enabled layers, not the whole world. If the needed layer is disabled, say so and offer to enable it. For follow-ups about the same set ("which of THOSE is closest?"), call analyst_query with followUp=true and only the new filter/sort.',
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
'FEED STATE CONTRACT — analyst_query and get_current_view_state carry feedProvenance from the same layerFeedState the Data Layers chips use (nominal/loading/degraded/stale/fallback/unavailable, or off). NEVER narrate a count as live or current when feedProvenance.overall is not nominal. Say the feedState in the same breath as the count ("12 flights in view, STALE"). If unavailable, do not invent a count; say the feed is unavailable. Fallback is valid data — name the fallback source, never call it an enable failure. Confirm only what the tool result returned.',
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, "near / nearby / how many aircraft" means the Contacts window: answer from contactsWindow in the tool result — those are the exact numbers on the user\'s panel. set_context_mode, analyst_query, and get_current_view_state carry it after Contacts settles. For "Open Contacts and tell me how many aircraft are within 250 km", call set_context_mode{mode:"contacts"} first and answer from contactsWindow.aircraft; do not answer from a pre-Contacts analyst query. analyst_query\'s own count measures currently-loaded records and is usually lower; never give it as the window count. CENTER PRECEDENCE for a nearby/how-many ask, in order: an explicit place in the question ("over Texas", "near Austin") always wins and ignores Contacts state; else the CONTACTS SUBJECT when Contacts is active and has one — a selected datacenter, dam, fire, or cable does NOT silently become the center; else an entity the user explicitly names ("around this datacenter"); else the current view, said aloud ("nothing is selected, so this is the current view"). With Contacts active but NO subject yet, use the view and say so; never read an empty panel. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "about 30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two different numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge ("a few", "less than a dozen"): if a tool returns 46, say 46. (5) When it matters, add once: counts cover loaded data, and the flights layer loads where you look.',
'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
@@ -48,6 +49,7 @@ function realtimeInstructions(
// string is the whole rollback.
'NAMED VIEWS are shorthand for tool calls you already have — there is no "mode" tool for them. Treat ONLY these as the shorthand: "infrastructure mode" / "the infrastructure view" / "show me global infrastructure" means three set_layer_visibility calls (local-datacenters, local-dams, telegeography-submarine-cables) plus zoom_to_globe; "environmental mode" / "earth watch" / "active events", said as the name of a view, means set_layer_visibility for local-firms and earthquakes plus zoom_to_globe. Anything vaguer is NOT this shorthand — an open-ended question about the world or the news is an ordinary question: answer it, or use analyst_query over the layers already on. Never switch a whole view on to answer a question nobody asked to see. When you do run one, make every call before speaking, then give one confirmation naming the resulting state; if the fires layer comes back unavailable because no FIRMS key is configured, say so plainly — the earthquakes still loaded. "Live contacts" and "space missions" are NOT this pattern: they stay set_context_mode{mode:"contacts"} and set_context_mode{mode:"space-missions"}.',
'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, snow, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any mention of "satellite" or "satellites" ALWAYS means the satellites DATA LAYER via set_layer_visibility, never a basemap. "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
+24 -11
View File
@@ -1,33 +1,46 @@
import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js';
import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js';
/**
* Requests/min/IP applied when GEV_RATELIMIT_OPENAI_PER_MIN is unset. Both
* places that already name a value agree on it: `.env.example` recommends 30
* and the Pinokio build ships 30 (pinokio/_ENVIRONMENT), so the packaged app
* is unaffected and only an unconfigured server changes. The HUD asks for a
* summary every 15s (HUD_SUMMARY_INTERVAL_MS), so an open tab spends 4/min and
* a voice session adds one token mint; the cap clears ordinary use several
* times over and bites only a caller working the endpoint far harder than the
* app does.
*/
const OPENAI_DEFAULT_PER_MIN = 30;
// Built LAZILY on first request, NOT at module load: `.env` values are applied to process.env later
// (the plugin config hook calls loadEnv → process.env, AFTER this module is imported), so reading
// process.env here at import time would always see them unset and silently stay unlimited even when
// process.env here at import time would always see them unset and fall back to the default even when
// configured via .env. Building on first request (like the OPENAI_API_KEY reads) sees the loaded env;
// the result is cached so the limiter's per-IP window state persists. `null` = unlimited (default).
// the result is cached so the limiter's per-IP window state persists. `null` = the explicit 0 opt-out.
let _openAiRateLimiter;
/** OpenAI cost endpoints (realtime/token + hud-summary). Null = unlimited (default). */
/** OpenAI cost endpoints (realtime/token + hud-summary). Null only when set to 0. */
function openAiRateLimiter() {
if (_openAiRateLimiter === undefined)
_openAiRateLimiter = makeOptInRateLimiter(
_openAiRateLimiter = makeCostRateLimiter(
process.env.GEV_RATELIMIT_OPENAI_PER_MIN,
OPENAI_DEFAULT_PER_MIN,
);
return _openAiRateLimiter;
}
/**
* Apply an opt-in limiter to a request, writing a 429 when over the cap.
* When `limiter` is null (unlimited, the default) this is a no-op returning
* `true`, so the handler proceeds exactly as before.
* Apply the limiter to a request, writing a 429 when over the cap.
* When `limiter` is null — the explicit `0` opt-out — this is a no-op
* returning `true`, so the handler proceeds unthrottled.
*
* @param {((key:string)=>boolean)|null} limiter
* @param {import('http').IncomingMessage} req
* @param {import('http').ServerResponse} res
* @returns {boolean} True if the request may proceed; false if a 429 was sent.
*/
function enforceOptInRateLimit(limiter, req, res) {
if (!limiter) return true; // unlimited (default) — no behavior change
function enforceRateLimit(limiter, req, res) {
if (!limiter) return true; // explicit 0 opt-out — unthrottled
if (limiter(clientKey(req))) return true;
res.statusCode = 429;
res.setHeader('Content-Type', 'application/json');
@@ -36,4 +49,4 @@ function enforceOptInRateLimit(limiter, req, res) {
return false;
}
export { enforceOptInRateLimit, openAiRateLimiter };
export { enforceRateLimit, openAiRateLimiter, OPENAI_DEFAULT_PER_MIN };
+5 -4
View File
@@ -1,4 +1,4 @@
import { enforceOptInRateLimit, openAiRateLimiter } from './rate-limit.js';
import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
import {
resolveVoiceModel,
isKnownVoiceTier,
@@ -177,6 +177,7 @@ function createRealtimeTokenHandler({
resolveOAuthAccessToken = () => readCodexOAuthAccessToken(),
environment = process.env,
models = {},
tools = GEV_REALTIME_TOOLS,
} = {}) {
return async (req, res) => {
res.setHeader('Cache-Control', 'no-store');
@@ -218,8 +219,8 @@ function createRealtimeTokenHandler({
return;
}
// Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). No-op when unset.
if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return;
// Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables.
if (!enforceRateLimit(openAiRateLimiter(), req, res)) return;
let bearer;
if (authMode === 'oauth') {
@@ -331,7 +332,7 @@ function createRealtimeTokenHandler({
output: { voice },
},
instructions: realtimeInstructions(annotationGuidance),
tools: GEV_REALTIME_TOOLS,
tools,
tool_choice: 'auto',
},
};
+12
View File
@@ -2,3 +2,15 @@ import { createActionTools } from '../../../src/voice/actionSchemas.js';
import { ACTION_DESCRIPTIONS } from './toolDescriptions.js';
export const GEV_REALTIME_TOOLS = createActionTools(ACTION_DESCRIPTIONS);
/**
* The session's tools: the app actions, then additional function tools
* whose names no action already uses.
*/
export function realtimeSessionTools(additional = []) {
const names = new Set(GEV_REALTIME_TOOLS.map((tool) => tool.name));
return [
...GEV_REALTIME_TOOLS,
...additional.filter((tool) => !names.has(tool.name)),
];
}