feat(camera): add globe rotation (#1225)

Add continuous globe rotation with direction and speed controls, viewport fit checks, camera ownership handoff, and voice refusal guards.
This commit is contained in:
manjunath22466 authored and GitHub committed 2026-10-23 14:51:42 -07:00
1 parent 8cdd184a90
commit b2fb4a261e
25 files changed
+3407 -36

No files matched your search

+229 -10
View File
@@ -6,6 +6,7 @@ import {
SUPERSEDED_RESPONSE_MEMORY,
shouldStopVoiceAfterRadioTool,
responseInstructionForToolResult,
isGlobeViewRefusal,
extractFunctionCalls,
callDedupeKeys,
parseArguments,
@@ -66,6 +67,11 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds = new Set();
this.globeRefusals = new Map();
this.globeAdmissionGates = new Map();
this.globeResets = new Map();
this.globeResponseRequests = new Map();
this.globeRequest = { failed: false, pending: new Set() };
this.activeToolAbortControllers = new Set();
this.narration = null;
this.narrationSequence = 0;
@@ -138,6 +144,7 @@ export class RealtimeTurns {
},
};
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
this.globeRequest = { failed: false, pending: new Set() };
this.requestUserTextResponse();
}
@@ -330,6 +337,7 @@ export class RealtimeTurns {
this.userTurnPending = true;
this.pendingResponseInstructions = null;
this.cancelRadioHandoff({ abortTools: true });
this.globeRequest = { failed: false, pending: new Set() };
this.setVoiceSpeaker('user');
// Point-and-ask: read the pointer as speech starts (open mic), or reuse
// the push-to-talk keydown snapshot, before the user audio commits.
@@ -384,6 +392,27 @@ export class RealtimeTurns {
}
const toolResponseId = payload.response_id || payload.response?.id || null;
// Follow-up responses retain reset ownership until fresh admitted input.
const arrivalRequest = this.globeRequest;
const globeRequest =
this.globeResponseRequests.get(toolResponseId) ?? arrivalRequest;
// Later siblings must wait for globe admission before they can mutate state.
// Other tool responses keep their existing concurrency.
let heldForAdmission = false;
while (toolResponseId && this.globeAdmissionGates.has(toolResponseId)) {
// Drop claimed duplicates before every wait, before their keys can expire.
if (
calls.every((call) =>
callDedupeKeys(call).some((key) => this.processedCalls.has(key)),
)
)
return;
heldForAdmission = true;
await this.globeAdmissionGates.get(toolResponseId).promise;
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
}
// A newer typed command superseded the response these calls belong to.
// They are stale intent — dispatching one would let the old turn mutate
// the map after the operator asked for something else.
@@ -394,7 +423,10 @@ export class RealtimeTurns {
// hazard `callDedupeKeys` is written to avoid). So each refused call gets
// a terminal output saying plainly that the turn moved on. No
// `response.create` follows — the deferred typed turn is the single answer.
if (this.isSupersededResponse(toolResponseId)) {
if (
this.isSupersededResponse(toolResponseId) ||
(heldForAdmission && arrivalRequest !== this.globeRequest)
) {
this.pruneProcessedCalls();
for (const call of calls) {
const keys = callDedupeKeys(call);
@@ -429,7 +461,19 @@ export class RealtimeTurns {
if (keys.some((key) => this.processedCalls.has(key))) continue;
keys.forEach((key) => this.processedCalls.set(key, performance.now()));
if (this.globeRefusals.get(toolResponseId)?.blocking) {
this.sendToolOutput(call.call_id || call.id, {
ok: false,
action: call.name,
skipped: true,
error: 'Not run: this response must wait for the Globe View reply.',
});
continue;
}
let result;
let globeAdmissionGate = null;
let globeResetGate = null;
const resultChannel = this.dc;
let radioHandoffEpochAtStart = this.radio.radioHandoffEpoch;
let toolController = null;
@@ -443,6 +487,80 @@ export class RealtimeTurns {
this.narration?.preamble();
try {
const parsedArguments = parseArguments(call.arguments);
if (
toolResponseId &&
call.name === 'move_camera' &&
String(parsedArguments.motion || '').toLowerCase() === 'rotate_globe'
) {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
globeAdmissionGate = { promise, resolve };
this.globeAdmissionGates.set(toolResponseId, globeAdmissionGate);
// OpenAI delivers sibling calls concurrently. A reset dispatched
// first must finish before rotation evaluates the camera's fit.
// Hold later siblings at admission while this rotation waits.
const resets = this.globeResets.get(toolResponseId);
if (
resets ||
globeRequest.failed ||
globeRequest.pending.size ||
arrivalRequest !== this.globeRequest
) {
while (resets?.pending.size || globeRequest.pending.size)
await Promise.all(
[
...new Set([
...(resets?.pending || []),
...globeRequest.pending,
]),
].map((gate) => gate.promise),
);
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
keys.forEach((key) =>
this.processedCalls.set(key, performance.now()),
);
const superseded =
this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest;
if (superseded || resets?.failed || globeRequest.failed) {
result = {
ok: false,
action: call.name,
reason: 'globe-reset-failed',
skipped: true,
...(superseded ? { superseded: true } : {}),
error: superseded
? 'Superseded by a newer command from the operator — this call was not run.'
: 'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
};
}
}
}
if (toolResponseId && call.name === 'zoom_to_globe') {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
let resets = this.globeResets.get(toolResponseId);
if (!resets) {
resets = { pending: new Set(), failed: false };
this.globeResets.set(toolResponseId, resets);
// Retain settled outcomes for later siblings, without evicting an
// active reset that another response still needs to await.
for (const [id, record] of this.globeResets) {
if (this.globeResets.size <= SUPERSEDED_RESPONSE_MEMORY) break;
if (!record.pending.size && id !== toolResponseId)
this.globeResets.delete(id);
}
}
globeResetGate = { promise, resolve, resets, request: globeRequest };
resets.pending.add(globeResetGate);
globeRequest.pending.add(globeResetGate);
}
const isRadioControlCall = call.name === 'control_radio';
const isRadioVisibilityCall =
call.name === 'set_layer_visibility' &&
@@ -499,7 +617,7 @@ export class RealtimeTurns {
authorityDomain: radioAuthorityDomain,
});
}
result = await (this.actionExecutor || this.runner)(
result ??= await (this.actionExecutor || this.runner)(
call.name,
parsedArguments,
{
@@ -515,7 +633,28 @@ export class RealtimeTurns {
radioHandoffEpochAtStart === this.radio.radioHandoffEpoch),
},
);
if (
globeAdmissionGate &&
this.globeResets.has(toolResponseId) &&
result?.reason === 'globe-view-required'
)
result = { ...result, reason: 'globe-reset-failed' };
if (!this.ownsConversation(eventChannel)) return;
if (
isGlobeViewRefusal(result) &&
toolResponseId &&
arrivalRequest === this.globeRequest &&
!this.isSupersededResponse(toolResponseId)
) {
this.globeRefusals.set(toolResponseId, {
result,
followupQueued: false,
blocking: !result.skipped,
});
while (this.globeRefusals.size > SUPERSEDED_RESPONSE_MEMORY) {
this.globeRefusals.delete(this.globeRefusals.keys().next().value);
}
}
if (result?.ok && result.radioPlaybackRequested) {
const sessionIsCurrent =
this.activeToolAbortControllers.has(toolController) &&
@@ -595,6 +734,23 @@ export class RealtimeTurns {
: {}),
};
} finally {
if (globeResetGate) {
const { resets, request } = globeResetGate;
if (result?.ok !== true || result.cancelled) {
resets.failed = true;
request.failed = true;
}
resets.pending.delete(globeResetGate);
request.pending.delete(globeResetGate);
globeResetGate.resolve();
}
if (globeAdmissionGate) {
if (
this.globeAdmissionGates.get(toolResponseId) === globeAdmissionGate
)
this.globeAdmissionGates.delete(toolResponseId);
globeAdmissionGate.resolve();
}
if (toolController) {
this.activeToolAbortControllers.delete(toolController);
this.radio.releaseTool(toolController);
@@ -671,13 +827,38 @@ export class RealtimeTurns {
});
}
if (!this.ownsConversation(eventChannel)) return;
// Keep the Radio handoff wording authoritative even when another tool
// result follows Radio in the same multi-intent response.
this.queueResponseCreate(
responseInstructionForToolResult(
this.radio.pendingRadioPlaybackResult || lastResult,
),
);
const refusal = this.globeRefusals.get(toolResponseId);
// A newer turn may arrive during admission or the viewport-image await.
// Preserve its reply; neither the refusal map nor lastResult may revive
// an old refusal follow-up after supersession.
if (
(this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest) &&
(refusal || isGlobeViewRefusal(lastResult) || lastResult?.superseded)
) {
// Complete an idle turn below, but leave a newer running tool's status
// authoritative while suppressing this obsolete refusal follow-up.
if (this.activeToolAbortControllers.size) return;
} else if (refusal) {
// Even an earlier sibling finishing late cannot replace the reset instruction.
if (!refusal.followupQueued) {
refusal.followupQueued = true;
this.queueResponseCreate({
instructions: responseInstructionForToolResult(refusal.result),
tool_choice: 'none',
});
}
} else {
// Keep Radio handoff wording authoritative for ordinary multi-tool replies.
const followupResult =
this.radio.pendingRadioPlaybackResult || lastResult;
const instructions = responseInstructionForToolResult(followupResult);
this.queueResponseCreate(
isGlobeViewRefusal(followupResult)
? { instructions, tool_choice: 'none' }
: instructions,
);
}
}
this.setStatus('listening', t('Ask or command'));
}
@@ -966,6 +1147,22 @@ export class RealtimeTurns {
this.userTurnPending = false;
this.activeResponseId =
payload.response?.id || payload.response_id || null;
// Late calls retain only their globe reset/request ownership. Ordinary
// tool admission and concurrency continue to use their existing policy.
if (
this.activeResponseId &&
!this.globeResponseRequests.has(this.activeResponseId)
) {
this.globeResponseRequests.set(
this.activeResponseId,
this.globeRequest,
);
while (this.globeResponseRequests.size > SUPERSEDED_RESPONSE_MEMORY)
this.globeResponseRequests.delete(
this.globeResponseRequests.keys().next().value,
);
}
this.setVoiceSpeaker('ai');
return;
}
@@ -1045,7 +1242,8 @@ export class RealtimeTurns {
const sent = this.sendRealtimeEvent(
{
type: 'response.create',
response: { instructions },
response:
typeof instructions === 'string' ? { instructions } : instructions,
},
'client.response_create.tool_followup',
);
@@ -1057,6 +1255,20 @@ export class RealtimeTurns {
controller.abort();
this.activeToolAbortControllers.clear();
this.radio.clearTools();
this.releaseGlobeResets();
}
releaseGlobeResets() {
for (const resets of this.globeResets.values()) {
if (!resets.pending.size) continue;
resets.failed = true;
for (const gate of resets.pending) {
gate.request.failed = true;
gate.request.pending.delete(gate);
gate.resolve();
}
resets.pending.clear();
}
}
reset() {
@@ -1068,6 +1280,13 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds.clear();
this.globeRefusals.clear();
for (const gate of this.globeAdmissionGates.values()) gate.resolve();
this.globeAdmissionGates.clear();
this.releaseGlobeResets();
this.globeResets.clear();
this.globeResponseRequests.clear();
this.globeRequest = { failed: false, pending: new Set() };
this.narration?.cancel();
this.assistantAudioPlaying = false;
this.speechResponseIds.clear();