diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 90d68d390..d73e7c899 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -620,7 +620,7 @@ Source factories have dedicated `layers//source` exports. ALPR and earth Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner. -The standalone app offers OpenAI and Gemini Live through an explicit provider chooser. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. +The standalone app offers OpenAI and Gemini Live through an explicit provider chooser. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 30 tools using separate description-only metadata. Description customization cannot replace argument types, enum values or required fields. diff --git a/server/providers/gemini/request.js b/server/providers/gemini/request.js index 0ce54d5ab..3dde288d6 100644 --- a/server/providers/gemini/request.js +++ b/server/providers/gemini/request.js @@ -17,6 +17,7 @@ export function isGeminiRequestOriginAllowed( typeof host !== 'string' || typeof origin !== 'string' || !host || + !origin || /[\s/?#@\\]/.test(host) ) return false; @@ -31,29 +32,14 @@ export function isGeminiRequestOriginAllowed( }).ok ) return false; + // The shared gate compares a present Origin exactly with this authority; + // this route also requires the Origin and an allowed Host. const boundHost = String(configuredHost || '') .trim() .toLowerCase(); - if ( - !isAllowedHost(host, resolveAllowedHosts(configuredAllowedHosts), [ - boundHost, - ]) - ) - return false; - try { - const authority = new URL(`${protocol}//${host}`); - const source = new URL(origin); - return ( - source.origin === authority.origin && - source.username === '' && - source.password === '' && - source.pathname === '/' && - source.search === '' && - source.hash === '' - ); - } catch { - return false; - } + return isAllowedHost(host, resolveAllowedHosts(configuredAllowedHosts), [ + boundHost, + ]); } /** Admit only the existing input gesture; callers cannot supply model/setup. */ diff --git a/src/voice/geminiSession.js b/src/voice/geminiSession.js index a44347ec8..04a573f76 100644 --- a/src/voice/geminiSession.js +++ b/src/voice/geminiSession.js @@ -12,8 +12,9 @@ import { readResponseJsonCapped } from '../sources/httpBody.js'; import { readLayerLifecycleSummary } from './layerSummary.js'; import { POINTER_OPEN_MIC_RECENT_MS, - pointerChip, - pointerContextItem, + pointerAnnouncement, + pointerTurnContext, + pointerTurnEvent, } from './pointerContext.js'; import { hasStructuredViewIdentity, @@ -32,7 +33,7 @@ const CONTROL_POLL_MS = 25; const MAX_GO_AWAY_MS = 10 * 60 * 1000; const MAX_MESSAGE_LENGTH = 3 * 1024 * 1024; const MAX_TOOL_CALLS = 16; -const MAX_SEEN_CALLS = 256; +const MAX_SEEN_CALLS = 4096; const MAX_CALL_ID_LENGTH = 1024; const MAX_MAP_EVENTS = 16; const MAX_MAP_CONTEXT_LENGTH = 16000; @@ -206,6 +207,8 @@ export class GeminiSessionController { // Live output has no response ID. A locally retired reply keeps ownership // of incoming output until its ordered server turnComplete boundary. this.replyPending = false; + // A reply Gemini started without a request; it gets no tool authority. + this.unrequestedReply = false; this.automaticTurnOpen = false; this.automaticActivityOwner = null; this.retiredAutomaticActivityOwner = null; @@ -1035,6 +1038,7 @@ export class GeminiSessionController { this.retiredReplyPending = true; } this.replyPending = false; + this.unrequestedReply = false; if (['push-to-talk', 'user-text'].includes(reason)) { this.automaticTurnOpen = false; this.automaticActivityOwner = null; @@ -1341,12 +1345,43 @@ export class GeminiSessionController { } } if (this.discardRetiredReply(message, epoch)) return; + // Push-to-talk owns every input boundary locally: a call that arrives with + // no input, reply or tool result outstanding was not requested, and nor is + // anything else in the reply it starts. Open-mic turns start on Gemini's + // server-side activity detection, which reports no boundary to gate on. + const unrequested = + this.input.pushToTalkMode && + !this.replyPending && + !this.userTurnPending && + !this.pendingTools; + let toolCalls = message.toolCall?.functionCalls || []; + if (toolCalls.length && (unrequested || this.unrequestedReply)) { + this.refuseUnrequestedTools(toolCalls); + toolCalls = []; + } + // In push-to-talk, a caption alone after its turn ended still belongs to + // that turn. It must not open a reply the next input would retire, + // discarding that input's own tool calls while it waits for a boundary + // that never comes. Open mic cannot tell it from a new reply's first + // caption, so there it still opens one. + const lateCaption = + this.input.pushToTalkMode && + !this.replyPending && + !this.userTurnPending && + !this.automaticTurnOpen && + !this.pendingTools && + !content?.modelTurn && + !content?.generationComplete && + !toolCalls.length; if ( content?.modelTurn || - content?.outputTranscription || + (content?.outputTranscription && !lateCaption) || content?.generationComplete || - message.toolCall + toolCalls.length ) { + // Output with nothing outstanding starts a reply nobody requested; a + // later call in it is refused like one that arrived first. + if (unrequested) this.unrequestedReply = true; // Once successor output is observed, an unattributed later interruption // cannot still claim the unobserved automatic predecessor's boundary. if (this.typedIntent) { @@ -1386,11 +1421,14 @@ export class GeminiSessionController { } this.modelSpoke = true; this.narration.audible(); + if (!messageCurrent()) { + settleFencedBoundary(); + return; + } } } } - if (message.toolCall?.functionCalls?.length) - this.enqueueTools(message.toolCall.functionCalls, epoch); + if (toolCalls.length) this.enqueueTools(toolCalls, epoch); if (content?.turnComplete) { if (this.typedIntent) { this.typedIntent.automaticPredecessor = null; @@ -1401,6 +1439,7 @@ export class GeminiSessionController { // pending tools and queued controls still gate presentation completion. this.userTurnPending = false; this.replyPending = false; + this.unrequestedReply = false; this.automaticTurnOpen = false; this.automaticActivityOwner = null; this.retiredAutomaticActivityOwner = null; @@ -1441,6 +1480,42 @@ export class GeminiSessionController { }); } + /** + * Answer unrequested calls without running them: every declaration is + * BLOCKING, so Live waits for a result. The reply stays tracked, so the next + * input retires it and its late boundary cannot close the user's turn. + */ + refuseUnrequestedTools(calls) { + const refused = new Map(); + for (const call of calls) + if ( + validCallId(call?.id) && + typeof call.name === 'string' && + !this.seenCalls.has(call.id) + ) + refused.set(call.id, call.name); + this.rememberCallIds([...refused.keys()]); + if (!refused.size) return; + this.unrequestedReply = true; + this.replyPending = true; + this.send({ + toolResponse: { + functionResponses: [...refused].map(([id, name]) => ({ + id, + name, + response: { + ok: false, + action: name, + changed: false, + error: 'No user request is active.', + responseGuidance: + 'Do not call another tool or claim an action ran. Wait for the user.', + }, + })), + }, + }); + } + rememberCallIds(ids) { const freshIds = new Set(ids.filter((id) => !this.seenCalls.has(id))); // Forgetting an old ID would let its replay acquire a new turn's authority. @@ -2076,42 +2151,25 @@ export class GeminiSessionController { snapshot = null; } if (!isCurrent()) return null; - const chip = pointerChip(snapshot); - this.pointerAnnounced = gesture && chip ? snapshot : null; - this.emit({ - type: 'pointer', - pointer: - gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null, - }); + const event = pointerTurnEvent(snapshot, gesture); + this.pointerAnnounced = event.announced; + this.emit({ type: 'pointer', pointer: event.pointer }); if (!isCurrent()) return null; - const item = !chip - ? this.pointerSentKey && this.pointerSentKey !== 'none' - ? { type: 'pointer_context', at, target: 'none' } - : null - : gesture - ? pointerContextItem(snapshot) - : { - type: 'pointer_context', - at, - pointing: snapshot.entity?.kind || 'ground', - }; - const { at: _at, ...identity } = item || {}; - const key = chip ? JSON.stringify(identity) : 'none'; - this.pendingPointerContext = - item && key !== this.pointerSentKey ? { item, key } : null; + this.pendingPointerContext = pointerTurnContext(snapshot, { + at, + gesture, + sentKey: this.pointerSentKey, + }); return snapshot; } /** Reveal an implicit referent only after the shared action runner uses it. */ announcePointer() { const snapshot = this.pointer?.activeSnapshot?.() || null; - const chip = pointerChip(snapshot); - if (!chip || snapshot === this.pointerAnnounced) return false; + const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced); + if (!pointer) return false; this.pointerAnnounced = snapshot; - this.emit({ - type: 'pointer', - pointer: { ...chip, screenPx: snapshot.screenPx }, - }); + this.emit({ type: 'pointer', pointer }); return true; } @@ -2398,6 +2456,7 @@ export class GeminiSessionController { this.pendingMapEvents = []; this.userTurnPending = false; this.replyPending = false; + this.unrequestedReply = false; this.automaticTurnOpen = false; this.automaticActivityOwner = null; this.retiredAutomaticActivityOwner = null; diff --git a/src/voice/geminiSession.test.mjs b/src/voice/geminiSession.test.mjs index 90b738786..b2f380b37 100644 --- a/src/voice/geminiSession.test.mjs +++ b/src/voice/geminiSession.test.mjs @@ -3193,3 +3193,208 @@ for (const segments of [['use foo as my voice provider', 'and zoom out']]) { assert.equal(h.timers.size, 0); }); } + +const zoomCall = (id) => ({ + toolCall: { + functionCalls: [ + { id, name: 'adjust_camera_zoom', args: { direction: 'in' } }, + ], + }, +}); +const toolResponses = (ws, from = 0) => + ws.sent + .slice(from) + .flatMap((message) => message.toolResponse?.functionResponses || []); + +async function completedPttTurn() { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + establishToolInput(h); + ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); + h.captures[0].drain(); + await tick(); + return { h, ws }; +} + +test('push-to-talk refuses an unrequested tool call and every call in its reply', async () => { + const { h, ws } = await completedPttTurn(); + const sent = ws.sent.length; + ws.message(zoomCall('unrequested')); + ws.message(zoomCall('follow-up')); + await h.c.toolQueue; + assert.deepEqual(h.calls, []); + assert.deepEqual( + toolResponses(ws, sent).map(({ id, response }) => [id, response.ok]), + [ + ['unrequested', false], + ['follow-up', false], + ], + ); + ws.message({ serverContent: { turnComplete: true } }); + establishToolInput(h, 'Zoom in'); + // A refused ID stays spent; only the newly requested call runs. + ws.message(zoomCall('unrequested')); + ws.message(zoomCall('requested')); + await h.c.toolQueue; + assert.deepEqual( + h.calls.map((call) => call.name), + ['adjust_camera_zoom'], + ); + assert.equal(toolResponses(ws, sent).length, 3); + h.c.stop(); +}); + +test('push-to-talk refuses a call after unprompted speech that started the reply', async () => { + const { h, ws } = await completedPttTurn(); + const sent = ws.sent.length; + ws.message({ + serverContent: { outputTranscription: { text: 'Let me zoom in' } }, + }); + ws.message(zoomCall('after-speech')); + await h.c.toolQueue; + assert.deepEqual(h.calls, []); + assert.deepEqual( + toolResponses(ws, sent).map(({ id, response }) => [id, response.ok]), + [['after-speech', false]], + ); + h.c.stop(); +}); + +test('a refusal does not hold back the completion of the reply still playing', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + establishToolInput(h); + ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); + ws.message(zoomCall('unrequested')); + h.captures[0].drain(); + await tick(); + assert.equal(h.events.filter((e) => e.type === 'completion').length, 1); + assert.deepEqual(h.calls, []); + h.c.stop(); +}); + +test('open microphone still runs a call Gemini makes without a local boundary', async () => { + const h = harness(); + await h.c.start(); + h.sockets[0].message(zoomCall('server-detected')); + await h.c.toolQueue; + assert.deepEqual( + h.calls.map((call) => call.name), + ['adjust_camera_zoom'], + ); + h.c.stop(); +}); + +test('a late caption after a completed push-to-talk turn cannot retire the next request', async () => { + const { h, ws } = await completedPttTurn(); + ws.message({ serverContent: { outputTranscription: { text: ' done.' } } }); + establishToolInput(h, 'Zoom in'); + ws.message(zoomCall('requested')); + await h.c.toolQueue; + h.advance(5_000); + assert.deepEqual( + h.calls.map((call) => call.name), + ['adjust_camera_zoom'], + ); + assert.equal(h.c.lastError, null); + h.c.stop(); +}); + +test('an open-mic reply that starts with a caption is retired by newer typed input', async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0]; + establishToolInput(h); + ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); + h.captures[0].drain(); + await tick(); + ws.message({ serverContent: { outputTranscription: { text: 'Zooming' } } }); + establishToolInput(h, 'Replacement request'); + ws.message(zoomCall('earlier-reply')); + ws.message({ serverContent: { turnComplete: true } }); + await h.c.toolQueue; + assert.deepEqual(h.calls, []); + h.c.stop(); +}); + +test('a refused reply boundary that arrives after new push-to-talk input cannot close it', async () => { + const { h, ws } = await completedPttTurn(); + const audio = h.captures[0]; + ws.message(zoomCall('unrequested')); + h.c.input.pushToTalkKeyHeld = true; + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { data: 'AAAB', mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + h.c.input.pushToTalkKeyHeld = false; + h.c.input.setMicrophoneEnabled(false); + const completions = h.events.filter((e) => e.type === 'completion').length; + ws.message({ serverContent: { turnComplete: true } }); + assert.equal( + h.events.filter((e) => e.type === 'completion').length, + completions, + ); + ws.message(zoomCall('requested')); + await h.c.toolQueue; + assert.deepEqual( + h.calls.map((call) => call.name), + ['adjust_camera_zoom'], + ); + h.c.stop(); +}); + +test('push-to-talk admits a call made while an earlier requested tool is still running', async () => { + const running = deferred(); + const h = harness({ + runAction: async (name) => { + h.calls.push({ name }); + if (h.calls.length === 1) await running.promise; + return { ok: true, action: name }; + }, + }); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + establishToolInput(h, 'Zoom in'); + ws.message(zoomCall('first')); + await tick(); + ws.message({ serverContent: { turnComplete: true } }); + ws.message(zoomCall('second')); + running.resolve(); + await h.c.toolQueue; + assert.equal(h.calls.length, 2); + assert.equal( + toolResponses(ws).some(({ response }) => response.ok === false), + false, + ); + h.c.stop(); +}); + +test('a narration callback that submits a successor keeps the old packet tools from running', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + establishToolInput(h, 'Zoom in'); + const audible = h.c.narration.audible.bind(h.c.narration); + let submitted = false; + h.c.narration.audible = () => { + audible(); + if (submitted) return; + submitted = true; + assert.equal(h.c.sendTextCommand('New request'), true); + }; + ws.message({ + serverContent: replyAudio('old'), + toolCall: { + functionCalls: [ + { id: 'old', name: 'adjust_camera_zoom', args: { direction: 'in' } }, + ], + }, + }); + await h.c.toolQueue; + assert.equal(submitted, true); + assert.deepEqual(h.calls, []); + h.c.stop(); +}); diff --git a/src/voice/pointerContext.js b/src/voice/pointerContext.js index 19ea6771b..3f9d09b6d 100644 --- a/src/voice/pointerContext.js +++ b/src/voice/pointerContext.js @@ -182,6 +182,63 @@ export function pointerChip(snapshot) { }; } +/** Dedupe key of the clearing pointer_context item. */ +export const POINTER_NONE_KEY = 'none'; + +/** + * The voice-card event for a turn's pointer: a gesture reveals its target at + * once; open-mic and typed turns reveal it only through announcePointer. + * @param {object|null} snapshot + * @param {boolean} gesture + * @returns {{announced: object|null, pointer: object|null}} + */ +export function pointerTurnEvent(snapshot, gesture) { + const chip = gesture ? pointerChip(snapshot) : null; + return chip + ? { announced: snapshot, pointer: { ...chip, screenPx: snapshot.screenPx } } + : { announced: null, pointer: null }; +} + +/** + * The pointer_context item a turn should send, or null when there is nothing + * new. Without a gesture the model learns only what kind of thing is pointed + * at, never which. Losing a sent pointer produces one clearing item. + * @param {object|null} snapshot + * @param {{at: string, gesture: boolean, sentKey: string|null}} turn + * @returns {{item: object, key: string}|null} + */ +export function pointerTurnContext(snapshot, { at, gesture, sentKey }) { + const chip = pointerChip(snapshot); + const item = !chip + ? sentKey && sentKey !== POINTER_NONE_KEY + ? { type: 'pointer_context', at, target: 'none' } + : null + : gesture + ? pointerContextItem(snapshot) + : { + type: 'pointer_context', + at, + pointing: snapshot.entity?.kind || 'ground', + }; + if (!item) return null; + const { at: _at, ...content } = item; + const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY; + return key === sentKey ? null : { item, key }; +} + +/** + * The voice-card event revealing a pointer a tool resolved, once per snapshot. + * @param {object|null} snapshot + * @param {object|null} announced Snapshot already shown. + * @returns {object|null} + */ +export function pointerAnnouncement(snapshot, announced) { + const chip = pointerChip(snapshot); + return chip && snapshot !== announced + ? { ...chip, screenPx: snapshot.screenPx } + : null; +} + /** * Track the pointer over one element (the globe canvas). * @param {object} options diff --git a/src/voice/pointerContext.test.mjs b/src/voice/pointerContext.test.mjs index 6c40cc0e9..7d8d56d9c 100644 --- a/src/voice/pointerContext.test.mjs +++ b/src/voice/pointerContext.test.mjs @@ -6,8 +6,13 @@ import { POINTER_HOLD_MS, buildPointerSnapshot, createPointerTracker, + POINTER_NONE_KEY, formatPointerContext, + pointerAnnouncement, pointerChip, + pointerContextItem, + pointerTurnContext, + pointerTurnEvent, } from './pointerContext.js'; function harness({ pick } = {}) { @@ -227,3 +232,65 @@ test('chip: "this" names an entity, "here" gives the ground coordinate', () => { assert.deepEqual(pointerChip(ground), { kind: 'here', label: '30.27, -97.74' }); assert.equal(pointerChip(null), null); }); + +test('pointer turn helpers reveal gesture targets, describe implicit ones by kind and dedupe repeats', () => { + const snapshot = { + fresh: true, + at: 'keydown', + entity: { kind: 'aircraft', layerId: 'flights', label: 'UAL1' }, + lat: 10, + lon: 20, + screen: { x: 5, y: 6 }, + screenPx: { x: 5, y: 6 }, + }; + const shown = pointerTurnEvent(snapshot, true); + assert.equal(shown.announced, snapshot); + assert.deepEqual(shown.pointer, { + kind: 'this', + label: 'UAL1', + screenPx: { x: 5, y: 6 }, + }); + assert.deepEqual(pointerTurnEvent(snapshot, false), { + announced: null, + pointer: null, + }); + + const gesture = pointerTurnContext(snapshot, { + at: 'keydown', + gesture: true, + sentKey: null, + }); + assert.deepEqual(gesture.item, pointerContextItem(snapshot)); + assert.equal( + pointerTurnContext(snapshot, { + at: 'keydown', + gesture: true, + sentKey: gesture.key, + }), + null, + ); + assert.deepEqual( + pointerTurnContext(snapshot, { at: 'text', gesture: false, sentKey: null }) + .item, + { type: 'pointer_context', at: 'text', pointing: 'aircraft' }, + ); + + const cleared = pointerTurnContext(null, { + at: 'text', + gesture: false, + sentKey: gesture.key, + }); + assert.deepEqual(cleared, { + item: { type: 'pointer_context', at: 'text', target: 'none' }, + key: POINTER_NONE_KEY, + }); + for (const sentKey of [null, POINTER_NONE_KEY]) + assert.equal( + pointerTurnContext(null, { at: 'text', gesture: false, sentKey }), + null, + ); + + assert.deepEqual(pointerAnnouncement(snapshot, null), shown.pointer); + assert.equal(pointerAnnouncement(snapshot, snapshot), null); + assert.equal(pointerAnnouncement(null, null), null); +}); diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index 86586ea01..06c504b20 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -10,14 +10,13 @@ import { TurnMetrics } from './turnMetrics.js'; import { createNarrationScheduler } from './narration.js'; import { createEarcon } from './earcon.js'; import { + POINTER_NONE_KEY, POINTER_OPEN_MIC_RECENT_MS, - pointerChip, - pointerContextItem, + pointerAnnouncement, + pointerTurnContext, + pointerTurnEvent, } from './pointerContext.js'; -/** Dedupe key of the clearing pointer_context item. */ -const POINTER_NONE_KEY = 'none'; - import { shouldPauseRadioForVoice } from './realtimeProtocol.js'; import { postDebugLog } from './realtimeDiagnostics.js'; @@ -287,31 +286,18 @@ export class GevRealtimeController extends RealtimeFacade { } catch { snapshot = null; } - const chip = pointerChip(snapshot); - this.pointerAnnounced = gesture && chip ? snapshot : null; - this.emitSessionEvent({ - type: 'pointer', - pointer: - gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null, + const event = pointerTurnEvent(snapshot, gesture); + this.pointerAnnounced = event.announced; + this.emitSessionEvent({ type: 'pointer', pointer: event.pointer }); + const context = pointerTurnContext(snapshot, { + at, + gesture, + sentKey: this.pointerSentKey, }); - const item = !chip - ? this.pointerSentKey && this.pointerSentKey !== POINTER_NONE_KEY - ? { type: 'pointer_context', at, target: 'none' } - : null - : gesture - ? pointerContextItem(snapshot) - : // What kind of thing, never which: no label or coordinates. - { - type: 'pointer_context', - at, - pointing: snapshot.entity?.kind || 'ground', - }; - if (!item) return snapshot; - const { at: _at, ...content } = item; - const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY; - if (key === this.pointerSentKey) return snapshot; - if (this._turns.notifyMapEvent(item)) { - this.pointerSentKey = key; + if (!context) return snapshot; + if (this._turns.notifyMapEvent(context.item)) { + this.pointerSentKey = context.key; + const chip = context.key !== POINTER_NONE_KEY; this.debugLog('pointer.context', { at, target: chip ? snapshot.target : 'none', @@ -327,13 +313,10 @@ export class GevRealtimeController extends RealtimeFacade { */ announcePointer() { const snapshot = this.pointer?.activeSnapshot?.() || null; - const chip = pointerChip(snapshot); - if (!chip || snapshot === this.pointerAnnounced) return false; + const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced); + if (!pointer) return false; this.pointerAnnounced = snapshot; - this.emitSessionEvent({ - type: 'pointer', - pointer: { ...chip, screenPx: snapshot.screenPx }, - }); + this.emitSessionEvent({ type: 'pointer', pointer }); return true; }