feat(voice): render the pointer reticle beside the voice control

PointerReticle keeps the turn's timers and tool tracking and presents
where the user pointed to the gev-pointer-reticle store, one entry per
reticle; the command dock teleports each to the body with the same
class, aria-hidden, hidden state, target and position. The voice QA
reads the card and reticle after Vue's flush, and the application guide
says leaving out the command dock leaves out the voice control and its
reticle.
This commit is contained in:
Sameh Khamis committed 2026-10-31 20:36:39 -07:00
1 parent f9bf782b9c
commit 0926cbc105
11 files changed
+215 -78

No files matched your search

+32
View File
@@ -0,0 +1,32 @@
import { defineStore } from 'pinia';
import { markRaw, shallowRef } from 'vue';
/**
* The voice pointer reticles as their view renders them. Each PointerReticle
* (src/voice/pointerReticle.js) presents where the user pointed while its
* voice turn is live.
*/
export const usePointerReticleStore = defineStore('gev-pointer-reticle', () => {
/**
* `[{owner, visible, left, top, target}]` in client pixels, one per
* reticle that has shown, `target` being 'entity' or 'ground'.
*/
const reticles = shallowRef([]);
/**
* @param {object} owner The reticle publishing.
* @param {?object} view Its fresh presentation, or null to remove it.
*/
function publish(owner, view) {
const index = reticles.value.findIndex((item) => item.owner === owner);
const next = reticles.value.slice();
if (!view) {
if (index < 0) return;
next.splice(index, 1);
} else if (index < 0) next.push(markRaw({ ...view, owner }));
else next[index] = markRaw({ ...view, owner });
reticles.value = next;
}
return { reticles, publish };
});
+7
View File
@@ -5,6 +5,7 @@ import { vClassToggle } from '../rail/classToggle.js';
import { useDisplayStore } from '../stores/display.js';
import { computed } from 'vue';
import { useLocationStore } from '../stores/location.js';
import { usePointerReticleStore } from '../stores/pointerReticle.js';
import { useVoiceStore } from '../stores/voice.js';
import PhoneTabBar from './panels/PhoneTabBar.vue';
import VoiceControl from './panels/VoiceControl.vue';
@@ -32,6 +33,9 @@ const dockOrder = computed(() => {
: ['control', 'location'];
});
// Where the user pointed while a voice turn is live (PointerReticle).
const { reticles } = storeToRefs(usePointerReticleStore());
function selectMapSource(chip) {
if (!chip.available) return;
void mapSourceControls.value?.select(chip.id).catch(() => {});
@@ -136,5 +140,8 @@ function selectMapSource(chip) {
</div>
</template>
<PhoneTabBar />
<Teleport to="body">
<div v-for="reticle in reticles" :key="reticle.owner.key" class="gev-pointer-reticle" aria-hidden="true" :hidden="!reticle.visible" :data-target="reticle.target" :style="{ left: `${reticle.left}px`, top: `${reticle.top}px` }"></div>
</Teleport>
</div>
</template>
+30 -29
View File
@@ -3,13 +3,17 @@
* live so the user sees what was understood before the answer.
*
* It is a fixed-position overlay (pointer-events: none) placed from the
* snapshot's canvas pixels, so it never touches the Cesium scene. The reticle
* marks where the user pointed, not a tracked world position: it leaves as
* soon as a tool moves the camera, when the reply completes, when the session
* ends, and after {@link RETICLE_MAX_MS} at most. Clean-UI and recording mode
* hide it through CSS.
* snapshot's canvas pixels, so it never touches the Cesium scene. It is
* presented to the pointer reticle store, which the command dock renders.
* The reticle marks where the user pointed, not a tracked world position: it
* leaves as soon as a tool moves the camera, when the reply completes, when
* the session ends, and after {@link RETICLE_MAX_MS} at most. Clean-UI and
* recording mode hide it through CSS.
*/
import { usePointerReticleStore } from '../ui/stores/pointerReticle.js';
import { applicationPinia } from '../ui/vueRuntime.js';
export const RETICLE_MAX_MS = 10000;
export const RETICLE_LINGER_MS = 1500;
@@ -26,23 +30,28 @@ const CAMERA_TOOLS = new Set([
'control_cockpit',
]);
// Keys each reticle's element in the view.
let reticleKeys = 0;
export class PointerReticle {
/**
* @param {object} options
* @param {Document} options.doc Document that owns the overlay.
* @param {() => {left: number, top: number}|null} options.rectOf Canvas
* client rect, read at show time.
* @param {object} [options.store] The pointer reticle store.
* @param {Function} [options.setTimer]
* @param {Function} [options.clearTimer]
*/
constructor({
doc,
rectOf,
store = usePointerReticleStore(applicationPinia()),
setTimer = (fn, ms) => setTimeout(fn, ms),
clearTimer = (id) => clearTimeout(id),
}) {
Object.assign(this, { doc, rectOf, setTimer, clearTimer });
this.node = null;
Object.assign(this, { rectOf, store, setTimer, clearTimer });
this.key = ++reticleKeys;
// What this reticle last presented; null until it first shows.
this.view = null;
this.timer = null;
// Tools running in this turn, keyed by call id (name when absent).
this.running = new Set();
@@ -50,17 +59,6 @@ export class PointerReticle {
this.destroyed = false;
}
ensureNode() {
if (this.node || !this.doc?.createElement) return this.node;
const node = this.doc.createElement('div');
node.className = 'gev-pointer-reticle';
node.setAttribute?.('aria-hidden', 'true');
node.hidden = true;
this.doc.body?.appendChild?.(node);
this.node = node;
return node;
}
schedule(ms) {
this.cancelTimer();
this.timer = this.setTimer(() => {
@@ -79,12 +77,14 @@ export class PointerReticle {
show(pointer) {
if (this.destroyed || !pointer?.screenPx) return;
const rect = this.rectOf?.();
const node = this.ensureNode();
if (!rect || !node) return;
node.style.left = `${Math.round(rect.left + pointer.screenPx.x)}px`;
node.style.top = `${Math.round(rect.top + pointer.screenPx.y)}px`;
node.dataset.target = pointer.kind === 'this' ? 'entity' : 'ground';
node.hidden = false;
if (!rect) return;
this.view = {
visible: true,
left: Math.round(rect.left + pointer.screenPx.x),
top: Math.round(rect.top + pointer.screenPx.y),
target: pointer.kind === 'this' ? 'entity' : 'ground',
};
this.store.publish(this, this.view);
this.visible = true;
this.running.clear();
this.schedule(RETICLE_MAX_MS);
@@ -95,7 +95,8 @@ export class PointerReticle {
this.running.clear();
if (!this.visible) return;
this.visible = false;
if (this.node) this.node.hidden = true;
this.view = { ...this.view, visible: false };
this.store.publish(this, this.view);
}
/** Follow provider-neutral session events. */
@@ -129,7 +130,7 @@ export class PointerReticle {
if (this.destroyed) return;
this.hide();
this.destroyed = true;
this.node?.remove?.();
this.node = null;
if (this.view) this.store.publish(this, null);
this.view = null;
}
}
+43 -38
View File
@@ -5,34 +5,27 @@ import { PointerReticle, RETICLE_LINGER_MS, RETICLE_MAX_MS } from './pointerReti
const css = readStylesheet(new URL('../../style.css', import.meta.url));
function fakeDoc() {
const appended = [];
function fakeStore() {
return {
appended,
body: { appendChild: (node) => appended.push(node) },
createElement: (tag) => ({
tag,
hidden: false,
style: {},
dataset: {},
attributes: {},
setAttribute(name, value) {
this.attributes[name] = value;
},
removed: false,
remove() {
this.removed = true;
},
}),
reticles: new Map(),
publishes: 0,
get reticle() {
return [...this.reticles.values()].at(-1) ?? null;
},
publish(owner, view) {
if (view) this.reticles.set(owner, view);
else this.reticles.delete(owner);
this.publishes += 1;
},
};
}
function harness() {
const doc = fakeDoc();
const store = fakeStore();
const timers = new Map();
let next = 1;
const reticle = new PointerReticle({
doc,
store,
rectOf: () => ({ left: 10, top: 20 }),
setTimer: (fn, ms) => {
const id = next++;
@@ -46,31 +39,26 @@ function harness() {
timers.delete(id);
timer.fn();
};
return { doc, timers, reticle, fire };
return { store, timers, reticle, fire };
}
const pointer = { kind: 'this', label: 'UPS793', screenPx: { x: 600, y: 200 } };
test('reticle: shows at the pointed pixel and caps its lifetime', () => {
const { doc, timers, reticle, fire } = harness();
const { store, timers, reticle, fire } = harness();
assert.equal(store.reticle, null);
reticle.handle({ type: 'pointer', pointer });
const node = doc.appended[0];
assert.equal(node.className, 'gev-pointer-reticle');
assert.equal(node.attributes['aria-hidden'], 'true');
assert.equal(node.hidden, false);
assert.equal(node.style.left, '610px');
assert.equal(node.style.top, '220px');
assert.equal(node.dataset.target, 'entity');
assert.deepEqual(store.reticle, { visible: true, left: 610, top: 220, target: 'entity' });
assert.equal([...timers.values()][0].ms, RETICLE_MAX_MS);
fire();
assert.equal(node.hidden, true);
assert.deepEqual(store.reticle, { visible: false, left: 610, top: 220, target: 'entity' });
});
test('reticle: leaves when a tool moves the camera, lingers after a reply', () => {
const { doc, timers, reticle, fire } = harness();
const { store, timers, reticle, fire } = harness();
reticle.handle({ type: 'pointer', pointer: { ...pointer, kind: 'here' } });
const node = doc.appended[0];
assert.equal(node.dataset.target, 'ground');
const hidden = () => !store.reticle.visible;
assert.equal(store.reticle.target, 'ground');
reticle.handle({ type: 'action-call', name: 'analyst_query', callId: 'c1' });
reticle.handle({ type: 'action-call', name: 'analyst_query', callId: 'c2' });
reticle.handle({ type: 'action-result', name: 'analyst_query', callId: 'c1' });
@@ -80,19 +68,36 @@ test('reticle: leaves when a tool moves the camera, lingers after a reply', () =
reticle.handle({ type: 'completion' });
assert.equal([...timers.values()][0].ms, RETICLE_LINGER_MS);
fire();
assert.equal(node.hidden, true);
assert.equal(hidden(), true);
reticle.handle({ type: 'pointer', pointer });
reticle.handle({ type: 'action-call', name: 'track_entity' });
assert.equal(node.hidden, true, 'camera tools hide it at once');
assert.equal(hidden(), true, 'camera tools hide it at once');
reticle.handle({ type: 'pointer', pointer });
reticle.handle({ type: 'state', state: 'idle' });
assert.equal(node.hidden, true, 'session end hides it');
assert.equal(hidden(), true, 'session end hides it');
assert.equal(timers.size, 0);
reticle.destroy();
reticle.destroy();
assert.equal(node.removed, true);
assert.equal(store.reticle, null, 'destroy withdraws the reticle');
const publishes = store.publishes;
reticle.handle({ type: 'pointer', pointer });
assert.equal(doc.appended.length, 1, 'destroyed reticle ignores events');
assert.equal(store.publishes, publishes, 'destroyed reticle ignores events');
});
test('reticle: each reticle keeps its own point', () => {
const store = fakeStore();
const rectOf = () => ({ left: 0, top: 0 });
const older = new PointerReticle({ store, rectOf, setTimer: () => 1, clearTimer() {} });
const newer = new PointerReticle({ store, rectOf, setTimer: () => 2, clearTimer() {} });
older.handle({ type: 'pointer', pointer });
newer.handle({ type: 'pointer', pointer: { ...pointer, screenPx: { x: 5, y: 6 } } });
newer.handle({ type: 'state', state: 'idle' });
assert.equal(store.reticles.get(older).visible, true, 'hiding the newer one leaves the older one');
newer.destroy();
assert.deepEqual([...store.reticles.keys()], [older]);
assert.notEqual(older.key, newer.key);
older.destroy();
assert.equal(store.reticles.size, 0);
});
test('reticle css: overlay is inert and hidden in Clean-UI and recording', () => {
+83
View File
@@ -0,0 +1,83 @@
// src/voice/pointerReticleView.test.mjs — the command dock renders each
// voice pointer reticle where its PointerReticle presents it.
import '../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createApp, nextTick } from 'vue';
import { createPinia } from 'pinia';
import { usePointerReticleStore } from '../ui/stores/pointerReticle.js';
import CommandDockView from '../ui/views/CommandDockView.vue';
import { PointerReticle } from './pointerReticle.js';
test('the reticle shows at the pointed pixel, hides, and leaves on destroy', async (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
document.body.replaceChildren();
const pinia = createPinia();
const app = createApp(CommandDockView).use(pinia);
app.mount(document.body.appendChild(document.createElement('div')));
t.after(() => app.unmount());
const node = () => document.querySelector('.gev-pointer-reticle');
const reticle = new PointerReticle({
rectOf: () => ({ left: 10, top: 20 }),
store: usePointerReticleStore(pinia),
});
await nextTick();
assert.equal(node(), null, 'nothing until the first point');
reticle.handle({
type: 'pointer',
pointer: { kind: 'this', screenPx: { x: 600, y: 200 } },
});
await nextTick();
const element = node();
assert.ok(element.parentElement === document.body);
assert.equal(element.getAttribute('aria-hidden'), 'true');
assert.equal(element.hidden, false);
assert.equal(element.style.left, '610px');
assert.equal(element.style.top, '220px');
assert.equal(element.dataset.target, 'entity');
reticle.handle({ type: 'state', state: 'idle' });
await nextTick();
assert.ok(node() === element, 'hiding keeps the element');
assert.equal(element.hidden, true);
reticle.handle({
type: 'pointer',
pointer: { kind: 'here', screenPx: { x: 1, y: 2 } },
});
await nextTick();
assert.equal(element.hidden, false);
assert.equal(element.dataset.target, 'ground');
reticle.destroy();
await nextTick();
assert.equal(node(), null);
});
test('two reticles show two elements', async (t) => {
document.body.replaceChildren();
const pinia = createPinia();
const app = createApp(CommandDockView).use(pinia);
app.mount(document.body.appendChild(document.createElement('div')));
t.after(() => app.unmount());
const store = usePointerReticleStore(pinia);
const rectOf = () => ({ left: 0, top: 0 });
const one = new PointerReticle({ rectOf, store });
const two = new PointerReticle({ rectOf, store });
t.after(() => {
one.destroy();
two.destroy();
});
const at = (x) => ({
type: 'pointer',
pointer: { kind: 'this', screenPx: { x, y: 0 } },
});
one.handle(at(1));
two.handle(at(2));
await nextTick();
const lefts = () =>
[...document.querySelectorAll('.gev-pointer-reticle')].map(
(node) => node.style.left,
);
assert.deepEqual(lefts(), ['1px', '2px']);
two.destroy();
await nextTick();
assert.deepEqual(lefts(), ['1px']);
});
+1 -3
View File
@@ -7,10 +7,8 @@ import { msg, registerUnsavedWork, t } from '../i18n/index.js';
function createDefaultReticle(pointer) {
const element = pointer?.element;
const doc = element?.ownerDocument;
if (!doc) return null;
if (!element?.ownerDocument) return null;
return new PointerReticle({
doc,
rectOf: () => element.getBoundingClientRect?.() || null,
});
}