diff --git a/docs/APPLICATION.md b/docs/APPLICATION.md index 91dbc99ed..8712c684f 100644 --- a/docs/APPLICATION.md +++ b/docs/APPLICATION.md @@ -209,7 +209,8 @@ such as `provider-settings`, and their hosts are removed. Leaving out `layer-panels` also leaves out the scene sharing dialogs it draws, and leaving out `scene-chrome` the before/after imagery dividers (the imagery still splits, with no line or handle to move it), the status chips, the toast and the ORBIT -indicator. An unknown +indicator, and leaving out `command-dock` the voice control and its pointer +reticle. An unknown view name is an error. A host that does not initialize Core's locale passes `localePicker: false` to `createApplicationControls` instead of editing the markup. Node tests that import application modules register the component diff --git a/docs/VUE-MIGRATION.md b/docs/VUE-MIGRATION.md index 60b6c4475..35719e4da 100644 --- a/docs/VUE-MIGRATION.md +++ b/docs/VUE-MIGRATION.md @@ -135,8 +135,10 @@ a feature supplying only its accent. what `createMapSourceControls()` publishes beside it, so `./ui/maps` needs no Vue; the location tray's city and landmark pills, mini status and search error render what `LocationControls` publishes to the - `gev-location` store, and the search field keeps its listeners), the - phone shell (`PhoneTabBar` renders the tabs and open sheet the shell + `gev-location` store, and the search field keeps its listeners; beside + the voice control, the dock teleports each voice pointer reticle to the + body where its `PointerReticle` presents it in the `gev-pointer-reticle` + store), the phone shell (`PhoneTabBar` renders the tabs and open sheet the shell publishes to the `gev-display` store, teleporting the tab bar and sheet backdrop to the end of the page from the first publish until the shell is destroyed; the shell takes their clicks from the document), the diff --git a/scripts/format-scope.json b/scripts/format-scope.json index d1498f08b..2460bbe99 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -744,6 +744,8 @@ "server/providers/openai/toolDescriptions.js", "src/voice/session.js", "src/voice/sessionCommands.js", + "src/ui/stores/pointerReticle.js", + "src/voice/pointerReticleView.test.mjs", "src/voice/realtimeSession.js", "src/voice/realtimeEvents.js", "src/voice/session.test.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index a115c18af..19be594b4 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -101,7 +101,8 @@ "src/ui/stores/sceneInteractions.js", "src/ui/views/panels/SceneInteractions.vue", "src/ui/stores/dataPacks.js", - "src/ui/views/panels/DataPacks.vue" + "src/ui/views/panels/DataPacks.vue", + "src/ui/stores/pointerReticle.js" ], "external": ["vue", "pinia"] }, @@ -2065,7 +2066,8 @@ "src/sources/requestDeadline.js", "src/i18n/marker.js", "src/ui/stores/voice.js", - "src/ui/vueRuntime.js" + "src/ui/vueRuntime.js", + "src/ui/stores/pointerReticle.js" ], "external": ["vue", "pinia"] }, @@ -3012,7 +3014,8 @@ "src/ui/stores/status.js", "src/ui/toast.js", "src/ui/stores/sceneInteractions.js", - "src/ui/stores/dataPacks.js" + "src/ui/stores/dataPacks.js", + "src/ui/stores/pointerReticle.js" ], "external": [ "hls.js", @@ -3979,7 +3982,8 @@ "src/ui/statusText.js", "src/voice/providerCapabilities.js", "src/ui/vueRuntime.js", - "src/ui/stores/voice.js" + "src/ui/stores/voice.js", + "src/ui/stores/pointerReticle.js" ], "external": ["vue", "pinia"] }, diff --git a/scripts/qa-voice-routing.mjs b/scripts/qa-voice-routing.mjs index 1c5c2eb83..b21b8e0f5 100644 --- a/scripts/qa-voice-routing.mjs +++ b/scripts/qa-voice-routing.mjs @@ -1023,7 +1023,7 @@ async function runDeixisBehavior(page, run, settle) { }; const beginAt = (x, y, said = null) => page.evaluate( - (px, py, text) => { + async (px, py, text) => { const voice = window.__gevVoiceCommands; if (text) { // What the session emits for a spoken turn: a new turn, then the caption. @@ -1043,6 +1043,8 @@ async function runDeixisBehavior(page, run, settle) { voice.pointer.hold('keydown'); const snapshot = voice.beginPointerTurn('keydown'); voice.pointer.releaseHold(); + // The voice card and the reticle render on Vue's next flush. + await Promise.resolve(); const chip = document.getElementById('gev-voice-card-pointer'); const reticle = document.querySelector('.gev-pointer-reticle'); return { diff --git a/src/ui/stores/pointerReticle.js b/src/ui/stores/pointerReticle.js new file mode 100644 index 000000000..5c663f601 --- /dev/null +++ b/src/ui/stores/pointerReticle.js @@ -0,0 +1,32 @@ +import { defineStore } from 'pinia'; +import { markRaw, shallowRef } from 'vue'; + +/** + * The voice pointer reticles as their view renders them. Each PointerReticle + * (src/voice/pointerReticle.js) presents where the user pointed while its + * voice turn is live. + */ +export const usePointerReticleStore = defineStore('gev-pointer-reticle', () => { + /** + * `[{owner, visible, left, top, target}]` in client pixels, one per + * reticle that has shown, `target` being 'entity' or 'ground'. + */ + const reticles = shallowRef([]); + + /** + * @param {object} owner The reticle publishing. + * @param {?object} view Its fresh presentation, or null to remove it. + */ + function publish(owner, view) { + const index = reticles.value.findIndex((item) => item.owner === owner); + const next = reticles.value.slice(); + if (!view) { + if (index < 0) return; + next.splice(index, 1); + } else if (index < 0) next.push(markRaw({ ...view, owner })); + else next[index] = markRaw({ ...view, owner }); + reticles.value = next; + } + + return { reticles, publish }; +}); diff --git a/src/ui/views/CommandDockView.vue b/src/ui/views/CommandDockView.vue index 56c1bde08..f6cd5219e 100644 --- a/src/ui/views/CommandDockView.vue +++ b/src/ui/views/CommandDockView.vue @@ -5,6 +5,7 @@ import { vClassToggle } from '../rail/classToggle.js'; import { useDisplayStore } from '../stores/display.js'; import { computed } from 'vue'; import { useLocationStore } from '../stores/location.js'; +import { usePointerReticleStore } from '../stores/pointerReticle.js'; import { useVoiceStore } from '../stores/voice.js'; import PhoneTabBar from './panels/PhoneTabBar.vue'; import VoiceControl from './panels/VoiceControl.vue'; @@ -32,6 +33,9 @@ const dockOrder = computed(() => { : ['control', 'location']; }); +// Where the user pointed while a voice turn is live (PointerReticle). +const { reticles } = storeToRefs(usePointerReticleStore()); + function selectMapSource(chip) { if (!chip.available) return; void mapSourceControls.value?.select(chip.id).catch(() => {}); @@ -136,5 +140,8 @@ function selectMapSource(chip) { + + + diff --git a/src/voice/pointerReticle.js b/src/voice/pointerReticle.js index 0702d7345..5a3a64fdc 100644 --- a/src/voice/pointerReticle.js +++ b/src/voice/pointerReticle.js @@ -3,13 +3,17 @@ * live so the user sees what was understood before the answer. * * It is a fixed-position overlay (pointer-events: none) placed from the - * snapshot's canvas pixels, so it never touches the Cesium scene. The reticle - * marks where the user pointed, not a tracked world position: it leaves as - * soon as a tool moves the camera, when the reply completes, when the session - * ends, and after {@link RETICLE_MAX_MS} at most. Clean-UI and recording mode - * hide it through CSS. + * snapshot's canvas pixels, so it never touches the Cesium scene. It is + * presented to the pointer reticle store, which the command dock renders. + * The reticle marks where the user pointed, not a tracked world position: it + * leaves as soon as a tool moves the camera, when the reply completes, when + * the session ends, and after {@link RETICLE_MAX_MS} at most. Clean-UI and + * recording mode hide it through CSS. */ +import { usePointerReticleStore } from '../ui/stores/pointerReticle.js'; +import { applicationPinia } from '../ui/vueRuntime.js'; + export const RETICLE_MAX_MS = 10000; export const RETICLE_LINGER_MS = 1500; @@ -26,23 +30,28 @@ const CAMERA_TOOLS = new Set([ 'control_cockpit', ]); +// Keys each reticle's element in the view. +let reticleKeys = 0; + export class PointerReticle { /** * @param {object} options - * @param {Document} options.doc Document that owns the overlay. * @param {() => {left: number, top: number}|null} options.rectOf Canvas * client rect, read at show time. + * @param {object} [options.store] The pointer reticle store. * @param {Function} [options.setTimer] * @param {Function} [options.clearTimer] */ constructor({ - doc, rectOf, + store = usePointerReticleStore(applicationPinia()), setTimer = (fn, ms) => setTimeout(fn, ms), clearTimer = (id) => clearTimeout(id), }) { - Object.assign(this, { doc, rectOf, setTimer, clearTimer }); - this.node = null; + Object.assign(this, { rectOf, store, setTimer, clearTimer }); + this.key = ++reticleKeys; + // What this reticle last presented; null until it first shows. + this.view = null; this.timer = null; // Tools running in this turn, keyed by call id (name when absent). this.running = new Set(); @@ -50,17 +59,6 @@ export class PointerReticle { this.destroyed = false; } - ensureNode() { - if (this.node || !this.doc?.createElement) return this.node; - const node = this.doc.createElement('div'); - node.className = 'gev-pointer-reticle'; - node.setAttribute?.('aria-hidden', 'true'); - node.hidden = true; - this.doc.body?.appendChild?.(node); - this.node = node; - return node; - } - schedule(ms) { this.cancelTimer(); this.timer = this.setTimer(() => { @@ -79,12 +77,14 @@ export class PointerReticle { show(pointer) { if (this.destroyed || !pointer?.screenPx) return; const rect = this.rectOf?.(); - const node = this.ensureNode(); - if (!rect || !node) return; - node.style.left = `${Math.round(rect.left + pointer.screenPx.x)}px`; - node.style.top = `${Math.round(rect.top + pointer.screenPx.y)}px`; - node.dataset.target = pointer.kind === 'this' ? 'entity' : 'ground'; - node.hidden = false; + if (!rect) return; + this.view = { + visible: true, + left: Math.round(rect.left + pointer.screenPx.x), + top: Math.round(rect.top + pointer.screenPx.y), + target: pointer.kind === 'this' ? 'entity' : 'ground', + }; + this.store.publish(this, this.view); this.visible = true; this.running.clear(); this.schedule(RETICLE_MAX_MS); @@ -95,7 +95,8 @@ export class PointerReticle { this.running.clear(); if (!this.visible) return; this.visible = false; - if (this.node) this.node.hidden = true; + this.view = { ...this.view, visible: false }; + this.store.publish(this, this.view); } /** Follow provider-neutral session events. */ @@ -129,7 +130,7 @@ export class PointerReticle { if (this.destroyed) return; this.hide(); this.destroyed = true; - this.node?.remove?.(); - this.node = null; + if (this.view) this.store.publish(this, null); + this.view = null; } } diff --git a/src/voice/pointerReticle.test.mjs b/src/voice/pointerReticle.test.mjs index 71a7071ad..b31e4d02f 100644 --- a/src/voice/pointerReticle.test.mjs +++ b/src/voice/pointerReticle.test.mjs @@ -5,34 +5,27 @@ import { PointerReticle, RETICLE_LINGER_MS, RETICLE_MAX_MS } from './pointerReti const css = readStylesheet(new URL('../../style.css', import.meta.url)); -function fakeDoc() { - const appended = []; +function fakeStore() { return { - appended, - body: { appendChild: (node) => appended.push(node) }, - createElement: (tag) => ({ - tag, - hidden: false, - style: {}, - dataset: {}, - attributes: {}, - setAttribute(name, value) { - this.attributes[name] = value; - }, - removed: false, - remove() { - this.removed = true; - }, - }), + reticles: new Map(), + publishes: 0, + get reticle() { + return [...this.reticles.values()].at(-1) ?? null; + }, + publish(owner, view) { + if (view) this.reticles.set(owner, view); + else this.reticles.delete(owner); + this.publishes += 1; + }, }; } function harness() { - const doc = fakeDoc(); + const store = fakeStore(); const timers = new Map(); let next = 1; const reticle = new PointerReticle({ - doc, + store, rectOf: () => ({ left: 10, top: 20 }), setTimer: (fn, ms) => { const id = next++; @@ -46,31 +39,26 @@ function harness() { timers.delete(id); timer.fn(); }; - return { doc, timers, reticle, fire }; + return { store, timers, reticle, fire }; } const pointer = { kind: 'this', label: 'UPS793', screenPx: { x: 600, y: 200 } }; test('reticle: shows at the pointed pixel and caps its lifetime', () => { - const { doc, timers, reticle, fire } = harness(); + const { store, timers, reticle, fire } = harness(); + assert.equal(store.reticle, null); reticle.handle({ type: 'pointer', pointer }); - const node = doc.appended[0]; - assert.equal(node.className, 'gev-pointer-reticle'); - assert.equal(node.attributes['aria-hidden'], 'true'); - assert.equal(node.hidden, false); - assert.equal(node.style.left, '610px'); - assert.equal(node.style.top, '220px'); - assert.equal(node.dataset.target, 'entity'); + assert.deepEqual(store.reticle, { visible: true, left: 610, top: 220, target: 'entity' }); assert.equal([...timers.values()][0].ms, RETICLE_MAX_MS); fire(); - assert.equal(node.hidden, true); + assert.deepEqual(store.reticle, { visible: false, left: 610, top: 220, target: 'entity' }); }); test('reticle: leaves when a tool moves the camera, lingers after a reply', () => { - const { doc, timers, reticle, fire } = harness(); + const { store, timers, reticle, fire } = harness(); reticle.handle({ type: 'pointer', pointer: { ...pointer, kind: 'here' } }); - const node = doc.appended[0]; - assert.equal(node.dataset.target, 'ground'); + const hidden = () => !store.reticle.visible; + assert.equal(store.reticle.target, 'ground'); reticle.handle({ type: 'action-call', name: 'analyst_query', callId: 'c1' }); reticle.handle({ type: 'action-call', name: 'analyst_query', callId: 'c2' }); reticle.handle({ type: 'action-result', name: 'analyst_query', callId: 'c1' }); @@ -80,19 +68,36 @@ test('reticle: leaves when a tool moves the camera, lingers after a reply', () = reticle.handle({ type: 'completion' }); assert.equal([...timers.values()][0].ms, RETICLE_LINGER_MS); fire(); - assert.equal(node.hidden, true); + assert.equal(hidden(), true); reticle.handle({ type: 'pointer', pointer }); reticle.handle({ type: 'action-call', name: 'track_entity' }); - assert.equal(node.hidden, true, 'camera tools hide it at once'); + assert.equal(hidden(), true, 'camera tools hide it at once'); reticle.handle({ type: 'pointer', pointer }); reticle.handle({ type: 'state', state: 'idle' }); - assert.equal(node.hidden, true, 'session end hides it'); + assert.equal(hidden(), true, 'session end hides it'); assert.equal(timers.size, 0); reticle.destroy(); reticle.destroy(); - assert.equal(node.removed, true); + assert.equal(store.reticle, null, 'destroy withdraws the reticle'); + const publishes = store.publishes; reticle.handle({ type: 'pointer', pointer }); - assert.equal(doc.appended.length, 1, 'destroyed reticle ignores events'); + assert.equal(store.publishes, publishes, 'destroyed reticle ignores events'); +}); + +test('reticle: each reticle keeps its own point', () => { + const store = fakeStore(); + const rectOf = () => ({ left: 0, top: 0 }); + const older = new PointerReticle({ store, rectOf, setTimer: () => 1, clearTimer() {} }); + const newer = new PointerReticle({ store, rectOf, setTimer: () => 2, clearTimer() {} }); + older.handle({ type: 'pointer', pointer }); + newer.handle({ type: 'pointer', pointer: { ...pointer, screenPx: { x: 5, y: 6 } } }); + newer.handle({ type: 'state', state: 'idle' }); + assert.equal(store.reticles.get(older).visible, true, 'hiding the newer one leaves the older one'); + newer.destroy(); + assert.deepEqual([...store.reticles.keys()], [older]); + assert.notEqual(older.key, newer.key); + older.destroy(); + assert.equal(store.reticles.size, 0); }); test('reticle css: overlay is inert and hidden in Clean-UI and recording', () => { diff --git a/src/voice/pointerReticleView.test.mjs b/src/voice/pointerReticleView.test.mjs new file mode 100644 index 000000000..71f8dacf3 --- /dev/null +++ b/src/voice/pointerReticleView.test.mjs @@ -0,0 +1,83 @@ +// src/voice/pointerReticleView.test.mjs — the command dock renders each +// voice pointer reticle where its PointerReticle presents it. +import '../testSupport/registerDom.mjs'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createApp, nextTick } from 'vue'; +import { createPinia } from 'pinia'; +import { usePointerReticleStore } from '../ui/stores/pointerReticle.js'; +import CommandDockView from '../ui/views/CommandDockView.vue'; +import { PointerReticle } from './pointerReticle.js'; + +test('the reticle shows at the pointed pixel, hides, and leaves on destroy', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + document.body.replaceChildren(); + const pinia = createPinia(); + const app = createApp(CommandDockView).use(pinia); + app.mount(document.body.appendChild(document.createElement('div'))); + t.after(() => app.unmount()); + const node = () => document.querySelector('.gev-pointer-reticle'); + const reticle = new PointerReticle({ + rectOf: () => ({ left: 10, top: 20 }), + store: usePointerReticleStore(pinia), + }); + await nextTick(); + assert.equal(node(), null, 'nothing until the first point'); + reticle.handle({ + type: 'pointer', + pointer: { kind: 'this', screenPx: { x: 600, y: 200 } }, + }); + await nextTick(); + const element = node(); + assert.ok(element.parentElement === document.body); + assert.equal(element.getAttribute('aria-hidden'), 'true'); + assert.equal(element.hidden, false); + assert.equal(element.style.left, '610px'); + assert.equal(element.style.top, '220px'); + assert.equal(element.dataset.target, 'entity'); + reticle.handle({ type: 'state', state: 'idle' }); + await nextTick(); + assert.ok(node() === element, 'hiding keeps the element'); + assert.equal(element.hidden, true); + reticle.handle({ + type: 'pointer', + pointer: { kind: 'here', screenPx: { x: 1, y: 2 } }, + }); + await nextTick(); + assert.equal(element.hidden, false); + assert.equal(element.dataset.target, 'ground'); + reticle.destroy(); + await nextTick(); + assert.equal(node(), null); +}); + +test('two reticles show two elements', async (t) => { + document.body.replaceChildren(); + const pinia = createPinia(); + const app = createApp(CommandDockView).use(pinia); + app.mount(document.body.appendChild(document.createElement('div'))); + t.after(() => app.unmount()); + const store = usePointerReticleStore(pinia); + const rectOf = () => ({ left: 0, top: 0 }); + const one = new PointerReticle({ rectOf, store }); + const two = new PointerReticle({ rectOf, store }); + t.after(() => { + one.destroy(); + two.destroy(); + }); + const at = (x) => ({ + type: 'pointer', + pointer: { kind: 'this', screenPx: { x, y: 0 } }, + }); + one.handle(at(1)); + two.handle(at(2)); + await nextTick(); + const lefts = () => + [...document.querySelectorAll('.gev-pointer-reticle')].map( + (node) => node.style.left, + ); + assert.deepEqual(lefts(), ['1px', '2px']); + two.destroy(); + await nextTick(); + assert.deepEqual(lefts(), ['1px']); +}); diff --git a/src/voice/sessionCommands.js b/src/voice/sessionCommands.js index 3edbadc36..8e0563b8c 100644 --- a/src/voice/sessionCommands.js +++ b/src/voice/sessionCommands.js @@ -7,10 +7,8 @@ import { msg, registerUnsavedWork, t } from '../i18n/index.js'; function createDefaultReticle(pointer) { const element = pointer?.element; - const doc = element?.ownerDocument; - if (!doc) return null; + if (!element?.ownerDocument) return null; return new PointerReticle({ - doc, rectOf: () => element.getBoundingClientRect?.() || null, }); }