mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-18 01:02:23 +00:00
Merge main into german/i18n-plurals-context
This commit is contained in:
commit
64b9cef14b
209 files changed
+35415
-701
No files matched your search
@@ -430,6 +430,14 @@ const schemas = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'open_stories',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'control_scene',
|
||||
parameters: {
|
||||
|
||||
@@ -59,7 +59,8 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
|
||||
// move_camera tilt takes an optional pitchDeg angle, described as
|
||||
// absolute with its -45 default, with steps sent as a direction.
|
||||
// show_street_level (map, source, photo) adds one tool.
|
||||
'0c27f62062001e6a07241cd301c3d357f55e9b74f66cd2904d16590510337468',
|
||||
// open_stories adds the Stories Explore entry action.
|
||||
'5a2deb65bef5db8fa49ee55dd66efb884e7c3ba97dbdcaad3fada087170d2225',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -157,6 +158,7 @@ test('all legacy action arguments are byte-identical after removing the delibera
|
||||
'find_imagery',
|
||||
'osm_query',
|
||||
'show_street_level',
|
||||
'open_stories',
|
||||
].includes(tool.name),
|
||||
);
|
||||
// Layer enums are generated from the voice layer manifest and pinned by
|
||||
|
||||
@@ -3,6 +3,7 @@ import { t, translateKnown } from '../i18n/index.js';
|
||||
import { msg } from '../i18n/marker.js';
|
||||
import { createGeminiAudio } from './geminiAudio.js';
|
||||
import { RealtimeInput } from './realtimeInput.js';
|
||||
import { isStoriesOpen } from './realtimeInputPolicy.js';
|
||||
import { RealtimeRadio } from './realtimeRadio.js';
|
||||
import { captureViewportImage } from './realtimeViewport.js';
|
||||
import { createNarrationScheduler } from './narration.js';
|
||||
@@ -652,7 +653,13 @@ export class GeminiSessionController {
|
||||
}
|
||||
|
||||
async start({ pushToTalk = false } = {}) {
|
||||
if (this.disposed || this.isActive() || this.signal?.aborted) return;
|
||||
if (
|
||||
isStoriesOpen() ||
|
||||
this.disposed ||
|
||||
this.isActive() ||
|
||||
this.signal?.aborted
|
||||
)
|
||||
return;
|
||||
const held = pushToTalk && this.input.pushToTalkKeyHeld;
|
||||
const spaceHeld = this.input.spaceKeyHeld;
|
||||
this.stop({ preserveStatus: true });
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
attachVoiceResult,
|
||||
} from './speech.js';
|
||||
import { resolveDeicticArgs } from './deixis.js';
|
||||
import { isStoriesOpen } from './realtimeInputPolicy.js';
|
||||
export { readLayerLifecycleSummary } from './layerSummary.js';
|
||||
import { defaultGeospatial } from '../search/defaults.js';
|
||||
import { msg } from '../i18n/index.js';
|
||||
@@ -298,6 +299,7 @@ export function createGevActionRunner({
|
||||
styleManager,
|
||||
dataManager,
|
||||
sceneDirector = null,
|
||||
openStories = null,
|
||||
annotations = null,
|
||||
placeSearch = unavailablePlaceSearch,
|
||||
floorServices = defaultFloorServices,
|
||||
@@ -451,6 +453,56 @@ export function createGevActionRunner({
|
||||
const current = () =>
|
||||
!runOptions.signal?.aborted &&
|
||||
(typeof runOptions.isCurrent !== 'function' || runOptions.isCurrent());
|
||||
if (name === 'open_stories') {
|
||||
const denied = { ok: false, action: name, applied: false };
|
||||
const cancelled = {
|
||||
...denied,
|
||||
code: 'CANCELLED',
|
||||
cancelled: true,
|
||||
error: 'This request was superseded.',
|
||||
};
|
||||
if (!current()) return cancelled;
|
||||
if (typeof openStories !== 'function')
|
||||
return {
|
||||
...denied,
|
||||
unavailable: true,
|
||||
error: 'Stories is unavailable in this host.',
|
||||
say: 'Stories is unavailable in this host.',
|
||||
};
|
||||
// Opening Stories can stop the voice session that requested it. The
|
||||
// action was applied even when that session is no longer current.
|
||||
const opened = await openStories({ isCurrent: current });
|
||||
if (opened === true)
|
||||
return {
|
||||
ok: true,
|
||||
action: name,
|
||||
applied: true,
|
||||
say: 'Stories is open.',
|
||||
};
|
||||
if (!current()) return cancelled;
|
||||
if (opened === 'already-open')
|
||||
return {
|
||||
ok: true,
|
||||
action: name,
|
||||
applied: false,
|
||||
say: 'Stories is already open.',
|
||||
};
|
||||
return {
|
||||
...denied,
|
||||
error: 'Stories could not be opened.',
|
||||
say: 'Stories could not be opened.',
|
||||
};
|
||||
}
|
||||
// Stories owns the camera and scenes. Leave the globe behind it alone.
|
||||
if (isStoriesOpen())
|
||||
return {
|
||||
ok: false,
|
||||
action: name,
|
||||
applied: false,
|
||||
refused: 'stories-open',
|
||||
error: 'Stories is open. Close Stories first.',
|
||||
say: 'Stories is open. Close Stories first to control the globe.',
|
||||
};
|
||||
// Step reports for voice narration and the voice card; never fatal.
|
||||
const reportProgress = (step, label = '') => {
|
||||
try {
|
||||
|
||||
@@ -258,6 +258,100 @@ test('zoom to globe adopts the shared visible reset route and returns its result
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test('voice changes nothing behind Stories while it is open', async () => {
|
||||
globalThis.window = globalThis.window || {
|
||||
clearTimeout,
|
||||
setTimeout,
|
||||
requestIdleCallback: null,
|
||||
};
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
};
|
||||
let calls = 0;
|
||||
let storiesVisible = true;
|
||||
const styleManager = {
|
||||
async resetToGlobeView() {
|
||||
calls += 1;
|
||||
return { ok: true };
|
||||
},
|
||||
};
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
styleManager,
|
||||
dataManager: { layers: new Map(), getAll: () => [] },
|
||||
openStories: () => (storiesVisible ? 'already-open' : true),
|
||||
});
|
||||
const classes = new Set(['stories-mode']);
|
||||
const priorDocument = globalThis.document;
|
||||
globalThis.document = {
|
||||
...(priorDocument ?? {}),
|
||||
body: { classList: { contains: (c) => classes.has(c) } },
|
||||
};
|
||||
try {
|
||||
const refused = await runner('zoom_to_globe');
|
||||
assert.equal(refused.ok, false);
|
||||
assert.equal(refused.refused, 'stories-open');
|
||||
const alreadyOpen = await runner('open_stories');
|
||||
assert.equal(alreadyOpen.ok, true);
|
||||
assert.equal(alreadyOpen.applied, false);
|
||||
assert.equal(calls, 0);
|
||||
// The input guard still owns the globe while the director settles.
|
||||
storiesVisible = false;
|
||||
const reopened = await runner('open_stories');
|
||||
assert.equal(reopened.ok, true);
|
||||
assert.equal(reopened.applied, true);
|
||||
assert.equal((await runner('zoom_to_globe')).refused, 'stories-open');
|
||||
classes.clear();
|
||||
await runner('zoom_to_globe');
|
||||
assert.equal(calls, 1, 'works again once Stories has closed');
|
||||
} finally {
|
||||
globalThis.document = priorDocument;
|
||||
}
|
||||
});
|
||||
|
||||
test('opening Stories only admits current requests and can stop its own session', async () => {
|
||||
const { viewer, styleManager } = createVoiceNavigationHarness();
|
||||
const first = new AbortController();
|
||||
let loaded = Promise.resolve();
|
||||
let admitted = 0;
|
||||
let opened = 0;
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
styleManager,
|
||||
dataManager: { layers: new Map(), getAll: () => [] },
|
||||
async openStories({ isCurrent }) {
|
||||
admitted += 1;
|
||||
await loaded;
|
||||
if (!isCurrent()) return false;
|
||||
opened += 1;
|
||||
first.abort();
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
const applied = await runner('open_stories', {}, { signal: first.signal });
|
||||
assert.equal(applied.ok, true);
|
||||
assert.equal(applied.applied, true);
|
||||
|
||||
const stale = await runner('open_stories', {}, { signal: first.signal });
|
||||
assert.equal(stale.cancelled, true);
|
||||
assert.equal(admitted, 1);
|
||||
|
||||
let finishLoading;
|
||||
loaded = new Promise((resolve) => { finishLoading = resolve; });
|
||||
const second = new AbortController();
|
||||
const pending = runner('open_stories', {}, { signal: second.signal });
|
||||
assert.equal(admitted, 2);
|
||||
second.abort();
|
||||
finishLoading();
|
||||
const canceled = await pending;
|
||||
assert.equal(canceled.cancelled, true);
|
||||
assert.equal(canceled.applied, false);
|
||||
assert.equal(opened, 1);
|
||||
});
|
||||
|
||||
test('dependent voice navigation waits for the destination viewport to arrive', async () => {
|
||||
globalThis.window = globalThis.window || {
|
||||
clearTimeout,
|
||||
|
||||
@@ -358,6 +358,34 @@ test('push-to-talk rechecks focus before the hold threshold claims voice', (t) =
|
||||
assert.equal(f.controller.spaceKeyHeld, false);
|
||||
});
|
||||
|
||||
test('opening Stories cancels a pending Space hold even when focus stays put', (t) => {
|
||||
const f = createPushToTalkFixture(t);
|
||||
const background = pushToTalkTarget();
|
||||
const classes = new Set();
|
||||
f.document.body = { classList: { contains: (name) => classes.has(name) } };
|
||||
|
||||
f.key('keydown', background);
|
||||
classes.add('stories-mode');
|
||||
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
|
||||
assert.deepEqual(f.starts, []);
|
||||
assert.deepEqual(f.radio, []);
|
||||
assert.equal(f.controller.spaceKeyHeld, false);
|
||||
assert.equal(f.timers.size, 0);
|
||||
|
||||
classes.delete('stories-mode');
|
||||
f.key('keydown', background);
|
||||
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
|
||||
assert.equal(f.microphone.enabled, true);
|
||||
f.key('keyup', background);
|
||||
assert.equal(f.microphone.enabled, false);
|
||||
|
||||
f.key('keydown', background);
|
||||
classes.add('stories-mode');
|
||||
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
|
||||
assert.equal(f.starts.length, 1);
|
||||
assert.equal(f.microphone.enabled, false);
|
||||
});
|
||||
|
||||
test('push-to-talk never claims a control-started hold after focus moves outside', (t) => {
|
||||
const f = createPushToTalkFixture(t);
|
||||
const control = pushToTalkTarget({ tagName: 'BUTTON' });
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import { isStoriesOpen } from './realtimeInputPolicy.js';
|
||||
|
||||
const DISCONNECT_GRACE_MS = 6000;
|
||||
|
||||
@@ -52,7 +53,8 @@ export class RealtimeConnection {
|
||||
return this.readStatus();
|
||||
}
|
||||
async start({ pushToTalk = false } = {}) {
|
||||
if (this.isActive() || this.lifetimeSignal?.aborted) return;
|
||||
if (isStoriesOpen() || this.isActive() || this.lifetimeSignal?.aborted)
|
||||
return;
|
||||
this.pauseRadioForVoice();
|
||||
const pushToTalkKeyHeld = pushToTalk && this.input.pushToTalkKeyHeld;
|
||||
const spaceKeyHeld = this.input.spaceKeyHeld;
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
PUSH_TO_TALK_HOLD_DELAY_MS,
|
||||
MICROPHONE_VISUALIZER_GATE,
|
||||
ASSISTANT_VISUALIZER_GATE,
|
||||
isStoriesOpen,
|
||||
isPushToTalkKey,
|
||||
SPACE_INTERACTIVE_SELECTOR,
|
||||
isInteractiveSpaceTarget,
|
||||
@@ -55,9 +56,11 @@ export class RealtimeInput {
|
||||
bindPushToTalkShortcut() {
|
||||
if (this.shortcutKeyDownHandler) return;
|
||||
this.shortcutKeyDownHandler = (event) => {
|
||||
// Space types in chat; it never talks while chat is open.
|
||||
// Space types in chat; it never talks while chat is open, nor while
|
||||
// Stories owns the keyboard (body.stories-mode).
|
||||
if (
|
||||
this.ui.state?.chat?.open === true ||
|
||||
isStoriesOpen() ||
|
||||
!shouldHandlePushToTalkKeyDown(event)
|
||||
)
|
||||
return;
|
||||
@@ -93,6 +96,10 @@ export class RealtimeInput {
|
||||
this.pushToTalkHoldTimer = null;
|
||||
if (holdGeneration !== this.pushToTalkHoldGeneration) return;
|
||||
if (!this.spaceKeyHeld || this.pushToTalkKeyHeld) return;
|
||||
if (isStoriesOpen()) {
|
||||
this.shortcutBlurHandler();
|
||||
return;
|
||||
}
|
||||
// A pointer or Tab focus change during the delay cancels the original
|
||||
// gesture rather than letting voice claim a key held for another owner.
|
||||
if (document.activeElement !== this.pushToTalkHoldFocusOwner) return;
|
||||
|
||||
@@ -9,6 +9,13 @@ export const MICROPHONE_VISUALIZER_GATE = 0.12;
|
||||
|
||||
export const ASSISTANT_VISUALIZER_GATE = 0.04;
|
||||
|
||||
/** Stories takes keyboard and microphone ownership away from voice. */
|
||||
export function isStoriesOpen() {
|
||||
return (
|
||||
globalThis.document?.body?.classList?.contains?.('stories-mode') ?? false
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether a keyboard event represents the hold-Space voice shortcut.
|
||||
* @param {KeyboardEvent|object|null} event
|
||||
|
||||
+12
-2
@@ -5,7 +5,12 @@
|
||||
* Events: state, transcript, action-call, progress, action-result, interruption,
|
||||
* completion, turn-metrics.
|
||||
*/
|
||||
export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||
export function createVoiceSession({
|
||||
createAdapter,
|
||||
runner,
|
||||
signal,
|
||||
canStart,
|
||||
}) {
|
||||
if (typeof createAdapter !== 'function' || typeof runner !== 'function')
|
||||
throw new TypeError('Voice requires an adapter factory and action runner');
|
||||
const lifetime = new AbortController();
|
||||
@@ -146,7 +151,12 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
async start(options) {
|
||||
if (disposed || !['idle', 'error'].includes(state)) return;
|
||||
if (
|
||||
disposed ||
|
||||
!['idle', 'error'].includes(state) ||
|
||||
canStart?.(options) === false
|
||||
)
|
||||
return;
|
||||
const attempt = ++startAttempt;
|
||||
emit({ type: 'state', state: 'connecting' });
|
||||
if (disposed || attempt !== startAttempt) return;
|
||||
|
||||
@@ -429,12 +429,13 @@ test('a late rejection cannot settle a newer turn card', async () => {
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
function startReentrancyFixture() {
|
||||
function startReentrancyFixture(canStart) {
|
||||
let starts = 0;
|
||||
let stops = 0;
|
||||
const settings = [];
|
||||
const session = createVoiceSession({
|
||||
runner: async () => ({ ok: true }),
|
||||
canStart,
|
||||
createAdapter: ({ emit }) => ({
|
||||
async start(options) {
|
||||
starts++;
|
||||
@@ -528,6 +529,28 @@ test('ordinary start forwards caller settings once and remains stoppable', async
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
test('start admission rejects microphone use before connecting and allows typed chat', async () => {
|
||||
const f = startReentrancyFixture((options) => options?.textOnly === true);
|
||||
const states = [];
|
||||
f.session.subscribe((event) => {
|
||||
if (event.type === 'state') states.push(event.state);
|
||||
});
|
||||
await f.session.start({ pushToTalk: true });
|
||||
assert.equal(f.starts(), 0);
|
||||
assert.equal(f.session.state, 'idle');
|
||||
assert.equal(states.includes('connecting'), false);
|
||||
|
||||
const settings = { textOnly: true };
|
||||
await f.session.start(settings);
|
||||
assert.equal(f.settings.length, 1);
|
||||
assert.equal(f.settings[0], settings);
|
||||
assert.equal(f.starts(), 1);
|
||||
assert.equal(f.session.state, 'listening');
|
||||
f.session.stop();
|
||||
assert.equal(f.session.state, 'idle');
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
for (const emitsIdle of [false, true]) {
|
||||
for (const outcome of ['normal', 'preserveStatus', 'throw']) {
|
||||
test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { nextTick } from 'vue';
|
||||
import { useOmnibarStore } from '../ui/stores/omnibar.js';
|
||||
import { applicationPinia } from '../ui/vueRuntime.js';
|
||||
import { bindAssistantPanelLayout, bindLayoutDock } from './panelLayout.js';
|
||||
import { bindChatControl } from './chatControl.js';
|
||||
import { bindVoiceMiniPreview } from './voiceMiniPreview.js';
|
||||
@@ -34,6 +36,7 @@ export function createVoiceCommands({
|
||||
supportedProviders,
|
||||
onProviderChange,
|
||||
onSessionEvent,
|
||||
canStartVoice,
|
||||
createCard = (options) => new VoiceCardControls(options),
|
||||
pointer = null,
|
||||
referents = null,
|
||||
@@ -49,6 +52,7 @@ export function createVoiceCommands({
|
||||
let panelMount;
|
||||
const session = createVoiceSession({
|
||||
runner,
|
||||
canStart: (options) => options?.textOnly || canStartVoice?.() !== false,
|
||||
signal,
|
||||
createAdapter: (hooks) => {
|
||||
if (panelHost) actionObserver = panelHost.observeActions(hooks);
|
||||
@@ -284,7 +288,7 @@ export function createVoiceCommands({
|
||||
void session.start({ pushToTalk: false });
|
||||
};
|
||||
const buttonHandler = () => {
|
||||
if (adapter.ignoreButtonClick?.()) return;
|
||||
if (canStartVoice?.() === false || adapter.ignoreButtonClick?.()) return;
|
||||
chat?.close();
|
||||
if (session.isActive() && adapter.getStartOptions?.().textOnly) {
|
||||
session.stop();
|
||||
@@ -332,5 +336,16 @@ export function createVoiceCommands({
|
||||
if (session.disposed) cleanup();
|
||||
else adapter.bindControls?.();
|
||||
window.__gevVoiceCommands = controls;
|
||||
// The omnibar runs the same actions directly, without a model.
|
||||
// A session already gone leaves the omnibar without a runner.
|
||||
if (!session.disposed) {
|
||||
const omnibar = useOmnibarStore(applicationPinia());
|
||||
omnibar.connectRunner((name, args) => runner(name, args));
|
||||
session.signal.addEventListener(
|
||||
'abort',
|
||||
() => omnibar.connectRunner(null),
|
||||
{ once: true },
|
||||
);
|
||||
}
|
||||
return controls;
|
||||
}
|
||||
@@ -1070,6 +1070,7 @@ const STEP_TITLES = Object.freeze({
|
||||
set_detection: msg('Adjust detection overlay'),
|
||||
set_map_stack: msg('Change map stack'),
|
||||
set_post_processing: msg('Adjust post-processing'),
|
||||
open_stories: msg('Stories'),
|
||||
control_scene: msg('Control scene'),
|
||||
control_cctv: msg('Control CCTV'),
|
||||
control_radio: msg('Control radio'),
|
||||
|
||||
Reference in new issue
Block a user