Merge main into german/i18n-plurals-context

This commit is contained in:
German Khokhlov committed 2026-11-17 19:29:53 +04:00
commit 64b9cef14b
209 files changed
+35415 -701

No files matched your search

+8
View File
@@ -430,6 +430,14 @@ const schemas = [
},
},
},
{
name: 'open_stories',
parameters: {
type: 'object',
additionalProperties: false,
properties: {},
},
},
{
name: 'control_scene',
parameters: {
+3 -1
View File
@@ -59,7 +59,8 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
// move_camera tilt takes an optional pitchDeg angle, described as
// absolute with its -45 default, with steps sent as a direction.
// show_street_level (map, source, photo) adds one tool.
'0c27f62062001e6a07241cd301c3d357f55e9b74f66cd2904d16590510337468',
// open_stories adds the Stories Explore entry action.
'5a2deb65bef5db8fa49ee55dd66efb884e7c3ba97dbdcaad3fada087170d2225',
);
});
@@ -157,6 +158,7 @@ test('all legacy action arguments are byte-identical after removing the delibera
'find_imagery',
'osm_query',
'show_street_level',
'open_stories',
].includes(tool.name),
);
// Layer enums are generated from the voice layer manifest and pinned by
+8 -1
View File
@@ -3,6 +3,7 @@ import { t, translateKnown } from '../i18n/index.js';
import { msg } from '../i18n/marker.js';
import { createGeminiAudio } from './geminiAudio.js';
import { RealtimeInput } from './realtimeInput.js';
import { isStoriesOpen } from './realtimeInputPolicy.js';
import { RealtimeRadio } from './realtimeRadio.js';
import { captureViewportImage } from './realtimeViewport.js';
import { createNarrationScheduler } from './narration.js';
@@ -652,7 +653,13 @@ export class GeminiSessionController {
}
async start({ pushToTalk = false } = {}) {
if (this.disposed || this.isActive() || this.signal?.aborted) return;
if (
isStoriesOpen() ||
this.disposed ||
this.isActive() ||
this.signal?.aborted
)
return;
const held = pushToTalk && this.input.pushToTalkKeyHeld;
const spaceHeld = this.input.spaceKeyHeld;
this.stop({ preserveStatus: true });
+52
View File
@@ -12,6 +12,7 @@ import {
attachVoiceResult,
} from './speech.js';
import { resolveDeicticArgs } from './deixis.js';
import { isStoriesOpen } from './realtimeInputPolicy.js';
export { readLayerLifecycleSummary } from './layerSummary.js';
import { defaultGeospatial } from '../search/defaults.js';
import { msg } from '../i18n/index.js';
@@ -298,6 +299,7 @@ export function createGevActionRunner({
styleManager,
dataManager,
sceneDirector = null,
openStories = null,
annotations = null,
placeSearch = unavailablePlaceSearch,
floorServices = defaultFloorServices,
@@ -451,6 +453,56 @@ export function createGevActionRunner({
const current = () =>
!runOptions.signal?.aborted &&
(typeof runOptions.isCurrent !== 'function' || runOptions.isCurrent());
if (name === 'open_stories') {
const denied = { ok: false, action: name, applied: false };
const cancelled = {
...denied,
code: 'CANCELLED',
cancelled: true,
error: 'This request was superseded.',
};
if (!current()) return cancelled;
if (typeof openStories !== 'function')
return {
...denied,
unavailable: true,
error: 'Stories is unavailable in this host.',
say: 'Stories is unavailable in this host.',
};
// Opening Stories can stop the voice session that requested it. The
// action was applied even when that session is no longer current.
const opened = await openStories({ isCurrent: current });
if (opened === true)
return {
ok: true,
action: name,
applied: true,
say: 'Stories is open.',
};
if (!current()) return cancelled;
if (opened === 'already-open')
return {
ok: true,
action: name,
applied: false,
say: 'Stories is already open.',
};
return {
...denied,
error: 'Stories could not be opened.',
say: 'Stories could not be opened.',
};
}
// Stories owns the camera and scenes. Leave the globe behind it alone.
if (isStoriesOpen())
return {
ok: false,
action: name,
applied: false,
refused: 'stories-open',
error: 'Stories is open. Close Stories first.',
say: 'Stories is open. Close Stories first to control the globe.',
};
// Step reports for voice narration and the voice card; never fatal.
const reportProgress = (step, label = '') => {
try {
+94
View File
@@ -258,6 +258,100 @@ test('zoom to globe adopts the shared visible reset route and returns its result
assert.equal(calls, 1);
});
test('voice changes nothing behind Stories while it is open', async () => {
globalThis.window = globalThis.window || {
clearTimeout,
setTimeout,
requestIdleCallback: null,
};
const viewer = {
clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: { moveEnd: { addEventListener() {} } },
};
let calls = 0;
let storiesVisible = true;
const styleManager = {
async resetToGlobeView() {
calls += 1;
return { ok: true };
},
};
const runner = createGevActionRunner({
viewer,
styleManager,
dataManager: { layers: new Map(), getAll: () => [] },
openStories: () => (storiesVisible ? 'already-open' : true),
});
const classes = new Set(['stories-mode']);
const priorDocument = globalThis.document;
globalThis.document = {
...(priorDocument ?? {}),
body: { classList: { contains: (c) => classes.has(c) } },
};
try {
const refused = await runner('zoom_to_globe');
assert.equal(refused.ok, false);
assert.equal(refused.refused, 'stories-open');
const alreadyOpen = await runner('open_stories');
assert.equal(alreadyOpen.ok, true);
assert.equal(alreadyOpen.applied, false);
assert.equal(calls, 0);
// The input guard still owns the globe while the director settles.
storiesVisible = false;
const reopened = await runner('open_stories');
assert.equal(reopened.ok, true);
assert.equal(reopened.applied, true);
assert.equal((await runner('zoom_to_globe')).refused, 'stories-open');
classes.clear();
await runner('zoom_to_globe');
assert.equal(calls, 1, 'works again once Stories has closed');
} finally {
globalThis.document = priorDocument;
}
});
test('opening Stories only admits current requests and can stop its own session', async () => {
const { viewer, styleManager } = createVoiceNavigationHarness();
const first = new AbortController();
let loaded = Promise.resolve();
let admitted = 0;
let opened = 0;
const runner = createGevActionRunner({
viewer,
styleManager,
dataManager: { layers: new Map(), getAll: () => [] },
async openStories({ isCurrent }) {
admitted += 1;
await loaded;
if (!isCurrent()) return false;
opened += 1;
first.abort();
return true;
},
});
const applied = await runner('open_stories', {}, { signal: first.signal });
assert.equal(applied.ok, true);
assert.equal(applied.applied, true);
const stale = await runner('open_stories', {}, { signal: first.signal });
assert.equal(stale.cancelled, true);
assert.equal(admitted, 1);
let finishLoading;
loaded = new Promise((resolve) => { finishLoading = resolve; });
const second = new AbortController();
const pending = runner('open_stories', {}, { signal: second.signal });
assert.equal(admitted, 2);
second.abort();
finishLoading();
const canceled = await pending;
assert.equal(canceled.cancelled, true);
assert.equal(canceled.applied, false);
assert.equal(opened, 1);
});
test('dependent voice navigation waits for the destination viewport to arrive', async () => {
globalThis.window = globalThis.window || {
clearTimeout,
+28
View File
@@ -358,6 +358,34 @@ test('push-to-talk rechecks focus before the hold threshold claims voice', (t) =
assert.equal(f.controller.spaceKeyHeld, false);
});
test('opening Stories cancels a pending Space hold even when focus stays put', (t) => {
const f = createPushToTalkFixture(t);
const background = pushToTalkTarget();
const classes = new Set();
f.document.body = { classList: { contains: (name) => classes.has(name) } };
f.key('keydown', background);
classes.add('stories-mode');
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
assert.deepEqual(f.starts, []);
assert.deepEqual(f.radio, []);
assert.equal(f.controller.spaceKeyHeld, false);
assert.equal(f.timers.size, 0);
classes.delete('stories-mode');
f.key('keydown', background);
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
assert.equal(f.microphone.enabled, true);
f.key('keyup', background);
assert.equal(f.microphone.enabled, false);
f.key('keydown', background);
classes.add('stories-mode');
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
assert.equal(f.starts.length, 1);
assert.equal(f.microphone.enabled, false);
});
test('push-to-talk never claims a control-started hold after focus moves outside', (t) => {
const f = createPushToTalkFixture(t);
const control = pushToTalkTarget({ tagName: 'BUTTON' });
+3 -1
View File
@@ -1,4 +1,5 @@
import { t } from '../i18n/index.js';
import { isStoriesOpen } from './realtimeInputPolicy.js';
const DISCONNECT_GRACE_MS = 6000;
@@ -52,7 +53,8 @@ export class RealtimeConnection {
return this.readStatus();
}
async start({ pushToTalk = false } = {}) {
if (this.isActive() || this.lifetimeSignal?.aborted) return;
if (isStoriesOpen() || this.isActive() || this.lifetimeSignal?.aborted)
return;
this.pauseRadioForVoice();
const pushToTalkKeyHeld = pushToTalk && this.input.pushToTalkKeyHeld;
const spaceKeyHeld = this.input.spaceKeyHeld;
+8 -1
View File
@@ -3,6 +3,7 @@ import {
PUSH_TO_TALK_HOLD_DELAY_MS,
MICROPHONE_VISUALIZER_GATE,
ASSISTANT_VISUALIZER_GATE,
isStoriesOpen,
isPushToTalkKey,
SPACE_INTERACTIVE_SELECTOR,
isInteractiveSpaceTarget,
@@ -55,9 +56,11 @@ export class RealtimeInput {
bindPushToTalkShortcut() {
if (this.shortcutKeyDownHandler) return;
this.shortcutKeyDownHandler = (event) => {
// Space types in chat; it never talks while chat is open.
// Space types in chat; it never talks while chat is open, nor while
// Stories owns the keyboard (body.stories-mode).
if (
this.ui.state?.chat?.open === true ||
isStoriesOpen() ||
!shouldHandlePushToTalkKeyDown(event)
)
return;
@@ -93,6 +96,10 @@ export class RealtimeInput {
this.pushToTalkHoldTimer = null;
if (holdGeneration !== this.pushToTalkHoldGeneration) return;
if (!this.spaceKeyHeld || this.pushToTalkKeyHeld) return;
if (isStoriesOpen()) {
this.shortcutBlurHandler();
return;
}
// A pointer or Tab focus change during the delay cancels the original
// gesture rather than letting voice claim a key held for another owner.
if (document.activeElement !== this.pushToTalkHoldFocusOwner) return;
+7
View File
@@ -9,6 +9,13 @@ export const MICROPHONE_VISUALIZER_GATE = 0.12;
export const ASSISTANT_VISUALIZER_GATE = 0.04;
/** Stories takes keyboard and microphone ownership away from voice. */
export function isStoriesOpen() {
return (
globalThis.document?.body?.classList?.contains?.('stories-mode') ?? false
);
}
/**
* Returns whether a keyboard event represents the hold-Space voice shortcut.
* @param {KeyboardEvent|object|null} event
+12 -2
View File
@@ -5,7 +5,12 @@
* Events: state, transcript, action-call, progress, action-result, interruption,
* completion, turn-metrics.
*/
export function createVoiceSession({ createAdapter, runner, signal }) {
export function createVoiceSession({
createAdapter,
runner,
signal,
canStart,
}) {
if (typeof createAdapter !== 'function' || typeof runner !== 'function')
throw new TypeError('Voice requires an adapter factory and action runner');
const lifetime = new AbortController();
@@ -146,7 +151,12 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
return () => listeners.delete(listener);
},
async start(options) {
if (disposed || !['idle', 'error'].includes(state)) return;
if (
disposed ||
!['idle', 'error'].includes(state) ||
canStart?.(options) === false
)
return;
const attempt = ++startAttempt;
emit({ type: 'state', state: 'connecting' });
if (disposed || attempt !== startAttempt) return;
+24 -1
View File
@@ -429,12 +429,13 @@ test('a late rejection cannot settle a newer turn card', async () => {
f.session.destroy();
});
function startReentrancyFixture() {
function startReentrancyFixture(canStart) {
let starts = 0;
let stops = 0;
const settings = [];
const session = createVoiceSession({
runner: async () => ({ ok: true }),
canStart,
createAdapter: ({ emit }) => ({
async start(options) {
starts++;
@@ -528,6 +529,28 @@ test('ordinary start forwards caller settings once and remains stoppable', async
f.session.destroy();
});
test('start admission rejects microphone use before connecting and allows typed chat', async () => {
const f = startReentrancyFixture((options) => options?.textOnly === true);
const states = [];
f.session.subscribe((event) => {
if (event.type === 'state') states.push(event.state);
});
await f.session.start({ pushToTalk: true });
assert.equal(f.starts(), 0);
assert.equal(f.session.state, 'idle');
assert.equal(states.includes('connecting'), false);
const settings = { textOnly: true };
await f.session.start(settings);
assert.equal(f.settings.length, 1);
assert.equal(f.settings[0], settings);
assert.equal(f.starts(), 1);
assert.equal(f.session.state, 'listening');
f.session.stop();
assert.equal(f.session.state, 'idle');
f.session.destroy();
});
for (const emitsIdle of [false, true]) {
for (const outcome of ['normal', 'preserveStatus', 'throw']) {
test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => {
+16 -1
View File
@@ -1,4 +1,6 @@
import { nextTick } from 'vue';
import { useOmnibarStore } from '../ui/stores/omnibar.js';
import { applicationPinia } from '../ui/vueRuntime.js';
import { bindAssistantPanelLayout, bindLayoutDock } from './panelLayout.js';
import { bindChatControl } from './chatControl.js';
import { bindVoiceMiniPreview } from './voiceMiniPreview.js';
@@ -34,6 +36,7 @@ export function createVoiceCommands({
supportedProviders,
onProviderChange,
onSessionEvent,
canStartVoice,
createCard = (options) => new VoiceCardControls(options),
pointer = null,
referents = null,
@@ -49,6 +52,7 @@ export function createVoiceCommands({
let panelMount;
const session = createVoiceSession({
runner,
canStart: (options) => options?.textOnly || canStartVoice?.() !== false,
signal,
createAdapter: (hooks) => {
if (panelHost) actionObserver = panelHost.observeActions(hooks);
@@ -284,7 +288,7 @@ export function createVoiceCommands({
void session.start({ pushToTalk: false });
};
const buttonHandler = () => {
if (adapter.ignoreButtonClick?.()) return;
if (canStartVoice?.() === false || adapter.ignoreButtonClick?.()) return;
chat?.close();
if (session.isActive() && adapter.getStartOptions?.().textOnly) {
session.stop();
@@ -332,5 +336,16 @@ export function createVoiceCommands({
if (session.disposed) cleanup();
else adapter.bindControls?.();
window.__gevVoiceCommands = controls;
// The omnibar runs the same actions directly, without a model.
// A session already gone leaves the omnibar without a runner.
if (!session.disposed) {
const omnibar = useOmnibarStore(applicationPinia());
omnibar.connectRunner((name, args) => runner(name, args));
session.signal.addEventListener(
'abort',
() => omnibar.connectRunner(null),
{ once: true },
);
}
return controls;
}
+1
View File
@@ -1070,6 +1070,7 @@ const STEP_TITLES = Object.freeze({
set_detection: msg('Adjust detection overlay'),
set_map_stack: msg('Change map stack'),
set_post_processing: msg('Adjust post-processing'),
open_stories: msg('Stories'),
control_scene: msg('Control scene'),
control_cctv: msg('Control CCTV'),
control_radio: msg('Control radio'),