Fix Stories navigation, sharing, and voice entry

This commit is contained in:
Manjunath committed 2026-11-16 20:12:10 +05:30
1 parent 5418438339
commit 9e9ce1db22
43 files changed
+744 -137

No files matched your search

+8
View File
@@ -430,6 +430,14 @@ const schemas = [
},
},
},
{
name: 'open_stories',
parameters: {
type: 'object',
additionalProperties: false,
properties: {},
},
},
{
name: 'control_scene',
parameters: {
+3 -1
View File
@@ -59,7 +59,8 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
// move_camera tilt takes an optional pitchDeg angle, described as
// absolute with its -45 default, with steps sent as a direction.
// show_street_level (map, source, photo) adds one tool.
'0c27f62062001e6a07241cd301c3d357f55e9b74f66cd2904d16590510337468',
// open_stories adds the Stories Explore entry action.
'5a2deb65bef5db8fa49ee55dd66efb884e7c3ba97dbdcaad3fada087170d2225',
);
});
@@ -157,6 +158,7 @@ test('all legacy action arguments are byte-identical after removing the delibera
'find_imagery',
'osm_query',
'show_street_level',
'open_stories',
].includes(tool.name),
);
// Layer enums are generated from the voice layer manifest and pinned by
+8 -1
View File
@@ -3,6 +3,7 @@ import { t, translateKnown } from '../i18n/index.js';
import { msg } from '../i18n/marker.js';
import { createGeminiAudio } from './geminiAudio.js';
import { RealtimeInput } from './realtimeInput.js';
import { isStoriesOpen } from './realtimeInputPolicy.js';
import { RealtimeRadio } from './realtimeRadio.js';
import { captureViewportImage } from './realtimeViewport.js';
import { createNarrationScheduler } from './narration.js';
@@ -652,7 +653,13 @@ export class GeminiSessionController {
}
async start({ pushToTalk = false } = {}) {
if (this.disposed || this.isActive() || this.signal?.aborted) return;
if (
isStoriesOpen() ||
this.disposed ||
this.isActive() ||
this.signal?.aborted
)
return;
const held = pushToTalk && this.input.pushToTalkKeyHeld;
const spaceHeld = this.input.spaceKeyHeld;
this.stop({ preserveStatus: true });
+47 -6
View File
@@ -8,6 +8,7 @@ import {
import { readLayerLifecycleSummary } from './layerSummary.js';
import { SPEECH_BUILDERS, attachVoiceResult } from './speech.js';
import { resolveDeicticArgs } from './deixis.js';
import { isStoriesOpen } from './realtimeInputPolicy.js';
export { readLayerLifecycleSummary } from './layerSummary.js';
import { defaultGeospatial } from '../search/defaults.js';
import { msg } from '../i18n/index.js';
@@ -294,6 +295,7 @@ export function createGevActionRunner({
styleManager,
dataManager,
sceneDirector = null,
openStories = null,
annotations = null,
placeSearch = unavailablePlaceSearch,
floorServices = defaultFloorServices,
@@ -444,9 +446,51 @@ export function createGevActionRunner({
runOptions = {},
) {
const args = rawArgs && typeof rawArgs === 'object' ? rawArgs : {};
// Stories covers GEV and owns its camera and scenes: an open-mic session
// must not fly, load or change what is behind it.
if (globalThis.document?.body?.classList?.contains?.('stories-mode'))
const current = () =>
!runOptions.signal?.aborted &&
(typeof runOptions.isCurrent !== 'function' || runOptions.isCurrent());
if (name === 'open_stories') {
const denied = { ok: false, action: name, applied: false };
const cancelled = {
...denied,
code: 'CANCELLED',
cancelled: true,
error: 'This request was superseded.',
};
if (!current()) return cancelled;
if (typeof openStories !== 'function')
return {
...denied,
unavailable: true,
error: 'Stories is unavailable in this host.',
say: 'Stories is unavailable in this host.',
};
// Opening Stories can stop the voice session that requested it. The
// action was applied even when that session is no longer current.
const opened = await openStories({ isCurrent: current });
if (opened === true)
return {
ok: true,
action: name,
applied: true,
say: 'Stories is open.',
};
if (!current()) return cancelled;
if (opened === 'already-open')
return {
ok: true,
action: name,
applied: false,
say: 'Stories is already open.',
};
return {
...denied,
error: 'Stories could not be opened.',
say: 'Stories could not be opened.',
};
}
// Stories owns the camera and scenes. Leave the globe behind it alone.
if (isStoriesOpen())
return {
ok: false,
action: name,
@@ -455,9 +499,6 @@ export function createGevActionRunner({
error: 'Stories is open. Close Stories first.',
say: 'Stories is open. Close Stories first to control the globe.',
};
const current = () =>
!runOptions.signal?.aborted &&
(typeof runOptions.isCurrent !== 'function' || runOptions.isCurrent());
// Step reports for voice narration and the voice card; never fatal.
const reportProgress = (step, label = '') => {
try {
+52
View File
@@ -270,6 +270,7 @@ test('voice changes nothing behind Stories while it is open', async () => {
camera: { moveEnd: { addEventListener() {} } },
};
let calls = 0;
let storiesVisible = true;
const styleManager = {
async resetToGlobeView() {
calls += 1;
@@ -280,6 +281,7 @@ test('voice changes nothing behind Stories while it is open', async () => {
viewer,
styleManager,
dataManager: { layers: new Map(), getAll: () => [] },
openStories: () => (storiesVisible ? 'already-open' : true),
});
const classes = new Set(['stories-mode']);
const priorDocument = globalThis.document;
@@ -291,7 +293,16 @@ test('voice changes nothing behind Stories while it is open', async () => {
const refused = await runner('zoom_to_globe');
assert.equal(refused.ok, false);
assert.equal(refused.refused, 'stories-open');
const alreadyOpen = await runner('open_stories');
assert.equal(alreadyOpen.ok, true);
assert.equal(alreadyOpen.applied, false);
assert.equal(calls, 0);
// The input guard still owns the globe while the director settles.
storiesVisible = false;
const reopened = await runner('open_stories');
assert.equal(reopened.ok, true);
assert.equal(reopened.applied, true);
assert.equal((await runner('zoom_to_globe')).refused, 'stories-open');
classes.clear();
await runner('zoom_to_globe');
assert.equal(calls, 1, 'works again once Stories has closed');
@@ -300,6 +311,47 @@ test('voice changes nothing behind Stories while it is open', async () => {
}
});
test('opening Stories only admits current requests and can stop its own session', async () => {
const { viewer, styleManager } = createVoiceNavigationHarness();
const first = new AbortController();
let loaded = Promise.resolve();
let admitted = 0;
let opened = 0;
const runner = createGevActionRunner({
viewer,
styleManager,
dataManager: { layers: new Map(), getAll: () => [] },
async openStories({ isCurrent }) {
admitted += 1;
await loaded;
if (!isCurrent()) return false;
opened += 1;
first.abort();
return true;
},
});
const applied = await runner('open_stories', {}, { signal: first.signal });
assert.equal(applied.ok, true);
assert.equal(applied.applied, true);
const stale = await runner('open_stories', {}, { signal: first.signal });
assert.equal(stale.cancelled, true);
assert.equal(admitted, 1);
let finishLoading;
loaded = new Promise((resolve) => { finishLoading = resolve; });
const second = new AbortController();
const pending = runner('open_stories', {}, { signal: second.signal });
assert.equal(admitted, 2);
second.abort();
finishLoading();
const canceled = await pending;
assert.equal(canceled.cancelled, true);
assert.equal(canceled.applied, false);
assert.equal(opened, 1);
});
test('dependent voice navigation waits for the destination viewport to arrive', async () => {
globalThis.window = globalThis.window || {
clearTimeout,
+28
View File
@@ -358,6 +358,34 @@ test('push-to-talk rechecks focus before the hold threshold claims voice', (t) =
assert.equal(f.controller.spaceKeyHeld, false);
});
test('opening Stories cancels a pending Space hold even when focus stays put', (t) => {
const f = createPushToTalkFixture(t);
const background = pushToTalkTarget();
const classes = new Set();
f.document.body = { classList: { contains: (name) => classes.has(name) } };
f.key('keydown', background);
classes.add('stories-mode');
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
assert.deepEqual(f.starts, []);
assert.deepEqual(f.radio, []);
assert.equal(f.controller.spaceKeyHeld, false);
assert.equal(f.timers.size, 0);
classes.delete('stories-mode');
f.key('keydown', background);
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
assert.equal(f.microphone.enabled, true);
f.key('keyup', background);
assert.equal(f.microphone.enabled, false);
f.key('keydown', background);
classes.add('stories-mode');
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
assert.equal(f.starts.length, 1);
assert.equal(f.microphone.enabled, false);
});
test('push-to-talk never claims a control-started hold after focus moves outside', (t) => {
const f = createPushToTalkFixture(t);
const control = pushToTalkTarget({ tagName: 'BUTTON' });
+3 -1
View File
@@ -1,4 +1,5 @@
import { t } from '../i18n/index.js';
import { isStoriesOpen } from './realtimeInputPolicy.js';
const DISCONNECT_GRACE_MS = 6000;
@@ -52,7 +53,8 @@ export class RealtimeConnection {
return this.readStatus();
}
async start({ pushToTalk = false } = {}) {
if (this.isActive() || this.lifetimeSignal?.aborted) return;
if (isStoriesOpen() || this.isActive() || this.lifetimeSignal?.aborted)
return;
this.pauseRadioForVoice();
const pushToTalkKeyHeld = pushToTalk && this.input.pushToTalkKeyHeld;
const spaceKeyHeld = this.input.spaceKeyHeld;
+6 -1
View File
@@ -3,6 +3,7 @@ import {
PUSH_TO_TALK_HOLD_DELAY_MS,
MICROPHONE_VISUALIZER_GATE,
ASSISTANT_VISUALIZER_GATE,
isStoriesOpen,
isPushToTalkKey,
SPACE_INTERACTIVE_SELECTOR,
isInteractiveSpaceTarget,
@@ -59,7 +60,7 @@ export class RealtimeInput {
// Stories owns the keyboard (body.stories-mode).
if (
this.ui.state?.chat?.open === true ||
globalThis.document?.body?.classList?.contains?.('stories-mode') ||
isStoriesOpen() ||
!shouldHandlePushToTalkKeyDown(event)
)
return;
@@ -95,6 +96,10 @@ export class RealtimeInput {
this.pushToTalkHoldTimer = null;
if (holdGeneration !== this.pushToTalkHoldGeneration) return;
if (!this.spaceKeyHeld || this.pushToTalkKeyHeld) return;
if (isStoriesOpen()) {
this.shortcutBlurHandler();
return;
}
// A pointer or Tab focus change during the delay cancels the original
// gesture rather than letting voice claim a key held for another owner.
if (document.activeElement !== this.pushToTalkHoldFocusOwner) return;
+7
View File
@@ -9,6 +9,13 @@ export const MICROPHONE_VISUALIZER_GATE = 0.12;
export const ASSISTANT_VISUALIZER_GATE = 0.04;
/** Stories takes keyboard and microphone ownership away from voice. */
export function isStoriesOpen() {
return (
globalThis.document?.body?.classList?.contains?.('stories-mode') ?? false
);
}
/**
* Returns whether a keyboard event represents the hold-Space voice shortcut.
* @param {KeyboardEvent|object|null} event
+12 -2
View File
@@ -5,7 +5,12 @@
* Events: state, transcript, action-call, progress, action-result, interruption,
* completion, turn-metrics.
*/
export function createVoiceSession({ createAdapter, runner, signal }) {
export function createVoiceSession({
createAdapter,
runner,
signal,
canStart,
}) {
if (typeof createAdapter !== 'function' || typeof runner !== 'function')
throw new TypeError('Voice requires an adapter factory and action runner');
const lifetime = new AbortController();
@@ -146,7 +151,12 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
return () => listeners.delete(listener);
},
async start(options) {
if (disposed || !['idle', 'error'].includes(state)) return;
if (
disposed ||
!['idle', 'error'].includes(state) ||
canStart?.(options) === false
)
return;
const attempt = ++startAttempt;
emit({ type: 'state', state: 'connecting' });
if (disposed || attempt !== startAttempt) return;
+14 -4
View File
@@ -429,12 +429,13 @@ test('a late rejection cannot settle a newer turn card', async () => {
f.session.destroy();
});
function startReentrancyFixture() {
function startReentrancyFixture(canStart) {
let starts = 0;
let stops = 0;
const settings = [];
const session = createVoiceSession({
runner: async () => ({ ok: true }),
canStart,
createAdapter: ({ emit }) => ({
async start(options) {
starts++;
@@ -515,9 +516,18 @@ test('Stop and explicit reentrant successor start acquire only for the successor
f.session.destroy();
});
test('ordinary start forwards caller settings once and remains stoppable', async () => {
const f = startReentrancyFixture();
const settings = { pushToTalk: true };
test('start admission rejects microphone use before connecting and allows typed chat', async () => {
const f = startReentrancyFixture((options) => options?.textOnly === true);
const states = [];
f.session.subscribe((event) => {
if (event.type === 'state') states.push(event.state);
});
await f.session.start({ pushToTalk: true });
assert.equal(f.starts(), 0);
assert.equal(f.session.state, 'idle');
assert.equal(states.includes('connecting'), false);
const settings = { textOnly: true };
await f.session.start(settings);
assert.equal(f.settings.length, 1);
assert.equal(f.settings[0], settings);
+3 -1
View File
@@ -34,6 +34,7 @@ export function createVoiceCommands({
supportedProviders,
onProviderChange,
onSessionEvent,
canStartVoice,
createCard = (options) => new VoiceCardControls(options),
pointer = null,
referents = null,
@@ -49,6 +50,7 @@ export function createVoiceCommands({
let panelMount;
const session = createVoiceSession({
runner,
canStart: (options) => options?.textOnly || canStartVoice?.() !== false,
signal,
createAdapter: (hooks) => {
if (panelHost) actionObserver = panelHost.observeActions(hooks);
@@ -284,7 +286,7 @@ export function createVoiceCommands({
void session.start({ pushToTalk: false });
};
const buttonHandler = () => {
if (adapter.ignoreButtonClick?.()) return;
if (canStartVoice?.() === false || adapter.ignoreButtonClick?.()) return;
chat?.close();
if (session.isActive() && adapter.getStartOptions?.().textOnly) {
session.stop();
+1
View File
@@ -1070,6 +1070,7 @@ const STEP_TITLES = Object.freeze({
set_detection: msg('Adjust detection overlay'),
set_map_stack: msg('Change map stack'),
set_post_processing: msg('Adjust post-processing'),
open_stories: msg('Stories'),
control_scene: msg('Control scene'),
control_cctv: msg('Control CCTV'),
control_radio: msg('Control radio'),