mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-19 01:32:19 +00:00
Merge branch 'main' into fix/i18n-installations-note
# Conflicts: # scripts/package-boundaries.json # src/layers/installations/controls.js
This commit is contained in:
commit
93b091d082
2063 files changed
+322399
-46202
No files matched your search
@@ -107,6 +107,36 @@ const schemas = [
|
||||
required: ['layerId'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'show_street_level',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
query: {
|
||||
type: 'string',
|
||||
maxLength: 160,
|
||||
},
|
||||
latitude: {
|
||||
type: 'number',
|
||||
minimum: -90,
|
||||
maximum: 90,
|
||||
},
|
||||
longitude: {
|
||||
type: 'number',
|
||||
minimum: -180,
|
||||
maximum: 180,
|
||||
},
|
||||
provider: {
|
||||
type: 'string',
|
||||
enum: ['google', 'mapillary', 'kartaview', 'panoramax'],
|
||||
},
|
||||
referent: {
|
||||
type: 'integer',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'adjust_camera_zoom',
|
||||
parameters: {
|
||||
@@ -400,6 +430,14 @@ const schemas = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'open_stories',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'control_scene',
|
||||
parameters: {
|
||||
|
||||
@@ -58,7 +58,9 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
|
||||
// control_cockpit states the own-layer entry rule (Contacts not required).
|
||||
// move_camera tilt takes an optional pitchDeg angle, described as
|
||||
// absolute with its -45 default, with steps sent as a direction.
|
||||
'6a0d154e68beb787c4e20f12d29a5c86577dc343f3e71d8b5651998ea45d8c3c',
|
||||
// show_street_level (map, source, photo) adds one tool.
|
||||
// open_stories adds the Stories Explore entry action.
|
||||
'5a2deb65bef5db8fa49ee55dd66efb884e7c3ba97dbdcaad3fada087170d2225',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -155,6 +157,8 @@ test('all legacy action arguments are byte-identical after removing the delibera
|
||||
'resolve_area',
|
||||
'find_imagery',
|
||||
'osm_query',
|
||||
'show_street_level',
|
||||
'open_stories',
|
||||
].includes(tool.name),
|
||||
);
|
||||
// Layer enums are generated from the voice layer manifest and pinned by
|
||||
|
||||
@@ -6,6 +6,7 @@ import { createControls as installationControls } from '../layers/installations/
|
||||
import { createQueries as transitQueries } from '../layers/transit/queries.js';
|
||||
import { createCyclonesLayer } from '../layers/cyclones/index.js';
|
||||
import { ANALYST_LAYERS } from '../data/analystEngine.js';
|
||||
import { fakeInput } from '../input/testDoubles.mjs';
|
||||
|
||||
/** Every declared analyst field must be a key the accessor actually emits. */
|
||||
function assertDeclaredFields(layerId, record) {
|
||||
@@ -183,16 +184,7 @@ test('cyclones flatten the storm position and stay empty while off', async () =>
|
||||
fetchedAt: Date.now(),
|
||||
}),
|
||||
},
|
||||
cesium: {
|
||||
ScreenSpaceEventType: { LEFT_CLICK: 'left' },
|
||||
ScreenSpaceEventHandler: class {
|
||||
setInputAction() {}
|
||||
destroy() {}
|
||||
isDestroyed() {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
},
|
||||
cesium: {},
|
||||
createRendering: () => ({
|
||||
setSnapshot: async () => true,
|
||||
setSelection() {},
|
||||
@@ -204,10 +196,12 @@ test('cyclones flatten the storm position and stay empty while off', async () =>
|
||||
}),
|
||||
hitTestOverlay: () => null,
|
||||
});
|
||||
layer.init({
|
||||
const viewer = {
|
||||
scene: { canvas: new EventTarget(), pick: () => null },
|
||||
camera: {},
|
||||
});
|
||||
};
|
||||
const input = fakeInput(viewer);
|
||||
layer.init(viewer);
|
||||
assert.deepEqual(layer.getAnalystRecords(), []);
|
||||
layer.enable();
|
||||
await layer.update();
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { createAreaStore, summarizeArea } from '../data/areaStore.js';
|
||||
import { findCamera } from '../render/rendererOf.js';
|
||||
import { createAreaResolver, foldLatinMarks } from '../data/areaResolver.js';
|
||||
import { circleRing, displayParts } from '../data/areaGeometry.js';
|
||||
import { findNaturalRegion } from '../data/naturalEarthRegions.js';
|
||||
@@ -176,12 +177,9 @@ export function createVoiceAreas({
|
||||
const picked = annotationResolver?.pickWorldFromScreen?.(viewer, 0.5, 0.5);
|
||||
if (Number.isFinite(picked?.lat) && Number.isFinite(picked?.lon))
|
||||
return picked;
|
||||
const carto = viewer?.camera?.positionCartographic;
|
||||
return carto
|
||||
? {
|
||||
lat: (carto.latitude * 180) / Math.PI,
|
||||
lon: (carto.longitude * 180) / Math.PI,
|
||||
}
|
||||
const pose = findCamera(viewer)?.pose();
|
||||
return Number.isFinite(pose?.lat) && Number.isFinite(pose?.lon)
|
||||
? { lat: pose.lat, lon: pose.lon }
|
||||
: null;
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,8 @@ import { createVoiceAreas, spokenArea } from './areaActions.js';
|
||||
import { pointInPreparedArea } from '../data/areaGeometry.js';
|
||||
import { presentResult } from './resultDisplay.js';
|
||||
import { findAdminCandidates as findBundledAdminCandidates } from '../data/adminBoundaries.js';
|
||||
import { bindRenderer } from '../render/rendererOf.js';
|
||||
import { fakeCameraPort } from '../testSupport/fakeCameraPort.mjs';
|
||||
|
||||
const square = (w, s, e, n) => [
|
||||
[w, s],
|
||||
@@ -314,6 +316,41 @@ test('an ambiguous name returns candidates for the model to ask about', async ()
|
||||
assert.equal(picked.display.source, 'Natural Earth');
|
||||
});
|
||||
|
||||
test('an ambiguous name is settled by the view: the ground at mid-screen, else where the camera stands', async () => {
|
||||
// Nothing under mid-screen (open sky, no pick): the camera's own position
|
||||
// over the Indian Punjab chooses it.
|
||||
const viewer = {};
|
||||
const camera = fakeCameraPort({ pose: { lon: 75.5, lat: 31 } });
|
||||
bindRenderer(viewer, { camera: camera.port });
|
||||
const { areas } = makeAreas({ viewer });
|
||||
const byCamera = await areas.resolveAreaAction({ query: 'Punjab' });
|
||||
assert.equal(byCamera.ok, true);
|
||||
assert.equal(byCamera.country, 'India');
|
||||
|
||||
// A camera that cannot say where it is settles nothing: the model asks.
|
||||
camera.setPose({ lon: NaN, lat: NaN });
|
||||
const unknown = await areas.resolveAreaAction({ query: 'Punjab' });
|
||||
assert.equal(unknown.needsClarification, true);
|
||||
|
||||
// The ground at mid-screen wins over the camera's position.
|
||||
const pakistan = PUNJABS[1].polygons[0][0];
|
||||
const lon = (pakistan[0][0] + pakistan[1][0]) / 2;
|
||||
const lat = (pakistan[0][1] + pakistan[2][1]) / 2;
|
||||
camera.setPose({ lon: 75.5, lat: 31 });
|
||||
const ladder = fakeLadder();
|
||||
const { areas: picking } = makeAreas({
|
||||
viewer,
|
||||
annotationResolver: {
|
||||
...ladder,
|
||||
pickWorldFromScreen: (v, x, y) =>
|
||||
v === viewer && x === 0.5 && y === 0.5 ? { lat, lon } : null,
|
||||
},
|
||||
});
|
||||
const byGround = await picking.resolveAreaAction({ query: 'Punjab' });
|
||||
assert.equal(byGround.ok, true);
|
||||
assert.equal(byGround.country, 'Pakistan');
|
||||
});
|
||||
|
||||
test('names reach the annotation outline ladder with the level as entity kind', async () => {
|
||||
const ladder = fakeLadder();
|
||||
const { areas } = makeAreas({ annotationResolver: ladder });
|
||||
|
||||
@@ -16,6 +16,7 @@ after(() => {
|
||||
function viewerFixture() {
|
||||
return {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
canvas: { addEventListener() {}, removeEventListener() {} },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
};
|
||||
|
||||
@@ -107,7 +107,7 @@ function initialState(panel) {
|
||||
t('Hold Space to speak · tap Space to activate focused controls'),
|
||||
},
|
||||
// A typed chat error reads as chat's, without the microphone hint.
|
||||
error: { text: '', typed: false },
|
||||
error: { text: '', typed: false, hint: true },
|
||||
// Connection and turn state beside the selected mode's indicator.
|
||||
phase: { text: '' },
|
||||
voicePanel: {
|
||||
|
||||
@@ -143,6 +143,8 @@ function resolveFly(args, { pointer, referents }) {
|
||||
entity: {
|
||||
layerId: target.layerId,
|
||||
id: target.id,
|
||||
// The answer names it, not its id ("near cam-9").
|
||||
...(target.label ? { label: target.label } : {}),
|
||||
...(hasPosition(target) ? { lat: target.lat, lon: target.lon } : {}),
|
||||
},
|
||||
},
|
||||
@@ -345,6 +347,7 @@ function resolveCropPixels(args, { imageFrame, cameraKey }) {
|
||||
const RESOLVERS = Object.freeze({
|
||||
track_entity: resolveTrack,
|
||||
fly_to_location: resolveFly,
|
||||
show_street_level: resolveFly,
|
||||
get_entity_context: resolveContext,
|
||||
analyst_query: resolveAnalyst,
|
||||
annotate_map: resolveAnnotate,
|
||||
|
||||
@@ -104,7 +104,7 @@ test('referents: numbered items resolve for track, fly and context', () => {
|
||||
const last = resolveDeicticArgs('track_entity', { query: 'x', referent: -1 }, { referents });
|
||||
assert.equal(last.args.query, 'ccc333', '-1 is the last one');
|
||||
const fly = resolveDeicticArgs('fly_to_location', { referent: 1 }, { referents });
|
||||
assert.deepEqual(fly.args, { entity: { layerId: 'flights', id: 'aaa111' } }, 'moving contacts are located at run time');
|
||||
assert.deepEqual(fly.args, { entity: { layerId: 'flights', id: 'aaa111', label: 'SWA12' } }, 'moving contacts are located at run time');
|
||||
const context = resolveDeicticArgs('get_entity_context', { referent: 3 }, { referents });
|
||||
assert.equal(context.args.scope, 'target');
|
||||
assert.equal(context.args.target.label, 'AAL9');
|
||||
@@ -180,3 +180,10 @@ test('numbered custom-labeled pins retain their resolved coordinates', () => {
|
||||
assert.deepEqual(resolveDeicticArgs('fly_to_location', { referent: 1 }, { referents }).args,
|
||||
{ latitude: 30.27, longitude: -97.74 });
|
||||
});
|
||||
|
||||
test('show_street_level: "street view of this" opens photos at the pointed ground point', () => {
|
||||
const out = resolveDeicticArgs('show_street_level', { query: 'pointer', provider: 'google' }, { pointer: ground });
|
||||
assert.deepEqual(out.args, { provider: 'google', latitude: 30.27, longitude: -97.74 });
|
||||
assert.equal(out.used.source, 'pointer');
|
||||
assert.deepEqual(resolveDeicticArgs('show_street_level', { query: 'Eiffel Tower' }, { pointer: ground }).args, { query: 'Eiffel Tower' });
|
||||
});
|
||||
@@ -14,6 +14,7 @@ import { DataLayerManager } from '../data/manager.js';
|
||||
const viewerFixture = () => ({
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
canvas: { addEventListener() {}, removeEventListener() {} },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
});
|
||||
const request = (ws, id, name, args = {}) =>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { bindFakeCesiumCamera } from '../testSupport/fakeCameraPort.mjs';
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
@@ -47,21 +48,20 @@ const outputs = (ws) =>
|
||||
function fixture({ holdFirst = false } = {}) {
|
||||
const state = { height: 1000, renders: 0, providerWrites: [], panels: [] };
|
||||
const position = { x: 6379137, y: 0, z: 0 };
|
||||
const viewer = {
|
||||
const viewer = bindFakeCesiumCamera({
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: {
|
||||
positionWC: position,
|
||||
positionCartographic: { height: 1000 },
|
||||
moveEnd: { addEventListener() {} },
|
||||
cancelFlight() {},
|
||||
zoomIn(n) {
|
||||
position.x -= n;
|
||||
state.height -= n;
|
||||
this.positionCartographic.height = state.height;
|
||||
},
|
||||
zoomOut(n) {
|
||||
position.x += n;
|
||||
state.height += n;
|
||||
// Looking straight down at the surface along -x.
|
||||
directionWC: { x: -1, y: 0, z: 0 },
|
||||
upWC: { x: 0, y: 0, z: 1 },
|
||||
rightWC: { x: 0, y: 1, z: 0 },
|
||||
setView({ destination }) {
|
||||
state.height -= position.x - destination.x;
|
||||
position.x = destination.x;
|
||||
this.positionCartographic.height = state.height;
|
||||
},
|
||||
},
|
||||
@@ -76,7 +76,7 @@ function fixture({ holdFirst = false } = {}) {
|
||||
state.renders++;
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager: { layers: new Map() },
|
||||
|
||||
+55
-13
@@ -3,6 +3,7 @@ import { t, translateKnown } from '../i18n/index.js';
|
||||
import { msg } from '../i18n/marker.js';
|
||||
import { createGeminiAudio } from './geminiAudio.js';
|
||||
import { RealtimeInput } from './realtimeInput.js';
|
||||
import { isStoriesOpen } from './realtimeInputPolicy.js';
|
||||
import { RealtimeRadio } from './realtimeRadio.js';
|
||||
import { captureViewportImage } from './realtimeViewport.js';
|
||||
import { createNarrationScheduler } from './narration.js';
|
||||
@@ -71,7 +72,9 @@ export function geminiSocketUrl({ token, websocketUrl }) {
|
||||
return url.href;
|
||||
}
|
||||
|
||||
// Known local-server failures are translated; unexpected diagnostics stay intact.
|
||||
// The local server's known failures, marked for translation. Each is kept as
|
||||
// its English catalog text, which the status words when it is shown, so a
|
||||
// language change words it again; unexpected diagnostics stay intact.
|
||||
function tokenErrorMessage(error) {
|
||||
switch (error) {
|
||||
case msg('Gemini voice could not connect. Try again.'):
|
||||
@@ -85,7 +88,6 @@ function tokenErrorMessage(error) {
|
||||
case msg('Token request is too large'):
|
||||
case msg('Use an empty JSON token request'):
|
||||
case msg('Method not allowed'):
|
||||
return t(error);
|
||||
default:
|
||||
return error;
|
||||
}
|
||||
@@ -359,18 +361,35 @@ export class GeminiSessionController {
|
||||
);
|
||||
}
|
||||
|
||||
setStatus(status, detail, recovery) {
|
||||
/**
|
||||
* @param {{missingKey?: boolean, params?: object}} [options] `missingKey`
|
||||
* is true when the server has no Gemini key: the control then drops its
|
||||
* microphone hint. `params` fill the placeholders of `detail`.
|
||||
*/
|
||||
setStatus(
|
||||
status,
|
||||
detail,
|
||||
recovery,
|
||||
{ missingKey = false, params = null } = {},
|
||||
) {
|
||||
this.status = status;
|
||||
if (status === 'listening' && this.input.pushToTalkMode)
|
||||
detail = this.input.pushToTalkKeyHeld
|
||||
? msg('Release Space to send')
|
||||
: msg('Hold Space to talk');
|
||||
// `detail` is worded in the current language; `sourceDetail` keeps the
|
||||
// message as given (English catalog text, with `sourceParams` for its
|
||||
// placeholders, when it is ours), so a view can word it again after a
|
||||
// language change.
|
||||
const event = {
|
||||
type: 'state',
|
||||
state: status,
|
||||
detail: translateKnown(detail),
|
||||
detail: params ? t(detail, params) : translateKnown(detail),
|
||||
...(typeof detail === 'string' && { sourceDetail: detail }),
|
||||
...(params && { sourceParams: params }),
|
||||
};
|
||||
if (recovery) event.recovery = translateKnown(recovery);
|
||||
if (missingKey) event.missingKey = true;
|
||||
this.emit(event);
|
||||
this.input.updateVoiceButtonLabel();
|
||||
if (['idle', 'connecting', 'error'].includes(status))
|
||||
@@ -584,12 +603,24 @@ export class GeminiSessionController {
|
||||
const data = await readResponseJsonCapped(response, 256 * 1024, signal);
|
||||
signal.throwIfAborted();
|
||||
if (!response.ok)
|
||||
throw new Error(
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
typeof data?.error === 'string'
|
||||
? tokenErrorMessage(data.error)
|
||||
: t('Gemini token request failed (HTTP {status})', {
|
||||
status: response.status,
|
||||
}),
|
||||
),
|
||||
typeof data?.code === 'string' ? { code: data.code } : {},
|
||||
// The status words this again in the current language when shown.
|
||||
typeof data?.error === 'string'
|
||||
? tokenErrorMessage(data.error)
|
||||
: t('Gemini token request failed (HTTP {status})', {
|
||||
status: response.status,
|
||||
}),
|
||||
? {}
|
||||
: {
|
||||
wording: {
|
||||
text: msg('Gemini token request failed (HTTP {status})'),
|
||||
params: { status: response.status },
|
||||
},
|
||||
},
|
||||
);
|
||||
if (
|
||||
typeof data?.model !== 'string' ||
|
||||
@@ -622,7 +653,13 @@ export class GeminiSessionController {
|
||||
}
|
||||
|
||||
async start({ pushToTalk = false } = {}) {
|
||||
if (this.disposed || this.isActive() || this.signal?.aborted) return;
|
||||
if (
|
||||
isStoriesOpen() ||
|
||||
this.disposed ||
|
||||
this.isActive() ||
|
||||
this.signal?.aborted
|
||||
)
|
||||
return;
|
||||
const held = pushToTalk && this.input.pushToTalkKeyHeld;
|
||||
const spaceHeld = this.input.spaceKeyHeld;
|
||||
this.stop({ preserveStatus: true });
|
||||
@@ -963,10 +1000,15 @@ export class GeminiSessionController {
|
||||
)
|
||||
: error?.name === 'TimeoutError'
|
||||
? msg('Gemini connection timed out. Try again.')
|
||||
: error?.message || msg('Gemini voice could not start');
|
||||
this.lastError = detail;
|
||||
: error?.wording?.text ||
|
||||
error?.message ||
|
||||
msg('Gemini voice could not start');
|
||||
this.lastError = error?.wording ? error.message : detail;
|
||||
this.stop({ preserveStatus: true });
|
||||
this.setStatus('error', detail, recovery);
|
||||
this.setStatus('error', detail, recovery, {
|
||||
missingKey: error?.code === 'API_KEY_MISSING',
|
||||
params: error?.wording?.params,
|
||||
});
|
||||
}
|
||||
|
||||
abortTools() {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { geminiSocketUrl } from './geminiSession.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { createPointerTracker } from './pointerContext.js';
|
||||
import { withToolCatalog } from './gevRealtime.js';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
@@ -3638,3 +3639,73 @@ test('cancelling a queued globe reset also skips its dependent rotation', async
|
||||
h.c.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('a missing server key reaches the error state flagged, other token failures do not', async () => {
|
||||
for (const [body, missingKey] of [
|
||||
[
|
||||
{
|
||||
error: 'Add a Gemini API key in Provider Settings',
|
||||
code: 'API_KEY_MISSING',
|
||||
},
|
||||
true,
|
||||
],
|
||||
[{ error: 'Gemini quota is unavailable. Try again later.' }, undefined],
|
||||
]) {
|
||||
const h = harness({
|
||||
fetchImpl: async () => Response.json(body, { status: 503 }),
|
||||
});
|
||||
await h.c.start();
|
||||
const error = h.events
|
||||
.filter((event) => event.type === 'state' && event.state === 'error')
|
||||
.at(-1);
|
||||
assert.equal(error?.detail, body.error);
|
||||
assert.equal(error?.missingKey, missingKey);
|
||||
assert.equal(h.sockets.length, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('an error keeps its English catalog text beside the worded detail', async (t2) => {
|
||||
t2.after(() => setLocale('en'));
|
||||
setLocale('es');
|
||||
const h = harness({
|
||||
fetchImpl: async () =>
|
||||
Response.json(
|
||||
{ error: 'Gemini quota is unavailable. Try again later.' },
|
||||
{ status: 503 },
|
||||
),
|
||||
});
|
||||
await h.c.start();
|
||||
const error = h.events
|
||||
.filter((event) => event.type === 'state' && event.state === 'error')
|
||||
.at(-1);
|
||||
assert.equal(
|
||||
error.sourceDetail,
|
||||
'Gemini quota is unavailable. Try again later.',
|
||||
);
|
||||
assert.equal(
|
||||
error.detail,
|
||||
t('Gemini quota is unavailable. Try again later.'),
|
||||
);
|
||||
assert.notEqual(error.detail, error.sourceDetail);
|
||||
});
|
||||
|
||||
test('a token failure without a message keeps its status for the next language', async (t2) => {
|
||||
t2.after(() => setLocale('en'));
|
||||
setLocale('es');
|
||||
const h = harness({
|
||||
fetchImpl: async () => Response.json({}, { status: 502 }),
|
||||
});
|
||||
await h.c.start();
|
||||
const error = h.events
|
||||
.filter((event) => event.type === 'state' && event.state === 'error')
|
||||
.at(-1);
|
||||
assert.equal(
|
||||
error.sourceDetail,
|
||||
'Gemini token request failed (HTTP {status})',
|
||||
);
|
||||
assert.deepEqual(error.sourceParams, { status: 502 });
|
||||
assert.equal(
|
||||
error.detail,
|
||||
t('Gemini token request failed (HTTP {status})', { status: 502 }),
|
||||
);
|
||||
});
|
||||
+744
-166
File diff suppressed because it is too large.
Load diff
+1293
-80
File diff suppressed because it is too large.
Load diff
@@ -5,6 +5,7 @@ import { createVoiceCommands } from './commands.js';
|
||||
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
|
||||
import { createPointerPicker } from './pointerPick.js';
|
||||
import { createReferentRegistry } from './referents.js';
|
||||
import { canvasOf } from '../render/rendererOf.js';
|
||||
export * from './realtimeController.js';
|
||||
|
||||
const ACTION_NAMES = new Set(GEV_ACTION_SCHEMAS.map((schema) => schema.name));
|
||||
@@ -32,7 +33,7 @@ export function withToolCatalog(runner, loadCatalog) {
|
||||
|
||||
/** Compose the standalone action runner with the voice controls. */
|
||||
export function initGevVoiceCommands(options) {
|
||||
const canvas = options.viewer?.scene?.canvas || null;
|
||||
const canvas = canvasOf(options.viewer);
|
||||
const cameraKey = () => cameraPoseKey(options.viewer);
|
||||
// Point-and-ask: the tracker and referents are shared by the runner (which
|
||||
// resolves 'pointer' and referent:n) and the controls (which snapshot the
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
// These helpers are DOM/WebRTC-free so they pin the screenshot down-scaling and
|
||||
// payload-byte estimation that keep an oversized dc.send from stranding a turn.
|
||||
import { test } from 'node:test';
|
||||
import { bindRenderer } from '../render/rendererOf.js';
|
||||
import { fakeSceneFrame } from '../testSupport/fakeCesiumScene.mjs';
|
||||
import assert from 'node:assert/strict';
|
||||
import { DataLayerManager } from '../data/manager.js';
|
||||
import { controlRadio as runControlRadio, createGevActionRunner as createActionRunner } from './gevActions.js';
|
||||
@@ -35,6 +37,15 @@ import {
|
||||
} from './gevRealtime.js';
|
||||
import { createVoiceCostTracker } from './voiceCost.js';
|
||||
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
|
||||
|
||||
/** A viewer over `scene` whose renderer's frame port is the scene's own loop. */
|
||||
function framed(scene) {
|
||||
const viewer = { scene };
|
||||
bindRenderer(viewer, { frame: fakeSceneFrame(viewer) });
|
||||
return viewer;
|
||||
}
|
||||
|
||||
test('push-to-talk recognizes Space by code or key', () => {
|
||||
assert.equal(PUSH_TO_TALK_HOLD_DELAY_MS, 500);
|
||||
@@ -347,6 +358,34 @@ test('push-to-talk rechecks focus before the hold threshold claims voice', (t) =
|
||||
assert.equal(f.controller.spaceKeyHeld, false);
|
||||
});
|
||||
|
||||
test('opening Stories cancels a pending Space hold even when focus stays put', (t) => {
|
||||
const f = createPushToTalkFixture(t);
|
||||
const background = pushToTalkTarget();
|
||||
const classes = new Set();
|
||||
f.document.body = { classList: { contains: (name) => classes.has(name) } };
|
||||
|
||||
f.key('keydown', background);
|
||||
classes.add('stories-mode');
|
||||
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
|
||||
assert.deepEqual(f.starts, []);
|
||||
assert.deepEqual(f.radio, []);
|
||||
assert.equal(f.controller.spaceKeyHeld, false);
|
||||
assert.equal(f.timers.size, 0);
|
||||
|
||||
classes.delete('stories-mode');
|
||||
f.key('keydown', background);
|
||||
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
|
||||
assert.equal(f.microphone.enabled, true);
|
||||
f.key('keyup', background);
|
||||
assert.equal(f.microphone.enabled, false);
|
||||
|
||||
f.key('keydown', background);
|
||||
classes.add('stories-mode');
|
||||
f.advance(PUSH_TO_TALK_HOLD_DELAY_MS);
|
||||
assert.equal(f.starts.length, 1);
|
||||
assert.equal(f.microphone.enabled, false);
|
||||
});
|
||||
|
||||
test('push-to-talk never claims a control-started hold after focus moves outside', (t) => {
|
||||
const f = createPushToTalkFixture(t);
|
||||
const control = pushToTalkTarget({ tagName: 'BUTTON' });
|
||||
@@ -1342,6 +1381,7 @@ test('generic same-response Radio visibility disable supersedes delayed Select',
|
||||
});
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
canvas: { addEventListener() {}, removeEventListener() {} },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
};
|
||||
@@ -1594,6 +1634,7 @@ test('Pause and Stop preserve independent dedicated and generic Radio ON across
|
||||
});
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
canvas: { addEventListener() {}, removeEventListener() {} },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
};
|
||||
@@ -2845,7 +2886,7 @@ test('hidden document yields no fresh frame — capture must not label a stale c
|
||||
};
|
||||
try {
|
||||
globalThis.document = { hidden: true };
|
||||
const fresh = await renderFreshCesiumFrame({ scene });
|
||||
const fresh = await renderFreshCesiumFrame(framed(scene));
|
||||
assert.equal(fresh, false, 'hidden capture reports non-fresh');
|
||||
assert.equal(requested, 0, 'no secret render restart while hidden');
|
||||
} finally {
|
||||
@@ -2862,7 +2903,7 @@ test('visible document with a rendering scene reports a fresh frame', async () =
|
||||
postRender: { addEventListener(listener) { fire = listener; return () => { fire = null; }; } },
|
||||
requestRender() { queueMicrotask(() => fire?.()); },
|
||||
};
|
||||
const fresh = await renderFreshCesiumFrame({ scene });
|
||||
const fresh = await renderFreshCesiumFrame(framed(scene));
|
||||
assert.equal(fresh, true);
|
||||
} finally {
|
||||
globalThis.document = originalDocument;
|
||||
@@ -2879,7 +2920,7 @@ test('a tab switch during the bounded render wait invalidates freshness', async
|
||||
postRender: { addEventListener(listener) { fire = listener; return () => { fire = null; }; } },
|
||||
requestRender() { doc.hidden = true; queueMicrotask(() => fire?.()); },
|
||||
};
|
||||
const fresh = await renderFreshCesiumFrame({ scene });
|
||||
const fresh = await renderFreshCesiumFrame(framed(scene));
|
||||
assert.equal(fresh, false, 'freshness rechecked after the await');
|
||||
} finally {
|
||||
globalThis.document = originalDocument;
|
||||
@@ -3708,3 +3749,76 @@ test('voice runs app actions itself and other tools through the catalog', async
|
||||
assert.deepEqual(disposals, [['removed']]);
|
||||
assert.equal(withToolCatalog(runner, undefined), runner);
|
||||
});
|
||||
|
||||
test('the controller forwards viewport escalation diagnostics to its debug log', async () => {
|
||||
const records = [];
|
||||
const controller = new GevRealtimeController({
|
||||
ui: createTestVoiceUi(),
|
||||
runner: async () => ({ ok: true }),
|
||||
radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} },
|
||||
});
|
||||
controller.debugLog = (event, detail) => records.push({ event, detail });
|
||||
controller.dc = { readyState: 'open', send() {}, close() {} };
|
||||
|
||||
assert.equal(
|
||||
await controller.sendVisualContextIfUseful({
|
||||
action: 'get_entity_context',
|
||||
scene: { basemap: { viewScale: 'regional' } },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(
|
||||
records.filter((record) => record.event.startsWith('viewport_context.')),
|
||||
[{
|
||||
event: 'viewport_context.decision',
|
||||
detail: { contextId: 1, send: false, reason: 'view-scale-not-local', viewScale: 'regional' },
|
||||
}],
|
||||
);
|
||||
});
|
||||
|
||||
test('a missing server key says where to add it, without the microphone hint, in every language', () => {
|
||||
const ui = createTestVoiceUi();
|
||||
const controller = new GevRealtimeController({
|
||||
ui,
|
||||
runner: async () => ({ ok: true }),
|
||||
radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} },
|
||||
});
|
||||
try {
|
||||
controller.reportError(
|
||||
'Realtime connection',
|
||||
Object.assign(new Error('OPENAI_API_KEY is not set'), { code: 'API_KEY_MISSING' }),
|
||||
);
|
||||
assert.equal(ui.state.error.text, 'Needs OPENAI_API_KEY — add it in Provider Settings');
|
||||
assert.equal(ui.state.error.hint, false);
|
||||
|
||||
setLocale('es');
|
||||
assert.notEqual(ui.state.error.text, 'Needs OPENAI_API_KEY — add it in Provider Settings');
|
||||
assert.match(ui.state.error.text, /OPENAI_API_KEY/);
|
||||
assert.equal(ui.state.error.hint, false);
|
||||
setLocale('en');
|
||||
|
||||
controller.reportError('Realtime connection', new Error('ICE failed'));
|
||||
assert.match(ui.state.error.text, /ICE failed/);
|
||||
assert.equal(ui.state.error.hint, true);
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
|
||||
test('a status from a shared helper is worded again after a language change', () => {
|
||||
const ui = createTestVoiceUi();
|
||||
const controller = new GevRealtimeController({
|
||||
ui,
|
||||
runner: async () => ({ ok: true }),
|
||||
radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} },
|
||||
});
|
||||
try {
|
||||
controller.setStatus('listening', 'Radio did not start');
|
||||
assert.equal(ui.state.detail.text, 'Radio did not start');
|
||||
setLocale('es');
|
||||
assert.equal(ui.state.detail.text, t('Radio did not start'));
|
||||
assert.notEqual(ui.state.detail.text, 'Radio did not start');
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
@@ -5,6 +5,8 @@ import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import * as Cesium from 'cesium';
|
||||
import { getBasemapLabelContext } from './gevActions.js';
|
||||
import { bindFakeCesiumCamera } from '../testSupport/fakeCameraPort.mjs';
|
||||
import { fakeSceneQuery } from '../testSupport/fakeCesiumScene.mjs';
|
||||
import {
|
||||
hudViewportSamples,
|
||||
placeLookupView,
|
||||
@@ -25,7 +27,12 @@ function createViewer({ latitude, longitude, heightM, groundM = 0 }) {
|
||||
const camera = {
|
||||
heading: 0,
|
||||
pitch: -Math.PI / 2,
|
||||
frustum: { fov: Math.PI / 3, aspectRatio: 1.5 },
|
||||
// Cesium's `fov` is the wider angle; `fovy` the vertical one.
|
||||
frustum: {
|
||||
fov: Math.PI / 3,
|
||||
fovy: 2 * Math.atan(Math.tan(Math.PI / 6) / 1.5),
|
||||
aspectRatio: 1.5,
|
||||
},
|
||||
pickEllipsoid(point) {
|
||||
return Cesium.Cartesian3.fromDegrees(
|
||||
state.longitude +
|
||||
@@ -63,7 +70,9 @@ function createViewer({ latitude, longitude, heightM, groundM = 0 }) {
|
||||
state.groundM,
|
||||
),
|
||||
};
|
||||
return { viewer: { camera, scene }, state, degreesPerPixel };
|
||||
const viewer = { camera, scene };
|
||||
bindFakeCesiumCamera(viewer, { query: fakeSceneQuery(viewer) });
|
||||
return { viewer, state, degreesPerPixel };
|
||||
}
|
||||
|
||||
/** Answers every lookup with a label naming the point it was asked about. */
|
||||
|
||||
@@ -714,13 +714,13 @@ test('without an operator Overpass the prompt never offers place search', () =>
|
||||
});
|
||||
|
||||
test('the Realtime prompt stays within its size budget', () => {
|
||||
// Instructions plus tool schemas, in characters, under the same 55,500
|
||||
// Instructions plus tool schemas, in characters, under the same 56,500
|
||||
// ceiling as the deixis prompt lint. Voice geometry adds resolve_area,
|
||||
// find_imagery and the analyst area scopes (osm_query only with an
|
||||
// operator Overpass).
|
||||
// operator Overpass); show_street_level adds its tool and routing rule.
|
||||
const size = (overpass) =>
|
||||
realtimeInstructions(undefined, { overpass }).length +
|
||||
JSON.stringify(realtimeTools({ overpass })).length;
|
||||
assert.ok(size(false) <= 55_500, `default prompt ${size(false)} chars`);
|
||||
assert.ok(size(true) <= 55_500, `with Overpass ${size(true)} chars`);
|
||||
assert.ok(size(false) <= 56_500, `default prompt ${size(false)} chars`);
|
||||
assert.ok(size(true) <= 56_500, `with Overpass ${size(true)} chars`);
|
||||
});
|
||||
@@ -302,7 +302,23 @@ const layers = [
|
||||
},
|
||||
{
|
||||
id: 'transit',
|
||||
aliases: ['buses', 'trains', 'public transit', 'trams', 'subway'],
|
||||
aliases: [
|
||||
'buses',
|
||||
'trains',
|
||||
'public transit',
|
||||
'trams',
|
||||
'subway',
|
||||
'trolleybus',
|
||||
'trolleybuses',
|
||||
'gondola',
|
||||
'gondolas',
|
||||
'cable car',
|
||||
'cable cars',
|
||||
'funicular',
|
||||
'funiculars',
|
||||
'monorail',
|
||||
'monorails',
|
||||
],
|
||||
context: false,
|
||||
query: {
|
||||
fields: {
|
||||
@@ -364,7 +380,12 @@ const layers = [
|
||||
},
|
||||
{
|
||||
id: 'street-level',
|
||||
aliases: ['street level', 'street-level imagery', 'mapillary'],
|
||||
aliases: [
|
||||
'street level',
|
||||
'street-level imagery',
|
||||
'mapillary',
|
||||
'street view',
|
||||
],
|
||||
context: false,
|
||||
noQuery: 'street-level imagery; no countable records',
|
||||
},
|
||||
@@ -457,8 +478,12 @@ export const VOICE_OFF_LAYERS = Object.freeze({
|
||||
// query for this layer wait until there is room.
|
||||
'drone-airspace':
|
||||
'informational airspace layer; no room in the prompt budget',
|
||||
// Same budget: a historical wreck archive waits for room too.
|
||||
shipwrecks: 'historical shipwreck archive; no room in the prompt budget',
|
||||
news: 'browsed in the News panel; voice tools are a later step',
|
||||
'wildlife-sightings':
|
||||
'community observation layer; no room in the prompt budget',
|
||||
rail: 'train layer; no room in the prompt budget, and "trains" still names transit',
|
||||
});
|
||||
|
||||
const TYPE_NAMES = Object.freeze({
|
||||
|
||||
@@ -119,6 +119,7 @@ test('spoken aliases resolve to real layers, keep every legacy phrase and never
|
||||
'weather radar': 'weather-radar', lightning: 'weather-lightning', 'military bases': 'military-installations',
|
||||
'recent imagery': 'recent-imagery', 'bike share': 'bikeshare', 'rocket launches': 'rocket-launches',
|
||||
'street level': 'street-level', mapillary: 'street-level',
|
||||
'street view': 'street-level',
|
||||
wind: 'wind', directions: 'directions', 'fire perimeters': 'fire-perimeters',
|
||||
'wildfire perimeters': 'fire-perimeters', 'burn areas': 'fire-perimeters',
|
||||
};
|
||||
|
||||
@@ -542,6 +542,7 @@ async function directActions(
|
||||
const runner = createGevActionRunner({
|
||||
viewer: {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
canvas: { addEventListener() {}, removeEventListener() {} },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
},
|
||||
|
||||
@@ -343,19 +343,11 @@ test('deixis prompt lint: one short rule, and every sentinel it names exists in
|
||||
tool('annotate_map').annotations.items.properties.target.description,
|
||||
/"pointer"/,
|
||||
);
|
||||
// Budget: the whole Realtime prompt (instructions + tools), measured in
|
||||
// characters. It was 56,308 after the area/imagery/OSM merge; stating each
|
||||
// policy once (WHERE precedence, AREAS routing, AFTER TOOLS speech) brought
|
||||
// it to 51,646. Voice geometry (resolve_area, find_imagery, osm_query and
|
||||
// the analyst area scopes) raised the ceiling to 55,500; tightening the
|
||||
// AREAS rule and the area tool descriptions left it at 55,103 as measured
|
||||
// below (every tool, osm_query included); main's Sea Marks and Ski Pistes
|
||||
// layers bring it to 55,210, and the tilt pitchDeg argument (with its -45
|
||||
// default) and main's cockpit entry rule to 55,339. integrationContract's
|
||||
// Overpass prompt is the tightest, at 55,499 (1 under). A new tool or rule
|
||||
// has to fit under this ceiling.
|
||||
// Budget: the whole Realtime prompt (instructions + every tool, osm_query
|
||||
// included), in characters. Every session pays for it in latency and cost,
|
||||
// so a new tool or rule has to fit under this ceiling or earn a raise.
|
||||
const size = instructions.length + JSON.stringify(GEV_REALTIME_TOOLS).length;
|
||||
assert.ok(size <= 55_500, `prompt size ${size}`);
|
||||
assert.ok(size <= 56_500, `prompt size ${size}`);
|
||||
});
|
||||
|
||||
test('after-tool speech allows returned items to answer requested lists and rankings', () => {
|
||||
|
||||
@@ -245,6 +245,31 @@ test('P2-7: duplicate callsigns and names resolve by canonical identity', () =>
|
||||
);
|
||||
});
|
||||
|
||||
test('vessels sharing an MMSI resolve by their vessel id', () => {
|
||||
const referents = createReferentRegistry();
|
||||
referents.recordResult('analyst_query', {
|
||||
ok: true,
|
||||
items: ['hull-a', 'hull-b'].map((vesselId) => ({
|
||||
layerKey: 'ais-live-vessels',
|
||||
id: 'EVER GIVEN',
|
||||
vesselId,
|
||||
mmsi: '353136000',
|
||||
name: 'EVER GIVEN',
|
||||
})),
|
||||
});
|
||||
assert.deepEqual(
|
||||
[1, 2].map(
|
||||
(n) =>
|
||||
resolveDeicticArgs(
|
||||
'track_entity',
|
||||
{ query: 'x', referent: n },
|
||||
{ referents },
|
||||
).args.query,
|
||||
),
|
||||
['hull-a', 'hull-b'],
|
||||
);
|
||||
});
|
||||
|
||||
test('P1-1: a result from before the last clear cannot repopulate the registry', () => {
|
||||
const referents = createReferentRegistry();
|
||||
const since = referents.generation();
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*
|
||||
* Provider-neutral and Cesium-free: the pick is injected.
|
||||
*/
|
||||
import { findCamera } from '../render/rendererOf.js';
|
||||
|
||||
/** Movement below this many CSS pixels is still the same dwell. */
|
||||
const POINTER_DWELL_PX = 12;
|
||||
@@ -55,23 +56,28 @@ const finite = finiteOrNull;
|
||||
|
||||
/**
|
||||
* Camera pose identity: position to 1 m, orientation to about 0.06 degrees.
|
||||
* A pixel read under one key names nothing under another. Reads plain
|
||||
* numbers only, so it needs no Cesium import.
|
||||
* @param {object} viewer Cesium viewer.
|
||||
* A pixel read under one key names nothing under another. Reads the
|
||||
* camera port's pose.
|
||||
* @param {object} viewer The scene object.
|
||||
* @returns {string|null}
|
||||
*/
|
||||
export function cameraPoseKey(viewer) {
|
||||
const camera = viewer?.camera;
|
||||
const p = camera?.positionWC;
|
||||
let pose = null;
|
||||
try {
|
||||
pose = findCamera(viewer)?.pose() ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const p = pose?.position;
|
||||
if (!p || !Number.isFinite(p.x)) return null;
|
||||
const angle = (value) => Math.round((Number(value) || 0) * 1000);
|
||||
return [
|
||||
Math.round(p.x),
|
||||
Math.round(p.y),
|
||||
Math.round(p.z),
|
||||
angle(camera.heading),
|
||||
angle(camera.pitch),
|
||||
angle(camera.roll),
|
||||
angle(pose.heading),
|
||||
angle(pose.pitch),
|
||||
angle(pose.roll),
|
||||
].join(',');
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { bindRenderer } from '../render/rendererOf.js';
|
||||
import { fakeSceneFrame } from '../testSupport/fakeCesiumScene.mjs';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
RealtimeViewport,
|
||||
@@ -6,6 +8,14 @@ import {
|
||||
renderFreshPointerFrame,
|
||||
} from './realtimeViewport.js';
|
||||
|
||||
|
||||
/** A viewer over `scene` whose renderer's frame port is the scene's own loop. */
|
||||
function framed(scene) {
|
||||
const viewer = { scene };
|
||||
bindRenderer(viewer, { frame: fakeSceneFrame(viewer) });
|
||||
return viewer;
|
||||
}
|
||||
|
||||
test('crop: a square around the pointer, clamped inside the canvas, in buffer pixels', () => {
|
||||
assert.deepEqual(
|
||||
pointerCropRect({
|
||||
@@ -110,7 +120,7 @@ test('crop: a slow render gets one bounded fresh-frame recovery request', async
|
||||
if (requests === 2) queueMicrotask(() => listener?.());
|
||||
},
|
||||
};
|
||||
assert.equal(await renderFreshPointerFrame({ scene }), true);
|
||||
assert.equal(await renderFreshPointerFrame(framed(scene)), true);
|
||||
assert.equal(requests, 2, 'only the pointer path retries once');
|
||||
assert.equal(listener, null, 'the successful listener is removed');
|
||||
} finally {
|
||||
@@ -129,7 +139,7 @@ test('crop: a hidden document never starts the recovery request', async () => {
|
||||
requests += 1;
|
||||
},
|
||||
};
|
||||
assert.equal(await renderFreshPointerFrame({ scene }), false);
|
||||
assert.equal(await renderFreshPointerFrame(framed(scene)), false);
|
||||
assert.equal(requests, 0);
|
||||
} finally {
|
||||
globalThis.document = originalDocument;
|
||||
|
||||
+18
-13
@@ -1,4 +1,5 @@
|
||||
import * as Cesium from 'cesium';
|
||||
import { pickAt as defaultPickAt } from '../input/dispatcher.js';
|
||||
import { canvasOf, findCamera } from '../render/rendererOf.js';
|
||||
import { resolvePickedContext } from '../data/pickedContext.js';
|
||||
import { cameraPoseKey } from './pointerContext.js';
|
||||
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
|
||||
@@ -10,34 +11,38 @@ function pointerRadiusKm(cameraHeightM) {
|
||||
return Math.round(Math.min(400, Math.max(5, heightKm * 0.75)));
|
||||
}
|
||||
|
||||
function cameraHeightM(viewer) {
|
||||
try {
|
||||
return findCamera(viewer)?.pose().height;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Cesium side of point-and-ask: one entity pick and one depth-aware world
|
||||
* The scene side of point-and-ask: one entity pick and one depth-aware world
|
||||
* pick at canvas CSS pixels. Called lazily by the pointer tracker (dwell and
|
||||
* snapshot only), never per mouse move.
|
||||
* @param {object} options
|
||||
* @param {object} options.viewer Cesium viewer.
|
||||
* @param {object} options.viewer The scene object.
|
||||
* @param {object} [options.dataManager] Layer lifecycle.
|
||||
* @param {Function} [options.pickWorldFromScreen] Normalized-pixel world pick.
|
||||
* @param {Function} [options.pickAt] Entity pick at canvas CSS pixels.
|
||||
* @returns {(x: number, y: number) => {entity: object|null, ground: object|null, radiusKm: number}|null}
|
||||
*/
|
||||
export function createPointerPicker({
|
||||
viewer,
|
||||
dataManager = null,
|
||||
pickWorldFromScreen = defaultAnnotationResolver.pickWorldFromScreen,
|
||||
pickAt = defaultPickAt,
|
||||
}) {
|
||||
return function pickPointer(x, y) {
|
||||
const scene = viewer?.scene;
|
||||
const canvas = scene?.canvas;
|
||||
if (!scene || !canvas) return null;
|
||||
const canvas = canvasOf(viewer);
|
||||
if (!canvas) return null;
|
||||
const width = canvas.clientWidth || canvas.width || 0;
|
||||
const height = canvas.clientHeight || canvas.height || 0;
|
||||
if (!width || !height) return null;
|
||||
let picked = null;
|
||||
try {
|
||||
picked = scene.pick(new Cesium.Cartesian2(x, y));
|
||||
} catch {
|
||||
picked = null;
|
||||
}
|
||||
const picked = pickAt(viewer, x, y);
|
||||
const entity = resolvePickedContext(picked, { dataManager });
|
||||
let ground = null;
|
||||
try {
|
||||
@@ -54,7 +59,7 @@ export function createPointerPicker({
|
||||
return {
|
||||
entity,
|
||||
ground,
|
||||
radiusKm: pointerRadiusKm(viewer.camera?.positionCartographic?.height),
|
||||
radiusKm: pointerRadiusKm(cameraHeightM(viewer)),
|
||||
cameraKey: cameraPoseKey(viewer),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -27,6 +27,7 @@ const CAMERA_TOOLS = new Set([
|
||||
'fly_route',
|
||||
'frame_overhead',
|
||||
'select_nearest_aircraft',
|
||||
'show_street_level',
|
||||
'control_cockpit',
|
||||
]);
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Imports intentionally target the candidate, never the immutable reference.
|
||||
import { bindFakeCesiumCamera } from '../testSupport/fakeCameraPort.mjs';
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
@@ -11,6 +12,7 @@ import { createGevActionRunner } from './gevActions.js';
|
||||
import { createPointerTracker } from './pointerContext.js';
|
||||
import { createReferentRegistry } from './referents.js';
|
||||
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
|
||||
const PROVIDER_KEY = 'gev.voice.provider';
|
||||
const settle = () => new Promise((resolve) => setImmediate(resolve));
|
||||
@@ -176,22 +178,20 @@ function compositionFixture(
|
||||
const canvas = new EventTarget();
|
||||
Object.assign(canvas, { clientWidth: 800, clientHeight: 600 });
|
||||
const position = { x: 6379137, y: 0, z: 0 };
|
||||
const viewer = {
|
||||
const viewer = bindFakeCesiumCamera({
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: {
|
||||
positionWC: position,
|
||||
positionCartographic: { height: state.height },
|
||||
moveEnd: { addEventListener() {} },
|
||||
cancelFlight() {},
|
||||
zoomIn(metres) {
|
||||
position.x -= metres;
|
||||
state.height -= metres;
|
||||
this.positionCartographic.height = state.height;
|
||||
state.zooms++;
|
||||
},
|
||||
zoomOut(metres) {
|
||||
position.x += metres;
|
||||
state.height += metres;
|
||||
// Looking straight down at the surface along -x.
|
||||
directionWC: { x: -1, y: 0, z: 0 },
|
||||
upWC: { x: 0, y: 0, z: 1 },
|
||||
rightWC: { x: 0, y: 1, z: 0 },
|
||||
setView({ destination }) {
|
||||
state.height -= position.x - destination.x;
|
||||
position.x = destination.x;
|
||||
this.positionCartographic.height = state.height;
|
||||
state.zooms++;
|
||||
},
|
||||
@@ -202,7 +202,7 @@ function compositionFixture(
|
||||
state.renders++;
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
const dataManager = { layers: new Map() };
|
||||
pointer = createPointerTracker({ element: null, now: () => 1000 });
|
||||
const referents = createReferentRegistry();
|
||||
@@ -871,3 +871,83 @@ test('a provider switch hands chat Settings to the new mount', async (t) => {
|
||||
assert.deepEqual(f.mounts[1].options.presentation, shown);
|
||||
assert.equal(f.facade.provider, next);
|
||||
});
|
||||
|
||||
test('a missing-key error hides the microphone hint; other errors keep it', async (t) => {
|
||||
const f = compositionFixture(t, {});
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const instance = f.instances.at(-1);
|
||||
const ui = f.views.at(-1);
|
||||
instance.hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'Add a Gemini API key in Provider Settings',
|
||||
missingKey: true,
|
||||
});
|
||||
assert.equal(
|
||||
ui.state.error.text,
|
||||
'Add a Gemini API key in Provider Settings',
|
||||
);
|
||||
assert.equal(ui.state.error.hint, false);
|
||||
instance.hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'Socket closed',
|
||||
});
|
||||
assert.equal(ui.state.error.text, 'Socket closed');
|
||||
assert.equal(ui.state.error.hint, true);
|
||||
});
|
||||
|
||||
test('a provider message is worded again after a language change', async (t2) => {
|
||||
t2.after(() => setLocale('en'));
|
||||
const f = compositionFixture(t2, {});
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const instance = f.instances.at(-1);
|
||||
const ui = f.views.at(-1);
|
||||
const quota = 'Gemini quota is unavailable. Try again later.';
|
||||
instance.hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: quota,
|
||||
sourceDetail: quota,
|
||||
});
|
||||
assert.equal(ui.state.error.text, quota);
|
||||
setLocale('es');
|
||||
assert.equal(ui.state.error.text, t(quota));
|
||||
assert.notEqual(ui.state.error.text, quota);
|
||||
setLocale('en');
|
||||
instance.hooks.emit({
|
||||
type: 'state',
|
||||
state: 'listening',
|
||||
detail: 'Ask or command',
|
||||
sourceDetail: 'Ask or command',
|
||||
});
|
||||
setLocale('es');
|
||||
assert.equal(ui.state.detail.text, t('Ask or command'));
|
||||
assert.notEqual(ui.state.detail.text, 'Ask or command');
|
||||
});
|
||||
|
||||
test('a provider message with placeholders is worded again with them', async (t2) => {
|
||||
t2.after(() => setLocale('en'));
|
||||
const f = compositionFixture(t2, {});
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const ui = f.views.at(-1);
|
||||
const text = 'Gemini token request failed (HTTP {status})';
|
||||
f.instances.at(-1).hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: t(text, { status: 502 }),
|
||||
sourceDetail: text,
|
||||
sourceParams: { status: 502 },
|
||||
});
|
||||
assert.equal(ui.state.error.text, 'Gemini token request failed (HTTP 502)');
|
||||
setLocale('es');
|
||||
assert.equal(ui.state.error.text, t(text, { status: 502 }));
|
||||
assert.match(ui.state.error.text, /502/);
|
||||
assert.notEqual(
|
||||
ui.state.error.text,
|
||||
'Gemini token request failed (HTTP 502)',
|
||||
);
|
||||
});
|
||||
@@ -39,8 +39,9 @@ export function createRealtimeBackend({
|
||||
if (!response.ok) {
|
||||
const reason =
|
||||
typeof data?.error === 'string' ? data.error : data?.error?.message;
|
||||
throw new Error(
|
||||
reason || `Realtime token failed: HTTP ${response.status}`,
|
||||
throw Object.assign(
|
||||
new Error(reason || `Realtime token failed: HTTP ${response.status}`),
|
||||
typeof data?.code === 'string' ? { code: data.code } : {},
|
||||
);
|
||||
}
|
||||
const token =
|
||||
|
||||
@@ -143,3 +143,17 @@ test('controller lifetime stops pending transport and releases resources through
|
||||
assert.equal(controller.stream, null);
|
||||
assert.equal(controller.isActive(), false);
|
||||
});
|
||||
|
||||
test('a missing server key keeps its code, so the UI can point to Provider Settings', async () => {
|
||||
const backend = createRealtimeBackend({
|
||||
tokenTransport: async () =>
|
||||
Response.json(
|
||||
{ error: 'OPENAI_API_KEY is not set', code: 'API_KEY_MISSING' },
|
||||
{ status: 503 },
|
||||
),
|
||||
});
|
||||
await assert.rejects(backend.requestToken(), {
|
||||
message: 'OPENAI_API_KEY is not set',
|
||||
code: 'API_KEY_MISSING',
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import { isStoriesOpen } from './realtimeInputPolicy.js';
|
||||
|
||||
const DISCONNECT_GRACE_MS = 6000;
|
||||
|
||||
@@ -52,7 +53,8 @@ export class RealtimeConnection {
|
||||
return this.readStatus();
|
||||
}
|
||||
async start({ pushToTalk = false } = {}) {
|
||||
if (this.isActive() || this.lifetimeSignal?.aborted) return;
|
||||
if (isStoriesOpen() || this.isActive() || this.lifetimeSignal?.aborted)
|
||||
return;
|
||||
this.pauseRadioForVoice();
|
||||
const pushToTalkKeyHeld = pushToTalk && this.input.pushToTalkKeyHeld;
|
||||
const spaceKeyHeld = this.input.spaceKeyHeld;
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
} from './pointerContext.js';
|
||||
|
||||
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
|
||||
import { postDebugLog } from './realtimeDiagnostics.js';
|
||||
import { missingVoiceKeyText, postDebugLog } from './realtimeDiagnostics.js';
|
||||
|
||||
export {
|
||||
readStoredVoiceTier,
|
||||
@@ -54,7 +54,7 @@ export {
|
||||
|
||||
import { createRealtimeBackend } from './realtimeBackend.js';
|
||||
|
||||
import { t } from '../i18n/index.js';
|
||||
import { t, translateKnown } from '../i18n/index.js';
|
||||
import { voiceStatusLabel } from './control.js';
|
||||
|
||||
// Machine states (dataset.status, CSS); voiceStatusLabel() translates labels.
|
||||
@@ -101,6 +101,7 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
|
||||
operations: {
|
||||
sendRealtimeEvent: (...args) => this.sendRealtimeEvent(...args),
|
||||
debugLog: (...args) => this.debugLog(...args),
|
||||
},
|
||||
});
|
||||
this._diagnostics = new RealtimeDiagnostics({
|
||||
@@ -406,9 +407,24 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
}
|
||||
}
|
||||
|
||||
setStatus(status, detail) {
|
||||
/**
|
||||
* @param {string} status
|
||||
* @param {string} [detail]
|
||||
* @param {{missingKey?: boolean}} [options] True when the error is a
|
||||
* missing server key: it is worded in the current language and shown
|
||||
* without the microphone hint.
|
||||
*/
|
||||
setStatus(status, detail, { missingKey = false } = {}) {
|
||||
this.status = status;
|
||||
this.emitSessionEvent({ type: 'state', state: status, detail });
|
||||
// A detail may be English catalog text (from the helpers this controller
|
||||
// shares with Gemini): it is worded in the current language when shown.
|
||||
this.emitSessionEvent({
|
||||
type: 'state',
|
||||
state: status,
|
||||
detail: translateKnown(detail),
|
||||
...(typeof detail === 'string' && { sourceDetail: detail }),
|
||||
...(missingKey && { missingKey: true }),
|
||||
});
|
||||
this.ui.update('root', {
|
||||
status,
|
||||
...(status === 'error' && { errorDismissed: false }),
|
||||
@@ -424,7 +440,7 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
? held
|
||||
? t('Release Space to send')
|
||||
: t('Hold Space to talk')
|
||||
: detail;
|
||||
: translateKnown(detail);
|
||||
const primaryDetail = () =>
|
||||
status === 'error'
|
||||
? t('VOICE UNAVAILABLE')
|
||||
@@ -434,8 +450,12 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
this.ui.update('error', {
|
||||
text: () =>
|
||||
status === 'error'
|
||||
? resolvedDetail() || t('Voice session could not be started.')
|
||||
? (missingKey && missingVoiceKeyText()) ||
|
||||
resolvedDetail() ||
|
||||
t('Voice session could not be started.')
|
||||
: '',
|
||||
// "Check microphone permission" is wrong advice for a missing key.
|
||||
hint: () => !(status === 'error' && missingKey),
|
||||
});
|
||||
if (status === 'idle' || status === 'connecting' || status === 'error') {
|
||||
this.setVoiceSpeaker('idle');
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
|
||||
/** The voice error when the server has no OpenAI key: where to add it. */
|
||||
export const missingVoiceKeyText = () =>
|
||||
t('Needs {envVars} — add it in Provider Settings', {
|
||||
envVars: 'OPENAI_API_KEY',
|
||||
});
|
||||
|
||||
const ERROR_LOG_LIMIT = 30;
|
||||
|
||||
const ERROR_STORAGE_KEY = 'gev-realtime-errors';
|
||||
@@ -83,6 +91,7 @@ function createErrorRecord(source, error, extra = {}) {
|
||||
String(error?.message || '').trim() ||
|
||||
'No browser error message supplied',
|
||||
errorDetail: rtcError?.errorDetail || null,
|
||||
code: typeof rtcError?.code === 'string' ? rtcError.code : null,
|
||||
sctpCauseCode: rtcError?.sctpCauseCode ?? null,
|
||||
receivedAlert: rtcError?.receivedAlert ?? null,
|
||||
sentAlert: rtcError?.sentAlert ?? null,
|
||||
@@ -91,6 +100,8 @@ function createErrorRecord(source, error, extra = {}) {
|
||||
}
|
||||
|
||||
function formatErrorForDisplay(record) {
|
||||
// A missing key is a setup step, not a connection fault: say where to add it.
|
||||
if (record.code === 'API_KEY_MISSING') return missingVoiceKeyText();
|
||||
const primary = [record.source, record.message].filter(Boolean).join(': ');
|
||||
const state = [
|
||||
record.errorDetail && `detail=${record.errorDetail}`,
|
||||
@@ -172,7 +183,11 @@ export class RealtimeDiagnostics {
|
||||
storeErrors(this.errors);
|
||||
console.error('[GEV Realtime]', record);
|
||||
this.debugLog('error', record);
|
||||
this.setStatus('error', formatErrorForDisplay(record));
|
||||
this.setStatus(
|
||||
'error',
|
||||
formatErrorForDisplay(record),
|
||||
record.code === 'API_KEY_MISSING' ? { missingKey: true } : undefined,
|
||||
);
|
||||
return record;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import { msg, t } from '../i18n/index.js';
|
||||
import {
|
||||
PUSH_TO_TALK_HOLD_DELAY_MS,
|
||||
MICROPHONE_VISUALIZER_GATE,
|
||||
ASSISTANT_VISUALIZER_GATE,
|
||||
isStoriesOpen,
|
||||
isPushToTalkKey,
|
||||
SPACE_INTERACTIVE_SELECTOR,
|
||||
isInteractiveSpaceTarget,
|
||||
@@ -55,9 +56,11 @@ export class RealtimeInput {
|
||||
bindPushToTalkShortcut() {
|
||||
if (this.shortcutKeyDownHandler) return;
|
||||
this.shortcutKeyDownHandler = (event) => {
|
||||
// Space types in chat; it never talks while chat is open.
|
||||
// Space types in chat; it never talks while chat is open, nor while
|
||||
// Stories owns the keyboard (body.stories-mode).
|
||||
if (
|
||||
this.ui.state?.chat?.open === true ||
|
||||
isStoriesOpen() ||
|
||||
!shouldHandlePushToTalkKeyDown(event)
|
||||
)
|
||||
return;
|
||||
@@ -93,6 +96,10 @@ export class RealtimeInput {
|
||||
this.pushToTalkHoldTimer = null;
|
||||
if (holdGeneration !== this.pushToTalkHoldGeneration) return;
|
||||
if (!this.spaceKeyHeld || this.pushToTalkKeyHeld) return;
|
||||
if (isStoriesOpen()) {
|
||||
this.shortcutBlurHandler();
|
||||
return;
|
||||
}
|
||||
// A pointer or Tab focus change during the delay cancels the original
|
||||
// gesture rather than letting voice claim a key held for another owner.
|
||||
if (document.activeElement !== this.pushToTalkHoldFocusOwner) return;
|
||||
@@ -124,7 +131,7 @@ export class RealtimeInput {
|
||||
this.ui.update('root', { pushToTalk: 'held' });
|
||||
this.setMicrophoneEnabled(true);
|
||||
if (this.status === 'listening')
|
||||
this.setStatus('listening', t('Release Space to send'));
|
||||
this.setStatus('listening', msg('Release Space to send'));
|
||||
} else {
|
||||
this.start({ pushToTalk: true });
|
||||
// start() performs a controlled stop() before connecting. Restore the
|
||||
@@ -201,7 +208,7 @@ export class RealtimeInput {
|
||||
if (!this.pushToTalkMode) return;
|
||||
this.setMicrophoneEnabled(false);
|
||||
if (this.status === 'listening')
|
||||
this.setStatus('listening', t('Hold Space to talk'));
|
||||
this.setStatus('listening', msg('Hold Space to talk'));
|
||||
else this.updateVoiceButtonLabel();
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,13 @@ export const MICROPHONE_VISUALIZER_GATE = 0.12;
|
||||
|
||||
export const ASSISTANT_VISUALIZER_GATE = 0.04;
|
||||
|
||||
/** Stories takes keyboard and microphone ownership away from voice. */
|
||||
export function isStoriesOpen() {
|
||||
return (
|
||||
globalThis.document?.body?.classList?.contains?.('stories-mode') ?? false
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether a keyboard event represents the hold-Space voice shortcut.
|
||||
* @param {KeyboardEvent|object|null} event
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import { msg } from '../i18n/index.js';
|
||||
import {
|
||||
silenceRadioForVoice,
|
||||
startPreparedRadioAfterPlaybackReady,
|
||||
@@ -261,7 +261,7 @@ export class RealtimeRadio {
|
||||
if (radioHandoff.result?.ok || radioHandoff.cancelled || !stillCurrent)
|
||||
return;
|
||||
if (this.dc?.readyState === 'open' && !this.userTurnPending) {
|
||||
this.setStatus('listening', t('Radio did not start'));
|
||||
this.setStatus('listening', msg('Radio did not start'));
|
||||
this.queueResponseCreate(
|
||||
'Say exactly one short correction: “The Radio station could not start. Voice is still on.”',
|
||||
);
|
||||
|
||||
+250
-79
@@ -3,6 +3,7 @@ import {
|
||||
hasStructuredViewIdentity,
|
||||
} from './realtimeProtocol.js';
|
||||
import { cameraPoseKey } from './pointerContext.js';
|
||||
import { canvasOf, findFrame, rendererOf } from '../render/rendererOf.js';
|
||||
|
||||
/** The live viewer's camera pose identity (Cesium-free read). */
|
||||
function readViewerCameraKey() {
|
||||
@@ -19,32 +20,28 @@ const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
|
||||
|
||||
export async function captureViewportImage() {
|
||||
const viewer = window.__godsEyeView?.viewer;
|
||||
const source =
|
||||
viewer?.scene?.canvas ||
|
||||
document.querySelector('#cesiumContainer .cesium-widget canvas');
|
||||
const source = canvasOf(viewer);
|
||||
if (!source || !source.width || !source.height) return null;
|
||||
// No fresh frame (hidden, or the bounded render wait timed out) → no
|
||||
// capture. The caller labels this image "Current"; a stale preserved
|
||||
// frame would feed the model old entities as current context. (perf
|
||||
// wave 2 fix)
|
||||
const fresh = await renderFreshCesiumFrame(viewer);
|
||||
if (!fresh) return null;
|
||||
|
||||
// Clamp BOTH dimensions by a total-pixel budget so tall portrait windows are
|
||||
// downscaled too (the old width-only clamp let them through — M13).
|
||||
const { width, height } = computeDownscale(
|
||||
const { width } = computeDownscale(
|
||||
source.width,
|
||||
source.height,
|
||||
VIEWPORT_MAX_PIXELS,
|
||||
);
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const ctx = canvas.getContext('2d');
|
||||
// No fresh frame (hidden, or the bounded render wait timed out) → no
|
||||
// capture. The caller labels this image "Current"; a stale preserved
|
||||
// frame would feed the model old entities as current context. (perf
|
||||
// wave 2 fix)
|
||||
const pixels = await readFreshFrame(viewer, renderFreshCesiumFrame, {
|
||||
scale: width / (source.clientWidth || source.width),
|
||||
});
|
||||
if (!pixels) return null;
|
||||
const ctx = canvasWith(pixels);
|
||||
if (!ctx) return null;
|
||||
const canvas = ctx.canvas;
|
||||
try {
|
||||
ctx.drawImage(source, 0, 0, width, height);
|
||||
if (isNearlyBlackFrame(ctx, width, height)) {
|
||||
if (isNearlyBlackFrame(ctx, canvas.width, canvas.height)) {
|
||||
console.warn('[GEV Voice] Skipped black Cesium viewport capture');
|
||||
return null;
|
||||
}
|
||||
@@ -72,6 +69,43 @@ const POINTER_CROP_MAX_AGE_MS = 20000;
|
||||
/** View scales where a crop shows buildings and terrain, not a smear. */
|
||||
const POINTER_CROP_SCALES = new Set(['local', 'city']);
|
||||
|
||||
/**
|
||||
* Reason-coded explanation of the existing visual escalation policy, for an
|
||||
* entity-context answer on an open channel (the caller checks both).
|
||||
* Pointer crops remain a separate path from full-frame screenshots.
|
||||
* These reasons are static diagnostics, never model-generated claims.
|
||||
*/
|
||||
export function viewportContextDecision(result) {
|
||||
const viewScale = result?.scene?.basemap?.viewScale ?? null;
|
||||
if (result?.scope === 'pointer') {
|
||||
if (result.selected)
|
||||
return { send: false, reason: 'pointer-identity-present', viewScale };
|
||||
if (!result.pointer)
|
||||
return { send: false, reason: 'pointer-missing', viewScale };
|
||||
// The crop path checks ground, freshness and scale itself, and logs why.
|
||||
return {
|
||||
send: true,
|
||||
kind: 'crop',
|
||||
reason: 'pointer-crop-eligible',
|
||||
viewScale,
|
||||
};
|
||||
}
|
||||
if (!shouldSendViewportImage(viewScale))
|
||||
return { send: false, reason: 'view-scale-not-local', viewScale };
|
||||
if (hasStructuredViewIdentity(result))
|
||||
return {
|
||||
send: false,
|
||||
reason: 'structured-identity-sufficient',
|
||||
viewScale,
|
||||
};
|
||||
return {
|
||||
send: true,
|
||||
kind: 'viewport',
|
||||
reason: 'structured-identity-missing',
|
||||
viewScale,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Square source rectangle around a canvas point, clamped inside the canvas,
|
||||
* in drawing-buffer pixels, plus where the point lands in the output image.
|
||||
@@ -128,38 +162,39 @@ export function pointerCropRect({
|
||||
* @returns {Promise<{dataUrl: string, rect: object}|null>} JPEG data URL and
|
||||
* the crop's normalized bounds in the full canvas, or null.
|
||||
*/
|
||||
async function capturePointerCrop(screenPx) {
|
||||
export async function capturePointerCrop(screenPx) {
|
||||
const viewer = window.__godsEyeView?.viewer;
|
||||
const source = viewer?.scene?.canvas;
|
||||
const source = canvasOf(viewer);
|
||||
if (!source?.width || !source?.height || !screenPx) return null;
|
||||
const fresh = await renderFreshPointerFrame(viewer);
|
||||
if (!fresh) return null;
|
||||
const cssWidth = source.clientWidth || source.width;
|
||||
const cssHeight = source.clientHeight || source.height;
|
||||
const rect = pointerCropRect({
|
||||
x: screenPx.x,
|
||||
y: screenPx.y,
|
||||
cssWidth: source.clientWidth || source.width,
|
||||
cssHeight: source.clientHeight || source.height,
|
||||
cssWidth,
|
||||
cssHeight,
|
||||
pixelWidth: source.width,
|
||||
pixelHeight: source.height,
|
||||
});
|
||||
if (!rect) return null;
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = rect.output;
|
||||
canvas.height = rect.output;
|
||||
const ctx = canvas.getContext('2d');
|
||||
// The crop in CSS pixels, as the renderer reads it.
|
||||
const toCssX = cssWidth / source.width;
|
||||
const toCssY = cssHeight / source.height;
|
||||
const crop = {
|
||||
x: rect.sx * toCssX,
|
||||
y: rect.sy * toCssY,
|
||||
width: rect.sw * toCssX,
|
||||
height: rect.sh * toCssY,
|
||||
};
|
||||
const pixels = await readFreshFrame(viewer, renderFreshPointerFrame, {
|
||||
crop,
|
||||
scale: rect.output / crop.width,
|
||||
});
|
||||
if (!pixels) return null;
|
||||
const ctx = squareOf(canvasWith(pixels), rect.output);
|
||||
if (!ctx) return null;
|
||||
const canvas = ctx.canvas;
|
||||
try {
|
||||
ctx.drawImage(
|
||||
source,
|
||||
rect.sx,
|
||||
rect.sy,
|
||||
rect.sw,
|
||||
rect.sh,
|
||||
0,
|
||||
0,
|
||||
rect.output,
|
||||
rect.output,
|
||||
);
|
||||
if (isNearlyBlackFrame(ctx, rect.output, rect.output)) return null;
|
||||
ctx.strokeStyle = 'rgba(0, 230, 255, 0.95)';
|
||||
ctx.lineWidth = 2;
|
||||
@@ -182,14 +217,70 @@ async function capturePointerCrop(screenPx) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The renderer's pixels of a fresh frame (`readPixels` with `options`), read
|
||||
* in the frame `render` waits for, so they are the frame just drawn on any
|
||||
* renderer; null with no fresh frame, or no renderer to read.
|
||||
* @param {object} viewer
|
||||
* @param {(viewer: object, onFrame: () => void) => Promise<boolean>} render
|
||||
* @param {{crop?: object, scale?: number}} options
|
||||
* @returns {Promise<ImageData|null>}
|
||||
*/
|
||||
async function readFreshFrame(viewer, render, options) {
|
||||
const renderer = rendererOf(viewer);
|
||||
if (typeof renderer?.readPixels !== 'function') return null;
|
||||
let pixels = null;
|
||||
const fresh = await render(viewer, () => {
|
||||
pixels = renderer.readPixels(options);
|
||||
});
|
||||
return fresh ? pixels : null;
|
||||
}
|
||||
|
||||
/** A 2D canvas holding `pixels`, or null where none can be made. */
|
||||
function canvasWith(pixels) {
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = pixels.width;
|
||||
canvas.height = pixels.height;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return null;
|
||||
try {
|
||||
ctx.putImageData(
|
||||
typeof ImageData === 'function' && !(pixels instanceof ImageData)
|
||||
? new ImageData(pixels.data, pixels.width, pixels.height)
|
||||
: pixels,
|
||||
0,
|
||||
0,
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
/**
|
||||
* `ctx`'s image as a `side` px square: the crop is read at one scale, so a
|
||||
* canvas stretched unevenly reads it a pixel or so off square.
|
||||
*/
|
||||
function squareOf(ctx, side) {
|
||||
if (!ctx || (ctx.canvas.width === side && ctx.canvas.height === side))
|
||||
return ctx;
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = side;
|
||||
canvas.height = side;
|
||||
const square = canvas.getContext('2d');
|
||||
if (!square) return null;
|
||||
square.drawImage(ctx.canvas, 0, 0, side, side);
|
||||
return square;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pointer crops get one bounded recovery request for a slow render loop.
|
||||
* Full-viewport captures intentionally keep their single 400 ms attempt.
|
||||
*/
|
||||
export async function renderFreshPointerFrame(viewer) {
|
||||
if (await renderFreshCesiumFrame(viewer)) return true;
|
||||
export async function renderFreshPointerFrame(viewer, onFrame) {
|
||||
if (await renderFreshCesiumFrame(viewer, onFrame)) return true;
|
||||
if (typeof document !== 'undefined' && document.hidden) return false;
|
||||
return renderFreshCesiumFrame(viewer);
|
||||
return renderFreshCesiumFrame(viewer, onFrame);
|
||||
}
|
||||
|
||||
// Scale (w, h) down so w*h <= maxPixels while preserving aspect ratio. Never
|
||||
@@ -222,15 +313,18 @@ export function estimateDataUrlBytes(dataUrl) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the canvas holds a CURRENT frame before capture.
|
||||
* Ensure the canvas holds a CURRENT frame before capture. `onFrame` runs
|
||||
* as that frame is drawn (its `postRender`), where a read sees it.
|
||||
* @param {object} viewer
|
||||
* @param {() => void} [onFrame]
|
||||
* @returns {Promise<boolean>} true only when a fresh frame was presented —
|
||||
* false while hidden (render loop suspended; a capture would be stale) or
|
||||
* when the bounded wait timed out. Callers must not label a non-fresh
|
||||
* canvas as current. (perf wave 2)
|
||||
*/
|
||||
export async function renderFreshCesiumFrame(viewer) {
|
||||
const scene = viewer?.scene;
|
||||
if (!scene) return false;
|
||||
export async function renderFreshCesiumFrame(viewer, onFrame) {
|
||||
const frame = findFrame(viewer);
|
||||
if (!frame) return false;
|
||||
// While the document is hidden the render loop is suspended — don't
|
||||
// secretly restart rendering for an optional screenshot, and don't pass
|
||||
// the stale preserved frame off as current.
|
||||
@@ -240,8 +334,14 @@ export async function renderFreshCesiumFrame(viewer) {
|
||||
// necessarily draw — request a frame and await its postRender (bounded),
|
||||
// which also covers the just-became-visible race.
|
||||
const rendered = new Promise((resolve) => {
|
||||
const remove = scene.postRender.addEventListener(() => {
|
||||
const remove = frame.on('postRender', () => {
|
||||
remove();
|
||||
try {
|
||||
onFrame?.();
|
||||
} catch {
|
||||
resolve(false);
|
||||
return;
|
||||
}
|
||||
resolve(true);
|
||||
});
|
||||
setTimeout(() => {
|
||||
@@ -249,7 +349,7 @@ export async function renderFreshCesiumFrame(viewer) {
|
||||
resolve(false);
|
||||
}, 400);
|
||||
});
|
||||
scene.requestRender?.();
|
||||
frame.requestRender('voice-capture');
|
||||
const fresh = await rendered;
|
||||
// A tab switch during the bounded wait invalidates freshness.
|
||||
if (typeof document !== 'undefined' && document.hidden) return false;
|
||||
@@ -299,6 +399,16 @@ export function isBenignViewportDeleteError(payload, pendingDeleteIds = null) {
|
||||
return code === 'item_not_found';
|
||||
}
|
||||
|
||||
/** Pairs one visual-context request's outcome with its decision. */
|
||||
function diagnosticsRecord(contextId = null) {
|
||||
return { contextId, logged: false };
|
||||
}
|
||||
|
||||
function reportOutcome(owner, diag, event, kind, reason) {
|
||||
diag.logged = true;
|
||||
owner.debugLog?.(event, { contextId: diag.contextId, kind, reason });
|
||||
}
|
||||
|
||||
/** Own the one retained viewport image and bounded pending deletion identities. */
|
||||
export class RealtimeViewport {
|
||||
constructor({
|
||||
@@ -319,6 +429,8 @@ export class RealtimeViewport {
|
||||
// Bumped by every capture request and every new user turn: only the
|
||||
// newest request of the current turn may publish.
|
||||
this.captureSeq = 0;
|
||||
// Diagnostics only: pairs each visual-context decision with its outcome.
|
||||
this.contextSeq = 0;
|
||||
this.lastPointerCrop = null;
|
||||
// What the retained image shows: `{kind:'viewport'|'crop', rect, cameraKey}`.
|
||||
// Pixel targeting (screenX/screenY) is read against it.
|
||||
@@ -330,28 +442,51 @@ export class RealtimeViewport {
|
||||
return this.readChannel();
|
||||
}
|
||||
|
||||
/**
|
||||
* Escalate an entity-context answer to an image when the gate allows it.
|
||||
* Logs one `viewport_context.decision`, then, for a send decision, exactly
|
||||
* one `.skipped` or `.result` outcome, even when a step throws. Every
|
||||
* event carries the same `contextId` and the image `kind`, so overlapping
|
||||
* requests pair up.
|
||||
*/
|
||||
async sendVisualContextIfUseful(result) {
|
||||
// Runs after every tool output. Only entity context on an open channel
|
||||
// can escalate, so anything else logs nothing and leaves the shared
|
||||
// debug-log budget alone.
|
||||
if (
|
||||
result?.action !== 'get_entity_context' ||
|
||||
!this.dc ||
|
||||
this.dc.readyState !== 'open'
|
||||
)
|
||||
return false;
|
||||
const viewScale = result.scene?.basemap?.viewScale;
|
||||
// Point-and-ask on bare ground: the crop around the pointer replaces the
|
||||
// whole-frame screenshot, and is pulled only when nothing was named.
|
||||
if (result.scope === 'pointer') {
|
||||
if (result.selected || !result.pointer) return false;
|
||||
return this.sendPointerCrop(this.readPointer(), { viewScale });
|
||||
const contextId = ++this.contextSeq;
|
||||
const decision = viewportContextDecision(result);
|
||||
this.debugLog?.('viewport_context.decision', { contextId, ...decision });
|
||||
if (!decision.send) return false;
|
||||
const diag = diagnosticsRecord(contextId);
|
||||
try {
|
||||
// #837's pointer crop is separate from the full viewport: preserve it.
|
||||
if (decision.kind === 'crop')
|
||||
return await this.sendPointerCrop(this.readPointer(), {
|
||||
viewScale: decision.viewScale,
|
||||
diag,
|
||||
});
|
||||
return await this.sendRetainedImage(
|
||||
() => this.capture(),
|
||||
"Current God's Eye View viewport screenshot. Read any clearly visible street, building, and place labels in the image and combine them with the structured nearbyPlaces, streetLabels, and scene context. Do not invent labels that are not legible.",
|
||||
'client.viewport_context',
|
||||
{ kind: 'viewport', diag },
|
||||
);
|
||||
} catch (error) {
|
||||
// The caller logs the error itself; close the pair, then rethrow.
|
||||
if (!diag.logged)
|
||||
this.debugLog?.('viewport_context.skipped', {
|
||||
contextId,
|
||||
kind: decision.kind,
|
||||
reason: 'error',
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
if (!shouldSendViewportImage(viewScale)) return false;
|
||||
if (hasStructuredViewIdentity(result)) return false;
|
||||
return this.sendRetainedImage(
|
||||
() => this.capture(),
|
||||
"Current God's Eye View viewport screenshot. Read any clearly visible street, building, and place labels in the image and combine them with the structured nearbyPlaces, streetLabels, and scene context. Do not invent labels that are not legible.",
|
||||
'client.viewport_context',
|
||||
{ kind: 'viewport' },
|
||||
);
|
||||
}
|
||||
|
||||
/** A new user turn: captures still in flight from the last one are dropped. */
|
||||
@@ -365,27 +500,39 @@ export class RealtimeViewport {
|
||||
* @param {object|null} snapshot Fresh pointer snapshot on bare ground.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async sendPointerCrop(snapshot, { viewScale } = {}) {
|
||||
if (!snapshot?.fresh || snapshot.target !== 'ground' || !snapshot.screenPx)
|
||||
async sendPointerCrop(
|
||||
snapshot,
|
||||
{ viewScale, diag = diagnosticsRecord() } = {},
|
||||
) {
|
||||
const skip = (reason) => {
|
||||
reportOutcome(this, diag, 'viewport_context.skipped', 'crop', reason);
|
||||
return false;
|
||||
};
|
||||
if (!snapshot?.fresh) return skip('pointer-not-fresh');
|
||||
if (snapshot.target !== 'ground') return skip('pointer-not-ground');
|
||||
if (!snapshot.screenPx) return skip('pointer-off-screen');
|
||||
// Too far out, the crop is a smear: every path shares this guard.
|
||||
if (!POINTER_CROP_SCALES.has(viewScale)) return false;
|
||||
if (snapshot === this.lastPointerCrop) return false;
|
||||
if (!POINTER_CROP_SCALES.has(viewScale))
|
||||
return skip('pointer-scale-not-useful');
|
||||
// Set before the capture finishes, so this also covers one in flight.
|
||||
if (snapshot === this.lastPointerCrop)
|
||||
return skip('pointer-already-requested');
|
||||
const capturedAt = Number(snapshot.capturedAt) || 0;
|
||||
if (this.now() - capturedAt > POINTER_CROP_MAX_AGE_MS) return false;
|
||||
if (this.now() - capturedAt > POINTER_CROP_MAX_AGE_MS)
|
||||
return skip('pointer-expired');
|
||||
// The snapshot's pixels only name its own camera pose.
|
||||
const snapshotCameraKey = snapshot.cameraKey;
|
||||
if (!snapshotCameraKey) return false;
|
||||
if (!snapshotCameraKey) return skip('pointer-pose-unknown');
|
||||
const samePose = () =>
|
||||
snapshotCameraKey === this.readCameraKey() &&
|
||||
this.now() - capturedAt <= POINTER_CROP_MAX_AGE_MS;
|
||||
if (!samePose()) return false;
|
||||
if (!samePose()) return skip('pose-changed');
|
||||
this.lastPointerCrop = snapshot;
|
||||
const sent = await this.sendRetainedImage(
|
||||
() => this.capturePointer(snapshot.screenPx),
|
||||
"Crop of the map around the user's pointer; the ring marks the exact spot they mean by this or here. Describe only what is clearly visible.",
|
||||
'client.pointer_crop',
|
||||
{ kind: 'crop', valid: samePose },
|
||||
{ kind: 'crop', valid: samePose, diag },
|
||||
);
|
||||
if (!sent && this.lastPointerCrop === snapshot) this.lastPointerCrop = null;
|
||||
return sent;
|
||||
@@ -393,22 +540,40 @@ export class RealtimeViewport {
|
||||
|
||||
/** Capture one image and make it the single retained image in context. */
|
||||
async sendRetainedImage(captureImage, text, label, frame = {}) {
|
||||
if (!this.dc || this.dc.readyState !== 'open') return false;
|
||||
const kind = frame.kind || 'viewport';
|
||||
const diag = frame.diag || diagnosticsRecord();
|
||||
const report = (event, reason) =>
|
||||
reportOutcome(this, diag, event, kind, reason);
|
||||
if (!this.dc || this.dc.readyState !== 'open') {
|
||||
report('viewport_context.skipped', 'channel-lost');
|
||||
return false;
|
||||
}
|
||||
const generation = this.generation;
|
||||
const request = ++this.captureSeq;
|
||||
const channel = this.dc;
|
||||
const captured = await captureImage();
|
||||
const imageUrl =
|
||||
typeof captured === 'string' ? captured : captured?.dataUrl || null;
|
||||
if (
|
||||
!imageUrl ||
|
||||
generation !== this.generation ||
|
||||
request !== this.captureSeq ||
|
||||
this.dc !== channel ||
|
||||
channel.readyState !== 'open' ||
|
||||
(typeof frame.valid === 'function' && !frame.valid())
|
||||
)
|
||||
if (!imageUrl) {
|
||||
report('viewport_context.skipped', 'capture-unavailable');
|
||||
return false;
|
||||
}
|
||||
// Same guards as before, checked in order so each drop names its cause:
|
||||
// reset() bumps both counters, so a session reset is tested first.
|
||||
const invalidated =
|
||||
generation !== this.generation
|
||||
? 'session-reset'
|
||||
: request !== this.captureSeq
|
||||
? 'superseded'
|
||||
: this.dc !== channel || channel.readyState !== 'open'
|
||||
? 'channel-lost'
|
||||
: typeof frame.valid === 'function' && !frame.valid()
|
||||
? 'frame-stale'
|
||||
: null;
|
||||
if (invalidated) {
|
||||
report('viewport_context.skipped', invalidated);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Keep at most one viewport screenshot in context. Images are the single
|
||||
// most expensive item (re-billed every turn they linger), so we proactively
|
||||
@@ -461,10 +626,16 @@ export class RealtimeViewport {
|
||||
// the item we just deleted as null and fall through so the caller still
|
||||
// issues queueResponseCreate WITHOUT the image, instead of stranding the turn.
|
||||
const sent = this.sendRealtimeEvent(contextEvent, label);
|
||||
// Report before the bookkeeping below, so a later throw cannot turn a
|
||||
// real send into a logged "error".
|
||||
report(
|
||||
'viewport_context.result',
|
||||
sent ? 'sent' : 'data-channel-send-failed',
|
||||
);
|
||||
this.lastViewportItemId = sent ? newItemId : null;
|
||||
this.retainedFrame = sent
|
||||
? {
|
||||
kind: frame.kind || 'viewport',
|
||||
kind,
|
||||
rect: captured?.rect || null,
|
||||
cameraKey: this.readCameraKey(),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
RealtimeViewport,
|
||||
viewportContextDecision,
|
||||
} from './realtimeViewport.js';
|
||||
|
||||
const localContext = (overrides = {}) => ({
|
||||
action: 'get_entity_context',
|
||||
scene: { basemap: { viewScale: 'local' } },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
test('viewport context reasons preserve the existing escalation gate', () => {
|
||||
assert.deepEqual(
|
||||
viewportContextDecision({
|
||||
action: 'get_entity_context',
|
||||
scene: { basemap: { viewScale: 'regional' } },
|
||||
}),
|
||||
{
|
||||
send: false,
|
||||
reason: 'view-scale-not-local',
|
||||
viewScale: 'regional',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
viewportContextDecision(
|
||||
localContext({
|
||||
selected: {
|
||||
id: 'earthquake:1',
|
||||
layerId: 'earthquakes',
|
||||
},
|
||||
}),
|
||||
),
|
||||
{
|
||||
send: false,
|
||||
reason: 'structured-identity-sufficient',
|
||||
viewScale: 'local',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(viewportContextDecision(localContext()), {
|
||||
send: true,
|
||||
kind: 'viewport',
|
||||
reason: 'structured-identity-missing',
|
||||
viewScale: 'local',
|
||||
});
|
||||
});
|
||||
|
||||
test('nearby structured place identity avoids visual escalation too', () => {
|
||||
assert.deepEqual(
|
||||
viewportContextDecision(
|
||||
localContext({
|
||||
scene: {
|
||||
basemap: {
|
||||
viewScale: 'local',
|
||||
nearbyPlaces: [{ name: 'Austin' }],
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
{
|
||||
send: false,
|
||||
reason: 'structured-identity-sufficient',
|
||||
viewScale: 'local',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('pointer ground selects the #837 crop path, not a full viewport', () => {
|
||||
const pointerContext = localContext({
|
||||
scope: 'pointer',
|
||||
pointer: { target: 'ground' },
|
||||
scene: { basemap: { viewScale: 'city' } },
|
||||
});
|
||||
assert.deepEqual(viewportContextDecision(pointerContext), {
|
||||
send: true,
|
||||
kind: 'crop',
|
||||
reason: 'pointer-crop-eligible',
|
||||
viewScale: 'city',
|
||||
});
|
||||
assert.deepEqual(
|
||||
viewportContextDecision({
|
||||
...pointerContext,
|
||||
scene: { basemap: { viewScale: 'global' } },
|
||||
}),
|
||||
{
|
||||
send: true,
|
||||
kind: 'crop',
|
||||
reason: 'pointer-crop-eligible',
|
||||
viewScale: 'global',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
viewportContextDecision({
|
||||
...pointerContext,
|
||||
selected: { id: 'building' },
|
||||
}),
|
||||
{
|
||||
send: false,
|
||||
reason: 'pointer-identity-present',
|
||||
viewScale: 'city',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
viewportContextDecision({ ...pointerContext, pointer: null }),
|
||||
{ send: false, reason: 'pointer-missing', viewScale: 'city' },
|
||||
);
|
||||
});
|
||||
|
||||
test('runtime dispatch preserves pointer crop and full image separation', async () => {
|
||||
const channel = { readyState: 'open' };
|
||||
const pointer = { fresh: true, target: 'ground' };
|
||||
const calls = [];
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
readPointer: () => pointer,
|
||||
operations: {
|
||||
debugLog: (event, detail) => calls.push({ event, detail }),
|
||||
},
|
||||
capture: async () => 'unused',
|
||||
});
|
||||
viewport.sendPointerCrop = async (snapshot, options) => {
|
||||
calls.push({ kind: 'crop', snapshot, options });
|
||||
return true;
|
||||
};
|
||||
viewport.sendRetainedImage = async () => {
|
||||
calls.push({ kind: 'viewport' });
|
||||
return true;
|
||||
};
|
||||
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(
|
||||
localContext({
|
||||
scope: 'pointer',
|
||||
pointer: { target: 'ground' },
|
||||
scene: { basemap: { viewScale: 'city' } },
|
||||
}),
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(calls.filter((call) => call.kind === 'crop').length, 1);
|
||||
assert.deepEqual(
|
||||
calls.find((call) => call.kind === 'crop').snapshot,
|
||||
pointer,
|
||||
);
|
||||
assert.equal(
|
||||
calls.find((call) => call.kind === 'crop').options.viewScale,
|
||||
'city',
|
||||
);
|
||||
assert.equal(calls.filter((call) => call.kind === 'viewport').length, 0);
|
||||
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(localContext()), true);
|
||||
assert.equal(calls.filter((call) => call.kind === 'viewport').length, 1);
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(
|
||||
localContext({ selected: { id: 'existing' } }),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(calls.filter((call) => call.kind === 'viewport').length, 1);
|
||||
assert.ok(
|
||||
calls.some(
|
||||
(call) =>
|
||||
call.event === 'viewport_context.decision' &&
|
||||
call.detail.reason === 'pointer-crop-eligible',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('closed channel never captures pointer or full viewport', async () => {
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => ({ readyState: 'closed' }),
|
||||
readPointer: () => {
|
||||
throw new Error('should not inspect pointer when channel is closed');
|
||||
},
|
||||
operations: {},
|
||||
});
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(
|
||||
localContext({ scope: 'pointer', pointer: { target: 'ground' } }),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(localContext()), false);
|
||||
});
|
||||
|
||||
test('capture failures and invalidated turns are reason-coded without sending', async () => {
|
||||
const channel = { readyState: 'open' };
|
||||
const diagnostics = [];
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
operations: {
|
||||
sendRealtimeEvent: () => {
|
||||
throw new Error('should not send unavailable image');
|
||||
},
|
||||
debugLog: (event, detail) => diagnostics.push({ event, detail }),
|
||||
},
|
||||
capture: async () => null,
|
||||
});
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(localContext()), false);
|
||||
assert.ok(
|
||||
diagnostics.some(
|
||||
(item) =>
|
||||
item.event === 'viewport_context.skipped' &&
|
||||
item.detail.reason === 'capture-unavailable',
|
||||
),
|
||||
);
|
||||
let release;
|
||||
viewport.capture = () =>
|
||||
new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const pending = viewport.sendVisualContextIfUseful(localContext());
|
||||
viewport.reset();
|
||||
release('data:image/jpeg;base64,AA==');
|
||||
assert.equal(await pending, false);
|
||||
assert.ok(
|
||||
diagnostics.some(
|
||||
(item) =>
|
||||
item.event === 'viewport_context.skipped' &&
|
||||
item.detail.reason === 'session-reset',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('successful retained send is logged without recording image contents', async () => {
|
||||
const channel = { readyState: 'open' };
|
||||
const diagnostics = [];
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
readCameraKey: () => 'camera:fixture',
|
||||
operations: {
|
||||
sendRealtimeEvent: () => true,
|
||||
debugLog: (event, detail) => diagnostics.push({ event, detail }),
|
||||
},
|
||||
capture: async () => 'data:image/jpeg;base64,AA==',
|
||||
});
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(localContext()), true);
|
||||
assert.ok(
|
||||
diagnostics.some(
|
||||
(item) =>
|
||||
item.event === 'viewport_context.result' &&
|
||||
item.detail.reason === 'sent',
|
||||
),
|
||||
);
|
||||
assert.equal(JSON.stringify(diagnostics).includes('data:image/jpeg'), false);
|
||||
});
|
||||
|
||||
const outcomeEvents = new Set([
|
||||
'viewport_context.skipped',
|
||||
'viewport_context.result',
|
||||
]);
|
||||
|
||||
function recordingViewport(options = {}) {
|
||||
const events = [];
|
||||
const channel = options.channel ?? { readyState: 'open' };
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
readCameraKey: () => 'camera:fixture',
|
||||
now: () => 1000,
|
||||
capturePointer: async () => ({
|
||||
dataUrl: 'data:image/jpeg;base64,AA==',
|
||||
rect: { x: 0, y: 0, width: 1, height: 1 },
|
||||
}),
|
||||
capture: async () => 'data:image/jpeg;base64,AA==',
|
||||
...options,
|
||||
operations: {
|
||||
sendRealtimeEvent: () => true,
|
||||
debugLog: (event, detail) => events.push({ event, detail }),
|
||||
...options.operations,
|
||||
},
|
||||
});
|
||||
return { viewport, events };
|
||||
}
|
||||
|
||||
const pointerGround = (overrides = {}) =>
|
||||
localContext({
|
||||
scope: 'pointer',
|
||||
pointer: { target: 'ground' },
|
||||
scene: { basemap: { viewScale: 'city' } },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const groundSnapshot = (overrides = {}) => ({
|
||||
fresh: true,
|
||||
target: 'ground',
|
||||
screenPx: { x: 10, y: 10 },
|
||||
capturedAt: 900,
|
||||
cameraKey: 'camera:fixture',
|
||||
...overrides,
|
||||
});
|
||||
|
||||
test('tool results other than entity context log nothing', async () => {
|
||||
const { viewport, events } = recordingViewport();
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful({ action: 'fly_to' }),
|
||||
false,
|
||||
);
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(null), false);
|
||||
assert.deepEqual(events, []);
|
||||
});
|
||||
|
||||
test('a crop decision that cannot send logs why, under the same id', async () => {
|
||||
const cases = [
|
||||
[{ fresh: false }, 'pointer-not-fresh'],
|
||||
[{ target: 'building' }, 'pointer-not-ground'],
|
||||
[{ screenPx: null }, 'pointer-off-screen'],
|
||||
[{ capturedAt: -50000 }, 'pointer-expired'],
|
||||
[{ cameraKey: null }, 'pointer-pose-unknown'],
|
||||
[{ cameraKey: 'camera:moved' }, 'pose-changed'],
|
||||
];
|
||||
for (const [override, reason] of cases) {
|
||||
const snapshot = groundSnapshot(override);
|
||||
const { viewport, events } = recordingViewport({
|
||||
readPointer: () => snapshot,
|
||||
});
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(pointerGround()),
|
||||
false,
|
||||
reason,
|
||||
);
|
||||
assert.equal(events.length, 2, reason);
|
||||
const [decision, outcome] = events;
|
||||
assert.equal(decision.event, 'viewport_context.decision');
|
||||
assert.equal(decision.detail.reason, 'pointer-crop-eligible');
|
||||
assert.deepEqual(outcome, {
|
||||
event: 'viewport_context.skipped',
|
||||
detail: { contextId: decision.detail.contextId, kind: 'crop', reason },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('a repeated pointer snapshot is logged as already requested', async () => {
|
||||
const snapshot = groundSnapshot();
|
||||
const { viewport, events } = recordingViewport({
|
||||
readPointer: () => snapshot,
|
||||
});
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(pointerGround()), true);
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(pointerGround()),
|
||||
false,
|
||||
);
|
||||
const outcomes = events.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.deepEqual(
|
||||
outcomes.map((item) => [item.detail.kind, item.detail.reason]),
|
||||
[
|
||||
['crop', 'sent'],
|
||||
['crop', 'pointer-already-requested'],
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('overlapping requests pair each outcome with its own decision', async () => {
|
||||
const releases = [];
|
||||
const { viewport, events } = recordingViewport({
|
||||
capture: () => new Promise((resolve) => releases.push(resolve)),
|
||||
});
|
||||
const first = viewport.sendVisualContextIfUseful(localContext());
|
||||
const second = viewport.sendVisualContextIfUseful(localContext());
|
||||
await Promise.resolve();
|
||||
releases[1]('data:image/jpeg;base64,AA==');
|
||||
releases[0]('data:image/jpeg;base64,AA==');
|
||||
assert.equal(await first, false);
|
||||
assert.equal(await second, true);
|
||||
const [firstId, secondId] = events
|
||||
.filter((item) => item.event === 'viewport_context.decision')
|
||||
.map((item) => item.detail.contextId);
|
||||
assert.notEqual(firstId, secondId);
|
||||
const outcomeFor = (id) =>
|
||||
events.filter(
|
||||
(item) => outcomeEvents.has(item.event) && item.detail.contextId === id,
|
||||
);
|
||||
assert.deepEqual(
|
||||
outcomeFor(firstId).map((item) => item.detail),
|
||||
[{ contextId: firstId, kind: 'viewport', reason: 'superseded' }],
|
||||
);
|
||||
assert.deepEqual(
|
||||
outcomeFor(secondId).map((item) => item.detail),
|
||||
[{ contextId: secondId, kind: 'viewport', reason: 'sent' }],
|
||||
);
|
||||
});
|
||||
|
||||
test('dropped captures name their cause', async () => {
|
||||
const reasonFor = async (context, interrupt, extra = {}) => {
|
||||
let release;
|
||||
const pending = () => new Promise((resolve) => (release = resolve));
|
||||
const { viewport, events } = recordingViewport({
|
||||
capture: pending,
|
||||
capturePointer: pending,
|
||||
...extra,
|
||||
});
|
||||
const sent = viewport.sendVisualContextIfUseful(context);
|
||||
await Promise.resolve();
|
||||
interrupt(viewport);
|
||||
release('data:image/jpeg;base64,AA==');
|
||||
assert.equal(await sent, false);
|
||||
return events
|
||||
.filter((item) => outcomeEvents.has(item.event))
|
||||
.map((item) => item.detail.reason);
|
||||
};
|
||||
|
||||
assert.deepEqual(
|
||||
await reasonFor(localContext(), (viewport) => viewport.beginTurn()),
|
||||
['superseded'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
await reasonFor(localContext(), (viewport) => viewport.reset()),
|
||||
['session-reset'],
|
||||
);
|
||||
const channel = { readyState: 'open' };
|
||||
assert.deepEqual(
|
||||
await reasonFor(localContext(), () => (channel.readyState = 'closed'), {
|
||||
channel,
|
||||
}),
|
||||
['channel-lost'],
|
||||
);
|
||||
let cameraKey = 'camera:fixture';
|
||||
const snapshot = groundSnapshot();
|
||||
assert.deepEqual(
|
||||
await reasonFor(pointerGround(), () => (cameraKey = 'camera:moved'), {
|
||||
readPointer: () => snapshot,
|
||||
readCameraKey: () => cameraKey,
|
||||
}),
|
||||
['frame-stale'],
|
||||
);
|
||||
});
|
||||
|
||||
test('every send decision is followed by exactly one outcome', async () => {
|
||||
const snapshots = [
|
||||
null,
|
||||
groundSnapshot(),
|
||||
groundSnapshot({ fresh: false }),
|
||||
groundSnapshot({ cameraKey: 'camera:moved' }),
|
||||
groundSnapshot({ capturedAt: -50000 }),
|
||||
];
|
||||
const contexts = [
|
||||
localContext(),
|
||||
localContext({ selected: { id: 'x', layerId: 'y' } }),
|
||||
pointerGround(),
|
||||
pointerGround({ scene: { basemap: { viewScale: 'global' } } }),
|
||||
pointerGround({ pointer: null }),
|
||||
];
|
||||
const captures = [
|
||||
async () => 'data:image/jpeg;base64,AA==',
|
||||
async () => null,
|
||||
];
|
||||
for (const snapshot of snapshots)
|
||||
for (const context of contexts)
|
||||
for (const capture of captures)
|
||||
for (const sendResult of [true, false]) {
|
||||
const { viewport, events } = recordingViewport({
|
||||
readPointer: () => snapshot,
|
||||
capture,
|
||||
capturePointer: capture,
|
||||
operations: { sendRealtimeEvent: () => sendResult },
|
||||
});
|
||||
await viewport.sendVisualContextIfUseful(context);
|
||||
const [decision, ...rest] = events;
|
||||
assert.equal(decision.event, 'viewport_context.decision');
|
||||
const outcomes = rest.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.equal(outcomes.length, decision.detail.send ? 1 : 0);
|
||||
for (const outcome of outcomes) {
|
||||
assert.equal(outcome.detail.contextId, decision.detail.contextId);
|
||||
assert.equal(outcome.detail.kind, decision.detail.kind);
|
||||
if (outcome.event === 'viewport_context.result')
|
||||
assert.equal(
|
||||
outcome.detail.reason,
|
||||
sendResult ? 'sent' : 'data-channel-send-failed',
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('a failed data-channel send is never logged as sent', async () => {
|
||||
for (const [sendResult, reason] of [
|
||||
[true, 'sent'],
|
||||
[false, 'data-channel-send-failed'],
|
||||
]) {
|
||||
const { viewport, events } = recordingViewport({
|
||||
operations: { sendRealtimeEvent: () => sendResult },
|
||||
});
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(localContext()),
|
||||
sendResult,
|
||||
);
|
||||
const outcomes = events.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.deepEqual(
|
||||
outcomes.map((item) => [item.event, item.detail.reason]),
|
||||
[['viewport_context.result', reason]],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('a step that throws still closes its decision, then rethrows', async () => {
|
||||
for (const [context, extra] of [
|
||||
[
|
||||
localContext(),
|
||||
{
|
||||
capture: async () => {
|
||||
throw new Error('capture boom');
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
pointerGround(),
|
||||
{
|
||||
readPointer: () => {
|
||||
throw new Error('pointer boom');
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
pointerGround(),
|
||||
{
|
||||
readPointer: () => groundSnapshot(),
|
||||
capturePointer: async () => {
|
||||
throw new Error('crop boom');
|
||||
},
|
||||
},
|
||||
],
|
||||
]) {
|
||||
const { viewport, events } = recordingViewport(extra);
|
||||
await assert.rejects(viewport.sendVisualContextIfUseful(context), /boom/);
|
||||
const [decision, ...rest] = events;
|
||||
assert.deepEqual(rest, [
|
||||
{
|
||||
event: 'viewport_context.skipped',
|
||||
detail: {
|
||||
contextId: decision.detail.contextId,
|
||||
kind: decision.detail.kind,
|
||||
reason: 'error',
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
test('direct calls log their defensive skips without a decision id', async () => {
|
||||
const open = recordingViewport();
|
||||
assert.equal(
|
||||
await open.viewport.sendPointerCrop(groundSnapshot(), {
|
||||
viewScale: 'global',
|
||||
}),
|
||||
false,
|
||||
);
|
||||
const closed = recordingViewport({ channel: { readyState: 'closed' } });
|
||||
assert.equal(
|
||||
await closed.viewport.sendPointerCrop(groundSnapshot(), {
|
||||
viewScale: 'city',
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
await closed.viewport.sendRetainedImage(
|
||||
async () => 'data:image/jpeg;base64,AA==',
|
||||
'text',
|
||||
'client.viewport_context',
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(
|
||||
[...open.events, ...closed.events].map((item) => item.detail),
|
||||
[
|
||||
{ contextId: null, kind: 'crop', reason: 'pointer-scale-not-useful' },
|
||||
{ contextId: null, kind: 'crop', reason: 'channel-lost' },
|
||||
{ contextId: null, kind: 'viewport', reason: 'channel-lost' },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('a throw after a real send still logs it as sent', async () => {
|
||||
let reads = 0;
|
||||
const { viewport, events } = recordingViewport({
|
||||
// The first read happens only after the image was handed to the channel.
|
||||
readCameraKey: () => {
|
||||
reads += 1;
|
||||
throw new Error('camera gone');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
viewport.sendVisualContextIfUseful(localContext()),
|
||||
/camera gone/,
|
||||
);
|
||||
assert.equal(reads, 1);
|
||||
const outcomes = events.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.deepEqual(
|
||||
outcomes.map((item) => [item.event, item.detail.reason]),
|
||||
[['viewport_context.result', 'sent']],
|
||||
);
|
||||
});
|
||||
|
||||
test('a second request for a snapshot still being captured is not sent twice', async () => {
|
||||
const releases = [];
|
||||
const snapshot = groundSnapshot();
|
||||
const { viewport, events } = recordingViewport({
|
||||
readPointer: () => snapshot,
|
||||
capturePointer: () => new Promise((resolve) => releases.push(resolve)),
|
||||
});
|
||||
const first = viewport.sendVisualContextIfUseful(pointerGround());
|
||||
const second = viewport.sendVisualContextIfUseful(pointerGround());
|
||||
await Promise.resolve();
|
||||
for (const release of releases)
|
||||
release({ dataUrl: 'data:image/jpeg;base64,AA==', rect: {} });
|
||||
assert.deepEqual([await first, await second], [true, false]);
|
||||
assert.equal(releases.length, 1);
|
||||
const outcomes = events.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.deepEqual(
|
||||
outcomes.map((item) => [item.detail.contextId, item.detail.reason]),
|
||||
[
|
||||
[2, 'pointer-already-requested'],
|
||||
[1, 'sent'],
|
||||
],
|
||||
);
|
||||
});
|
||||
+10
-3
@@ -23,10 +23,17 @@ function finite(value) {
|
||||
|
||||
function normalize(entry, index) {
|
||||
if (!entry || typeof entry !== 'object') return null;
|
||||
// Identity is the canonical key (transponder hex, MMSI, NORAD id); a
|
||||
// Identity is the canonical key (transponder hex, vessel id, NORAD id); a
|
||||
// display `id` such as a callsign is only a label, and two contacts can
|
||||
// share one.
|
||||
const id = entry.icao24 ?? entry.mmsi ?? entry.noradId ?? entry.id ?? null;
|
||||
// share one. Vessels can share an MMSI too, so a vessel's key is its
|
||||
// `vesselId` when the row names one.
|
||||
const id =
|
||||
entry.icao24 ??
|
||||
entry.vesselId ??
|
||||
entry.mmsi ??
|
||||
entry.noradId ??
|
||||
entry.id ??
|
||||
null;
|
||||
const label = String(
|
||||
entry.label ?? entry.callsign ?? entry.name ?? entry.id ?? id ?? '',
|
||||
).trim();
|
||||
|
||||
+12
-2
@@ -5,7 +5,12 @@
|
||||
* Events: state, transcript, action-call, progress, action-result, interruption,
|
||||
* completion, turn-metrics.
|
||||
*/
|
||||
export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||
export function createVoiceSession({
|
||||
createAdapter,
|
||||
runner,
|
||||
signal,
|
||||
canStart,
|
||||
}) {
|
||||
if (typeof createAdapter !== 'function' || typeof runner !== 'function')
|
||||
throw new TypeError('Voice requires an adapter factory and action runner');
|
||||
const lifetime = new AbortController();
|
||||
@@ -146,7 +151,12 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
async start(options) {
|
||||
if (disposed || !['idle', 'error'].includes(state)) return;
|
||||
if (
|
||||
disposed ||
|
||||
!['idle', 'error'].includes(state) ||
|
||||
canStart?.(options) === false
|
||||
)
|
||||
return;
|
||||
const attempt = ++startAttempt;
|
||||
emit({ type: 'state', state: 'connecting' });
|
||||
if (disposed || attempt !== startAttempt) return;
|
||||
|
||||
@@ -429,12 +429,13 @@ test('a late rejection cannot settle a newer turn card', async () => {
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
function startReentrancyFixture() {
|
||||
function startReentrancyFixture(canStart) {
|
||||
let starts = 0;
|
||||
let stops = 0;
|
||||
const settings = [];
|
||||
const session = createVoiceSession({
|
||||
runner: async () => ({ ok: true }),
|
||||
canStart,
|
||||
createAdapter: ({ emit }) => ({
|
||||
async start(options) {
|
||||
starts++;
|
||||
@@ -528,6 +529,28 @@ test('ordinary start forwards caller settings once and remains stoppable', async
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
test('start admission rejects microphone use before connecting and allows typed chat', async () => {
|
||||
const f = startReentrancyFixture((options) => options?.textOnly === true);
|
||||
const states = [];
|
||||
f.session.subscribe((event) => {
|
||||
if (event.type === 'state') states.push(event.state);
|
||||
});
|
||||
await f.session.start({ pushToTalk: true });
|
||||
assert.equal(f.starts(), 0);
|
||||
assert.equal(f.session.state, 'idle');
|
||||
assert.equal(states.includes('connecting'), false);
|
||||
|
||||
const settings = { textOnly: true };
|
||||
await f.session.start(settings);
|
||||
assert.equal(f.settings.length, 1);
|
||||
assert.equal(f.settings[0], settings);
|
||||
assert.equal(f.starts(), 1);
|
||||
assert.equal(f.session.state, 'listening');
|
||||
f.session.stop();
|
||||
assert.equal(f.session.state, 'idle');
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
for (const emitsIdle of [false, true]) {
|
||||
for (const outcome of ['normal', 'preserveStatus', 'throw']) {
|
||||
test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { nextTick } from 'vue';
|
||||
import { useOmnibarStore } from '../ui/stores/omnibar.js';
|
||||
import { applicationPinia } from '../ui/vueRuntime.js';
|
||||
import { bindAssistantPanelLayout, bindLayoutDock } from './panelLayout.js';
|
||||
import { bindChatControl } from './chatControl.js';
|
||||
import { bindVoiceMiniPreview } from './voiceMiniPreview.js';
|
||||
@@ -7,7 +9,7 @@ import { createVoiceSession, isStoppable } from './session.js';
|
||||
import { VoiceCardControls } from './voiceCard.js';
|
||||
import { PointerReticle } from './pointerReticle.js';
|
||||
import { VOICE_PROVIDERS } from './providerCapabilities.js';
|
||||
import { t } from '../i18n/index.js';
|
||||
import { t, translateKnown } from '../i18n/index.js';
|
||||
|
||||
function createDefaultReticle(pointer) {
|
||||
const element = pointer?.element;
|
||||
@@ -34,6 +36,7 @@ export function createVoiceCommands({
|
||||
supportedProviders,
|
||||
onProviderChange,
|
||||
onSessionEvent,
|
||||
canStartVoice,
|
||||
createCard = (options) => new VoiceCardControls(options),
|
||||
pointer = null,
|
||||
referents = null,
|
||||
@@ -49,6 +52,7 @@ export function createVoiceCommands({
|
||||
let panelMount;
|
||||
const session = createVoiceSession({
|
||||
runner,
|
||||
canStart: (options) => options?.textOnly || canStartVoice?.() !== false,
|
||||
signal,
|
||||
createAdapter: (hooks) => {
|
||||
if (panelHost) actionObserver = panelHost.observeActions(hooks);
|
||||
@@ -114,11 +118,20 @@ export function createVoiceCommands({
|
||||
// States are machine values; only the label is translated.
|
||||
ui.update('status', { text: () => voiceStatusLabel(event.state) });
|
||||
// The portable session reports 'Voice off' in English: show the idle
|
||||
// label instead. Other details are provider messages, shown as they come.
|
||||
// label instead. Other details are provider messages; one that keeps its
|
||||
// English catalog text is worded again in the current language.
|
||||
const idle = event.state === 'idle';
|
||||
const detail = event.detail === 'Voice off' ? '' : event.detail;
|
||||
const source = event.sourceDetail ?? event.detail;
|
||||
const detail = () =>
|
||||
source === 'Voice off'
|
||||
? ''
|
||||
: event.sourceDetail
|
||||
? event.sourceParams
|
||||
? t(event.sourceDetail, event.sourceParams)
|
||||
: translateKnown(event.sourceDetail)
|
||||
: event.detail;
|
||||
ui.update('detail', {
|
||||
text: () => detail || (idle ? t('Voice off') : t('Voice active')),
|
||||
text: () => detail() || (idle ? t('Voice off') : t('Voice active')),
|
||||
});
|
||||
// A typed chat session is not the microphone.
|
||||
ui.update('button', {
|
||||
@@ -127,8 +140,10 @@ export function createVoiceCommands({
|
||||
ui.update('error', {
|
||||
text: () =>
|
||||
event.state === 'error'
|
||||
? event.detail || t('Voice could not be started.')
|
||||
? detail() || t('Voice could not be started.')
|
||||
: '',
|
||||
// "Check microphone permission" is wrong advice for a missing key.
|
||||
hint: () => !(event.state === 'error' && event.missingKey === true),
|
||||
});
|
||||
});
|
||||
// Captions, plan and result card nested in the control; it only reads
|
||||
@@ -273,7 +288,7 @@ export function createVoiceCommands({
|
||||
void session.start({ pushToTalk: false });
|
||||
};
|
||||
const buttonHandler = () => {
|
||||
if (adapter.ignoreButtonClick?.()) return;
|
||||
if (canStartVoice?.() === false || adapter.ignoreButtonClick?.()) return;
|
||||
chat?.close();
|
||||
if (session.isActive() && adapter.getStartOptions?.().textOnly) {
|
||||
session.stop();
|
||||
@@ -321,5 +336,16 @@ export function createVoiceCommands({
|
||||
if (session.disposed) cleanup();
|
||||
else adapter.bindControls?.();
|
||||
window.__gevVoiceCommands = controls;
|
||||
// The omnibar runs the same actions directly, without a model.
|
||||
// A session already gone leaves the omnibar without a runner.
|
||||
if (!session.disposed) {
|
||||
const omnibar = useOmnibarStore(applicationPinia());
|
||||
omnibar.connectRunner((name, args) => runner(name, args));
|
||||
session.signal.addEventListener(
|
||||
'abort',
|
||||
() => omnibar.connectRunner(null),
|
||||
{ once: true },
|
||||
);
|
||||
}
|
||||
return controls;
|
||||
}
|
||||
+226
-5
@@ -24,6 +24,7 @@
|
||||
|
||||
import { voiceLayer } from './layerManifest.js';
|
||||
import {
|
||||
formatDate,
|
||||
formatNumber,
|
||||
getLocale,
|
||||
msg,
|
||||
@@ -178,9 +179,27 @@ const KNOWN_ERRORS = new Map(
|
||||
),
|
||||
msg('Could not arrive at the requested place'),
|
||||
msg('The nearest airborne aircraft could not be selected'),
|
||||
// show_street_level
|
||||
msg('Latitude must be within ±90° and longitude within ±180°'),
|
||||
msg('Street Level is unavailable'),
|
||||
msg('No street-level imagery source is configured'),
|
||||
msg('Street Level could not be enabled'),
|
||||
msg('Street Level changed while it looked for imagery'),
|
||||
msg('Aim the view at a street'),
|
||||
].map((text) => [text, text]),
|
||||
);
|
||||
|
||||
/** The imagery filters an empty street-level result names, as its chips say them. */
|
||||
function shownImageryFilters(label) {
|
||||
return [
|
||||
label.includes('360°') ? '360°' : null,
|
||||
label.includes('flat-photo') ? t('FLAT') : null,
|
||||
label.includes('SINCE') ? t('SINCE') : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
function noneWithinView(target, radius) {
|
||||
switch (target) {
|
||||
case 'flights':
|
||||
@@ -258,6 +277,69 @@ const ERROR_PATTERNS = [
|
||||
/^No (\S+) within (\d+(?:\.\d+)?) km of the current view$/,
|
||||
([, target, km]) => noneWithinView(target, shownKm(km)),
|
||||
],
|
||||
// show_street_level: the filter clause first, around the reason it extends.
|
||||
[
|
||||
/^(.+) with the (.+ filters?) on$/,
|
||||
([, reason, filters]) =>
|
||||
t('{reason} (filters on: {filters})', {
|
||||
reason: shownError(reason),
|
||||
filters: shownImageryFilters(filters),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^(.+) is not a street-level imagery source here$/,
|
||||
([, source]) =>
|
||||
t('{source} is not a street-level imagery source here', { source }),
|
||||
],
|
||||
[
|
||||
/^(.+) needs its key$/,
|
||||
([, source]) =>
|
||||
t('{source} needs its key', { source: translateKnown(source) }),
|
||||
],
|
||||
[
|
||||
/^(.+) needs the (.+) map, which is not available$/,
|
||||
([, source, map]) =>
|
||||
t('{source} needs the {map} map, which is not available', {
|
||||
source: translateKnown(source),
|
||||
map: map === 'photoreal' ? 'Google 3D' : map,
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^(.+) could not be switched on$/,
|
||||
([, source]) =>
|
||||
t('{source} could not be switched on', {
|
||||
source: translateKnown(source),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^(.+) is not available on this map$/,
|
||||
([, source]) =>
|
||||
t('{source} is not available on this map', {
|
||||
source: translateKnown(source),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^(.+) imagery could not be opened$/,
|
||||
([, source]) =>
|
||||
t('{source} imagery could not be opened', {
|
||||
source: translateKnown(source),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^(.+) has no position to look for imagery at$/,
|
||||
([, place]) =>
|
||||
t('{place} has no position to look for imagery at', {
|
||||
place: localizePlaceName(place),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^No (.+) imagery near (.+)$/,
|
||||
([, source, place]) =>
|
||||
t('No {source} imagery near {place}', {
|
||||
source: translateKnown(source),
|
||||
place: localizePlaceName(place),
|
||||
}),
|
||||
],
|
||||
];
|
||||
|
||||
/** A tool error as the card shows it. */
|
||||
@@ -501,7 +583,7 @@ function frameOverhead(result) {
|
||||
.filter((entry) => entry?.label || entry?.id)
|
||||
.map((entry, index) => ({
|
||||
n: index + 1,
|
||||
id: entry.icao24 || entry.mmsi || entry.id || null,
|
||||
id: entry.icao24 || entry.id || entry.mmsi || null,
|
||||
label: spokenLabel(entry.label || entry.id),
|
||||
layerId: result.layerId,
|
||||
}));
|
||||
@@ -594,6 +676,98 @@ function selectNearestAircraft(result) {
|
||||
};
|
||||
}
|
||||
|
||||
/** Month and year an image was captured, or ''. */
|
||||
function capturedMonth(capturedAt) {
|
||||
const ms = Number(capturedAt);
|
||||
if (!Number.isFinite(ms) || ms <= 0) return '';
|
||||
try {
|
||||
// Dates are UTC months (Google: the first of the month).
|
||||
return formatDate(new Date(ms), {
|
||||
month: 'short',
|
||||
year: 'numeric',
|
||||
timeZone: 'UTC',
|
||||
});
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function showStreetLevel(result) {
|
||||
// `location` is the flight's own result on a failed flight; its label is
|
||||
// the place, never the object.
|
||||
const place = spokenLabel(
|
||||
result?.label ||
|
||||
(typeof result?.location === 'string'
|
||||
? result.location
|
||||
: result?.location?.label) ||
|
||||
'there',
|
||||
);
|
||||
const source = spokenLabel(result?.providerName || 'Street Level');
|
||||
// The card in the interface language; the spoken line stays English.
|
||||
const shownSource = result?.providerName
|
||||
? spokenLabel(result.providerName)
|
||||
: t('Street Level');
|
||||
// The view centre is a phrase, not a place name: the card says it in the
|
||||
// interface language.
|
||||
const shown = result?.atViewCentre
|
||||
? t('the view centre')
|
||||
: spokenLabel(localizePlaceName(result?.label || '')) || place;
|
||||
if (result?.cancelled) return null;
|
||||
if (!result?.ok) {
|
||||
let say;
|
||||
if (result?.stage === 'location') say = `Couldn't find ${place}.`;
|
||||
else if (result?.stage === 'map')
|
||||
say = `${source} needs the Google 3D map, which isn't available.`;
|
||||
else if (result?.stage === 'key')
|
||||
say = result?.keyRejected
|
||||
? `${source} refused its key.`
|
||||
: `${source} needs its key.`;
|
||||
else if (result?.stage === 'layer') say = "Couldn't turn on Street Level.";
|
||||
else if (result?.stage === 'view')
|
||||
say = 'Street-level photos are hidden in this view. Leave it first.';
|
||||
else if (result?.stage === 'imagery' && result?.aim === 'sky')
|
||||
say = 'Aim the middle of the view at a street first.';
|
||||
else if (result?.stage === 'imagery' && result?.filter?.label)
|
||||
say = `No ${source} imagery near ${place} with the ${result.filter.label} on.`;
|
||||
else if (result?.stage === 'imagery' && result?.noImagery === true)
|
||||
say = `No ${source} imagery near ${place}.`;
|
||||
// A service or viewer failure is not missing coverage.
|
||||
else if (result?.stage === 'imagery')
|
||||
say = `${source} could not open imagery near ${place}.`;
|
||||
else say = `Couldn't open ${source}.`;
|
||||
return {
|
||||
say,
|
||||
display: {
|
||||
title: shownSource,
|
||||
lines: errorLines(result),
|
||||
sources: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (result.armed === true)
|
||||
return {
|
||||
say: sentence(`${source} is ready at ${place}: press OPEN to load it`),
|
||||
display: {
|
||||
title: shownSource,
|
||||
lines: [shown, t('Press OPEN to load the photo')],
|
||||
sources: [{ label: shownSource }],
|
||||
},
|
||||
};
|
||||
const title = spokenLabel(result.title || '', 60);
|
||||
const month = capturedMonth(result.capturedAt);
|
||||
return {
|
||||
say: sentence(`${source} at ${title || place}`),
|
||||
display: {
|
||||
title: shownSource,
|
||||
lines: [
|
||||
title || shown,
|
||||
month ? t('Captured {month}', { month }) : null,
|
||||
].filter(Boolean),
|
||||
sources: [{ label: shownSource }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const SPOKEN_ROUTE = Object.freeze({
|
||||
between: (origin, destination) => `${origin} to ${destination}`,
|
||||
from: (origin) => `from ${origin}`,
|
||||
@@ -801,6 +975,7 @@ export const SPEECH_BUILDERS = Object.freeze({
|
||||
fly_to_location: flyToLocation,
|
||||
frame_overhead: frameOverhead,
|
||||
select_nearest_aircraft: selectNearestAircraft,
|
||||
show_street_level: showStreetLevel,
|
||||
get_entity_context: getEntityContext,
|
||||
get_current_view_state: getCurrentViewState,
|
||||
annotate_map: annotateMap,
|
||||
@@ -856,6 +1031,14 @@ function annotationTargets(args) {
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** Street-level sources as a plan step names them. */
|
||||
const STREET_LEVEL_SOURCE_NAMES = Object.freeze({
|
||||
google: 'Street View',
|
||||
mapillary: 'Mapillary',
|
||||
kartaview: 'KartaView',
|
||||
panoramax: 'Panoramax',
|
||||
});
|
||||
|
||||
// frame_overhead targets as the schema names them.
|
||||
function frameStepLabel(target) {
|
||||
switch (target) {
|
||||
@@ -887,6 +1070,7 @@ const STEP_TITLES = Object.freeze({
|
||||
set_detection: msg('Adjust detection overlay'),
|
||||
set_map_stack: msg('Change map stack'),
|
||||
set_post_processing: msg('Adjust post-processing'),
|
||||
open_stories: msg('Stories'),
|
||||
control_scene: msg('Control scene'),
|
||||
control_cctv: msg('Control CCTV'),
|
||||
control_radio: msg('Control radio'),
|
||||
@@ -927,6 +1111,29 @@ export function planStepLabel(name, args = {}) {
|
||||
? t('Nearest aircraft to {place}', { place: shownPlace(place, 32) })
|
||||
: t('Nearest aircraft to Here');
|
||||
}
|
||||
case 'show_street_level': {
|
||||
const source = STREET_LEVEL_SOURCE_NAMES[args.provider];
|
||||
const query = String(args.query || '').trim();
|
||||
// A pointed or numbered thing is found later (deixis): no place yet.
|
||||
if (query.toLowerCase() === 'pointer' || args.referent != null)
|
||||
return source || t('Street Level');
|
||||
// "This spot" and "the view centre" are phrases, not place names:
|
||||
// each is a message of its own, worded in the grammar of the language.
|
||||
if (query) {
|
||||
const place = shownPlace(query, 32);
|
||||
return source
|
||||
? t('{source} at {place}', { source, place })
|
||||
: t('Street level at {place}', { place });
|
||||
}
|
||||
// Coordinates count only as a pair, as show_street_level reads them.
|
||||
if (args.latitude != null && args.longitude != null)
|
||||
return source
|
||||
? t('{source} at this spot', { source })
|
||||
: t('Street level at this spot');
|
||||
return source
|
||||
? t('{source} at the view centre', { source })
|
||||
: t('Street level at the view centre');
|
||||
}
|
||||
case 'set_layer_visibility': {
|
||||
const on = args.enabled !== false;
|
||||
const label =
|
||||
@@ -988,6 +1195,8 @@ export function narrationLabel(name, args = {}) {
|
||||
presetPlaceName(args.locationQuery || args.locationId || ''),
|
||||
32,
|
||||
);
|
||||
case 'show_street_level':
|
||||
return shownPlace(args.query || '', 32);
|
||||
case 'resolve_area':
|
||||
return displayName(args.query || '', 32);
|
||||
case 'find_imagery':
|
||||
@@ -999,6 +1208,9 @@ export function narrationLabel(name, args = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
/** The place show_street_level reports when it names none. */
|
||||
export const VIEW_CENTRE_LABEL = 'the view centre';
|
||||
|
||||
/** On-screen labels for progress steps. */
|
||||
const STEP_LABELS = Object.freeze({
|
||||
resolve: msg('Finding places'),
|
||||
@@ -1008,6 +1220,8 @@ const STEP_LABELS = Object.freeze({
|
||||
layer: msg('Turning on layer'),
|
||||
refresh: msg('Loading aircraft'),
|
||||
nearest: msg('Picking nearest'),
|
||||
map: msg('Switching map'),
|
||||
imagery: msg('Opening imagery'),
|
||||
draw: msg('Drawing outline'),
|
||||
catalog: msg('Searching imagery catalog'),
|
||||
drape: msg('Loading imagery'),
|
||||
@@ -1021,10 +1235,11 @@ export function progressStepLabel(step, label) {
|
||||
: step
|
||||
? spokenLabel(step, 24)
|
||||
: t('Working');
|
||||
// The join is a catalog string so each language spaces its colon.
|
||||
return label
|
||||
? t('{name}: {state}', { name: base, state: spokenLabel(label, 32) })
|
||||
: base;
|
||||
// The join is a catalog string so each language spaces its colon. The
|
||||
// view centre is a phrase show_street_level reports in English, not a name.
|
||||
const state =
|
||||
label === VIEW_CENTRE_LABEL ? t('the view centre') : spokenLabel(label, 32);
|
||||
return label ? t('{name}: {state}', { name: base, state }) : base;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1056,6 +1271,12 @@ export function progressLine(step, label) {
|
||||
return 'Loading aircraft there.';
|
||||
case 'nearest':
|
||||
return 'Picking the nearest aircraft.';
|
||||
case 'map':
|
||||
return place === 'photoreal'
|
||||
? 'Switching to Google 3D.'
|
||||
: 'Switching the map.';
|
||||
case 'imagery':
|
||||
return place ? `Looking for imagery at ${place}.` : null;
|
||||
case 'still':
|
||||
return place ? `Still working on ${place}.` : 'Still working on it.';
|
||||
default:
|
||||
|
||||
+282
-1
@@ -7,10 +7,11 @@ import {
|
||||
progressLine,
|
||||
progressStepLabel,
|
||||
spokenLabel,
|
||||
VIEW_CENTRE_LABEL,
|
||||
} from './speech.js';
|
||||
import { presentResult } from './resultDisplay.js';
|
||||
import { countWords, findHedges } from './speechLint.js';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
|
||||
function envelope(name, result, args) {
|
||||
return attachVoiceResult(name, result, args);
|
||||
@@ -536,6 +537,238 @@ test('labels are bounded and cleaned before they are spoken', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('the street-level card says "the view centre" in the interface language', () => {
|
||||
setLocale('de');
|
||||
try {
|
||||
const out = envelope('show_street_level', {
|
||||
ok: true,
|
||||
providerName: 'Mapillary',
|
||||
label: 'the view centre',
|
||||
atViewCentre: true,
|
||||
});
|
||||
assert.equal(out.display.lines[0], 'die Bildmitte');
|
||||
assert.match(out.say, /the view centre/, 'speech stays English');
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
|
||||
test('street-level failure cards are in the interface language; speech stays English', () => {
|
||||
setLocale('de');
|
||||
try {
|
||||
const card = (error, extra = {}) =>
|
||||
envelope('show_street_level', {
|
||||
ok: false,
|
||||
action: 'show_street_level',
|
||||
providerName: 'Google Street View',
|
||||
label: 'Paris',
|
||||
error,
|
||||
...extra,
|
||||
});
|
||||
const lines = (error, extra) => card(error, extra).display.lines;
|
||||
assert.deepEqual(lines('Google Street View needs its key', { stage: 'key' }), [
|
||||
'Google Street View braucht seinen Schlüssel',
|
||||
]);
|
||||
assert.deepEqual(
|
||||
lines('Google Street View needs the photoreal map, which is not available', { stage: 'map' }),
|
||||
['Google Street View braucht die Karte „Google 3D“, die nicht verfügbar ist'],
|
||||
);
|
||||
assert.deepEqual(lines('Mapillary could not be switched on', { stage: 'provider' }), [
|
||||
'Mapillary konnte nicht eingeschaltet werden',
|
||||
]);
|
||||
assert.deepEqual(lines('Street Level could not be enabled', { stage: 'layer' }), [
|
||||
'Straßenebene konnte nicht eingeschaltet werden',
|
||||
]);
|
||||
assert.deepEqual(
|
||||
lines('Latitude must be within ±90° and longitude within ±180°', { stage: 'location' }),
|
||||
['Der Breitengrad muss innerhalb von ±90° und der Längengrad innerhalb von ±180° liegen'],
|
||||
);
|
||||
assert.deepEqual(lines('Mapillary imagery could not be opened', { stage: 'imagery' }), [
|
||||
'Bilder von Mapillary konnten nicht geöffnet werden',
|
||||
]);
|
||||
// The filter clause wraps the reason, already in the interface language,
|
||||
// and names the filters as their chips do.
|
||||
const filtered = card('No Mapillary imagery near Paris with the 360° and SINCE filters on', {
|
||||
stage: 'imagery',
|
||||
noImagery: true,
|
||||
filter: { label: '360° and SINCE filters' },
|
||||
});
|
||||
assert.deepEqual(filtered.display.lines, [
|
||||
'Keine Bilder von Mapillary in der Nähe von Paris (aktive Filter: 360°, SEIT)',
|
||||
]);
|
||||
assert.match(filtered.say, /with the 360° and SINCE filters on/, 'speech stays English');
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
|
||||
test('street-level speech names the source and the place, and says which step failed', () => {
|
||||
const ok = envelope('show_street_level', {
|
||||
ok: true,
|
||||
action: 'show_street_level',
|
||||
provider: 'google',
|
||||
providerName: 'Google Street View',
|
||||
label: '10th Street, Sacramento',
|
||||
title: '1000 10th St, Sacramento, CA',
|
||||
capturedAt: Date.UTC(2024, 4, 1), // Google dates: the first of the month, UTC
|
||||
});
|
||||
assert.equal(ok.say, 'Google Street View at 1000 10th St, Sacramento, CA.');
|
||||
assertSpeakable(ok.say);
|
||||
assert.equal(ok.display.title, 'Google Street View');
|
||||
assert.equal(ok.display.lines[1], 'Captured May 2024');
|
||||
|
||||
const stages = {
|
||||
location: "Couldn't find Nowhere.",
|
||||
map: "Google Street View needs the Google 3D map, which isn't available.",
|
||||
key: 'Google Street View needs its key.',
|
||||
layer: "Couldn't turn on Street Level.",
|
||||
// A service or viewer failure, not missing coverage.
|
||||
imagery: 'Google Street View could not open imagery near Nowhere.',
|
||||
};
|
||||
for (const [stage, say] of Object.entries(stages)) {
|
||||
const out = envelope('show_street_level', {
|
||||
ok: false,
|
||||
stage,
|
||||
label: 'Nowhere',
|
||||
providerName: 'Google Street View',
|
||||
error: 'detail',
|
||||
});
|
||||
assert.equal(out.say, say, stage);
|
||||
assertSpeakable(out.say);
|
||||
assert.deepEqual(out.display.lines, ['detail']);
|
||||
}
|
||||
const empty = envelope('show_street_level', {
|
||||
ok: false,
|
||||
stage: 'imagery',
|
||||
noImagery: true,
|
||||
label: 'Nowhere',
|
||||
providerName: 'Mapillary',
|
||||
});
|
||||
assert.equal(empty.say, 'No Mapillary imagery near Nowhere.');
|
||||
const sky = envelope('show_street_level', {
|
||||
ok: false,
|
||||
stage: 'imagery',
|
||||
aim: 'sky',
|
||||
atViewCentre: true,
|
||||
label: 'the view centre',
|
||||
providerName: 'Mapillary',
|
||||
});
|
||||
assert.equal(sky.say, 'Aim the middle of the view at a street first.');
|
||||
assertSpeakable(sky.say);
|
||||
const armed = envelope('show_street_level', {
|
||||
ok: true,
|
||||
armed: true,
|
||||
providerName: 'Google Street View',
|
||||
label: '10th St, Sacramento',
|
||||
});
|
||||
assert.equal(armed.say, 'Google Street View is ready at 10th St, Sacramento: press OPEN to load it.');
|
||||
assertSpeakable(armed.say);
|
||||
assert.deepEqual(armed.display.lines, ['10th St, Sacramento', 'Press OPEN to load the photo']);
|
||||
const unfound = envelope('show_street_level', {
|
||||
ok: false,
|
||||
stage: 'location',
|
||||
location: { ok: false, action: 'fly_to_location', label: 'Nowhere Town' },
|
||||
});
|
||||
assert.equal(unfound.say, "Couldn't find Nowhere Town.");
|
||||
assert.equal(unfound.display.title, 'Street Level');
|
||||
const refused = envelope('show_street_level', {
|
||||
ok: false,
|
||||
stage: 'key',
|
||||
keyRejected: true,
|
||||
providerName: 'Google Street View',
|
||||
});
|
||||
assert.equal(refused.say, 'Google Street View refused its key.');
|
||||
assert.equal(
|
||||
envelope('show_street_level', { ok: false, cancelled: true }).say,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', {
|
||||
query: 'Eiffel Tower',
|
||||
provider: 'google',
|
||||
}),
|
||||
'Street View at Eiffel Tower',
|
||||
);
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', {
|
||||
query: 'Times Square',
|
||||
provider: 'mapillary',
|
||||
}),
|
||||
'Mapillary at Times Square',
|
||||
);
|
||||
assert.equal(
|
||||
progressStepLabel('imagery', 'Eiffel Tower'),
|
||||
'Opening imagery: Eiffel Tower',
|
||||
);
|
||||
assert.equal(progressLine('map', 'photoreal'), 'Switching to Google 3D.');
|
||||
assert.equal(
|
||||
progressLine('imagery', 'Eiffel Tower'),
|
||||
'Looking for imagery at Eiffel Tower.',
|
||||
);
|
||||
});
|
||||
|
||||
test('the capture month is a UTC month whatever the timezone, and labels name the place', async () => {
|
||||
const { execFileSync } = await import('node:child_process');
|
||||
const speech = new URL('./speech.js', import.meta.url).href;
|
||||
// Google dates a panorama 2024-05 as the first of the month, UTC. West of
|
||||
// Greenwich that instant is still April; the card must still say May.
|
||||
const out = execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
'--input-type=module',
|
||||
'-e',
|
||||
`import { attachVoiceResult } from ${JSON.stringify(speech)};
|
||||
const r = attachVoiceResult('show_street_level', { ok: true, providerName: 'Google Street View', label: 'x', capturedAt: Date.UTC(2024, 4, 1) });
|
||||
process.stdout.write(r.display.lines[1]);`,
|
||||
],
|
||||
{ env: { ...process.env, TZ: 'America/Los_Angeles' }, encoding: 'utf8' },
|
||||
);
|
||||
assert.equal(out, 'Captured May 2024');
|
||||
const { narrationLabel } = await import('./speech.js');
|
||||
assert.equal(narrationLabel('show_street_level', { query: 'Eiffel Tower' }), 'Eiffel Tower');
|
||||
assert.equal(narrationLabel('show_street_level', {}), '');
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', { latitude: 1, longitude: 2 }),
|
||||
'Street level at this spot',
|
||||
);
|
||||
assert.equal(planStepLabel('show_street_level', {}), 'Street level at the view centre');
|
||||
});
|
||||
|
||||
test('the plan step names KartaView as a source', () => {
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', {
|
||||
query: 'Alexanderplatz',
|
||||
provider: 'kartaview',
|
||||
}),
|
||||
'KartaView at Alexanderplatz',
|
||||
);
|
||||
});
|
||||
|
||||
test('an empty street-level search names the imagery filter that hid the photos', () => {
|
||||
const out = envelope('show_street_level', {
|
||||
ok: false,
|
||||
stage: 'imagery',
|
||||
label: 'Alexanderplatz',
|
||||
providerName: 'KartaView',
|
||||
filter: { pano: 'pano', label: '360° filter' },
|
||||
});
|
||||
assert.equal(
|
||||
out.say,
|
||||
'No KartaView imagery near Alexanderplatz with the 360° filter on.',
|
||||
);
|
||||
assertSpeakable(out.say);
|
||||
});
|
||||
|
||||
test('the plan step names Panoramax as a source', () => {
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', {
|
||||
query: 'Montmartre',
|
||||
provider: 'panoramax',
|
||||
}),
|
||||
'Panoramax at Montmartre',
|
||||
);
|
||||
});
|
||||
|
||||
test('progress step labels join through the catalog colon', () => {
|
||||
setLocale('fr');
|
||||
try {
|
||||
@@ -547,3 +780,51 @@ test('progress step labels join through the catalog colon', () => {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
|
||||
test('the street-level card words "the view centre" and "this spot" as phrases, not names', () => {
|
||||
setLocale('de');
|
||||
try {
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', { provider: 'mapillary' }),
|
||||
'Mapillary in der Bildmitte',
|
||||
);
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', { latitude: 1, longitude: 2 }),
|
||||
'Straßenebene an dieser Stelle',
|
||||
);
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', { query: 'Alexanderplatz', provider: 'kartaview' }),
|
||||
'KartaView: Alexanderplatz',
|
||||
);
|
||||
assert.equal(
|
||||
progressStepLabel('imagery', VIEW_CENTRE_LABEL),
|
||||
`${t('Opening imagery')}: die Bildmitte`,
|
||||
);
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
|
||||
test('the street-level plan step names the place as show_street_level finds it', () => {
|
||||
// A pointed or numbered thing is located later, by deixis.
|
||||
assert.equal(planStepLabel('show_street_level', { referent: 2 }), 'Street Level');
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', { referent: 2, provider: 'mapillary' }),
|
||||
'Mapillary',
|
||||
);
|
||||
assert.equal(planStepLabel('show_street_level', { query: ' Pointer ' }), 'Street Level');
|
||||
// One coordinate alone is no spot: the action looks at the view centre.
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', { latitude: 10 }),
|
||||
'Street level at the view centre',
|
||||
);
|
||||
setLocale('de');
|
||||
try {
|
||||
assert.equal(
|
||||
planStepLabel('show_street_level', { referent: 1 }),
|
||||
'Straßenebene',
|
||||
);
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { bindRenderer } from '../render/rendererOf.js';
|
||||
import {
|
||||
captureViewportImage,
|
||||
capturePointerCrop,
|
||||
} from './realtimeViewport.js';
|
||||
|
||||
/**
|
||||
* A page with a bound renderer whose canvas must never be read directly:
|
||||
* its pixels come only through `readPixels`, as the frame it just drew.
|
||||
*/
|
||||
function page({ cssWidth = 800, cssHeight = 600, dpr = 2, dprY = dpr } = {}) {
|
||||
const canvas = {
|
||||
width: cssWidth * dpr,
|
||||
height: cssHeight * dprY,
|
||||
clientWidth: cssWidth,
|
||||
clientHeight: cssHeight,
|
||||
};
|
||||
let fire = null;
|
||||
const reads = [];
|
||||
let drawn = false;
|
||||
const renderer = {
|
||||
canvas,
|
||||
frame: {
|
||||
on(phase, cb) {
|
||||
if (phase !== 'postRender') return () => {};
|
||||
fire = cb;
|
||||
return () => (fire = null);
|
||||
},
|
||||
requestRender() {
|
||||
queueMicrotask(() => {
|
||||
drawn = true;
|
||||
fire?.();
|
||||
drawn = false;
|
||||
});
|
||||
},
|
||||
},
|
||||
readPixels(options) {
|
||||
reads.push({ ...options, duringFrame: drawn });
|
||||
const width = Math.round(
|
||||
(options.crop?.width ?? cssWidth) * options.scale,
|
||||
);
|
||||
const height = Math.round(
|
||||
(options.crop?.height ?? cssHeight) * options.scale,
|
||||
);
|
||||
return {
|
||||
width,
|
||||
height,
|
||||
data: new Uint8ClampedArray(width * height * 4).fill(200),
|
||||
};
|
||||
},
|
||||
};
|
||||
const viewer = {};
|
||||
bindRenderer(viewer, renderer);
|
||||
const put = [];
|
||||
const encoded = [];
|
||||
const document = {
|
||||
hidden: false,
|
||||
createElement() {
|
||||
const out = {
|
||||
width: 0,
|
||||
height: 0,
|
||||
getContext: () => ctx,
|
||||
toDataURL: (type) => {
|
||||
encoded.push([out.width, out.height]);
|
||||
return `data:${type};base64,AAAA`;
|
||||
},
|
||||
};
|
||||
const ctx = {
|
||||
canvas: out,
|
||||
putImageData: (image, x, y) =>
|
||||
put.push([image.width, image.height, x, y]),
|
||||
drawImage(source) {
|
||||
assert.notEqual(
|
||||
source,
|
||||
canvas,
|
||||
'the renderer’s canvas is never drawn',
|
||||
);
|
||||
},
|
||||
getImageData: (x, y, w, h) => ({
|
||||
data: new Uint8ClampedArray(w * h * 4).fill(200),
|
||||
}),
|
||||
beginPath() {},
|
||||
arc() {},
|
||||
stroke() {},
|
||||
};
|
||||
return out;
|
||||
},
|
||||
querySelector: () => canvas,
|
||||
};
|
||||
return { viewer, reads, put, encoded, document };
|
||||
}
|
||||
|
||||
function withPage(p, fn) {
|
||||
const saved = [globalThis.window, globalThis.document];
|
||||
globalThis.window = { __godsEyeView: { viewer: p.viewer } };
|
||||
globalThis.document = p.document;
|
||||
return Promise.resolve(fn()).finally(() => {
|
||||
[globalThis.window, globalThis.document] = saved;
|
||||
});
|
||||
}
|
||||
|
||||
test('the viewport image is the renderer’s readPixels of the fresh frame, as it is drawn', async () => {
|
||||
const p = page();
|
||||
const image = await withPage(p, () => captureViewportImage());
|
||||
assert.match(image, /^data:image\/jpeg;base64,/);
|
||||
assert.equal(p.reads.length, 1);
|
||||
const [read] = p.reads;
|
||||
assert.equal(read.duringFrame, true, 'read in the frame it waited for');
|
||||
assert.equal(read.crop, undefined, 'the whole canvas');
|
||||
// 1600 × 1200 drawing-buffer pixels, cut to the 1200 × 900 budget.
|
||||
assert.deepEqual(p.put, [[1200, 900, 0, 0]]);
|
||||
});
|
||||
|
||||
test('the pointer crop is the renderer’s readPixels around the pointer, as the frame is drawn', async () => {
|
||||
const p = page();
|
||||
const crop = await withPage(p, () => capturePointerCrop({ x: 400, y: 300 }));
|
||||
assert.ok(crop?.dataUrl);
|
||||
assert.deepEqual(crop.rect, { x: 0.18, y: 44 / 600, w: 0.64, h: 512 / 600 });
|
||||
const [read] = p.reads;
|
||||
assert.equal(read.duringFrame, true);
|
||||
assert.deepEqual(read.crop, { x: 144, y: 44, width: 512, height: 512 });
|
||||
assert.deepEqual(p.put, [[512, 512, 0, 0]]);
|
||||
});
|
||||
|
||||
test('the pointer crop is its full square even where the canvas is stretched unevenly', async () => {
|
||||
// Drawing-buffer pixels three to a CSS pixel across and 0.3 down: the
|
||||
// crop read back is a row taller than it is wide.
|
||||
const p = page({ dpr: 3, dprY: 0.3 });
|
||||
const crop = await withPage(p, () => capturePointerCrop({ x: 400, y: 300 }));
|
||||
assert.ok(crop?.dataUrl);
|
||||
assert.deepEqual(p.encoded, [[512, 512]], 'a 512 px square');
|
||||
});
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
fakeRenderer,
|
||||
fakeThumbnails,
|
||||
} from '../layers/recentImagery/testDoubles.mjs';
|
||||
import { bindFakeCesiumCamera } from '../testSupport/fakeCameraPort.mjs';
|
||||
|
||||
globalThis.window = globalThis.window || {
|
||||
clearTimeout,
|
||||
@@ -148,10 +149,12 @@ export function harness({
|
||||
const enabled = new Set(on);
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
canvas: { addEventListener() {}, removeEventListener() {} },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: {
|
||||
moveEnd: { addEventListener() {} },
|
||||
// The view: a box over Kathmandu, in radians.
|
||||
// The view: a box over Kathmandu, in radians, as the fake Cesium
|
||||
// camera gives it; the camera port answers in degrees.
|
||||
computeViewRectangle: () => ({
|
||||
west: (85.2 * Math.PI) / 180,
|
||||
south: (27.6 * Math.PI) / 180,
|
||||
@@ -165,6 +168,8 @@ export function harness({
|
||||
},
|
||||
},
|
||||
};
|
||||
// The actions read the view through the camera port.
|
||||
bindFakeCesiumCamera(viewer);
|
||||
const referents = createReferentRegistry();
|
||||
let snapshot = pointer;
|
||||
const osmCalls = [];
|
||||
|
||||
Reference in new issue
Block a user