mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 08:12:24 +00:00
Merge commit '0ceee8df40b97c6c693e86a4443794cd4ed1ff35' into HEAD
This commit is contained in:
commit
f878efe22d
306 files changed
+39352
-1624
No files matched your search
@@ -107,7 +107,7 @@ function initialState(panel) {
|
||||
t('Hold Space to speak · tap Space to activate focused controls'),
|
||||
},
|
||||
// A typed chat error reads as chat's, without the microphone hint.
|
||||
error: { text: '', typed: false },
|
||||
error: { text: '', typed: false, hint: true },
|
||||
// Connection and turn state beside the selected mode's indicator.
|
||||
phase: { text: '' },
|
||||
voicePanel: {
|
||||
|
||||
+47
-12
@@ -71,7 +71,9 @@ export function geminiSocketUrl({ token, websocketUrl }) {
|
||||
return url.href;
|
||||
}
|
||||
|
||||
// Known local-server failures are translated; unexpected diagnostics stay intact.
|
||||
// The local server's known failures, marked for translation. Each is kept as
|
||||
// its English catalog text, which the status words when it is shown, so a
|
||||
// language change words it again; unexpected diagnostics stay intact.
|
||||
function tokenErrorMessage(error) {
|
||||
switch (error) {
|
||||
case msg('Gemini voice could not connect. Try again.'):
|
||||
@@ -85,7 +87,6 @@ function tokenErrorMessage(error) {
|
||||
case msg('Token request is too large'):
|
||||
case msg('Use an empty JSON token request'):
|
||||
case msg('Method not allowed'):
|
||||
return t(error);
|
||||
default:
|
||||
return error;
|
||||
}
|
||||
@@ -359,18 +360,35 @@ export class GeminiSessionController {
|
||||
);
|
||||
}
|
||||
|
||||
setStatus(status, detail, recovery) {
|
||||
/**
|
||||
* @param {{missingKey?: boolean, params?: object}} [options] `missingKey`
|
||||
* is true when the server has no Gemini key: the control then drops its
|
||||
* microphone hint. `params` fill the placeholders of `detail`.
|
||||
*/
|
||||
setStatus(
|
||||
status,
|
||||
detail,
|
||||
recovery,
|
||||
{ missingKey = false, params = null } = {},
|
||||
) {
|
||||
this.status = status;
|
||||
if (status === 'listening' && this.input.pushToTalkMode)
|
||||
detail = this.input.pushToTalkKeyHeld
|
||||
? msg('Release Space to send')
|
||||
: msg('Hold Space to talk');
|
||||
// `detail` is worded in the current language; `sourceDetail` keeps the
|
||||
// message as given (English catalog text, with `sourceParams` for its
|
||||
// placeholders, when it is ours), so a view can word it again after a
|
||||
// language change.
|
||||
const event = {
|
||||
type: 'state',
|
||||
state: status,
|
||||
detail: translateKnown(detail),
|
||||
detail: params ? t(detail, params) : translateKnown(detail),
|
||||
...(typeof detail === 'string' && { sourceDetail: detail }),
|
||||
...(params && { sourceParams: params }),
|
||||
};
|
||||
if (recovery) event.recovery = translateKnown(recovery);
|
||||
if (missingKey) event.missingKey = true;
|
||||
this.emit(event);
|
||||
this.input.updateVoiceButtonLabel();
|
||||
if (['idle', 'connecting', 'error'].includes(status))
|
||||
@@ -584,12 +602,24 @@ export class GeminiSessionController {
|
||||
const data = await readResponseJsonCapped(response, 256 * 1024, signal);
|
||||
signal.throwIfAborted();
|
||||
if (!response.ok)
|
||||
throw new Error(
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
typeof data?.error === 'string'
|
||||
? tokenErrorMessage(data.error)
|
||||
: t('Gemini token request failed (HTTP {status})', {
|
||||
status: response.status,
|
||||
}),
|
||||
),
|
||||
typeof data?.code === 'string' ? { code: data.code } : {},
|
||||
// The status words this again in the current language when shown.
|
||||
typeof data?.error === 'string'
|
||||
? tokenErrorMessage(data.error)
|
||||
: t('Gemini token request failed (HTTP {status})', {
|
||||
status: response.status,
|
||||
}),
|
||||
? {}
|
||||
: {
|
||||
wording: {
|
||||
text: msg('Gemini token request failed (HTTP {status})'),
|
||||
params: { status: response.status },
|
||||
},
|
||||
},
|
||||
);
|
||||
if (
|
||||
typeof data?.model !== 'string' ||
|
||||
@@ -963,10 +993,15 @@ export class GeminiSessionController {
|
||||
)
|
||||
: error?.name === 'TimeoutError'
|
||||
? msg('Gemini connection timed out. Try again.')
|
||||
: error?.message || msg('Gemini voice could not start');
|
||||
this.lastError = detail;
|
||||
: error?.wording?.text ||
|
||||
error?.message ||
|
||||
msg('Gemini voice could not start');
|
||||
this.lastError = error?.wording ? error.message : detail;
|
||||
this.stop({ preserveStatus: true });
|
||||
this.setStatus('error', detail, recovery);
|
||||
this.setStatus('error', detail, recovery, {
|
||||
missingKey: error?.code === 'API_KEY_MISSING',
|
||||
params: error?.wording?.params,
|
||||
});
|
||||
}
|
||||
|
||||
abortTools() {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { geminiSocketUrl } from './geminiSession.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { createPointerTracker } from './pointerContext.js';
|
||||
import { withToolCatalog } from './gevRealtime.js';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
@@ -3638,3 +3639,73 @@ test('cancelling a queued globe reset also skips its dependent rotation', async
|
||||
h.c.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('a missing server key reaches the error state flagged, other token failures do not', async () => {
|
||||
for (const [body, missingKey] of [
|
||||
[
|
||||
{
|
||||
error: 'Add a Gemini API key in Provider Settings',
|
||||
code: 'API_KEY_MISSING',
|
||||
},
|
||||
true,
|
||||
],
|
||||
[{ error: 'Gemini quota is unavailable. Try again later.' }, undefined],
|
||||
]) {
|
||||
const h = harness({
|
||||
fetchImpl: async () => Response.json(body, { status: 503 }),
|
||||
});
|
||||
await h.c.start();
|
||||
const error = h.events
|
||||
.filter((event) => event.type === 'state' && event.state === 'error')
|
||||
.at(-1);
|
||||
assert.equal(error?.detail, body.error);
|
||||
assert.equal(error?.missingKey, missingKey);
|
||||
assert.equal(h.sockets.length, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('an error keeps its English catalog text beside the worded detail', async (t2) => {
|
||||
t2.after(() => setLocale('en'));
|
||||
setLocale('es');
|
||||
const h = harness({
|
||||
fetchImpl: async () =>
|
||||
Response.json(
|
||||
{ error: 'Gemini quota is unavailable. Try again later.' },
|
||||
{ status: 503 },
|
||||
),
|
||||
});
|
||||
await h.c.start();
|
||||
const error = h.events
|
||||
.filter((event) => event.type === 'state' && event.state === 'error')
|
||||
.at(-1);
|
||||
assert.equal(
|
||||
error.sourceDetail,
|
||||
'Gemini quota is unavailable. Try again later.',
|
||||
);
|
||||
assert.equal(
|
||||
error.detail,
|
||||
t('Gemini quota is unavailable. Try again later.'),
|
||||
);
|
||||
assert.notEqual(error.detail, error.sourceDetail);
|
||||
});
|
||||
|
||||
test('a token failure without a message keeps its status for the next language', async (t2) => {
|
||||
t2.after(() => setLocale('en'));
|
||||
setLocale('es');
|
||||
const h = harness({
|
||||
fetchImpl: async () => Response.json({}, { status: 502 }),
|
||||
});
|
||||
await h.c.start();
|
||||
const error = h.events
|
||||
.filter((event) => event.type === 'state' && event.state === 'error')
|
||||
.at(-1);
|
||||
assert.equal(
|
||||
error.sourceDetail,
|
||||
'Gemini token request failed (HTTP {status})',
|
||||
);
|
||||
assert.deepEqual(error.sourceParams, { status: 502 });
|
||||
assert.equal(
|
||||
error.detail,
|
||||
t('Gemini token request failed (HTTP {status})', { status: 502 }),
|
||||
);
|
||||
});
|
||||
+11
-4
@@ -2842,7 +2842,7 @@ async function trackEntity(
|
||||
() => {
|
||||
let trackedOk = false;
|
||||
if (family.kind === 'vessel') {
|
||||
trackedOk = !!module.selectById?.(found.mmsi);
|
||||
trackedOk = !!module.selectById?.(found.id ?? found.mmsi);
|
||||
flyToLandmark(viewer, found.latitude, found.longitude, {
|
||||
range: 6000,
|
||||
pitch: -45,
|
||||
@@ -4503,8 +4503,13 @@ function locateLayerEntity(dataManager, { layerId, id, lat, lon }) {
|
||||
?.module?.getAnalystRecords?.()
|
||||
.filter(
|
||||
(item) =>
|
||||
String(item.icao24 ?? item.mmsi ?? item.noradId ?? item.id) ===
|
||||
String(id),
|
||||
String(
|
||||
item.icao24 ??
|
||||
item.vesselId ??
|
||||
item.mmsi ??
|
||||
item.noradId ??
|
||||
item.id,
|
||||
) === String(id),
|
||||
) || [];
|
||||
// Bikeshare uses station names as IDs: retain the coordinate identity
|
||||
// even if a namesake disappears. Moving records use their fresh fix.
|
||||
@@ -5844,6 +5849,7 @@ function cameraViewBox(viewer) {
|
||||
/** Identity fields every compact analyst item keeps when present. */
|
||||
const ANALYST_IDENTITY_FIELDS = Object.freeze([
|
||||
'icao24',
|
||||
'vesselId',
|
||||
'mmsi',
|
||||
'registration',
|
||||
'label',
|
||||
@@ -5861,6 +5867,7 @@ const ANALYST_CANONICAL_FIELDS = new Set([
|
||||
'layerKey',
|
||||
'id',
|
||||
'icao24',
|
||||
'vesselId',
|
||||
'mmsi',
|
||||
'noradId',
|
||||
'registration',
|
||||
@@ -5873,7 +5880,7 @@ const ANALYST_CANONICAL_FIELDS = new Set([
|
||||
* follow-up can fly to it), distance, and that layer's manifest fields with
|
||||
* times as ISO text. The full record set stays engine-side for follow-ups.
|
||||
*
|
||||
* `icao24`/`mmsi` ride along because the tool instructions tell the model to
|
||||
* `icao24`/`vesselId`/`mmsi` ride along because the tool instructions tell the model to
|
||||
* hand this result straight to track_entity, and `id` is a DISPLAY label
|
||||
* (callsign, else registration, else hex). A callsign-less contact therefore
|
||||
* handed track_entity a tail number the lookup could not resolve, and the
|
||||
|
||||
@@ -35,6 +35,7 @@ import {
|
||||
} from './gevRealtime.js';
|
||||
import { createVoiceCostTracker } from './voiceCost.js';
|
||||
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
|
||||
test('push-to-talk recognizes Space by code or key', () => {
|
||||
assert.equal(PUSH_TO_TALK_HOLD_DELAY_MS, 500);
|
||||
@@ -3708,3 +3709,76 @@ test('voice runs app actions itself and other tools through the catalog', async
|
||||
assert.deepEqual(disposals, [['removed']]);
|
||||
assert.equal(withToolCatalog(runner, undefined), runner);
|
||||
});
|
||||
|
||||
test('the controller forwards viewport escalation diagnostics to its debug log', async () => {
|
||||
const records = [];
|
||||
const controller = new GevRealtimeController({
|
||||
ui: createTestVoiceUi(),
|
||||
runner: async () => ({ ok: true }),
|
||||
radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} },
|
||||
});
|
||||
controller.debugLog = (event, detail) => records.push({ event, detail });
|
||||
controller.dc = { readyState: 'open', send() {}, close() {} };
|
||||
|
||||
assert.equal(
|
||||
await controller.sendVisualContextIfUseful({
|
||||
action: 'get_entity_context',
|
||||
scene: { basemap: { viewScale: 'regional' } },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(
|
||||
records.filter((record) => record.event.startsWith('viewport_context.')),
|
||||
[{
|
||||
event: 'viewport_context.decision',
|
||||
detail: { contextId: 1, send: false, reason: 'view-scale-not-local', viewScale: 'regional' },
|
||||
}],
|
||||
);
|
||||
});
|
||||
|
||||
test('a missing server key says where to add it, without the microphone hint, in every language', () => {
|
||||
const ui = createTestVoiceUi();
|
||||
const controller = new GevRealtimeController({
|
||||
ui,
|
||||
runner: async () => ({ ok: true }),
|
||||
radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} },
|
||||
});
|
||||
try {
|
||||
controller.reportError(
|
||||
'Realtime connection',
|
||||
Object.assign(new Error('OPENAI_API_KEY is not set'), { code: 'API_KEY_MISSING' }),
|
||||
);
|
||||
assert.equal(ui.state.error.text, 'Needs OPENAI_API_KEY — add it in Provider Settings');
|
||||
assert.equal(ui.state.error.hint, false);
|
||||
|
||||
setLocale('es');
|
||||
assert.notEqual(ui.state.error.text, 'Needs OPENAI_API_KEY — add it in Provider Settings');
|
||||
assert.match(ui.state.error.text, /OPENAI_API_KEY/);
|
||||
assert.equal(ui.state.error.hint, false);
|
||||
setLocale('en');
|
||||
|
||||
controller.reportError('Realtime connection', new Error('ICE failed'));
|
||||
assert.match(ui.state.error.text, /ICE failed/);
|
||||
assert.equal(ui.state.error.hint, true);
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
|
||||
test('a status from a shared helper is worded again after a language change', () => {
|
||||
const ui = createTestVoiceUi();
|
||||
const controller = new GevRealtimeController({
|
||||
ui,
|
||||
runner: async () => ({ ok: true }),
|
||||
radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} },
|
||||
});
|
||||
try {
|
||||
controller.setStatus('listening', 'Radio did not start');
|
||||
assert.equal(ui.state.detail.text, 'Radio did not start');
|
||||
setLocale('es');
|
||||
assert.equal(ui.state.detail.text, t('Radio did not start'));
|
||||
assert.notEqual(ui.state.detail.text, 'Radio did not start');
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
@@ -462,6 +462,9 @@ export const VOICE_OFF_LAYERS = Object.freeze({
|
||||
// query for this layer wait until there is room.
|
||||
'drone-airspace':
|
||||
'informational airspace layer; no room in the prompt budget',
|
||||
// Same budget: a historical wreck archive waits for room too.
|
||||
shipwrecks: 'historical shipwreck archive; no room in the prompt budget',
|
||||
news: 'browsed in the News panel; voice tools are a later step',
|
||||
'wildlife-sightings':
|
||||
'community observation layer; no room in the prompt budget',
|
||||
});
|
||||
|
||||
@@ -245,6 +245,31 @@ test('P2-7: duplicate callsigns and names resolve by canonical identity', () =>
|
||||
);
|
||||
});
|
||||
|
||||
test('vessels sharing an MMSI resolve by their vessel id', () => {
|
||||
const referents = createReferentRegistry();
|
||||
referents.recordResult('analyst_query', {
|
||||
ok: true,
|
||||
items: ['hull-a', 'hull-b'].map((vesselId) => ({
|
||||
layerKey: 'ais-live-vessels',
|
||||
id: 'EVER GIVEN',
|
||||
vesselId,
|
||||
mmsi: '353136000',
|
||||
name: 'EVER GIVEN',
|
||||
})),
|
||||
});
|
||||
assert.deepEqual(
|
||||
[1, 2].map(
|
||||
(n) =>
|
||||
resolveDeicticArgs(
|
||||
'track_entity',
|
||||
{ query: 'x', referent: n },
|
||||
{ referents },
|
||||
).args.query,
|
||||
),
|
||||
['hull-a', 'hull-b'],
|
||||
);
|
||||
});
|
||||
|
||||
test('P1-1: a result from before the last clear cannot repopulate the registry', () => {
|
||||
const referents = createReferentRegistry();
|
||||
const since = referents.generation();
|
||||
|
||||
@@ -11,6 +11,7 @@ import { createGevActionRunner } from './gevActions.js';
|
||||
import { createPointerTracker } from './pointerContext.js';
|
||||
import { createReferentRegistry } from './referents.js';
|
||||
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
|
||||
const PROVIDER_KEY = 'gev.voice.provider';
|
||||
const settle = () => new Promise((resolve) => setImmediate(resolve));
|
||||
@@ -871,3 +872,83 @@ test('a provider switch hands chat Settings to the new mount', async (t) => {
|
||||
assert.deepEqual(f.mounts[1].options.presentation, shown);
|
||||
assert.equal(f.facade.provider, next);
|
||||
});
|
||||
|
||||
test('a missing-key error hides the microphone hint; other errors keep it', async (t) => {
|
||||
const f = compositionFixture(t, {});
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const instance = f.instances.at(-1);
|
||||
const ui = f.views.at(-1);
|
||||
instance.hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'Add a Gemini API key in Provider Settings',
|
||||
missingKey: true,
|
||||
});
|
||||
assert.equal(
|
||||
ui.state.error.text,
|
||||
'Add a Gemini API key in Provider Settings',
|
||||
);
|
||||
assert.equal(ui.state.error.hint, false);
|
||||
instance.hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'Socket closed',
|
||||
});
|
||||
assert.equal(ui.state.error.text, 'Socket closed');
|
||||
assert.equal(ui.state.error.hint, true);
|
||||
});
|
||||
|
||||
test('a provider message is worded again after a language change', async (t2) => {
|
||||
t2.after(() => setLocale('en'));
|
||||
const f = compositionFixture(t2, {});
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const instance = f.instances.at(-1);
|
||||
const ui = f.views.at(-1);
|
||||
const quota = 'Gemini quota is unavailable. Try again later.';
|
||||
instance.hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: quota,
|
||||
sourceDetail: quota,
|
||||
});
|
||||
assert.equal(ui.state.error.text, quota);
|
||||
setLocale('es');
|
||||
assert.equal(ui.state.error.text, t(quota));
|
||||
assert.notEqual(ui.state.error.text, quota);
|
||||
setLocale('en');
|
||||
instance.hooks.emit({
|
||||
type: 'state',
|
||||
state: 'listening',
|
||||
detail: 'Ask or command',
|
||||
sourceDetail: 'Ask or command',
|
||||
});
|
||||
setLocale('es');
|
||||
assert.equal(ui.state.detail.text, t('Ask or command'));
|
||||
assert.notEqual(ui.state.detail.text, 'Ask or command');
|
||||
});
|
||||
|
||||
test('a provider message with placeholders is worded again with them', async (t2) => {
|
||||
t2.after(() => setLocale('en'));
|
||||
const f = compositionFixture(t2, {});
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const ui = f.views.at(-1);
|
||||
const text = 'Gemini token request failed (HTTP {status})';
|
||||
f.instances.at(-1).hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: t(text, { status: 502 }),
|
||||
sourceDetail: text,
|
||||
sourceParams: { status: 502 },
|
||||
});
|
||||
assert.equal(ui.state.error.text, 'Gemini token request failed (HTTP 502)');
|
||||
setLocale('es');
|
||||
assert.equal(ui.state.error.text, t(text, { status: 502 }));
|
||||
assert.match(ui.state.error.text, /502/);
|
||||
assert.notEqual(
|
||||
ui.state.error.text,
|
||||
'Gemini token request failed (HTTP 502)',
|
||||
);
|
||||
});
|
||||
@@ -39,8 +39,9 @@ export function createRealtimeBackend({
|
||||
if (!response.ok) {
|
||||
const reason =
|
||||
typeof data?.error === 'string' ? data.error : data?.error?.message;
|
||||
throw new Error(
|
||||
reason || `Realtime token failed: HTTP ${response.status}`,
|
||||
throw Object.assign(
|
||||
new Error(reason || `Realtime token failed: HTTP ${response.status}`),
|
||||
typeof data?.code === 'string' ? { code: data.code } : {},
|
||||
);
|
||||
}
|
||||
const token =
|
||||
|
||||
@@ -143,3 +143,17 @@ test('controller lifetime stops pending transport and releases resources through
|
||||
assert.equal(controller.stream, null);
|
||||
assert.equal(controller.isActive(), false);
|
||||
});
|
||||
|
||||
test('a missing server key keeps its code, so the UI can point to Provider Settings', async () => {
|
||||
const backend = createRealtimeBackend({
|
||||
tokenTransport: async () =>
|
||||
Response.json(
|
||||
{ error: 'OPENAI_API_KEY is not set', code: 'API_KEY_MISSING' },
|
||||
{ status: 503 },
|
||||
),
|
||||
});
|
||||
await assert.rejects(backend.requestToken(), {
|
||||
message: 'OPENAI_API_KEY is not set',
|
||||
code: 'API_KEY_MISSING',
|
||||
});
|
||||
});
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
} from './pointerContext.js';
|
||||
|
||||
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
|
||||
import { postDebugLog } from './realtimeDiagnostics.js';
|
||||
import { missingVoiceKeyText, postDebugLog } from './realtimeDiagnostics.js';
|
||||
|
||||
export {
|
||||
readStoredVoiceTier,
|
||||
@@ -54,7 +54,7 @@ export {
|
||||
|
||||
import { createRealtimeBackend } from './realtimeBackend.js';
|
||||
|
||||
import { t } from '../i18n/index.js';
|
||||
import { t, translateKnown } from '../i18n/index.js';
|
||||
import { voiceStatusLabel } from './control.js';
|
||||
|
||||
// Machine states (dataset.status, CSS); voiceStatusLabel() translates labels.
|
||||
@@ -101,6 +101,7 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
|
||||
operations: {
|
||||
sendRealtimeEvent: (...args) => this.sendRealtimeEvent(...args),
|
||||
debugLog: (...args) => this.debugLog(...args),
|
||||
},
|
||||
});
|
||||
this._diagnostics = new RealtimeDiagnostics({
|
||||
@@ -406,9 +407,24 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
}
|
||||
}
|
||||
|
||||
setStatus(status, detail) {
|
||||
/**
|
||||
* @param {string} status
|
||||
* @param {string} [detail]
|
||||
* @param {{missingKey?: boolean}} [options] True when the error is a
|
||||
* missing server key: it is worded in the current language and shown
|
||||
* without the microphone hint.
|
||||
*/
|
||||
setStatus(status, detail, { missingKey = false } = {}) {
|
||||
this.status = status;
|
||||
this.emitSessionEvent({ type: 'state', state: status, detail });
|
||||
// A detail may be English catalog text (from the helpers this controller
|
||||
// shares with Gemini): it is worded in the current language when shown.
|
||||
this.emitSessionEvent({
|
||||
type: 'state',
|
||||
state: status,
|
||||
detail: translateKnown(detail),
|
||||
...(typeof detail === 'string' && { sourceDetail: detail }),
|
||||
...(missingKey && { missingKey: true }),
|
||||
});
|
||||
this.ui.update('root', {
|
||||
status,
|
||||
...(status === 'error' && { errorDismissed: false }),
|
||||
@@ -424,7 +440,7 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
? held
|
||||
? t('Release Space to send')
|
||||
: t('Hold Space to talk')
|
||||
: detail;
|
||||
: translateKnown(detail);
|
||||
const primaryDetail = () =>
|
||||
status === 'error'
|
||||
? t('VOICE UNAVAILABLE')
|
||||
@@ -434,8 +450,12 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
this.ui.update('error', {
|
||||
text: () =>
|
||||
status === 'error'
|
||||
? resolvedDetail() || t('Voice session could not be started.')
|
||||
? (missingKey && missingVoiceKeyText()) ||
|
||||
resolvedDetail() ||
|
||||
t('Voice session could not be started.')
|
||||
: '',
|
||||
// "Check microphone permission" is wrong advice for a missing key.
|
||||
hint: () => !(status === 'error' && missingKey),
|
||||
});
|
||||
if (status === 'idle' || status === 'connecting' || status === 'error') {
|
||||
this.setVoiceSpeaker('idle');
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
|
||||
/** The voice error when the server has no OpenAI key: where to add it. */
|
||||
export const missingVoiceKeyText = () =>
|
||||
t('Needs {envVars} — add it in Provider Settings', {
|
||||
envVars: 'OPENAI_API_KEY',
|
||||
});
|
||||
|
||||
const ERROR_LOG_LIMIT = 30;
|
||||
|
||||
const ERROR_STORAGE_KEY = 'gev-realtime-errors';
|
||||
@@ -83,6 +91,7 @@ function createErrorRecord(source, error, extra = {}) {
|
||||
String(error?.message || '').trim() ||
|
||||
'No browser error message supplied',
|
||||
errorDetail: rtcError?.errorDetail || null,
|
||||
code: typeof rtcError?.code === 'string' ? rtcError.code : null,
|
||||
sctpCauseCode: rtcError?.sctpCauseCode ?? null,
|
||||
receivedAlert: rtcError?.receivedAlert ?? null,
|
||||
sentAlert: rtcError?.sentAlert ?? null,
|
||||
@@ -91,6 +100,8 @@ function createErrorRecord(source, error, extra = {}) {
|
||||
}
|
||||
|
||||
function formatErrorForDisplay(record) {
|
||||
// A missing key is a setup step, not a connection fault: say where to add it.
|
||||
if (record.code === 'API_KEY_MISSING') return missingVoiceKeyText();
|
||||
const primary = [record.source, record.message].filter(Boolean).join(': ');
|
||||
const state = [
|
||||
record.errorDetail && `detail=${record.errorDetail}`,
|
||||
@@ -172,7 +183,11 @@ export class RealtimeDiagnostics {
|
||||
storeErrors(this.errors);
|
||||
console.error('[GEV Realtime]', record);
|
||||
this.debugLog('error', record);
|
||||
this.setStatus('error', formatErrorForDisplay(record));
|
||||
this.setStatus(
|
||||
'error',
|
||||
formatErrorForDisplay(record),
|
||||
record.code === 'API_KEY_MISSING' ? { missingKey: true } : undefined,
|
||||
);
|
||||
return record;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import { msg, t } from '../i18n/index.js';
|
||||
import {
|
||||
PUSH_TO_TALK_HOLD_DELAY_MS,
|
||||
MICROPHONE_VISUALIZER_GATE,
|
||||
@@ -124,7 +124,7 @@ export class RealtimeInput {
|
||||
this.ui.update('root', { pushToTalk: 'held' });
|
||||
this.setMicrophoneEnabled(true);
|
||||
if (this.status === 'listening')
|
||||
this.setStatus('listening', t('Release Space to send'));
|
||||
this.setStatus('listening', msg('Release Space to send'));
|
||||
} else {
|
||||
this.start({ pushToTalk: true });
|
||||
// start() performs a controlled stop() before connecting. Restore the
|
||||
@@ -201,7 +201,7 @@ export class RealtimeInput {
|
||||
if (!this.pushToTalkMode) return;
|
||||
this.setMicrophoneEnabled(false);
|
||||
if (this.status === 'listening')
|
||||
this.setStatus('listening', t('Hold Space to talk'));
|
||||
this.setStatus('listening', msg('Hold Space to talk'));
|
||||
else this.updateVoiceButtonLabel();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import { msg } from '../i18n/index.js';
|
||||
import {
|
||||
silenceRadioForVoice,
|
||||
startPreparedRadioAfterPlaybackReady,
|
||||
@@ -261,7 +261,7 @@ export class RealtimeRadio {
|
||||
if (radioHandoff.result?.ok || radioHandoff.cancelled || !stillCurrent)
|
||||
return;
|
||||
if (this.dc?.readyState === 'open' && !this.userTurnPending) {
|
||||
this.setStatus('listening', t('Radio did not start'));
|
||||
this.setStatus('listening', msg('Radio did not start'));
|
||||
this.queueResponseCreate(
|
||||
'Say exactly one short correction: “The Radio station could not start. Voice is still on.”',
|
||||
);
|
||||
|
||||
+140
-32
@@ -72,6 +72,43 @@ const POINTER_CROP_MAX_AGE_MS = 20000;
|
||||
/** View scales where a crop shows buildings and terrain, not a smear. */
|
||||
const POINTER_CROP_SCALES = new Set(['local', 'city']);
|
||||
|
||||
/**
|
||||
* Reason-coded explanation of the existing visual escalation policy, for an
|
||||
* entity-context answer on an open channel (the caller checks both).
|
||||
* Pointer crops remain a separate path from full-frame screenshots.
|
||||
* These reasons are static diagnostics, never model-generated claims.
|
||||
*/
|
||||
export function viewportContextDecision(result) {
|
||||
const viewScale = result?.scene?.basemap?.viewScale ?? null;
|
||||
if (result?.scope === 'pointer') {
|
||||
if (result.selected)
|
||||
return { send: false, reason: 'pointer-identity-present', viewScale };
|
||||
if (!result.pointer)
|
||||
return { send: false, reason: 'pointer-missing', viewScale };
|
||||
// The crop path checks ground, freshness and scale itself, and logs why.
|
||||
return {
|
||||
send: true,
|
||||
kind: 'crop',
|
||||
reason: 'pointer-crop-eligible',
|
||||
viewScale,
|
||||
};
|
||||
}
|
||||
if (!shouldSendViewportImage(viewScale))
|
||||
return { send: false, reason: 'view-scale-not-local', viewScale };
|
||||
if (hasStructuredViewIdentity(result))
|
||||
return {
|
||||
send: false,
|
||||
reason: 'structured-identity-sufficient',
|
||||
viewScale,
|
||||
};
|
||||
return {
|
||||
send: true,
|
||||
kind: 'viewport',
|
||||
reason: 'structured-identity-missing',
|
||||
viewScale,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Square source rectangle around a canvas point, clamped inside the canvas,
|
||||
* in drawing-buffer pixels, plus where the point lands in the output image.
|
||||
@@ -299,6 +336,16 @@ export function isBenignViewportDeleteError(payload, pendingDeleteIds = null) {
|
||||
return code === 'item_not_found';
|
||||
}
|
||||
|
||||
/** Pairs one visual-context request's outcome with its decision. */
|
||||
function diagnosticsRecord(contextId = null) {
|
||||
return { contextId, logged: false };
|
||||
}
|
||||
|
||||
function reportOutcome(owner, diag, event, kind, reason) {
|
||||
diag.logged = true;
|
||||
owner.debugLog?.(event, { contextId: diag.contextId, kind, reason });
|
||||
}
|
||||
|
||||
/** Own the one retained viewport image and bounded pending deletion identities. */
|
||||
export class RealtimeViewport {
|
||||
constructor({
|
||||
@@ -319,6 +366,8 @@ export class RealtimeViewport {
|
||||
// Bumped by every capture request and every new user turn: only the
|
||||
// newest request of the current turn may publish.
|
||||
this.captureSeq = 0;
|
||||
// Diagnostics only: pairs each visual-context decision with its outcome.
|
||||
this.contextSeq = 0;
|
||||
this.lastPointerCrop = null;
|
||||
// What the retained image shows: `{kind:'viewport'|'crop', rect, cameraKey}`.
|
||||
// Pixel targeting (screenX/screenY) is read against it.
|
||||
@@ -330,28 +379,51 @@ export class RealtimeViewport {
|
||||
return this.readChannel();
|
||||
}
|
||||
|
||||
/**
|
||||
* Escalate an entity-context answer to an image when the gate allows it.
|
||||
* Logs one `viewport_context.decision`, then, for a send decision, exactly
|
||||
* one `.skipped` or `.result` outcome, even when a step throws. Every
|
||||
* event carries the same `contextId` and the image `kind`, so overlapping
|
||||
* requests pair up.
|
||||
*/
|
||||
async sendVisualContextIfUseful(result) {
|
||||
// Runs after every tool output. Only entity context on an open channel
|
||||
// can escalate, so anything else logs nothing and leaves the shared
|
||||
// debug-log budget alone.
|
||||
if (
|
||||
result?.action !== 'get_entity_context' ||
|
||||
!this.dc ||
|
||||
this.dc.readyState !== 'open'
|
||||
)
|
||||
return false;
|
||||
const viewScale = result.scene?.basemap?.viewScale;
|
||||
// Point-and-ask on bare ground: the crop around the pointer replaces the
|
||||
// whole-frame screenshot, and is pulled only when nothing was named.
|
||||
if (result.scope === 'pointer') {
|
||||
if (result.selected || !result.pointer) return false;
|
||||
return this.sendPointerCrop(this.readPointer(), { viewScale });
|
||||
const contextId = ++this.contextSeq;
|
||||
const decision = viewportContextDecision(result);
|
||||
this.debugLog?.('viewport_context.decision', { contextId, ...decision });
|
||||
if (!decision.send) return false;
|
||||
const diag = diagnosticsRecord(contextId);
|
||||
try {
|
||||
// #837's pointer crop is separate from the full viewport: preserve it.
|
||||
if (decision.kind === 'crop')
|
||||
return await this.sendPointerCrop(this.readPointer(), {
|
||||
viewScale: decision.viewScale,
|
||||
diag,
|
||||
});
|
||||
return await this.sendRetainedImage(
|
||||
() => this.capture(),
|
||||
"Current God's Eye View viewport screenshot. Read any clearly visible street, building, and place labels in the image and combine them with the structured nearbyPlaces, streetLabels, and scene context. Do not invent labels that are not legible.",
|
||||
'client.viewport_context',
|
||||
{ kind: 'viewport', diag },
|
||||
);
|
||||
} catch (error) {
|
||||
// The caller logs the error itself; close the pair, then rethrow.
|
||||
if (!diag.logged)
|
||||
this.debugLog?.('viewport_context.skipped', {
|
||||
contextId,
|
||||
kind: decision.kind,
|
||||
reason: 'error',
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
if (!shouldSendViewportImage(viewScale)) return false;
|
||||
if (hasStructuredViewIdentity(result)) return false;
|
||||
return this.sendRetainedImage(
|
||||
() => this.capture(),
|
||||
"Current God's Eye View viewport screenshot. Read any clearly visible street, building, and place labels in the image and combine them with the structured nearbyPlaces, streetLabels, and scene context. Do not invent labels that are not legible.",
|
||||
'client.viewport_context',
|
||||
{ kind: 'viewport' },
|
||||
);
|
||||
}
|
||||
|
||||
/** A new user turn: captures still in flight from the last one are dropped. */
|
||||
@@ -365,27 +437,39 @@ export class RealtimeViewport {
|
||||
* @param {object|null} snapshot Fresh pointer snapshot on bare ground.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async sendPointerCrop(snapshot, { viewScale } = {}) {
|
||||
if (!snapshot?.fresh || snapshot.target !== 'ground' || !snapshot.screenPx)
|
||||
async sendPointerCrop(
|
||||
snapshot,
|
||||
{ viewScale, diag = diagnosticsRecord() } = {},
|
||||
) {
|
||||
const skip = (reason) => {
|
||||
reportOutcome(this, diag, 'viewport_context.skipped', 'crop', reason);
|
||||
return false;
|
||||
};
|
||||
if (!snapshot?.fresh) return skip('pointer-not-fresh');
|
||||
if (snapshot.target !== 'ground') return skip('pointer-not-ground');
|
||||
if (!snapshot.screenPx) return skip('pointer-off-screen');
|
||||
// Too far out, the crop is a smear: every path shares this guard.
|
||||
if (!POINTER_CROP_SCALES.has(viewScale)) return false;
|
||||
if (snapshot === this.lastPointerCrop) return false;
|
||||
if (!POINTER_CROP_SCALES.has(viewScale))
|
||||
return skip('pointer-scale-not-useful');
|
||||
// Set before the capture finishes, so this also covers one in flight.
|
||||
if (snapshot === this.lastPointerCrop)
|
||||
return skip('pointer-already-requested');
|
||||
const capturedAt = Number(snapshot.capturedAt) || 0;
|
||||
if (this.now() - capturedAt > POINTER_CROP_MAX_AGE_MS) return false;
|
||||
if (this.now() - capturedAt > POINTER_CROP_MAX_AGE_MS)
|
||||
return skip('pointer-expired');
|
||||
// The snapshot's pixels only name its own camera pose.
|
||||
const snapshotCameraKey = snapshot.cameraKey;
|
||||
if (!snapshotCameraKey) return false;
|
||||
if (!snapshotCameraKey) return skip('pointer-pose-unknown');
|
||||
const samePose = () =>
|
||||
snapshotCameraKey === this.readCameraKey() &&
|
||||
this.now() - capturedAt <= POINTER_CROP_MAX_AGE_MS;
|
||||
if (!samePose()) return false;
|
||||
if (!samePose()) return skip('pose-changed');
|
||||
this.lastPointerCrop = snapshot;
|
||||
const sent = await this.sendRetainedImage(
|
||||
() => this.capturePointer(snapshot.screenPx),
|
||||
"Crop of the map around the user's pointer; the ring marks the exact spot they mean by this or here. Describe only what is clearly visible.",
|
||||
'client.pointer_crop',
|
||||
{ kind: 'crop', valid: samePose },
|
||||
{ kind: 'crop', valid: samePose, diag },
|
||||
);
|
||||
if (!sent && this.lastPointerCrop === snapshot) this.lastPointerCrop = null;
|
||||
return sent;
|
||||
@@ -393,22 +477,40 @@ export class RealtimeViewport {
|
||||
|
||||
/** Capture one image and make it the single retained image in context. */
|
||||
async sendRetainedImage(captureImage, text, label, frame = {}) {
|
||||
if (!this.dc || this.dc.readyState !== 'open') return false;
|
||||
const kind = frame.kind || 'viewport';
|
||||
const diag = frame.diag || diagnosticsRecord();
|
||||
const report = (event, reason) =>
|
||||
reportOutcome(this, diag, event, kind, reason);
|
||||
if (!this.dc || this.dc.readyState !== 'open') {
|
||||
report('viewport_context.skipped', 'channel-lost');
|
||||
return false;
|
||||
}
|
||||
const generation = this.generation;
|
||||
const request = ++this.captureSeq;
|
||||
const channel = this.dc;
|
||||
const captured = await captureImage();
|
||||
const imageUrl =
|
||||
typeof captured === 'string' ? captured : captured?.dataUrl || null;
|
||||
if (
|
||||
!imageUrl ||
|
||||
generation !== this.generation ||
|
||||
request !== this.captureSeq ||
|
||||
this.dc !== channel ||
|
||||
channel.readyState !== 'open' ||
|
||||
(typeof frame.valid === 'function' && !frame.valid())
|
||||
)
|
||||
if (!imageUrl) {
|
||||
report('viewport_context.skipped', 'capture-unavailable');
|
||||
return false;
|
||||
}
|
||||
// Same guards as before, checked in order so each drop names its cause:
|
||||
// reset() bumps both counters, so a session reset is tested first.
|
||||
const invalidated =
|
||||
generation !== this.generation
|
||||
? 'session-reset'
|
||||
: request !== this.captureSeq
|
||||
? 'superseded'
|
||||
: this.dc !== channel || channel.readyState !== 'open'
|
||||
? 'channel-lost'
|
||||
: typeof frame.valid === 'function' && !frame.valid()
|
||||
? 'frame-stale'
|
||||
: null;
|
||||
if (invalidated) {
|
||||
report('viewport_context.skipped', invalidated);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Keep at most one viewport screenshot in context. Images are the single
|
||||
// most expensive item (re-billed every turn they linger), so we proactively
|
||||
@@ -461,10 +563,16 @@ export class RealtimeViewport {
|
||||
// the item we just deleted as null and fall through so the caller still
|
||||
// issues queueResponseCreate WITHOUT the image, instead of stranding the turn.
|
||||
const sent = this.sendRealtimeEvent(contextEvent, label);
|
||||
// Report before the bookkeeping below, so a later throw cannot turn a
|
||||
// real send into a logged "error".
|
||||
report(
|
||||
'viewport_context.result',
|
||||
sent ? 'sent' : 'data-channel-send-failed',
|
||||
);
|
||||
this.lastViewportItemId = sent ? newItemId : null;
|
||||
this.retainedFrame = sent
|
||||
? {
|
||||
kind: frame.kind || 'viewport',
|
||||
kind,
|
||||
rect: captured?.rect || null,
|
||||
cameraKey: this.readCameraKey(),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
RealtimeViewport,
|
||||
viewportContextDecision,
|
||||
} from './realtimeViewport.js';
|
||||
|
||||
const localContext = (overrides = {}) => ({
|
||||
action: 'get_entity_context',
|
||||
scene: { basemap: { viewScale: 'local' } },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
test('viewport context reasons preserve the existing escalation gate', () => {
|
||||
assert.deepEqual(
|
||||
viewportContextDecision({
|
||||
action: 'get_entity_context',
|
||||
scene: { basemap: { viewScale: 'regional' } },
|
||||
}),
|
||||
{
|
||||
send: false,
|
||||
reason: 'view-scale-not-local',
|
||||
viewScale: 'regional',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
viewportContextDecision(
|
||||
localContext({
|
||||
selected: {
|
||||
id: 'earthquake:1',
|
||||
layerId: 'earthquakes',
|
||||
},
|
||||
}),
|
||||
),
|
||||
{
|
||||
send: false,
|
||||
reason: 'structured-identity-sufficient',
|
||||
viewScale: 'local',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(viewportContextDecision(localContext()), {
|
||||
send: true,
|
||||
kind: 'viewport',
|
||||
reason: 'structured-identity-missing',
|
||||
viewScale: 'local',
|
||||
});
|
||||
});
|
||||
|
||||
test('nearby structured place identity avoids visual escalation too', () => {
|
||||
assert.deepEqual(
|
||||
viewportContextDecision(
|
||||
localContext({
|
||||
scene: {
|
||||
basemap: {
|
||||
viewScale: 'local',
|
||||
nearbyPlaces: [{ name: 'Austin' }],
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
{
|
||||
send: false,
|
||||
reason: 'structured-identity-sufficient',
|
||||
viewScale: 'local',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('pointer ground selects the #837 crop path, not a full viewport', () => {
|
||||
const pointerContext = localContext({
|
||||
scope: 'pointer',
|
||||
pointer: { target: 'ground' },
|
||||
scene: { basemap: { viewScale: 'city' } },
|
||||
});
|
||||
assert.deepEqual(viewportContextDecision(pointerContext), {
|
||||
send: true,
|
||||
kind: 'crop',
|
||||
reason: 'pointer-crop-eligible',
|
||||
viewScale: 'city',
|
||||
});
|
||||
assert.deepEqual(
|
||||
viewportContextDecision({
|
||||
...pointerContext,
|
||||
scene: { basemap: { viewScale: 'global' } },
|
||||
}),
|
||||
{
|
||||
send: true,
|
||||
kind: 'crop',
|
||||
reason: 'pointer-crop-eligible',
|
||||
viewScale: 'global',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
viewportContextDecision({
|
||||
...pointerContext,
|
||||
selected: { id: 'building' },
|
||||
}),
|
||||
{
|
||||
send: false,
|
||||
reason: 'pointer-identity-present',
|
||||
viewScale: 'city',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
viewportContextDecision({ ...pointerContext, pointer: null }),
|
||||
{ send: false, reason: 'pointer-missing', viewScale: 'city' },
|
||||
);
|
||||
});
|
||||
|
||||
test('runtime dispatch preserves pointer crop and full image separation', async () => {
|
||||
const channel = { readyState: 'open' };
|
||||
const pointer = { fresh: true, target: 'ground' };
|
||||
const calls = [];
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
readPointer: () => pointer,
|
||||
operations: {
|
||||
debugLog: (event, detail) => calls.push({ event, detail }),
|
||||
},
|
||||
capture: async () => 'unused',
|
||||
});
|
||||
viewport.sendPointerCrop = async (snapshot, options) => {
|
||||
calls.push({ kind: 'crop', snapshot, options });
|
||||
return true;
|
||||
};
|
||||
viewport.sendRetainedImage = async () => {
|
||||
calls.push({ kind: 'viewport' });
|
||||
return true;
|
||||
};
|
||||
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(
|
||||
localContext({
|
||||
scope: 'pointer',
|
||||
pointer: { target: 'ground' },
|
||||
scene: { basemap: { viewScale: 'city' } },
|
||||
}),
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(calls.filter((call) => call.kind === 'crop').length, 1);
|
||||
assert.deepEqual(
|
||||
calls.find((call) => call.kind === 'crop').snapshot,
|
||||
pointer,
|
||||
);
|
||||
assert.equal(
|
||||
calls.find((call) => call.kind === 'crop').options.viewScale,
|
||||
'city',
|
||||
);
|
||||
assert.equal(calls.filter((call) => call.kind === 'viewport').length, 0);
|
||||
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(localContext()), true);
|
||||
assert.equal(calls.filter((call) => call.kind === 'viewport').length, 1);
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(
|
||||
localContext({ selected: { id: 'existing' } }),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(calls.filter((call) => call.kind === 'viewport').length, 1);
|
||||
assert.ok(
|
||||
calls.some(
|
||||
(call) =>
|
||||
call.event === 'viewport_context.decision' &&
|
||||
call.detail.reason === 'pointer-crop-eligible',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('closed channel never captures pointer or full viewport', async () => {
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => ({ readyState: 'closed' }),
|
||||
readPointer: () => {
|
||||
throw new Error('should not inspect pointer when channel is closed');
|
||||
},
|
||||
operations: {},
|
||||
});
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(
|
||||
localContext({ scope: 'pointer', pointer: { target: 'ground' } }),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(localContext()), false);
|
||||
});
|
||||
|
||||
test('capture failures and invalidated turns are reason-coded without sending', async () => {
|
||||
const channel = { readyState: 'open' };
|
||||
const diagnostics = [];
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
operations: {
|
||||
sendRealtimeEvent: () => {
|
||||
throw new Error('should not send unavailable image');
|
||||
},
|
||||
debugLog: (event, detail) => diagnostics.push({ event, detail }),
|
||||
},
|
||||
capture: async () => null,
|
||||
});
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(localContext()), false);
|
||||
assert.ok(
|
||||
diagnostics.some(
|
||||
(item) =>
|
||||
item.event === 'viewport_context.skipped' &&
|
||||
item.detail.reason === 'capture-unavailable',
|
||||
),
|
||||
);
|
||||
let release;
|
||||
viewport.capture = () =>
|
||||
new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const pending = viewport.sendVisualContextIfUseful(localContext());
|
||||
viewport.reset();
|
||||
release('data:image/jpeg;base64,AA==');
|
||||
assert.equal(await pending, false);
|
||||
assert.ok(
|
||||
diagnostics.some(
|
||||
(item) =>
|
||||
item.event === 'viewport_context.skipped' &&
|
||||
item.detail.reason === 'session-reset',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('successful retained send is logged without recording image contents', async () => {
|
||||
const channel = { readyState: 'open' };
|
||||
const diagnostics = [];
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
readCameraKey: () => 'camera:fixture',
|
||||
operations: {
|
||||
sendRealtimeEvent: () => true,
|
||||
debugLog: (event, detail) => diagnostics.push({ event, detail }),
|
||||
},
|
||||
capture: async () => 'data:image/jpeg;base64,AA==',
|
||||
});
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(localContext()), true);
|
||||
assert.ok(
|
||||
diagnostics.some(
|
||||
(item) =>
|
||||
item.event === 'viewport_context.result' &&
|
||||
item.detail.reason === 'sent',
|
||||
),
|
||||
);
|
||||
assert.equal(JSON.stringify(diagnostics).includes('data:image/jpeg'), false);
|
||||
});
|
||||
|
||||
const outcomeEvents = new Set([
|
||||
'viewport_context.skipped',
|
||||
'viewport_context.result',
|
||||
]);
|
||||
|
||||
function recordingViewport(options = {}) {
|
||||
const events = [];
|
||||
const channel = options.channel ?? { readyState: 'open' };
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
readCameraKey: () => 'camera:fixture',
|
||||
now: () => 1000,
|
||||
capturePointer: async () => ({
|
||||
dataUrl: 'data:image/jpeg;base64,AA==',
|
||||
rect: { x: 0, y: 0, width: 1, height: 1 },
|
||||
}),
|
||||
capture: async () => 'data:image/jpeg;base64,AA==',
|
||||
...options,
|
||||
operations: {
|
||||
sendRealtimeEvent: () => true,
|
||||
debugLog: (event, detail) => events.push({ event, detail }),
|
||||
...options.operations,
|
||||
},
|
||||
});
|
||||
return { viewport, events };
|
||||
}
|
||||
|
||||
const pointerGround = (overrides = {}) =>
|
||||
localContext({
|
||||
scope: 'pointer',
|
||||
pointer: { target: 'ground' },
|
||||
scene: { basemap: { viewScale: 'city' } },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const groundSnapshot = (overrides = {}) => ({
|
||||
fresh: true,
|
||||
target: 'ground',
|
||||
screenPx: { x: 10, y: 10 },
|
||||
capturedAt: 900,
|
||||
cameraKey: 'camera:fixture',
|
||||
...overrides,
|
||||
});
|
||||
|
||||
test('tool results other than entity context log nothing', async () => {
|
||||
const { viewport, events } = recordingViewport();
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful({ action: 'fly_to' }),
|
||||
false,
|
||||
);
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(null), false);
|
||||
assert.deepEqual(events, []);
|
||||
});
|
||||
|
||||
test('a crop decision that cannot send logs why, under the same id', async () => {
|
||||
const cases = [
|
||||
[{ fresh: false }, 'pointer-not-fresh'],
|
||||
[{ target: 'building' }, 'pointer-not-ground'],
|
||||
[{ screenPx: null }, 'pointer-off-screen'],
|
||||
[{ capturedAt: -50000 }, 'pointer-expired'],
|
||||
[{ cameraKey: null }, 'pointer-pose-unknown'],
|
||||
[{ cameraKey: 'camera:moved' }, 'pose-changed'],
|
||||
];
|
||||
for (const [override, reason] of cases) {
|
||||
const snapshot = groundSnapshot(override);
|
||||
const { viewport, events } = recordingViewport({
|
||||
readPointer: () => snapshot,
|
||||
});
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(pointerGround()),
|
||||
false,
|
||||
reason,
|
||||
);
|
||||
assert.equal(events.length, 2, reason);
|
||||
const [decision, outcome] = events;
|
||||
assert.equal(decision.event, 'viewport_context.decision');
|
||||
assert.equal(decision.detail.reason, 'pointer-crop-eligible');
|
||||
assert.deepEqual(outcome, {
|
||||
event: 'viewport_context.skipped',
|
||||
detail: { contextId: decision.detail.contextId, kind: 'crop', reason },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('a repeated pointer snapshot is logged as already requested', async () => {
|
||||
const snapshot = groundSnapshot();
|
||||
const { viewport, events } = recordingViewport({
|
||||
readPointer: () => snapshot,
|
||||
});
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(pointerGround()), true);
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(pointerGround()),
|
||||
false,
|
||||
);
|
||||
const outcomes = events.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.deepEqual(
|
||||
outcomes.map((item) => [item.detail.kind, item.detail.reason]),
|
||||
[
|
||||
['crop', 'sent'],
|
||||
['crop', 'pointer-already-requested'],
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('overlapping requests pair each outcome with its own decision', async () => {
|
||||
const releases = [];
|
||||
const { viewport, events } = recordingViewport({
|
||||
capture: () => new Promise((resolve) => releases.push(resolve)),
|
||||
});
|
||||
const first = viewport.sendVisualContextIfUseful(localContext());
|
||||
const second = viewport.sendVisualContextIfUseful(localContext());
|
||||
await Promise.resolve();
|
||||
releases[1]('data:image/jpeg;base64,AA==');
|
||||
releases[0]('data:image/jpeg;base64,AA==');
|
||||
assert.equal(await first, false);
|
||||
assert.equal(await second, true);
|
||||
const [firstId, secondId] = events
|
||||
.filter((item) => item.event === 'viewport_context.decision')
|
||||
.map((item) => item.detail.contextId);
|
||||
assert.notEqual(firstId, secondId);
|
||||
const outcomeFor = (id) =>
|
||||
events.filter(
|
||||
(item) => outcomeEvents.has(item.event) && item.detail.contextId === id,
|
||||
);
|
||||
assert.deepEqual(
|
||||
outcomeFor(firstId).map((item) => item.detail),
|
||||
[{ contextId: firstId, kind: 'viewport', reason: 'superseded' }],
|
||||
);
|
||||
assert.deepEqual(
|
||||
outcomeFor(secondId).map((item) => item.detail),
|
||||
[{ contextId: secondId, kind: 'viewport', reason: 'sent' }],
|
||||
);
|
||||
});
|
||||
|
||||
test('dropped captures name their cause', async () => {
|
||||
const reasonFor = async (context, interrupt, extra = {}) => {
|
||||
let release;
|
||||
const pending = () => new Promise((resolve) => (release = resolve));
|
||||
const { viewport, events } = recordingViewport({
|
||||
capture: pending,
|
||||
capturePointer: pending,
|
||||
...extra,
|
||||
});
|
||||
const sent = viewport.sendVisualContextIfUseful(context);
|
||||
await Promise.resolve();
|
||||
interrupt(viewport);
|
||||
release('data:image/jpeg;base64,AA==');
|
||||
assert.equal(await sent, false);
|
||||
return events
|
||||
.filter((item) => outcomeEvents.has(item.event))
|
||||
.map((item) => item.detail.reason);
|
||||
};
|
||||
|
||||
assert.deepEqual(
|
||||
await reasonFor(localContext(), (viewport) => viewport.beginTurn()),
|
||||
['superseded'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
await reasonFor(localContext(), (viewport) => viewport.reset()),
|
||||
['session-reset'],
|
||||
);
|
||||
const channel = { readyState: 'open' };
|
||||
assert.deepEqual(
|
||||
await reasonFor(localContext(), () => (channel.readyState = 'closed'), {
|
||||
channel,
|
||||
}),
|
||||
['channel-lost'],
|
||||
);
|
||||
let cameraKey = 'camera:fixture';
|
||||
const snapshot = groundSnapshot();
|
||||
assert.deepEqual(
|
||||
await reasonFor(pointerGround(), () => (cameraKey = 'camera:moved'), {
|
||||
readPointer: () => snapshot,
|
||||
readCameraKey: () => cameraKey,
|
||||
}),
|
||||
['frame-stale'],
|
||||
);
|
||||
});
|
||||
|
||||
test('every send decision is followed by exactly one outcome', async () => {
|
||||
const snapshots = [
|
||||
null,
|
||||
groundSnapshot(),
|
||||
groundSnapshot({ fresh: false }),
|
||||
groundSnapshot({ cameraKey: 'camera:moved' }),
|
||||
groundSnapshot({ capturedAt: -50000 }),
|
||||
];
|
||||
const contexts = [
|
||||
localContext(),
|
||||
localContext({ selected: { id: 'x', layerId: 'y' } }),
|
||||
pointerGround(),
|
||||
pointerGround({ scene: { basemap: { viewScale: 'global' } } }),
|
||||
pointerGround({ pointer: null }),
|
||||
];
|
||||
const captures = [
|
||||
async () => 'data:image/jpeg;base64,AA==',
|
||||
async () => null,
|
||||
];
|
||||
for (const snapshot of snapshots)
|
||||
for (const context of contexts)
|
||||
for (const capture of captures)
|
||||
for (const sendResult of [true, false]) {
|
||||
const { viewport, events } = recordingViewport({
|
||||
readPointer: () => snapshot,
|
||||
capture,
|
||||
capturePointer: capture,
|
||||
operations: { sendRealtimeEvent: () => sendResult },
|
||||
});
|
||||
await viewport.sendVisualContextIfUseful(context);
|
||||
const [decision, ...rest] = events;
|
||||
assert.equal(decision.event, 'viewport_context.decision');
|
||||
const outcomes = rest.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.equal(outcomes.length, decision.detail.send ? 1 : 0);
|
||||
for (const outcome of outcomes) {
|
||||
assert.equal(outcome.detail.contextId, decision.detail.contextId);
|
||||
assert.equal(outcome.detail.kind, decision.detail.kind);
|
||||
if (outcome.event === 'viewport_context.result')
|
||||
assert.equal(
|
||||
outcome.detail.reason,
|
||||
sendResult ? 'sent' : 'data-channel-send-failed',
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('a failed data-channel send is never logged as sent', async () => {
|
||||
for (const [sendResult, reason] of [
|
||||
[true, 'sent'],
|
||||
[false, 'data-channel-send-failed'],
|
||||
]) {
|
||||
const { viewport, events } = recordingViewport({
|
||||
operations: { sendRealtimeEvent: () => sendResult },
|
||||
});
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful(localContext()),
|
||||
sendResult,
|
||||
);
|
||||
const outcomes = events.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.deepEqual(
|
||||
outcomes.map((item) => [item.event, item.detail.reason]),
|
||||
[['viewport_context.result', reason]],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('a step that throws still closes its decision, then rethrows', async () => {
|
||||
for (const [context, extra] of [
|
||||
[
|
||||
localContext(),
|
||||
{
|
||||
capture: async () => {
|
||||
throw new Error('capture boom');
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
pointerGround(),
|
||||
{
|
||||
readPointer: () => {
|
||||
throw new Error('pointer boom');
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
pointerGround(),
|
||||
{
|
||||
readPointer: () => groundSnapshot(),
|
||||
capturePointer: async () => {
|
||||
throw new Error('crop boom');
|
||||
},
|
||||
},
|
||||
],
|
||||
]) {
|
||||
const { viewport, events } = recordingViewport(extra);
|
||||
await assert.rejects(viewport.sendVisualContextIfUseful(context), /boom/);
|
||||
const [decision, ...rest] = events;
|
||||
assert.deepEqual(rest, [
|
||||
{
|
||||
event: 'viewport_context.skipped',
|
||||
detail: {
|
||||
contextId: decision.detail.contextId,
|
||||
kind: decision.detail.kind,
|
||||
reason: 'error',
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
test('direct calls log their defensive skips without a decision id', async () => {
|
||||
const open = recordingViewport();
|
||||
assert.equal(
|
||||
await open.viewport.sendPointerCrop(groundSnapshot(), {
|
||||
viewScale: 'global',
|
||||
}),
|
||||
false,
|
||||
);
|
||||
const closed = recordingViewport({ channel: { readyState: 'closed' } });
|
||||
assert.equal(
|
||||
await closed.viewport.sendPointerCrop(groundSnapshot(), {
|
||||
viewScale: 'city',
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
await closed.viewport.sendRetainedImage(
|
||||
async () => 'data:image/jpeg;base64,AA==',
|
||||
'text',
|
||||
'client.viewport_context',
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(
|
||||
[...open.events, ...closed.events].map((item) => item.detail),
|
||||
[
|
||||
{ contextId: null, kind: 'crop', reason: 'pointer-scale-not-useful' },
|
||||
{ contextId: null, kind: 'crop', reason: 'channel-lost' },
|
||||
{ contextId: null, kind: 'viewport', reason: 'channel-lost' },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('a throw after a real send still logs it as sent', async () => {
|
||||
let reads = 0;
|
||||
const { viewport, events } = recordingViewport({
|
||||
// The first read happens only after the image was handed to the channel.
|
||||
readCameraKey: () => {
|
||||
reads += 1;
|
||||
throw new Error('camera gone');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
viewport.sendVisualContextIfUseful(localContext()),
|
||||
/camera gone/,
|
||||
);
|
||||
assert.equal(reads, 1);
|
||||
const outcomes = events.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.deepEqual(
|
||||
outcomes.map((item) => [item.event, item.detail.reason]),
|
||||
[['viewport_context.result', 'sent']],
|
||||
);
|
||||
});
|
||||
|
||||
test('a second request for a snapshot still being captured is not sent twice', async () => {
|
||||
const releases = [];
|
||||
const snapshot = groundSnapshot();
|
||||
const { viewport, events } = recordingViewport({
|
||||
readPointer: () => snapshot,
|
||||
capturePointer: () => new Promise((resolve) => releases.push(resolve)),
|
||||
});
|
||||
const first = viewport.sendVisualContextIfUseful(pointerGround());
|
||||
const second = viewport.sendVisualContextIfUseful(pointerGround());
|
||||
await Promise.resolve();
|
||||
for (const release of releases)
|
||||
release({ dataUrl: 'data:image/jpeg;base64,AA==', rect: {} });
|
||||
assert.deepEqual([await first, await second], [true, false]);
|
||||
assert.equal(releases.length, 1);
|
||||
const outcomes = events.filter((item) => outcomeEvents.has(item.event));
|
||||
assert.deepEqual(
|
||||
outcomes.map((item) => [item.detail.contextId, item.detail.reason]),
|
||||
[
|
||||
[2, 'pointer-already-requested'],
|
||||
[1, 'sent'],
|
||||
],
|
||||
);
|
||||
});
|
||||
+10
-3
@@ -23,10 +23,17 @@ function finite(value) {
|
||||
|
||||
function normalize(entry, index) {
|
||||
if (!entry || typeof entry !== 'object') return null;
|
||||
// Identity is the canonical key (transponder hex, MMSI, NORAD id); a
|
||||
// Identity is the canonical key (transponder hex, vessel id, NORAD id); a
|
||||
// display `id` such as a callsign is only a label, and two contacts can
|
||||
// share one.
|
||||
const id = entry.icao24 ?? entry.mmsi ?? entry.noradId ?? entry.id ?? null;
|
||||
// share one. Vessels can share an MMSI too, so a vessel's key is its
|
||||
// `vesselId` when the row names one.
|
||||
const id =
|
||||
entry.icao24 ??
|
||||
entry.vesselId ??
|
||||
entry.mmsi ??
|
||||
entry.noradId ??
|
||||
entry.id ??
|
||||
null;
|
||||
const label = String(
|
||||
entry.label ?? entry.callsign ?? entry.name ?? entry.id ?? id ?? '',
|
||||
).trim();
|
||||
|
||||
@@ -7,7 +7,7 @@ import { createVoiceSession, isStoppable } from './session.js';
|
||||
import { VoiceCardControls } from './voiceCard.js';
|
||||
import { PointerReticle } from './pointerReticle.js';
|
||||
import { VOICE_PROVIDERS } from './providerCapabilities.js';
|
||||
import { t } from '../i18n/index.js';
|
||||
import { t, translateKnown } from '../i18n/index.js';
|
||||
|
||||
function createDefaultReticle(pointer) {
|
||||
const element = pointer?.element;
|
||||
@@ -114,11 +114,20 @@ export function createVoiceCommands({
|
||||
// States are machine values; only the label is translated.
|
||||
ui.update('status', { text: () => voiceStatusLabel(event.state) });
|
||||
// The portable session reports 'Voice off' in English: show the idle
|
||||
// label instead. Other details are provider messages, shown as they come.
|
||||
// label instead. Other details are provider messages; one that keeps its
|
||||
// English catalog text is worded again in the current language.
|
||||
const idle = event.state === 'idle';
|
||||
const detail = event.detail === 'Voice off' ? '' : event.detail;
|
||||
const source = event.sourceDetail ?? event.detail;
|
||||
const detail = () =>
|
||||
source === 'Voice off'
|
||||
? ''
|
||||
: event.sourceDetail
|
||||
? event.sourceParams
|
||||
? t(event.sourceDetail, event.sourceParams)
|
||||
: translateKnown(event.sourceDetail)
|
||||
: event.detail;
|
||||
ui.update('detail', {
|
||||
text: () => detail || (idle ? t('Voice off') : t('Voice active')),
|
||||
text: () => detail() || (idle ? t('Voice off') : t('Voice active')),
|
||||
});
|
||||
// A typed chat session is not the microphone.
|
||||
ui.update('button', {
|
||||
@@ -127,8 +136,10 @@ export function createVoiceCommands({
|
||||
ui.update('error', {
|
||||
text: () =>
|
||||
event.state === 'error'
|
||||
? event.detail || t('Voice could not be started.')
|
||||
? detail() || t('Voice could not be started.')
|
||||
: '',
|
||||
// "Check microphone permission" is wrong advice for a missing key.
|
||||
hint: () => !(event.state === 'error' && event.missingKey === true),
|
||||
});
|
||||
});
|
||||
// Captions, plan and result card nested in the control; it only reads
|
||||
|
||||
+1
-1
@@ -502,7 +502,7 @@ function frameOverhead(result) {
|
||||
.filter((entry) => entry?.label || entry?.id)
|
||||
.map((entry, index) => ({
|
||||
n: index + 1,
|
||||
id: entry.icao24 || entry.mmsi || entry.id || null,
|
||||
id: entry.icao24 || entry.id || entry.mmsi || null,
|
||||
label: spokenLabel(entry.label || entry.id),
|
||||
layerId: result.layerId,
|
||||
}));
|
||||
|
||||
Reference in new issue
Block a user