mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 02:44:02 +00:00
feat(voice): harden Gemini Live provider integration
This commit is contained in:
1 parent
d4492d8f8c
commit
4fde920665
102 files changed
+28528
-1581
No files matched your search
+13
-1
@@ -56,6 +56,7 @@ KEY_SETUP_EXTERNAL_KEYS=()
|
||||
[[ -n "${GOOGLE_MAPS_SERVER_API_KEY:-}" ]] && KEY_SETUP_EXTERNAL_KEYS+=(GOOGLE_MAPS_SERVER_API_KEY)
|
||||
[[ -n "${CESIUM_ION_TOKEN:-}" ]] && KEY_SETUP_EXTERNAL_KEYS+=(CESIUM_ION_TOKEN)
|
||||
[[ -n "${OPENAI_API_KEY:-}" ]] && KEY_SETUP_EXTERNAL_KEYS+=(OPENAI_API_KEY)
|
||||
[[ -n "${GEMINI_API_KEY:-}" ]] && KEY_SETUP_EXTERNAL_KEYS+=(GEMINI_API_KEY)
|
||||
[[ -n "${AISSTREAM_API_KEY:-}" ]] && KEY_SETUP_EXTERNAL_KEYS+=(AISSTREAM_API_KEY)
|
||||
[[ -n "${FIRMS_MAP_KEY:-}" ]] && KEY_SETUP_EXTERNAL_KEYS+=(FIRMS_MAP_KEY)
|
||||
[[ -n "${TOMTOM_API_KEY:-}" ]] && KEY_SETUP_EXTERNAL_KEYS+=(TOMTOM_API_KEY)
|
||||
@@ -233,6 +234,9 @@ resolve_opensky_credentials
|
||||
# Add to Keychain with e.g.:
|
||||
# security add-generic-password -U -s "openai-api" -a "api-key" -w
|
||||
OPENAI_API_KEY="${OPENAI_API_KEY:-$(read_dotenv_value "OPENAI_API_KEY")}"
|
||||
# Gemini uses its own voice credential, never the Google Maps key or Keychain.
|
||||
GEMINI_API_KEY="${GEMINI_API_KEY:-$(read_dotenv_value "GEMINI_API_KEY")}"
|
||||
GEMINI_LIVE_MODEL="${GEMINI_LIVE_MODEL:-$(read_dotenv_value "GEMINI_LIVE_MODEL")}"
|
||||
AISSTREAM_API_KEY="${AISSTREAM_API_KEY:-$(read_dotenv_value "AISSTREAM_API_KEY")}"
|
||||
CESIUM_ION_TOKEN="${CESIUM_ION_TOKEN:-$(read_dotenv_value "CESIUM_ION_TOKEN")}"
|
||||
LL2_API_TOKEN="${LL2_API_TOKEN:-$(read_dotenv_value "LL2_API_TOKEN")}"
|
||||
@@ -347,7 +351,13 @@ case "${OPENSKY_AUTH_MODE}" in
|
||||
echo "OpenSky auth: disabled (anonymous mode)"
|
||||
;;
|
||||
esac
|
||||
[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key (voice + HUD summary): not set — GEV MIC off unless a Codex ChatGPT sign-in is selected in Provider Settings (experimental)"
|
||||
[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key (voice + HUD summary): not set"
|
||||
[[ -n "${GEMINI_API_KEY}" ]] && echo "Gemini key (voice): configured" || echo "Gemini key (voice): not set"
|
||||
if [[ -n "${OPENAI_API_KEY}" || -n "${GEMINI_API_KEY}" ]]; then
|
||||
echo "Voice control: available"
|
||||
else
|
||||
echo "Voice control: add an OpenAI or Gemini key, or select a configured OpenAI provider in Provider Settings"
|
||||
fi
|
||||
[[ -n "${AISSTREAM_API_KEY}" ]] && echo "AISStream key (live vessels): configured" || echo "AISStream key (live vessels): not set — ships layer empty"
|
||||
if [[ -n "${GOOGLE_MAPS_API_KEY}" ]]; then
|
||||
echo "Startup map: Google Photorealistic 3D Tiles (direct)"
|
||||
@@ -415,6 +425,8 @@ put_env_if_set OPENSKY_CLIENT_SECRET "${OPENSKY_CLIENT_SECRET}"
|
||||
put_env_if_set OPENSKY_USERNAME "${OPENSKY_USERNAME}"
|
||||
put_env_if_set OPENSKY_PASSWORD "${OPENSKY_PASSWORD}"
|
||||
put_env_if_set OPENAI_API_KEY "${OPENAI_API_KEY}"
|
||||
put_env_if_set GEMINI_API_KEY "${GEMINI_API_KEY}"
|
||||
put_env_if_set GEMINI_LIVE_MODEL "${GEMINI_LIVE_MODEL}"
|
||||
put_env_if_set AISSTREAM_API_KEY "${AISSTREAM_API_KEY}"
|
||||
put_env_if_set CESIUM_ION_TOKEN "${CESIUM_ION_TOKEN}"
|
||||
put_env_if_set TOMTOM_API_KEY "${TOMTOM_API_KEY}"
|
||||
|
||||
@@ -1054,5 +1054,24 @@
|
||||
"src/tooling/webAppIcons.test.mjs",
|
||||
"src/ui/flyToDataSource.js",
|
||||
"src/ui/flyToDataSource.test.mjs",
|
||||
"src/data/gbfsProxyRateLimit.test.mjs"
|
||||
"src/data/gbfsProxyRateLimit.test.mjs",
|
||||
"src/voice/commands.test.mjs",
|
||||
"src/voice/providerLifetime.test.mjs",
|
||||
"src/voice/providerCapabilities.js",
|
||||
"src/voice/providerCapabilities.test.mjs",
|
||||
"src/voice/providerIntent.js",
|
||||
"src/voice/providerIntent.test.mjs",
|
||||
"src/voice/providerIntentFragments.js",
|
||||
"src/voice/providerIntentFragments.test.mjs",
|
||||
"src/voice/toolExecutionPolicy.js",
|
||||
"src/voice/toolExecutionPolicy.test.mjs",
|
||||
"src/voice/userTurnIntent.js",
|
||||
"src/voice/userTurnIntent.test.mjs",
|
||||
"scripts/qa-gemini-voice.mjs",
|
||||
"src/geminiServer.test.mjs",
|
||||
"src/voice/geminiAudio.test.mjs",
|
||||
"src/voice/geminiAutomaticVad.test.mjs",
|
||||
"src/voice/geminiSession.test.mjs",
|
||||
"src/voice/integrationOwnership.test.mjs",
|
||||
"src/voice/providerCommands.test.mjs"
|
||||
]
|
||||
@@ -313,6 +313,7 @@
|
||||
"server/providers/openai/debug-log.js",
|
||||
"server/providers/openai/hud-summary.js",
|
||||
"server/providers/openai/instructions.js",
|
||||
"src/voice/providerCapabilities.js",
|
||||
"server/providers/openai/rate-limit.js",
|
||||
"server/providers/openai/realtime.js",
|
||||
"server/providers/openai/tools.js",
|
||||
@@ -1408,6 +1409,10 @@
|
||||
"src/voice/layerSummary.js",
|
||||
"src/voice/voiceCost.js",
|
||||
"src/voice/commands.js",
|
||||
"src/voice/providerCommands.js",
|
||||
"src/voice/geminiSession.js",
|
||||
"src/voice/geminiAudio.js",
|
||||
"src/voice/geminiCaptureWorklet.js",
|
||||
"src/voice/session.js",
|
||||
"src/voice/sessionCommands.js",
|
||||
"src/voice/realtimeSession.js",
|
||||
@@ -1434,7 +1439,12 @@
|
||||
"src/voice/narration.js",
|
||||
"src/voice/earcon.js",
|
||||
"src/voice/pointerContext.js",
|
||||
"src/voice/pointerReticle.js"
|
||||
"src/voice/pointerReticle.js",
|
||||
"src/voice/providerCapabilities.js",
|
||||
"src/voice/providerIntent.js",
|
||||
"src/voice/providerIntentFragments.js",
|
||||
"src/voice/userTurnIntent.js",
|
||||
"src/voice/toolExecutionPolicy.js"
|
||||
],
|
||||
"external": []
|
||||
},
|
||||
@@ -2011,6 +2021,10 @@
|
||||
"src/ui/visualPresets.js",
|
||||
"src/ui/visualSettings.js",
|
||||
"src/voice/commands.js",
|
||||
"src/voice/providerCommands.js",
|
||||
"src/voice/geminiSession.js",
|
||||
"src/voice/geminiAudio.js",
|
||||
"src/voice/geminiCaptureWorklet.js",
|
||||
"src/voice/control.js",
|
||||
"src/voice/gevActions.js",
|
||||
"src/voice/speech.js",
|
||||
@@ -2261,7 +2275,6 @@
|
||||
"src/data/cameraArrival.js",
|
||||
"src/layers/alpr/floors.js",
|
||||
"src/data/militaryNames.js",
|
||||
"src/sources/requestDeadline.js",
|
||||
"src/data/local_data/osm_military_names/names.json",
|
||||
"src/layers/installations/namedMarkers.js",
|
||||
"src/data/infrastructureOverlayEntry.js",
|
||||
@@ -2273,6 +2286,11 @@
|
||||
"src/data/cameraFraming.js",
|
||||
"src/data/geoImportPins.js",
|
||||
"src/data/geoImportImages.js",
|
||||
"src/voice/providerCapabilities.js",
|
||||
"src/voice/providerIntent.js",
|
||||
"src/voice/providerIntentFragments.js",
|
||||
"src/voice/userTurnIntent.js",
|
||||
"src/voice/toolExecutionPolicy.js",
|
||||
"src/app/layers/airQuality.js",
|
||||
"src/layers/airQuality/index.js",
|
||||
"src/layers/airQuality/model.js",
|
||||
@@ -2822,7 +2840,6 @@
|
||||
"src/data/cameraArrival.js",
|
||||
"src/layers/alpr/floors.js",
|
||||
"src/data/militaryNames.js",
|
||||
"src/sources/requestDeadline.js",
|
||||
"src/data/local_data/osm_military_names/names.json",
|
||||
"src/layers/installations/namedMarkers.js",
|
||||
"src/data/infrastructureOverlayEntry.js",
|
||||
|
||||
@@ -11,6 +11,8 @@ export const PINOKIO_CONFIG_FIELDS = Object.freeze([
|
||||
'GOOGLE_MAPS_SERVER_API_KEY',
|
||||
'CESIUM_ION_TOKEN',
|
||||
'OPENAI_API_KEY',
|
||||
'GEMINI_API_KEY',
|
||||
'GEMINI_LIVE_MODEL',
|
||||
'AISSTREAM_API_KEY',
|
||||
'FIRMS_MAP_KEY',
|
||||
'TOMTOM_API_KEY',
|
||||
@@ -18,6 +20,7 @@ export const PINOKIO_CONFIG_FIELDS = Object.freeze([
|
||||
'OPENSKY_CLIENT_SECRET',
|
||||
'LL2_API_TOKEN',
|
||||
'GEV_RATELIMIT_OPENAI_PER_MIN',
|
||||
'GEV_RATELIMIT_GEMINI_PER_MIN',
|
||||
'GEV_RATELIMIT_GOOGLE_PER_MIN',
|
||||
'PINOKIO_SHARE_CLOUDFLARE',
|
||||
'PINOKIO_SHARE_LOCAL',
|
||||
@@ -26,6 +29,7 @@ export const PINOKIO_CONFIG_FIELDS = Object.freeze([
|
||||
|
||||
const PINOKIO_DEFAULTS = Object.freeze({
|
||||
GEV_RATELIMIT_OPENAI_PER_MIN: '30',
|
||||
GEV_RATELIMIT_GEMINI_PER_MIN: '30',
|
||||
GEV_RATELIMIT_GOOGLE_PER_MIN: '120',
|
||||
PINOKIO_SHARE_CLOUDFLARE: 'false',
|
||||
PINOKIO_SHARE_LOCAL: 'false',
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env node
|
||||
/** Real Gemini acceptance: feed a WAV through Chromium's microphone and await spoken output.
|
||||
* Example: node scripts/qa-gemini-voice.mjs --wav path/to/question.wav --expect-response 'four|4'
|
||||
* Requires a configured Gemini key in the running app; never reads the permanent key.
|
||||
*/
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const option = (name, fallback) => {
|
||||
const index = args.indexOf(name);
|
||||
return index < 0 ? fallback : args[index + 1];
|
||||
};
|
||||
const appUrl = new URL(option('--url', 'http://127.0.0.1:43981/'));
|
||||
appUrl.searchParams.set('welcome', '0');
|
||||
const wav = option('--wav');
|
||||
if (!wav)
|
||||
throw new Error(
|
||||
'Pass --wav with a PCM WAV containing the spoken test request.',
|
||||
);
|
||||
const wavPath = path.resolve(wav);
|
||||
const fixture = await fs.readFile(wavPath);
|
||||
if (
|
||||
fixture.toString('ascii', 0, 4) !== 'RIFF' ||
|
||||
fixture.toString('ascii', 8, 12) !== 'WAVE'
|
||||
)
|
||||
throw new Error('The microphone fixture must be a WAV file.');
|
||||
const expectedResponse = option('--expect-response', 'four|4');
|
||||
const expectedInput = option('--expect-input', 'two plus two|2 plus 2|2 \\+ 2');
|
||||
const requiredAction = option('--expect-action', '');
|
||||
const minimumHeight = Number(option('--min-height', '0'));
|
||||
const outDir = path.resolve(option('--out', 'qa-shots/gemini-voice'));
|
||||
await fs.mkdir(outDir, { recursive: true });
|
||||
const browser = await puppeteer.launch({
|
||||
headless: !args.includes('--headed'),
|
||||
args: [
|
||||
'--use-fake-device-for-media-stream',
|
||||
`--use-file-for-fake-audio-capture=${wavPath}%noloop`,
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--window-size=1440,900',
|
||||
],
|
||||
});
|
||||
let result;
|
||||
try {
|
||||
const context = browser.defaultBrowserContext();
|
||||
await context.overridePermissions(appUrl.origin, ['microphone']);
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1440, height: 900, deviceScaleFactor: 1 });
|
||||
// Only the test input device is substituted. GEV still captures this browser
|
||||
// MediaStream through its real AudioWorklet and calls the real Gemini API.
|
||||
if (args.includes('--virtual-mic')) {
|
||||
await page.evaluateOnNewDocument((base64) => {
|
||||
const original = navigator.mediaDevices.getUserMedia.bind(
|
||||
navigator.mediaDevices,
|
||||
);
|
||||
navigator.mediaDevices.getUserMedia = async (constraints) => {
|
||||
if (!constraints.audio || constraints.video)
|
||||
return original(constraints);
|
||||
const context = new AudioContext();
|
||||
await context.resume();
|
||||
const bytes = Uint8Array.from(atob(base64), (char) =>
|
||||
char.charCodeAt(0),
|
||||
);
|
||||
const buffer = await context.decodeAudioData(bytes.buffer);
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
const destination = context.createMediaStreamDestination();
|
||||
source.connect(destination);
|
||||
const stream = destination.stream;
|
||||
const track = stream.getAudioTracks()[0];
|
||||
const stop = track.stop.bind(track);
|
||||
track.stop = () => {
|
||||
try {
|
||||
source.stop();
|
||||
} catch {
|
||||
/* Already ended. */
|
||||
}
|
||||
source.disconnect();
|
||||
destination.disconnect();
|
||||
void context.close().catch(() => {});
|
||||
stop();
|
||||
};
|
||||
source.start(context.currentTime + 0.25);
|
||||
return stream;
|
||||
};
|
||||
}, fixture.toString('base64'));
|
||||
}
|
||||
const tokenStatuses = [];
|
||||
page.on('response', (response) => {
|
||||
if (new URL(response.url()).pathname === '/api/gemini/token')
|
||||
tokenStatuses.push(response.status());
|
||||
});
|
||||
await page.goto(appUrl.href, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const loading = document.getElementById('loading-screen');
|
||||
return (
|
||||
window.__godsEyeView?.voiceCommands &&
|
||||
document.getElementById('gev-voice-provider') &&
|
||||
(!loading ||
|
||||
getComputedStyle(loading).display === 'none' ||
|
||||
Number(getComputedStyle(loading).opacity) === 0)
|
||||
);
|
||||
},
|
||||
{ timeout: 90000 },
|
||||
);
|
||||
await page.select('#gev-voice-provider', 'gemini');
|
||||
await page.evaluate(() => {
|
||||
window.__geminiAcceptance = {
|
||||
transcripts: [],
|
||||
inputSegments: [],
|
||||
actions: [],
|
||||
};
|
||||
window.__godsEyeView.voiceCommands.session.subscribe((event) => {
|
||||
if (
|
||||
event.type === 'transcript' &&
|
||||
event.role === 'user' &&
|
||||
event.attribution === 'unverified' &&
|
||||
event.delta === true &&
|
||||
event.final === false &&
|
||||
typeof event.text === 'string'
|
||||
)
|
||||
window.__geminiAcceptance.inputSegments.push(event.text);
|
||||
if (event.type === 'transcript' && event.final)
|
||||
window.__geminiAcceptance.transcripts.push({
|
||||
role: event.role,
|
||||
text: event.text,
|
||||
});
|
||||
if (event.type === 'action-result')
|
||||
window.__geminiAcceptance.actions.push({
|
||||
name: event.name,
|
||||
ok: event.result?.ok === true,
|
||||
});
|
||||
});
|
||||
});
|
||||
const began = Date.now();
|
||||
await page.click('#gev-voice-button');
|
||||
let timedOut = false;
|
||||
try {
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const voice = window.__godsEyeView.voiceCommands;
|
||||
const d = voice.getDiagnostics();
|
||||
return (
|
||||
d.status === 'error' ||
|
||||
(d.completedTurns > 0 &&
|
||||
d.audio?.outputSamples > 0 &&
|
||||
(d.audio.discardedOutputSamples > 0 ||
|
||||
(d.audio.completedOutputSamples === d.audio.outputSamples &&
|
||||
d.audio.queuedSeconds < 0.05)))
|
||||
);
|
||||
},
|
||||
{ timeout: 75000 },
|
||||
);
|
||||
} catch {
|
||||
timedOut = true;
|
||||
}
|
||||
const proof = await page.evaluate(() => ({
|
||||
...window.__geminiAcceptance,
|
||||
diagnostics: window.__godsEyeView.voiceCommands.getDiagnostics(),
|
||||
provider: document.querySelector('#gev-voice-provider').value,
|
||||
detail: document.querySelector('#gev-voice-detail').textContent,
|
||||
cameraHeightM:
|
||||
window.__godsEyeView.viewer.camera.positionCartographic.height,
|
||||
}));
|
||||
// Fixture-level caption evidence only: these unordered native segments do
|
||||
// not identify a command, authorize a tool, or form a final user transcript.
|
||||
const userText = proof.inputSegments.join('');
|
||||
const assistantText = proof.transcripts
|
||||
.filter((item) => item.role === 'assistant')
|
||||
.map((item) => item.text)
|
||||
.join(' ');
|
||||
// A completed Radio handoff releases Gemini's speaker and microphone.
|
||||
// Closed audio is valid only when the successful handoff is already idle.
|
||||
const completedRadioHandoff =
|
||||
proof.actions.some(
|
||||
(action) => action.name === 'control_radio' && action.ok,
|
||||
) &&
|
||||
proof.diagnostics.status === 'idle' &&
|
||||
!proof.diagnostics.connected &&
|
||||
!proof.diagnostics.microphoneActive &&
|
||||
proof.diagnostics.audio?.resourcesReleased === true;
|
||||
const checks = {
|
||||
geminiSelected: proof.provider === 'gemini',
|
||||
tokenGranted: tokenStatuses.includes(200),
|
||||
heardInput: new RegExp(expectedInput, 'i').test(userText),
|
||||
expectedReply: new RegExp(expectedResponse, 'i').test(assistantText),
|
||||
realMicrophoneFrames:
|
||||
proof.diagnostics.audio?.inputChunks > 0 &&
|
||||
proof.diagnostics.audio?.nonSilentInputChunks > 0,
|
||||
playedReply:
|
||||
proof.diagnostics.audio?.outputSamples > 0 &&
|
||||
proof.diagnostics.audio.completedOutputSamples ===
|
||||
proof.diagnostics.audio.outputSamples &&
|
||||
proof.diagnostics.audio.discardedOutputSamples === 0 &&
|
||||
proof.diagnostics.audio.queuedSeconds < 0.05 &&
|
||||
(proof.diagnostics.audio.contextState === 'running' ||
|
||||
(completedRadioHandoff &&
|
||||
proof.diagnostics.audio.contextState === 'closed')),
|
||||
completed:
|
||||
!timedOut &&
|
||||
proof.diagnostics.completedTurns > 0 &&
|
||||
proof.diagnostics.status !== 'error',
|
||||
action:
|
||||
!requiredAction ||
|
||||
proof.actions.some(
|
||||
(action) => action.name === requiredAction && action.ok,
|
||||
),
|
||||
camera: !minimumHeight || proof.cameraHeightM >= minimumHeight,
|
||||
};
|
||||
await page.screenshot({ path: path.join(outDir, 'response.png') });
|
||||
// Stop is idempotent; toggling the button could restart an idle session
|
||||
// after Radio has taken the speaker between proof collection and cleanup.
|
||||
await page.evaluate(() => window.__godsEyeView.voiceCommands.stop());
|
||||
const stopped = await page.evaluate(() =>
|
||||
window.__godsEyeView.voiceCommands.getDiagnostics(),
|
||||
);
|
||||
checks.stopped =
|
||||
stopped.status === 'idle' &&
|
||||
!stopped.connected &&
|
||||
!stopped.microphoneActive;
|
||||
result = {
|
||||
ok: Object.values(checks).every(Boolean),
|
||||
microphoneSource: args.includes('--virtual-mic')
|
||||
? 'Web Audio virtual microphone from macOS WAV'
|
||||
: 'Chromium WAV capture device',
|
||||
checks,
|
||||
elapsedMs: Date.now() - began,
|
||||
fixtureSha256: createHash('sha256').update(fixture).digest('hex'),
|
||||
tokenStatuses,
|
||||
...proof,
|
||||
stopped,
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(outDir, 'result.json'),
|
||||
JSON.stringify(result, null, 2) + '\n',
|
||||
);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
console.log(`VERDICT: ${result.ok ? 'PASS' : 'FAIL'}`);
|
||||
process.exitCode = result.ok ? 0 : 1;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
@@ -298,11 +298,11 @@ async function main() {
|
||||
);
|
||||
console.log('PASS: header drag lifts CCTV out of the rail');
|
||||
|
||||
// Park near the top: the voice dock would cover the south corners once
|
||||
// the loop below has grown the window.
|
||||
// Park high enough that the voice provider row cannot cover the southwest
|
||||
// handle after the loop below has grown the window.
|
||||
{
|
||||
const header = await headerPoint();
|
||||
await drag(header, 0, 120 - header.y);
|
||||
await drag(header, 0, 40 - header.y);
|
||||
}
|
||||
|
||||
const limits = {
|
||||
@@ -346,6 +346,28 @@ async function main() {
|
||||
assert.deepEqual(driftedEdges(before, after, 'nw', 2), []);
|
||||
console.log('PASS: minimum size holds from both corners');
|
||||
|
||||
// Minimum-size clamps do not move the window away from the top edge.
|
||||
// Drag it down through the real header before proving north-edge growth.
|
||||
{
|
||||
const header = await headerPoint();
|
||||
await drag(header, 0, 140 - header.y);
|
||||
}
|
||||
for (const dir of ['n', 'ne']) {
|
||||
before = await readPanel();
|
||||
const { dx, dy } = growthDelta(dir, 40);
|
||||
const expected = resizeBox(before, dir, dx, dy, limits);
|
||||
assert.ok(
|
||||
expected.top <= before.top - 38,
|
||||
`${dir} has room to grow north`,
|
||||
);
|
||||
await drag(handlePoint(before, dir), dx, dy);
|
||||
after = await readPanel();
|
||||
for (const key of ['left', 'top', 'width', 'height'])
|
||||
near(after[key], expected[key], 2, `${dir} north ${key}`);
|
||||
assert.deepEqual(driftedEdges(before, after, dir, 2), [], `${dir} pins`);
|
||||
}
|
||||
console.log('PASS: north edges grow from the minimum size');
|
||||
|
||||
const saved = await readPanel();
|
||||
const record = JSON.parse(saved.stored);
|
||||
assert.equal(record.floating, true);
|
||||
|
||||
@@ -2182,7 +2182,7 @@ async function runBehaviorLayer() {
|
||||
?.get('military-awareness')
|
||||
?.module?.getContextSnapshot?.();
|
||||
const cohort = snap?.cohorts?.find((c) => c.id === 'flights');
|
||||
return cohort ? cohort.count : null;
|
||||
return Number.isFinite(cohort?.count) ? cohort.count : 'unknown';
|
||||
});
|
||||
const windowBlock = r?.contactsWindow || null;
|
||||
report(
|
||||
|
||||
@@ -16,6 +16,7 @@ export const CREDENTIALS = Object.freeze([
|
||||
{ name: 'GOOGLE_MAPS_SERVER_API_KEY', label: 'Google Places / Street View server', keychain: [] },
|
||||
{ name: 'CESIUM_ION_TOKEN', label: 'Cesium ion', keychain: [['cesium-ion', 'token']] },
|
||||
{ name: 'OPENAI_API_KEY', label: 'OpenAI voice', keychain: [['openai-api', 'api-key']] },
|
||||
{ name: 'GEMINI_API_KEY', label: 'Gemini voice', keychain: [] },
|
||||
{ name: 'AISSTREAM_API_KEY', label: 'AISStream vessels', keychain: [['aisstream-api', 'api-key']] },
|
||||
{ name: 'FIRMS_MAP_KEY', label: 'NASA FIRMS fires', keychain: [['firms-map', 'map-key']] },
|
||||
{ name: 'TOMTOM_API_KEY', label: 'TomTom traffic', keychain: [['tomtom-api', 'api-key']] },
|
||||
@@ -176,9 +177,9 @@ export function buildCapabilitySummary(
|
||||
? 'Google Photorealistic 3D Tiles through Cesium ion; Bing and world-terrain stacks available'
|
||||
: 'Esri World Imagery (keyless satellite basemap) with keyless terrain',
|
||||
flights,
|
||||
voice: configured('OPENAI_API_KEY')
|
||||
voice: configured('OPENAI_API_KEY') || configured('GEMINI_API_KEY')
|
||||
? 'available'
|
||||
: 'off until an OpenAI key is added, or a Codex ChatGPT sign-in is selected in Provider Settings (experimental)',
|
||||
: 'off until an OpenAI or Gemini key is added, or a configured OpenAI provider is selected in Provider Settings',
|
||||
vessels: configured('AISSTREAM_API_KEY') ? 'live AISStream feed' : 'off until an AISStream key is added',
|
||||
fires: configured('FIRMS_MAP_KEY') ? 'live NASA FIRMS feed' : 'off until a FIRMS key is added',
|
||||
traffic: configured('TOMTOM_API_KEY') ? 'live TomTom flow' : 'built-in traffic simulation',
|
||||
|
||||
@@ -82,7 +82,7 @@ export const CORE_PHRASES = Object.freeze([
|
||||
{ phrase: 'Show me the nearest traffic camera', expect: 'control_cctv', typedArgs: [where('control_cctv', (a) => a.action === 'nearest')] },
|
||||
{ phrase: 'Turn on the camera viewsheds', expect: 'control_cctv', typedArgs: [where('control_cctv', (a) => a.action === 'viewshed' && a.enabled !== false)] },
|
||||
{ phrase: 'Play a news radio station near Austin', expect: 'control_radio', args: { action: 'select', category: 'news', locationId: 'austin' }, typedArgs: [where('control_radio', (a) => a.action === 'select' && a.category === 'news' && a.locationId === 'austin')] },
|
||||
{ phrase: 'Turn on the radio', expect: 'control_radio', args: { action: 'play' }, typedArgs: [where('control_radio', (a) => a.action === 'play')] },
|
||||
{ phrase: 'Turn on the radio', expect: 'control_radio', args: { action: 'enable' }, typedArgs: [where('control_radio', (a) => a.action === 'enable')] },
|
||||
{ phrase: 'Set the radio volume to thirty percent', expect: 'control_radio', args: { action: 'volume', volumePct: 30 } },
|
||||
{ phrase: 'Pause the radio', expect: 'control_radio', args: { action: 'pause' }, typedArgs: [where('control_radio', (a) => a.action === 'pause')] },
|
||||
{ phrase: 'Stop the radio', expect: 'control_radio', args: { action: 'stop' }, typedArgs: [where('control_radio', (a) => a.action === 'stop')] },
|
||||
@@ -110,7 +110,7 @@ export const CORE_PHRASES = Object.freeze([
|
||||
{
|
||||
phrase: 'Go to full planet view and then turn on the radio',
|
||||
expect: ['zoom_to_globe', 'control_radio'],
|
||||
argsByTool: { control_radio: { action: 'play' } },
|
||||
argsByTool: { control_radio: { action: 'enable' } },
|
||||
},
|
||||
|
||||
// — camera verbs (tools #23/#24) —
|
||||
@@ -507,8 +507,8 @@ export const DIALOGUES = Object.freeze([
|
||||
{
|
||||
id: 'radio-then-volume',
|
||||
turns: ['Turn on the radio', 'Turn it down to twenty percent'],
|
||||
results: { control_radio: [{ ok: true, prepared: true, action: 'play', station: 'KUT 90.5 Austin' }] },
|
||||
first: { expect: 'control_radio', typedArgs: [where('control_radio', (a) => a.action === 'play')] },
|
||||
results: { control_radio: [{ ok: true, enabled: true, action: 'enable', audioState: 'stopped' }] },
|
||||
first: { expect: 'control_radio', typedArgs: [where('control_radio', (a) => a.action === 'enable')] },
|
||||
second: { expect: 'control_radio', typedArgs: [where('control_radio', (a) => a.action === 'volume' && a.volumePct === 20)] },
|
||||
},
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user