fix(voice): recover idle timers and retire panel audio observers

src/voice/control.js

@@ -49,7 +49,7 @@ export function createVoiceControl({ reset = false, panel = false } = {}) {
         <div id="gev-voice-status">${voiceStatusLabel('idle')}</div>
         ${
           panel
-            ? ''
+            ? `<span id="gev-voice-cost-warning" class="gev-voice-cost-value gev-voice-cost-warning" hidden role="status" aria-live="polite"></span>`
             : `<div class="gev-voice-cost">
           <button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="${t('Voice model tier — applies next session')}">${t('STD')}</button>
           <span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="${t('Estimated session cost')}">~$0.00</span>
@@ -193,6 +193,7 @@ export function createVoiceControl({ reset = false, panel = false } = {}) {
     errorDetail: root.querySelector('#gev-voice-error-detail'),
     tierButton: root.querySelector('#gev-voice-tier'),
     costValue: root.querySelector('#gev-voice-cost-value'),
+    costWarning: root.querySelector('#gev-voice-cost-warning'),
     card: root.querySelector('#gev-voice-card'),
   };
 }

src/voice/panel/nativeActivity.js

@@ -36,6 +36,33 @@ export function observeNativeActivity({
   let audio = null,
     version = 0;
   const restores = [];
+  const audioRestores = [];
+  const pendingOutput = new Map();
+  let untrackedOutput = !outputKnown;
+  function retireAudio() {
+    audio = null;
+    for (const restore of audioRestores.splice(0).reverse()) restore();
+  }
+  function refreshOutputKnowledge() {
+    outputKnown = !untrackedOutput && pendingOutput.size === 0;
+  }
+  function hasNoAudioOutput(response) {
+    return (
+      ['completed', 'cancelled', 'failed', 'incomplete'].includes(
+        response?.status,
+      ) &&
+      Array.isArray(response.output) &&
+      response.output.every(
+        (item) =>
+          item?.type === 'function_call' ||
+          (item?.type === 'message' &&
+            Array.isArray(item.content) &&
+            item.content.every((part) =>
+              ['text', 'output_text', 'refusal'].includes(part?.type),
+            )),
+      )
+    );
+  }
   function snapshot() {
     const turns = nativeProvider === 'openai' ? controller?._turns : controller;
     return {
@@ -72,7 +99,7 @@ export function observeNativeActivity({
       } catch {}
     }
   }
-  function wrap(object, key, after) {
+  function wrap(object, key, after, ownedRestores = restores) {
     const original = object?.[key];
     if (typeof original !== 'function') return;
     function wrapped(...args) {
@@ -86,7 +113,7 @@ export function observeNativeActivity({
       return result;
     }
     object[key] = wrapped;
-    restores.push(() => {
+    ownedRestores.push(() => {
       if (object[key] === wrapped) object[key] = original;
     });
   }
@@ -104,9 +131,11 @@ export function observeNativeActivity({
         (state === 'connecting' && wasConnected)
       ) {
         version++;
-        audio = null;
+        retireAudio();
         inputBusy = false;
         outputKnown = cleanNativeOutput();
+        untrackedOutput = !outputKnown;
+        pendingOutput.clear();
       }
     }
     notify();
@@ -127,13 +156,41 @@ export function observeNativeActivity({
           inputBusy = true;
         if (event?.type === 'input_audio_buffer.speech_stopped')
           inputBusy = false;
-        // A previous turn's drain does not prove the next transport emits output events.
-        if (event?.type === 'response.created') outputKnown = false;
+        const type = event?.type;
+        const id = event?.response_id || event?.response?.id;
+        // Keep each unresolved response separate: a later silent reply cannot
+        // establish that an earlier audio reply has drained.
+        if (type === 'response.created') {
+          if (id && pendingOutput.size < 64)
+            pendingOutput.set(id, { audioSeen: false });
+          else untrackedOutput = true;
+        }
+        const pending = pendingOutput.get(id);
         if (
-          event?.type === 'output_audio_buffer.started' ||
-          event?.type === 'output_audio_buffer.cleared'
+          pending &&
+          (type?.startsWith('response.output_audio') ||
+            type?.startsWith('response.audio') ||
+            ['audio', 'output_audio'].includes(event?.part?.type))
         )
-          outputKnown = true;
+          pending.audioSeen = true;
+        if (
+          type === 'response.done' &&
+          pending &&
+          !pending.audioSeen &&
+          hasNoAudioOutput(event.response)
+        )
+          pendingOutput.delete(id);
+        if (
+          type === 'output_audio_buffer.started' ||
+          type === 'output_audio_buffer.cleared'
+        ) {
+          // Native playback events establish the transport's output lifecycle.
+          // Generation completion alone never establishes audio drainage.
+          untrackedOutput = false;
+          if (id) pendingOutput.delete(id);
+          else pendingOutput.clear();
+        }
+        refreshOutputKnowledge();
         notify();
       });
     } else {
@@ -183,12 +240,23 @@ export function observeNativeActivity({
         forwarded[0] = decorated;
         current = Reflect.apply(original, this, forwarded);
         if (!dead && stamp === version) {
+          retireAudio();
           audio = current;
           inputBusy = false;
           for (const key of ['play', 'clearPlayback', 'close'])
-            wrap(current, key, () => {
-              if (audio === current && controller.audio === current) notify();
-            });
+            wrap(
+              current,
+              key,
+              () => {
+                if (audio !== current) return;
+                if (key === 'close') {
+                  retireAudio();
+                  inputBusy = false;
+                }
+                if (controller.audio === current) notify();
+              },
+              audioRestores,
+            );
         }
         return current;
       }
@@ -203,7 +271,9 @@ export function observeNativeActivity({
     if (dead) return;
     dead = true;
     version++;
-    for (const restore of restores.reverse()) restore();
+    retireAudio();
+    pendingOutput.clear();
+    for (const restore of restores.splice(0).reverse()) restore();
     signal?.removeEventListener('abort', destroy);
   }
   signal?.addEventListener('abort', destroy, { once: true });

src/voice/panel/nativeActivity.test.mjs

@@ -471,3 +471,202 @@ test('native typed input observation preserves receiver/result, refuses failed s
   old.call(controller, 'late');
   assert.equal(inputs, 1);
 });
+
+test('explicit silent OpenAI completion rearms expiry, including cancelled and failed responses', () => {
+  for (const status of ['completed', 'cancelled', 'failed', 'incomplete']) {
+    const turns = new RealtimeTurns({});
+    const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
+    let idle;
+    let timer;
+    let expired = 0;
+    const observer = observeNativeActivity({
+      adapter: { controller },
+      nativeProvider: 'openai',
+      onChange: () => idle?.update(),
+    });
+    idle = createVoiceInactivity({
+      readActivity: observer.snapshot,
+      readPending: () => false,
+      setTimer(callback) {
+        timer = callback;
+        return 1;
+      },
+      clearTimer() {
+        timer = null;
+      },
+      onExpire: () => expired++,
+    });
+    controller.status = 'listening';
+    controller.dc = {};
+    observer.handle({ type: 'state', state: 'listening' });
+    assert.equal(typeof timer, 'function');
+    turns.responseActive = true;
+    turns.observeSpeech({
+      type: 'response.created',
+      response: { id: 'silent' },
+    });
+    assert.equal(timer, null);
+    turns.observeSpeech({
+      type: 'response.done',
+      response: {
+        id: 'silent',
+        status,
+        output:
+          status === 'completed'
+            ? [{ type: 'function_call', name: 'set_panel_open' }]
+            : [],
+      },
+    });
+    // The provider updates generation state after forwarding the raw event.
+    assert.equal(timer, null);
+    turns.responseActive = false;
+    observer.handle({ type: 'completion' });
+    assert.equal(observer.snapshot().known, true);
+    assert.equal(typeof timer, 'function');
+    timer();
+    assert.equal(expired, 1);
+    idle.destroy();
+    observer.destroy();
+  }
+});
+
+test('silent OpenAI replies cannot clear unresolved older audio or incomplete output metadata', () => {
+  const turns = new RealtimeTurns({});
+  const observer = observeNativeActivity({
+    adapter: {
+      controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
+    },
+    nativeProvider: 'openai',
+  });
+  const created = (id) =>
+    turns.observeSpeech({ type: 'response.created', response: { id } });
+  const done = (id, output) =>
+    turns.observeSpeech({
+      type: 'response.done',
+      response: { id, status: 'completed', output },
+    });
+  created('audio');
+  turns.observeSpeech({
+    type: 'response.output_audio.delta',
+    response_id: 'audio',
+  });
+  created('silent');
+  done('silent', []);
+  assert.equal(observer.snapshot().known, false);
+  done('audio', []); // Even contradictory completion metadata cannot erase audio evidence.
+  assert.equal(observer.snapshot().known, false);
+  turns.observeSpeech({
+    type: 'output_audio_buffer.started',
+    response_id: 'audio',
+  });
+  assert.equal(observer.snapshot().outputBusy, true);
+  done('silent', []);
+  assert.equal(observer.snapshot().outputBusy, true);
+  turns.observeSpeech({
+    type: 'output_audio_buffer.stopped',
+    response_id: 'audio',
+  });
+  assert.equal(observer.snapshot().outputBusy, false);
+  created('unknown');
+  done('unknown', undefined);
+  assert.equal(observer.snapshot().known, false);
+  done('unobserved', []);
+  assert.equal(observer.snapshot().known, false);
+  observer.destroy();
+});
+
+test('overlapping silent replies resolve independently and audio output items remain unknown until playback', () => {
+  const turns = new RealtimeTurns({});
+  const observer = observeNativeActivity({
+    adapter: {
+      controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
+    },
+    nativeProvider: 'openai',
+  });
+  for (const id of ['a', 'b'])
+    turns.observeSpeech({ type: 'response.created', response: { id } });
+  for (const id of ['a', 'b']) {
+    turns.observeSpeech({
+      type: 'response.done',
+      response: {
+        id,
+        status: 'completed',
+        output: [
+          { type: 'message', content: [{ type: 'text', text: 'Done' }] },
+        ],
+      },
+    });
+    assert.equal(observer.snapshot().known, id === 'b');
+  }
+  turns.observeSpeech({ type: 'response.created', response: { id: 'audio' } });
+  turns.observeSpeech({
+    type: 'response.done',
+    response: {
+      id: 'audio',
+      status: 'completed',
+      output: [{ type: 'message', content: [{ type: 'audio' }] }],
+    },
+  });
+  assert.equal(observer.snapshot().known, false);
+  observer.destroy();
+});
+
+test('silent completion cannot establish output knowledge when observer attached mid-session', () => {
+  const turns = new RealtimeTurns({});
+  const observer = observeNativeActivity({
+    adapter: {
+      controller: { status: 'listening', dc: {}, _turns: turns, _input: {} },
+    },
+    nativeProvider: 'openai',
+  });
+  turns.observeSpeech({ type: 'response.created', response: { id: 'new' } });
+  turns.observeSpeech({
+    type: 'response.done',
+    response: { id: 'new', status: 'completed', output: [] },
+  });
+  assert.equal(observer.snapshot().known, false);
+  observer.destroy();
+});
+
+test('Gemini restores each retired audio immediately across repeated same-mount restarts', () => {
+  const originals = new Map();
+  const controller = {
+    input: {},
+    createAudio() {
+      const audio = { pending: 0, play() {}, clearPlayback() {}, close() {} };
+      originals.set(audio, { ...audio });
+      return audio;
+    },
+    handleMessage() {},
+  };
+  const observer = observeNativeActivity({
+    adapter: { controller },
+    nativeProvider: 'gemini',
+  });
+  const assertRestored = (audio) => {
+    for (const key of ['play', 'clearPlayback', 'close'])
+      assert.equal(audio[key], originals.get(audio)[key]);
+  };
+  for (let i = 0; i < 5; i++) {
+    observer.handle({ type: 'state', state: 'connecting' });
+    controller.audio = controller.createAudio({});
+    const audio = controller.audio;
+    observer.handle({ type: 'state', state: 'listening' });
+    assert.notEqual(audio.close, originals.get(audio).close);
+    audio.close();
+    assertRestored(audio);
+    controller.audio = null;
+    observer.handle({ type: 'state', state: 'idle' });
+  }
+  controller.audio = controller.createAudio({});
+  const replaced = controller.audio;
+  controller.audio = controller.createAudio({});
+  assertRestored(replaced);
+  const current = controller.audio;
+  const replacement = () => {};
+  current.play = replacement;
+  observer.handle({ type: 'state', state: 'error' });
+  assert.equal(current.play, replacement);
+  assert.equal(current.close, originals.get(current).close);
+  observer.destroy();
+});

src/voice/realtimeCost.js

@@ -46,6 +46,17 @@ export class RealtimeCost {
    */
   syncCostUi() {
     const state = this.costTracker.state();
+    const syncWarning = () => {
+      const warning = this.ui?.costWarning;
+      const value = this.ui?.costValue;
+      if (!warning || !value) return;
+      warning.hidden = !['warn', 'cap', 'unknown'].includes(
+        value.dataset.level,
+      );
+      warning.textContent = warning.hidden ? '' : value.textContent;
+      warning.title = value.title;
+      warning.dataset.level = value.dataset.level;
+    };
     const pendingTier = resolveVoiceModel(this.voiceTier).tier;
     const isMini = pendingTier === 'mini';
     if (this.ui?.tierButton) {
@@ -91,6 +102,7 @@ export class RealtimeCost {
                 'Usage is incomplete because a response was still in flight when the session ended.',
               )
             : '');
+        syncWarning();
         return;
       }
       this.ui.costValue.hidden = false;
@@ -114,6 +126,7 @@ export class RealtimeCost {
               'Estimate is incomplete — a response was still in flight when the session ended, so its usage was never reported.',
             )
           : '');
+      syncWarning();
     }
   }
 

src/voice/realtimeCostPanel.test.mjs

@@ -0,0 +1,100 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createVoiceControl } from './control.js';
+import { RealtimeCost } from './realtimeCost.js';
+import { createVoiceCostTracker, resolveVoiceModel } from './voiceCost.js';
+import { installPanelDom } from './panel/testDom.mjs';
+
+function harness(t) {
+  installPanelDom(t);
+  const ui = createVoiceControl({ panel: true });
+  const stops = [];
+  const statuses = [];
+  const cost = new RealtimeCost({
+    readUi: () => ui,
+    readStatus: () => 'listening',
+    operations: {
+      isActive: () => true,
+      stop: (options) => stops.push(options),
+      setStatus: (...args) => statuses.push(args),
+      debugLog() {},
+    },
+  });
+  cost.cloudVoiceAuth = 'api-key';
+  cost.costTracker = createVoiceCostTracker({
+    tier: 'standard',
+    limits: { warnUsd: 2, capUsd: 5 },
+  });
+  const recordThreeDollars = () => {
+    const tokens = Math.ceil(
+      (3 * 1_000_000) / resolveVoiceModel('standard').rates.textOutput,
+    );
+    const state = cost.costTracker.record({
+      output_tokens: tokens,
+      output_token_details: { text_tokens: tokens },
+    });
+    cost.applyCost(state);
+  };
+  t.mock.method(console, 'warn', () => {});
+  return { cost, ui, stops, statuses, recordThreeDollars };
+}
+
+test('soft cost warning is visible outside closed Settings, while ordinary spend stays inside', (t) => {
+  const { cost, ui, recordThreeDollars } = harness(t);
+  cost.syncCostUi();
+  assert.equal(ui.voicePanel.hidden, true);
+  assert.equal(ui.costValue.checkVisibility(), false);
+  assert.equal(ui.costWarning.hidden, true);
+  assert.equal(ui.voicePanel.contains(ui.costWarning), false);
+  assert.equal(ui.root.querySelectorAll('#gev-voice-cost-value').length, 1);
+
+  recordThreeDollars();
+  assert.equal(ui.costWarning.checkVisibility(), true);
+  assert.equal(ui.costWarning.dataset.level, 'warn');
+  assert.equal(ui.costWarning.textContent, ui.costValue.textContent);
+  assert.equal(ui.costWarning.title, ui.costValue.title);
+  assert.match(ui.costWarning.title, /Warns at/);
+  assert.equal(
+    ui.voicePanel.hidden,
+    true,
+    'warning never forces Settings open',
+  );
+
+  cost.prepareSession();
+  cost.syncCostUi();
+  assert.equal(
+    ui.costWarning.hidden,
+    true,
+    'new session clears the old warning',
+  );
+  assert.equal(ui.costWarning.textContent, '');
+});
+
+test('closed Settings cannot hide the hard cap or suppress its ordinary stop and status', (t) => {
+  const { ui, stops, statuses, recordThreeDollars } = harness(t);
+  recordThreeDollars();
+  assert.equal(stops.length, 0);
+  recordThreeDollars();
+  assert.equal(ui.costWarning.checkVisibility(), true);
+  assert.equal(ui.costWarning.dataset.level, 'cap');
+  assert.deepEqual(stops, [{ preserveStatus: true }]);
+  assert.equal(statuses.at(-1)[0], 'idle');
+  assert.match(statuses.at(-1)[1], /Session ended — cost cap/);
+});
+
+test('unknown OAuth cost remains visible outside Settings and clears on a fresh API meter', (t) => {
+  const { cost, ui } = harness(t);
+  cost.sessionCloudVoiceAuth = 'oauth';
+  cost.syncCostUi();
+  assert.equal(ui.costWarning.checkVisibility(), true);
+  assert.equal(ui.costWarning.dataset.level, 'unknown');
+  assert.equal(ui.costWarning.textContent, 'COST UNKNOWN');
+  assert.match(
+    ui.costWarning.title,
+    /USD cost and the API spend cap are unavailable/,
+  );
+  assert.equal(ui.voicePanel.hidden, true);
+  cost.sessionCloudVoiceAuth = 'api-key';
+  cost.syncCostUi();
+  assert.equal(ui.costWarning.hidden, true);
+});