gods-eye-view/src/voice/turnMetrics.js

import { countWords, findHedges } from './speechLint.js';

/** Quiet period after the last audio or response before a turn is closed. */
const TURN_SETTLE_MS = 2500;

/**
 * Per-turn latency and speech spans for one voice session.
 *
 * A turn opens at the user's end of speech (or a typed command) and closes
 * when the next turn starts, the session stops, or everything has been quiet
 * for TURN_SETTLE_MS with no response or tool outstanding. The closed span is
 * handed to `emit` once. Times are monotonic milliseconds from `now`.
 *
 * Span fields: eou_ms (speech start → end of speech), first_text_ms (end of
 * speech → first streamed transcript, i.e. generation), first_audio_ms (end
 * of speech → first playback event), max_silence_ms (longest gap with nothing
 * audible before the final audio), tools[{name, ms, ok}], preamble (assistant
 * speech before a tool call in the same response), progress_lines,
 * spoken_words, answer_words, hedge_hits, interruptions, and deltas
 * {count, first_ms, last_ms} in place of logging every streamed delta.
 *
 * Playback timing comes from `output_audio_buffer.*` events. Only on a
 * transport that never sends them does the first transcript delta stand in
 * for audible speech; once one arrives, a provisional transcript timeline is
 * replaced and later turns ignore transcript timing.
 */
export class TurnMetrics {
  constructor({
    emit,
    now = () => performance.now(),
    setTimer = (fn, ms) => setTimeout(fn, ms),
    clearTimer = (id) => clearTimeout(id),
    settleMs = TURN_SETTLE_MS,
  } = {}) {
    Object.assign(this, { emit, now, setTimer, clearTimer, settleMs });
    this.turn = null;
    this.sequence = 0;
    this.settleTimer = null;
    /** Sticky: this transport reports playback with buffer events. */
    this.playbackEvents = false;
  }

  /** Begin a turn for typed input; the command is its end of speech. */
  userText() {
    this.close('next_turn');
    this.open('text');
    this.turn.eouAt = this.now();
  }

  /** Feed every provider event; unknown types are ignored. */
  serverEvent(payload) {
    const type = payload?.type;
    if (!type) return;
    if (type === 'input_audio_buffer.speech_started') {
      if (this.turn?.audible) this.turn.interruptions++;
      this.close('next_turn');
      this.open('voice');
      this.turn.speechStartAt = this.now();
      return;
    }
    const turn = this.turn;
    if (!turn) return;
    if (type.endsWith('.delta')) {
      const at = this.now();
      turn.deltaCount++;
      if (turn.firstDeltaAt === null) turn.firstDeltaAt = at;
      turn.lastDeltaAt = at;
    }
    if (type === 'input_audio_buffer.speech_stopped') {
      turn.eouAt = this.now();
      return;
    }
    if (type === 'response.created') {
      turn.activeResponses++;
      this.cancelSettle();
      return;
    }
    if (type === 'output_audio_buffer.started') {
      this.audioStarted('buffer');
      return;
    }
    if (
      type === 'output_audio_buffer.stopped' ||
      type === 'output_audio_buffer.cleared'
    ) {
      this.audioStopped();
      return;
    }
    if (type === 'response.output_audio_transcript.delta') {
      if (turn.firstTextAt === null) turn.firstTextAt = this.now();
      // Transports without buffer events: the first transcript delta is the
      // closest available proxy for audible speech.
      if (!this.playbackEvents && !turn.bufferEvents)
        this.audioStarted('transcript');
      return;
    }
    if (type === 'response.done') {
      turn.activeResponses = Math.max(0, turn.activeResponses - 1);
      this.responseDone(payload.response || {});
      if (!turn.bufferEvents && turn.audible) this.audioStopped();
      this.scheduleSettle();
    }
  }

  /** A tool began executing. Returns a token for toolEnd. */
  toolStart(name) {
    if (!this.turn) this.open('tool');
    this.cancelSettle();
    const entry = { name, startedAt: this.now(), ms: null, ok: null };
    this.turn.tools.push(entry);
    return entry;
  }

  /** A tool finished; `ok` mirrors its result. */
  toolEnd(entry, ok) {
    if (!entry || entry.ms !== null) return;
    entry.ms = Math.round(this.now() - entry.startedAt);
    entry.ok = Boolean(ok);
    this.scheduleSettle();
  }

  /** Narration produced a spoken progress line or an earcon. */
  narration(kind) {
    if (!this.turn) return;
    if (kind === 'progress') this.turn.progressLines++;
    if (kind === 'earcon') this.turn.earcons++;
  }

  /** The user cut the assistant off explicitly (Space barge-in). */
  bargeIn() {
    if (!this.turn) return;
    this.turn.interruptions++;
    this.turn.bargeIns++;
  }

  /** Close the current turn now (session stop). */
  flush(reason = 'stop') {
    this.close(reason);
  }

  open(kind) {
    this.cancelSettle();
    this.turn = {
      id: ++this.sequence,
      kind,
      openedAt: this.now(),
      speechStartAt: null,
      eouAt: null,
      firstAudioAt: null,
      firstAudioSource: null,
      audible: false,
      audibleSince: null,
      lastAudioStop: null,
      gaps: [],
      bufferEvents: false,
      activeResponses: 0,
      tools: [],
      firstTextAt: null,
      deltaCount: 0,
      firstDeltaAt: null,
      lastDeltaAt: null,
      preamble: false,
      preambleText: '',
      progressLines: 0,
      earcons: 0,
      answerText: [],
      progressText: [],
      interruptions: 0,
      bargeIns: 0,
      responses: 0,
    };
  }

  audioStarted(source) {
    const turn = this.turn;
    if (source === 'buffer') {
      turn.bufferEvents = true;
      this.playbackEvents = true;
      // Authoritative playback replaces a provisional transcript timeline.
      if (turn.firstAudioSource === 'transcript') {
        turn.firstAudioAt = null;
        turn.firstAudioSource = null;
        turn.gaps = [];
        turn.audible = false;
        turn.lastAudioStop = null;
      }
    }
    if (turn.audible) return;
    const at = this.now();
    this.cancelSettle();
    if (turn.firstAudioAt === null) {
      turn.firstAudioAt = at;
      turn.firstAudioSource = source;
      if (turn.eouAt !== null) turn.gaps.push(at - turn.eouAt);
    } else if (turn.lastAudioStop !== null) {
      turn.gaps.push(at - turn.lastAudioStop);
    }
    turn.audible = true;
    turn.audibleSince = at;
  }

  audioStopped() {
    const turn = this.turn;
    if (!turn?.audible) return;
    turn.audible = false;
    turn.lastAudioStop = this.now();
    this.scheduleSettle();
  }

  responseDone(response) {
    const turn = this.turn;
    turn.responses++;
    const output = Array.isArray(response.output) ? response.output : [];
    const isProgress = response.metadata?.gev === 'progress';
    let sawCall = false;
    for (const item of output) {
      if (item?.type === 'function_call') {
        sawCall = true;
        continue;
      }
      if (item?.type !== 'message') continue;
      const text = messageText(item);
      if (!text) continue;
      if (isProgress) {
        turn.progressText.push(text);
      } else if (
        !sawCall &&
        output.some((other) => other?.type === 'function_call')
      ) {
        turn.preamble = true;
        turn.preambleText = [turn.preambleText, text].filter(Boolean).join(' ');
      } else {
        turn.answerText.push(text);
      }
    }
  }

  scheduleSettle() {
    const turn = this.turn;
    if (!turn || turn.audible || turn.activeResponses > 0) return;
    if (turn.tools.some((tool) => tool.ms === null)) return;
    this.cancelSettle();
    this.settleTimer = this.setTimer(() => {
      this.settleTimer = null;
      this.close('settled');
    }, this.settleMs);
  }

  cancelSettle() {
    if (this.settleTimer === null) return;
    this.clearTimer(this.settleTimer);
    this.settleTimer = null;
  }

  close(reason) {
    this.cancelSettle();
    const turn = this.turn;
    if (!turn) return;
    this.turn = null;
    if (turn.audible) turn.lastAudioStop = this.now();
    const round = (value) => (value === null ? null : Math.round(value));
    const answer = turn.answerText.join(' ');
    const allText = [turn.preambleText, ...turn.progressText, answer].join(' ');
    const sinceEou = (at) =>
      at === null || turn.eouAt === null ? null : round(at - turn.eouAt);
    const span = {
      turn: turn.id,
      kind: turn.kind,
      reason,
      eou_ms:
        turn.speechStartAt !== null && turn.eouAt !== null
          ? round(turn.eouAt - turn.speechStartAt)
          : null,
      first_audio_ms:
        turn.eouAt !== null && turn.firstAudioAt !== null
          ? round(turn.firstAudioAt - turn.eouAt)
          : null,
      first_audio_source: turn.firstAudioSource,
      first_text_ms: sinceEou(turn.firstTextAt),
      deltas: {
        count: turn.deltaCount,
        first_ms: sinceEou(turn.firstDeltaAt),
        last_ms: sinceEou(turn.lastDeltaAt),
      },
      max_silence_ms: turn.gaps.length ? round(Math.max(...turn.gaps)) : null,
      no_audio: turn.firstAudioAt === null,
      tools: turn.tools.map(({ name, ms, ok }) => ({ name, ms, ok })),
      preamble: turn.preamble,
      preamble_text: turn.preambleText || null,
      progress_lines: turn.progressLines,
      earcons: turn.earcons,
      responses: turn.responses,
      spoken_words: countWords(allText),
      answer_words: countWords(answer),
      answer_text: answer || null,
      hedge_hits: findHedges(allText),
      interruptions: turn.interruptions,
      barge_ins: turn.bargeIns,
    };
    try {
      this.emit?.(span);
    } catch {
      /* Metrics never interrupt voice. */
    }
  }
}

function messageText(item) {
  return (Array.isArray(item.content) ? item.content : [])
    .map((part) => part?.transcript ?? part?.text ?? '')
    .filter(Boolean)
    .join(' ')
    .trim();
}