evOo-local/packages/core/src/streaming-voice.ts

import { speakable } from "./speech-text.js";

export const DEFAULT_ENDPOINT_SILENCE_MS = 120;
export const DEFAULT_VAD_TICK_MS = 20;
export const DEFAULT_SILENCE_RMS = 0.035;
export const DEFAULT_SAMPLE_RATE = 16_000;
export const DEFAULT_MIN_SPEECH_FRAMES = 3;
export const MAX_STREAMING_AUDIO_BYTES = 16 * 1024 * 1024;

export interface StreamingVoiceFrame {
  pcm16: Int16Array;
  sampleRate: number;
  timestamp: number;
}

export interface StreamingTranscriptDelta {
  text: string;
  isFinal: boolean;
  confidence?: number;
}

export function float32ToInt16(input: Float32Array): Int16Array {
  const output = new Int16Array(input.length);
  for (let i = 0; i < input.length; i += 1) {
    const s = Math.max(-1, Math.min(1, input[i] ?? 0));
    output[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
  }
  return output;
}

export function int16ToFloat32(input: Int16Array): Float32Array {
  const output = new Float32Array(input.length);
  for (let i = 0; i < input.length; i += 1) {
    const s = input[i] ?? 0;
    output[i] = s < 0 ? s / 0x8000 : s / 0x7fff;
  }
  return output;
}

export function calculateRms(pcm16: Int16Array): number {
  if (pcm16.length === 0) return 0;
  let sum = 0;
  for (let i = 0; i < pcm16.length; i += 1) {
    const norm = (pcm16[i] ?? 0) / 32768;
    sum += norm * norm;
  }
  return Math.sqrt(sum / pcm16.length);
}

export interface StreamingVadConfig {
  endpointSilenceMs?: number;
  silenceThresholdRms?: number;
  minSpeechFrames?: number;
  frameDurationMs?: number;
}

export interface StreamingVadDecision {
  isSpeech: boolean;
  energy: number;
  endpointReached: boolean;
  consecutiveSilenceMs: number;
}

export class StreamingVad {
  private readonly endpointSilenceMs: number;
  private readonly silenceThresholdRms: number;
  private readonly minSpeechFrames: number;
  private readonly frameDurationMs: number;

  private speechFrames = 0;
  private hasSpoken = false;
  private silentDurationMs = 0;
  private endpointFired = false;

  constructor(config: StreamingVadConfig = {}) {
    this.endpointSilenceMs = config.endpointSilenceMs ?? DEFAULT_ENDPOINT_SILENCE_MS;
    this.silenceThresholdRms = config.silenceThresholdRms ?? DEFAULT_SILENCE_RMS;
    this.minSpeechFrames = config.minSpeechFrames ?? DEFAULT_MIN_SPEECH_FRAMES;
    this.frameDurationMs = config.frameDurationMs ?? DEFAULT_VAD_TICK_MS;
  }

  processFrame(frame: StreamingVoiceFrame): StreamingVadDecision {
    const rms = calculateRms(frame.pcm16);
    const energy = Math.min(1, Math.max(0, rms / 0.12));

    if (rms > this.silenceThresholdRms) {
      this.speechFrames += 1;
      this.silentDurationMs = 0;
      const isConfirmed = this.speechFrames >= this.minSpeechFrames;
      if (isConfirmed) {
        this.hasSpoken = true;
        this.endpointFired = false;
      }
      return {
        isSpeech: isConfirmed,
        energy,
        endpointReached: false,
        consecutiveSilenceMs: 0,
      };
    }

    this.speechFrames = 0;
    if (this.hasSpoken && !this.endpointFired) {
      this.silentDurationMs += this.frameDurationMs;
      if (this.silentDurationMs >= this.endpointSilenceMs) {
        this.endpointFired = true;
        return {
          isSpeech: false,
          energy,
          endpointReached: true,
          consecutiveSilenceMs: this.silentDurationMs,
        };
      }
    }

    return {
      isSpeech: false,
      energy,
      endpointReached: false,
      consecutiveSilenceMs: this.silentDurationMs,
    };
  }

  reset(): void {
    this.speechFrames = 0;
    this.hasSpoken = false;
    this.silentDurationMs = 0;
    this.endpointFired = false;
  }
}

export interface ClauseSegmenterOptions {
  minWords?: number;
  maxWords?: number;
  cleanSpeech?: boolean;
}

const ABBREVIATIONS = /\b(?:e\.g|i\.e|etc|vs|dr|mr|mrs|ms|prof|no|approx|st|dept)\.?$/i;

function countWords(str: string): number {
  const trimmed = str.trim();
  if (!trimmed) return 0;
  return trimmed.split(/\s+/).length;
}

function isAbbreviationOrDecimal(prefix: string, matchIndex: number, text: string): boolean {
  // Check decimal: preceding char is digit and following char is digit
  if (matchIndex > 0 && matchIndex + 1 < text.length) {
    const prevChar = text[matchIndex - 1];
    const nextChar = text[matchIndex + 1];
    if (prevChar && nextChar && /\d/.test(prevChar) && /\d/.test(nextChar)) {
      return true;
    }
  }

  // Check abbreviation at the end of prefix
  const lastWord = prefix.trim().split(/\s+/).pop() ?? "";
  return ABBREVIATIONS.test(lastWord);
}

interface CodeFenceRange {
  start: number;
  end: number;
  closed: boolean;
}

function getCodeFenceRanges(text: string): CodeFenceRange[] {
  const ranges: CodeFenceRange[] = [];
  const regex = /(`{3,}|~{3,})[^\n]*\n[\s\S]*?(?:\1|$)/g;
  let match = regex.exec(text);
  while (match !== null) {
    const start = match.index;
    const content = match[0];
    const marker = match[1];
    const firstNl = content.indexOf("\n");
    const rest = content.slice(firstNl + 1);
    const closed = marker ? rest.includes(marker) : false;
    ranges.push({ start, end: start + content.length, closed });
    match = regex.exec(text);
  }
  const pending = /(`{3,}|~{3,})[^\n]*$/.exec(text);
  if (pending && (ranges.length === 0 || pending.index >= (ranges[ranges.length - 1]?.end ?? 0))) {
    ranges.push({ start: pending.index, end: text.length, closed: false });
  }
  return ranges;
}

export function isInsideCodeFence(text: string, index: number): boolean {
  const ranges = getCodeFenceRanges(text);
  return ranges.some((r) => index >= r.start && index < r.end);
}

export class ClauseSegmenter {
  private buffer = "";
  private readonly minWords: number;
  private readonly maxWords: number;
  private readonly cleanSpeech: boolean;

  constructor(options: ClauseSegmenterOptions = {}) {
    this.minWords = options.minWords ?? 4;
    this.maxWords = options.maxWords ?? 8;
    this.cleanSpeech = options.cleanSpeech ?? true;
  }

  push(chunk: string): string[] {
    this.buffer += chunk;
    const emitted: string[] = [];

    while (this.buffer.length > 0) {
      const ranges = getCodeFenceRanges(this.buffer);
      let cutIndex = -1;
      let cutLen = 0;

      // Scan through buffer for potential cut points
      // We look for terminal punctuation [.!?], clause punctuation [,;:—–], newlines, or word count threshold
      let i = 0;
      let rangeIdx = 0;
      while (i < this.buffer.length) {
        while (rangeIdx < ranges.length && i >= (ranges[rangeIdx]?.end ?? 0)) {
          rangeIdx += 1;
        }
        const currRange = ranges[rangeIdx];
        if (currRange && i >= currRange.start && i < currRange.end) {
          if (currRange.closed) {
            const wordsBefore = countWords(this.buffer.slice(0, currRange.start));
            if (wordsBefore >= this.maxWords) break;
            cutIndex = currRange.end;
            cutLen = 0;
            break;
          }
          i = currRange.end;
          continue;
        }

        const char = this.buffer[i];

        if (char === "." || char === "!" || char === "?") {
          // Terminal sentence candidate
          const prefix = this.buffer.slice(0, i + 1);
          if (!isAbbreviationOrDecimal(prefix, i, this.buffer)) {
            // Check if followed by whitespace or at end with next char space
            const nextIdx = i + 1;
            if (nextIdx < this.buffer.length && /\s/.test(this.buffer[nextIdx]!)) {
              cutIndex = i;
              cutLen = 1;
              break;
            }
          }
        } else if (
          char === "," ||
          char === ";" ||
          char === ":" ||
          char === "—" ||
          char === "–" ||
          char === "\n"
        ) {
          // Clause punctuation candidate - check if we have enough words
          const prefix = this.buffer.slice(0, i);
          const words = countWords(prefix);
          const nextIdx = i + 1;
          if (
            words >= this.minWords &&
            nextIdx < this.buffer.length &&
            /\s/.test(this.buffer[nextIdx]!)
          ) {
            cutIndex = i;
            cutLen = 1;
            break;
          }
        }

        i += 1;
      }

      // If no punctuation cut found, check if word count exceeds maxWords
      if (cutIndex === -1) {
        const openRange = ranges.find((r) => !r.closed);
        const limitIndex = openRange ? openRange.start : this.buffer.length;
        const prefixToConsider = this.buffer.slice(0, limitIndex);
        const words = countWords(prefixToConsider);
        if (words >= this.maxWords) {
          // Find the whitespace after the maxWords-th word or after minWords
          const matches = [...prefixToConsider.matchAll(/\s+/g)];
          const validMatches = matches.filter((m) => {
            if (m.index === undefined) return false;
            return !ranges.some((r) => m.index! >= r.start && m.index! < r.end);
          });
          if (validMatches.length >= this.minWords) {
            const chosenMatch = validMatches[Math.min(validMatches.length - 1, this.maxWords - 1)];
            if (chosenMatch && chosenMatch.index !== undefined) {
              cutIndex = chosenMatch.index;
              cutLen = chosenMatch[0].length;
            }
          }
        }
      }

      if (cutIndex === -1) {
        break;
      }

      const rawSegment = this.buffer.slice(0, cutIndex + (cutLen > 1 ? 0 : cutLen)).trim();
      this.buffer = this.buffer.slice(cutIndex + cutLen).trimStart();

      if (rawSegment) {
        const clean = this.cleanSpeech ? speakable(rawSegment) : rawSegment;
        if (clean) {
          emitted.push(clean);
        }
      }
    }

    return emitted;
  }

  flush(): string[] {
    const raw = this.buffer.trim();
    this.buffer = "";
    if (!raw) return [];
    const clean = this.cleanSpeech ? speakable(raw) : raw;
    return clean ? [clean] : [];
  }

  reset(): void {
    this.buffer = "";
  }
}

export interface StreamingVoiceClient {
  startStream(): Promise<void>;
  sendFrame(frame: StreamingVoiceFrame): void;
  onTranscriptDelta(cb: (delta: string, isFinal: boolean) => void): void;
  onPlaybackChunk(cb: (chunk: Uint8Array) => void): void;
  commitStream?(): Promise<void>;
  interrupt(): void;
  dispose?(): void;
}