import { speakable } from "./speech-text.js"; export const DEFAULT_ENDPOINT_SILENCE_MS = 120; export const DEFAULT_VAD_TICK_MS = 20; export const DEFAULT_SILENCE_RMS = 0.035; export const DEFAULT_SAMPLE_RATE = 16_000; export const DEFAULT_MIN_SPEECH_FRAMES = 3; export const MAX_STREAMING_AUDIO_BYTES = 16 * 1024 * 1024; export interface StreamingVoiceFrame { pcm16: Int16Array; sampleRate: number; timestamp: number; } export interface StreamingTranscriptDelta { text: string; isFinal: boolean; confidence?: number; } export function float32ToInt16(input: Float32Array): Int16Array { const output = new Int16Array(input.length); for (let i = 0; i < input.length; i += 1) { const s = Math.max(-1, Math.min(1, input[i] ?? 0)); output[i] = s < 0 ? s * 0x8000 : s * 0x7fff; } return output; } export function int16ToFloat32(input: Int16Array): Float32Array { const output = new Float32Array(input.length); for (let i = 0; i < input.length; i += 1) { const s = input[i] ?? 0; output[i] = s < 0 ? s / 0x8000 : s / 0x7fff; } return output; } export function calculateRms(pcm16: Int16Array): number { if (pcm16.length === 0) return 0; let sum = 0; for (let i = 0; i < pcm16.length; i += 1) { const norm = (pcm16[i] ?? 0) / 32768; sum += norm * norm; } return Math.sqrt(sum / pcm16.length); } export interface StreamingVadConfig { endpointSilenceMs?: number; silenceThresholdRms?: number; minSpeechFrames?: number; frameDurationMs?: number; } export interface StreamingVadDecision { isSpeech: boolean; energy: number; endpointReached: boolean; consecutiveSilenceMs: number; } export class StreamingVad { private readonly endpointSilenceMs: number; private readonly silenceThresholdRms: number; private readonly minSpeechFrames: number; private readonly frameDurationMs: number; private speechFrames = 0; private hasSpoken = false; private silentDurationMs = 0; private endpointFired = false; constructor(config: StreamingVadConfig = {}) { this.endpointSilenceMs = config.endpointSilenceMs ?? DEFAULT_ENDPOINT_SILENCE_MS; this.silenceThresholdRms = config.silenceThresholdRms ?? DEFAULT_SILENCE_RMS; this.minSpeechFrames = config.minSpeechFrames ?? DEFAULT_MIN_SPEECH_FRAMES; this.frameDurationMs = config.frameDurationMs ?? DEFAULT_VAD_TICK_MS; } processFrame(frame: StreamingVoiceFrame): StreamingVadDecision { const rms = calculateRms(frame.pcm16); const energy = Math.min(1, Math.max(0, rms / 0.12)); if (rms > this.silenceThresholdRms) { this.speechFrames += 1; this.silentDurationMs = 0; const isConfirmed = this.speechFrames >= this.minSpeechFrames; if (isConfirmed) { this.hasSpoken = true; this.endpointFired = false; } return { isSpeech: isConfirmed, energy, endpointReached: false, consecutiveSilenceMs: 0, }; } this.speechFrames = 0; if (this.hasSpoken && !this.endpointFired) { this.silentDurationMs += this.frameDurationMs; if (this.silentDurationMs >= this.endpointSilenceMs) { this.endpointFired = true; return { isSpeech: false, energy, endpointReached: true, consecutiveSilenceMs: this.silentDurationMs, }; } } return { isSpeech: false, energy, endpointReached: false, consecutiveSilenceMs: this.silentDurationMs, }; } reset(): void { this.speechFrames = 0; this.hasSpoken = false; this.silentDurationMs = 0; this.endpointFired = false; } } export interface ClauseSegmenterOptions { minWords?: number; maxWords?: number; cleanSpeech?: boolean; } const ABBREVIATIONS = /\b(?:e\.g|i\.e|etc|vs|dr|mr|mrs|ms|prof|no|approx|st|dept)\.?$/i; function countWords(str: string): number { const trimmed = str.trim(); if (!trimmed) return 0; return trimmed.split(/\s+/).length; } function isAbbreviationOrDecimal(prefix: string, matchIndex: number, text: string): boolean { // Check decimal: preceding char is digit and following char is digit if (matchIndex > 0 && matchIndex + 1 < text.length) { const prevChar = text[matchIndex - 1]; const nextChar = text[matchIndex + 1]; if (prevChar && nextChar && /\d/.test(prevChar) && /\d/.test(nextChar)) { return true; } } // Check abbreviation at the end of prefix const lastWord = prefix.trim().split(/\s+/).pop() ?? ""; return ABBREVIATIONS.test(lastWord); } interface CodeFenceRange { start: number; end: number; closed: boolean; } function getCodeFenceRanges(text: string): CodeFenceRange[] { const ranges: CodeFenceRange[] = []; const regex = /(`{3,}|~{3,})[^\n]*\n[\s\S]*?(?:\1|$)/g; let match = regex.exec(text); while (match !== null) { const start = match.index; const content = match[0]; const marker = match[1]; const firstNl = content.indexOf("\n"); const rest = content.slice(firstNl + 1); const closed = marker ? rest.includes(marker) : false; ranges.push({ start, end: start + content.length, closed }); match = regex.exec(text); } const pending = /(`{3,}|~{3,})[^\n]*$/.exec(text); if (pending && (ranges.length === 0 || pending.index >= (ranges[ranges.length - 1]?.end ?? 0))) { ranges.push({ start: pending.index, end: text.length, closed: false }); } return ranges; } export function isInsideCodeFence(text: string, index: number): boolean { const ranges = getCodeFenceRanges(text); return ranges.some((r) => index >= r.start && index < r.end); } export class ClauseSegmenter { private buffer = ""; private readonly minWords: number; private readonly maxWords: number; private readonly cleanSpeech: boolean; constructor(options: ClauseSegmenterOptions = {}) { this.minWords = options.minWords ?? 4; this.maxWords = options.maxWords ?? 8; this.cleanSpeech = options.cleanSpeech ?? true; } push(chunk: string): string[] { this.buffer += chunk; const emitted: string[] = []; while (this.buffer.length > 0) { const ranges = getCodeFenceRanges(this.buffer); let cutIndex = -1; let cutLen = 0; // Scan through buffer for potential cut points // We look for terminal punctuation [.!?], clause punctuation [,;:—–], newlines, or word count threshold let i = 0; let rangeIdx = 0; while (i < this.buffer.length) { while (rangeIdx < ranges.length && i >= (ranges[rangeIdx]?.end ?? 0)) { rangeIdx += 1; } const currRange = ranges[rangeIdx]; if (currRange && i >= currRange.start && i < currRange.end) { if (currRange.closed) { const wordsBefore = countWords(this.buffer.slice(0, currRange.start)); if (wordsBefore >= this.maxWords) break; cutIndex = currRange.end; cutLen = 0; break; } i = currRange.end; continue; } const char = this.buffer[i]; if (char === "." || char === "!" || char === "?") { // Terminal sentence candidate const prefix = this.buffer.slice(0, i + 1); if (!isAbbreviationOrDecimal(prefix, i, this.buffer)) { // Check if followed by whitespace or at end with next char space const nextIdx = i + 1; if (nextIdx < this.buffer.length && /\s/.test(this.buffer[nextIdx]!)) { cutIndex = i; cutLen = 1; break; } } } else if ( char === "," || char === ";" || char === ":" || char === "—" || char === "–" || char === "\n" ) { // Clause punctuation candidate - check if we have enough words const prefix = this.buffer.slice(0, i); const words = countWords(prefix); const nextIdx = i + 1; if ( words >= this.minWords && nextIdx < this.buffer.length && /\s/.test(this.buffer[nextIdx]!) ) { cutIndex = i; cutLen = 1; break; } } i += 1; } // If no punctuation cut found, check if word count exceeds maxWords if (cutIndex === -1) { const openRange = ranges.find((r) => !r.closed); const limitIndex = openRange ? openRange.start : this.buffer.length; const prefixToConsider = this.buffer.slice(0, limitIndex); const words = countWords(prefixToConsider); if (words >= this.maxWords) { // Find the whitespace after the maxWords-th word or after minWords const matches = [...prefixToConsider.matchAll(/\s+/g)]; const validMatches = matches.filter((m) => { if (m.index === undefined) return false; return !ranges.some((r) => m.index! >= r.start && m.index! < r.end); }); if (validMatches.length >= this.minWords) { const chosenMatch = validMatches[Math.min(validMatches.length - 1, this.maxWords - 1)]; if (chosenMatch && chosenMatch.index !== undefined) { cutIndex = chosenMatch.index; cutLen = chosenMatch[0].length; } } } } if (cutIndex === -1) { break; } const rawSegment = this.buffer.slice(0, cutIndex + (cutLen > 1 ? 0 : cutLen)).trim(); this.buffer = this.buffer.slice(cutIndex + cutLen).trimStart(); if (rawSegment) { const clean = this.cleanSpeech ? speakable(rawSegment) : rawSegment; if (clean) { emitted.push(clean); } } } return emitted; } flush(): string[] { const raw = this.buffer.trim(); this.buffer = ""; if (!raw) return []; const clean = this.cleanSpeech ? speakable(raw) : raw; return clean ? [clean] : []; } reset(): void { this.buffer = ""; } } export interface StreamingVoiceClient { startStream(): Promise; sendFrame(frame: StreamingVoiceFrame): void; onTranscriptDelta(cb: (delta: string, isFinal: boolean) => void): void; onPlaybackChunk(cb: (chunk: Uint8Array) => void): void; commitStream?(): Promise; interrupt(): void; dispose?(): void; }