evOo-local/apps/web/src/lib/dictation.ts

import { t } from "@lingui/core/macro";
import { readBoundedResponseBytes } from "@rakazo/core";
import { selectedSpaceId, withSpaceHeaders } from "./rpc.js";

export type DictationMode = "hold" | "endpoint";

export type DictationSnapshot = {
  status: "idle" | "listening" | "transcribing";
  transcript: string;
  /** Normalized mic energy 0–1 for voice presence visuals. */
  energy: number;
  error?: string;
};

type SpeechRecognitionCtor = new () => SpeechRecognitionLike;

interface SpeechRecognitionLike {
  continuous: boolean;
  interimResults: boolean;
  lang: string;
  onresult: ((event: { resultIndex: number; results: SpeechRecognitionResultList }) => void) | null;
  onerror: ((event: { error?: string }) => void) | null;
  onend: (() => void) | null;
  start(): void;
  stop(): void;
  abort(): void;
}

const IDLE: DictationSnapshot = { status: "idle", transcript: "", energy: 0 };
const ENDPOINT_TICK_MS = 80;
const SILENCE_RMS = 0.035;
export const TRANSCRIPTION_RESPONSE_TIMEOUT_MS = 70_000;
export const MAX_TRANSCRIPTION_RESPONSE_BYTES = 64 * 1024;
const endpointUnsupported = () =>
  t`This browser can't detect when you stop talking. Use Chrome, the desktop app, or hold-to-talk in the composer.`;
const micUnavailable = () =>
  t`Microphone access is not available in this browser. Use Chrome, the desktop app, or check system settings.`;

/** Browsers hide the microphone and speech recognition outside HTTPS and localhost. */
function insecureContext(): boolean {
  return typeof window !== "undefined" && window.isSecureContext === false;
}

export function webSpeechAvailable(): boolean {
  return Boolean(speechRecognitionCtor());
}

function speechRecognitionCtor(): SpeechRecognitionCtor | undefined {
  if (typeof window === "undefined") return undefined;
  const host = window as Window & {
    SpeechRecognition?: SpeechRecognitionCtor;
    webkitSpeechRecognition?: SpeechRecognitionCtor;
  };
  return host.SpeechRecognition ?? host.webkitSpeechRecognition;
}

function audioContextCtor(): (new () => AudioContext) | undefined {
  if (typeof AudioContext === "function") return AudioContext;
  if (typeof window === "undefined") return undefined;
  const host = window as Window & { webkitAudioContext?: typeof AudioContext };
  return host.webkitAudioContext;
}

export class Dictation {
  private snapshot: DictationSnapshot = IDLE;
  private watchers = new Set<(s: DictationSnapshot) => void>();
  private recognition: SpeechRecognitionLike | null = null;
  private media: MediaRecorder | null = null;
  private chunks: Blob[] = [];
  private token = 0;
  private silenceTimer: ReturnType<typeof setTimeout> | undefined;
  private transcribeAbort: AbortController | null = null;
  private audioContext: AudioContext | null = null;
  private vadTimer: ReturnType<typeof setInterval> | undefined;
  private micStream: MediaStream | null = null;
  private onFinal: ((text: string) => void) | null = null;

  subscribe(fn: (s: DictationSnapshot) => void): () => void {
    this.watchers.add(fn);
    fn(this.snapshot);
    return () => {
      this.watchers.delete(fn);
    };
  }

  get state(): DictationSnapshot {
    return this.snapshot;
  }

  private set(next: DictationSnapshot) {
    this.snapshot = next;
    for (const watcher of [...this.watchers]) watcher(next);
  }

  stop(reason: "submit" | "cancel" | "replace" = "cancel") {
    this.token += 1;
    this.transcribeAbort?.abort();
    this.transcribeAbort = null;
    this.stopVad();
    clearTimeout(this.silenceTimer);
    this.silenceTimer = undefined;
    const rec = this.recognition;
    this.recognition = null;
    if (rec) {
      rec.onresult = null;
      rec.onerror = null;
      rec.onend = null;
      try {
        if (reason === "submit") rec.stop();
        else rec.abort();
      } catch {
        // already stopped
      }
    }
    const media = this.media;
    this.media = null;
    if (media && media.state !== "inactive") {
      try {
        media.stop();
      } catch {
        // already stopped
      }
    }
    this.chunks = [];
    this.onFinal = null;
    this.stopMicStream();
    if (this.snapshot.status !== "idle" || this.snapshot.error) this.set(IDLE);
  }

  async listen(opts: {
    mode: DictationMode;
    transcribe?: boolean;
    endpointMs?: number;
    onFinal: (text: string) => void;
  }): Promise<void> {
    this.stop("replace");
    const mine = this.token;
    const spaceId = selectedSpaceId();
    this.onFinal = opts.onFinal;
    if (insecureContext()) {
      this.set({
        ...IDLE,
        error: t`Voice input needs a secure connection. Open this address with https://.`,
      });
      return;
    }
    this.set({ status: "listening", transcript: "", energy: 0 });
    if (webSpeechAvailable()) {
      const stream = await this.openMic(mine);
      if (!stream) return;
      this.micStream = stream;
      this.startMicEnergy(stream, mine);
      this.listenWebSpeech(opts.mode, opts.endpointMs ?? 850, mine);
      return;
    }
    if (opts.transcribe) {
      await this.listenRecorder(mine, opts.mode, opts.endpointMs ?? 850, spaceId);
      return;
    }
    this.set({
      ...IDLE,
      error: t`This browser can't recognize speech itself. Connect a voice provider with transcription, or use Safari, Chrome or the desktop app.`,
    });
  }

  private listenWebSpeech(mode: DictationMode, endpointMs: number, mine: number) {
    const Ctor = speechRecognitionCtor();
    if (!Ctor) return;
    const rec = new Ctor();
    rec.continuous = true;
    rec.interimResults = true;
    rec.lang = navigator.language || "en-US";
    rec.onresult = (event) => {
      if (this.token !== mine) return;
      let transcript = "";
      for (let i = 0; i < event.results.length; i += 1) {
        transcript += event.results[i]?.[0]?.transcript ?? "";
      }
      this.set({
        status: "listening",
        transcript: transcript.trim(),
        energy: this.snapshot.energy,
      });
      if (mode !== "endpoint") return;
      clearTimeout(this.silenceTimer);
      this.silenceTimer = setTimeout(() => {
        if (this.token !== mine) return;
        const text = this.snapshot.transcript.trim();
        this.finish(text, mine);
      }, endpointMs);
    };
    rec.onerror = (event) => {
      if (this.token !== mine) return;
      if (event.error === "aborted" || event.error === "no-speech") return;
      this.set({
        ...IDLE,
        error: dictationFailed(event.error),
      });
    };
    rec.onend = () => {
      if (this.token !== mine) return;
      if (this.snapshot.status !== "listening") return;
      if (mode === "hold") {
        this.finish(this.snapshot.transcript, mine);
        return;
      }
      const text = this.snapshot.transcript.trim();
      if (text) {
        this.finish(text, mine);
        return;
      }
      try {
        rec.start();
      } catch {
        this.set({ ...IDLE, error: t`Dictation ended unexpectedly.` });
      }
    };
    this.recognition = rec;
    rec.start();
  }

  private async openMic(mine: number): Promise<MediaStream | null> {
    if (!navigator.mediaDevices?.getUserMedia) {
      if (this.token !== mine) return null;
      this.set({ ...IDLE, error: micUnavailable() });
      return null;
    }
    try {
      return await navigator.mediaDevices.getUserMedia({ audio: true });
    } catch (error) {
      if (this.token !== mine) return null;
      this.set({ ...IDLE, error: micErrorMessage(error) });
      return null;
    }
  }

  private async listenRecorder(
    mine: number,
    mode: DictationMode,
    endpointMs: number,
    spaceId: string | null,
  ) {
    const stream = await this.openMic(mine);
    if (!stream) return;
    if (this.token !== mine) {
      for (const track of stream.getTracks()) track.stop();
      return;
    }
    if (mode === "endpoint" && !this.armSilence(stream, mine, endpointMs)) {
      for (const track of stream.getTracks()) track.stop();
      this.set({ ...IDLE, error: endpointUnsupported() });
      return;
    }
    const media = new MediaRecorder(stream);
    this.media = media;
    this.chunks = [];
    media.ondataavailable = (event) => {
      if (this.token !== mine) return;
      if (event.data.size) this.chunks.push(event.data);
    };
    media.onstop = () => {
      for (const track of stream.getTracks()) track.stop();
      if (this.token !== mine) return;
      this.stopVad();
      void this.transcribeChunks(mine, spaceId);
    };
    media.start(mode === "endpoint" ? 250 : undefined);
  }

  private startMicEnergy(stream: MediaStream, mine: number): boolean {
    const Ctor = audioContextCtor();
    if (!Ctor) return false;
    try {
      const ctx = new Ctor();
      const source = ctx.createMediaStreamSource(stream);
      const analyser = ctx.createAnalyser();
      analyser.fftSize = 2048;
      source.connect(analyser);
      this.audioContext = ctx;
      if (ctx.state === "suspended") void ctx.resume();
      const data = new Uint8Array(analyser.fftSize);
      this.vadTimer = setInterval(() => {
        if (this.token !== mine) return;
        const energy = rmsEnergyFromAnalyser(analyser, data);
        if (this.snapshot.energy !== energy) {
          this.set({ ...this.snapshot, energy });
        }
      }, ENDPOINT_TICK_MS);
      return true;
    } catch {
      this.stopVad();
      return false;
    }
  }

  private armSilence(stream: MediaStream, mine: number, endpointMs: number): boolean {
    const Ctor = audioContextCtor();
    if (!Ctor) return false;
    try {
      const ctx = new Ctor();
      const source = ctx.createMediaStreamSource(stream);
      const analyser = ctx.createAnalyser();
      analyser.fftSize = 2048;
      source.connect(analyser);
      this.audioContext = ctx;
      if (ctx.state === "suspended") void ctx.resume();
      const data = new Uint8Array(analyser.fftSize);
      let heardSpeech = false;
      let silentFor = 0;
      this.vadTimer = setInterval(() => {
        const media = this.media;
        if (this.token !== mine || !media || media.state === "inactive") return;
        const rms = rmsFromAnalyser(analyser, data);
        const energy = Math.min(1, Math.max(0, rms / 0.12));
        if (this.snapshot.energy !== energy) {
          this.set({ ...this.snapshot, energy });
        }
        if (rms > SILENCE_RMS) {
          heardSpeech = true;
          silentFor = 0;
          return;
        }
        if (!heardSpeech) return;
        silentFor += ENDPOINT_TICK_MS;
        if (silentFor < endpointMs) return;
        this.stopVad();
        try {
          media.stop();
        } catch {
          // already stopped
        }
      }, ENDPOINT_TICK_MS);
      return true;
    } catch {
      this.stopVad();
      return false;
    }
  }

  private stopVad() {
    clearInterval(this.vadTimer);
    this.vadTimer = undefined;
    const ctx = this.audioContext;
    this.audioContext = null;
    if (ctx) void ctx.close().catch(() => undefined);
  }

  private stopMicStream() {
    const stream = this.micStream;
    this.micStream = null;
    if (!stream) return;
    for (const track of stream.getTracks()) track.stop();
  }

  private async transcribeChunks(mine: number, spaceId: string | null) {
    if (this.token !== mine) return;
    const blob = new Blob(this.chunks, { type: this.chunks[0]?.type || "audio/webm" });
    this.chunks = [];
    if (!blob.size) {
      this.set(IDLE);
      return;
    }
    this.set({ status: "transcribing", transcript: this.snapshot.transcript, energy: 0 });
    const abort = new AbortController();
    this.transcribeAbort = abort;
    const timer = setTimeout(
      () => abort.abort(new Error("Transcription request timed out.")),
      TRANSCRIPTION_RESPONSE_TIMEOUT_MS,
    );
    try {
      const audioBase64 = await blobToBase64(blob);
      if (this.token !== mine) return;
      const res = await withAbort(
        fetch("/api/voice/transcribe", {
          method: "POST",
          headers: withSpaceHeaders({ "content-type": "application/json" }, spaceId),
          credentials: "include",
          body: JSON.stringify({ audioBase64, mimeType: blob.type }),
          signal: abort.signal,
        }),
        abort.signal,
      );
      if (this.token !== mine) {
        cancelResponse(res);
        return;
      }
      const body = await readTranscriptionBody(res, abort.signal);
      if (this.token !== mine) return;
      if (!res.ok) {
        this.set({ ...IDLE, error: body.error ?? t`Could not transcribe that recording.` });
        return;
      }
      this.finish(body.text ?? "", mine);
    } catch (error) {
      // User cancel bumps token in stop() before aborting, so a matching token
      // means the deadline timer fired. Browsers may reject fetch as AbortError
      // instead of signal.reason; surface the timeout either way.
      if (this.token !== mine) return;
      const timedOut =
        (abort.signal.reason instanceof Error &&
          abort.signal.reason.message === "Transcription request timed out.") ||
        (error instanceof Error && error.message === "Transcription request timed out.");
      if (timedOut || (error instanceof Error && error.name === "AbortError")) {
        this.set({ ...IDLE, error: t`Transcription request timed out.` });
        return;
      }
      this.set({
        ...IDLE,
        error: error instanceof Error ? error.message : t`Could not transcribe that recording.`,
      });
    } finally {
      clearTimeout(timer);
      if (this.transcribeAbort === abort) this.transcribeAbort = null;
    }
  }

  submitHold() {
    if (this.media && this.media.state !== "inactive") {
      this.media.stop();
      return;
    }
    this.finish(this.snapshot.transcript, this.token);
  }

  private finish(text: string, mine: number) {
    if (this.token !== mine) return;
    const trimmed = text.trim();
    const onFinal = this.onFinal;
    this.stop("submit");
    if (trimmed) onFinal?.(trimmed);
  }
}

async function blobToBase64(blob: Blob): Promise<string> {
  const buffer = await blob.arrayBuffer();
  const bytes = new Uint8Array(buffer);
  let binary = "";
  for (const byte of bytes) binary += String.fromCharCode(byte);
  return btoa(binary);
}

async function readTranscriptionBody(
  response: Response,
  signal: AbortSignal,
): Promise<{ text?: string; error?: string }> {
  const declared = Number(response.headers.get("content-length") ?? 0);
  if (Number.isFinite(declared) && declared > MAX_TRANSCRIPTION_RESPONSE_BYTES) {
    cancelResponse(response);
    throw new Error("Transcription response is too large.");
  }
  const bytes = await readBoundedResponseBytes(response, {
    maxBytes: MAX_TRANSCRIPTION_RESPONSE_BYTES,
    tooLargeMessage: "Transcription response is too large.",
    read: (operation) => withAbort(operation(), signal),
  });
  try {
    const parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown;
    return parsed && typeof parsed === "object"
      ? (parsed as { text?: string; error?: string })
      : {};
  } catch {
    return {};
  }
}

function withAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
  if (signal.aborted) return Promise.reject(signal.reason ?? new Error("Request aborted."));
  return new Promise<T>((resolve, reject) => {
    const onAbort = () => reject(signal.reason ?? new Error("Request aborted."));
    signal.addEventListener("abort", onAbort, { once: true });
    promise.then(
      (value) => {
        signal.removeEventListener("abort", onAbort);
        resolve(value);
      },
      (error: unknown) => {
        signal.removeEventListener("abort", onAbort);
        // Prefer abort reason over a bare AbortError from fetch/read.
        reject(signal.aborted ? (signal.reason ?? error) : error);
      },
    );
  });
}

function cancelResponse(response: Response): void {
  try {
    void Promise.resolve(response.body?.cancel()).catch(() => undefined);
  } catch {
    // Response cleanup is best-effort.
  }
}

function rmsFromAnalyser(analyser: AnalyserNode, data: Uint8Array): number {
  analyser.getByteTimeDomainData(data as unknown as Uint8Array<ArrayBuffer>);
  let sum = 0;
  for (const sample of data) {
    const n = (sample - 128) / 128;
    sum += n * n;
  }
  return Math.sqrt(sum / data.length);
}

function rmsEnergyFromAnalyser(analyser: AnalyserNode, data: Uint8Array): number {
  return Math.min(1, Math.max(0, rmsFromAnalyser(analyser, data) / 0.12));
}

function micErrorMessage(error: unknown): string {
  if (error instanceof DOMException) {
    if (error.name === "NotAllowedError") {
      return t`Microphone access denied. Allow the microphone in your browser or system settings, then try again.`;
    }
    if (error.name === "NotFoundError") {
      return t`No microphone found. Connect a microphone and try again.`;
    }
  }
  return error instanceof Error ? error.message : t`Microphone failed`;
}

function dictationFailed(code: string | undefined): string {
  // Safari reports a blocked recognizer (Siri & Dictation off) as not-allowed.
  if (code === "not-allowed" || code === "service-not-allowed") {
    return t`Speech recognition is blocked. Allow the microphone and turn on dictation in your system settings.`;
  }
  return code ? t`Dictation failed: ${code}` : t`Dictation failed.`;
}

export const dictation = new Dictation();