evOo-local/apps/mobile/lib/voice.ts

import type { VoiceInfo } from "@rakazo/contracts";
import { ensureAiDataConsent, readBoundedResponseBytes, voiceIdForLocale } from "@rakazo/core";
import { File, Paths } from "expo-file-system";
import { promptAiConsent } from "./ai-consent";
import type { ApiRequestContext } from "./api";
import { captureApiRequestContext, rpc } from "./api";
import { t } from "./i18n";

type SpeechOptions = { voiceId?: string; botId?: string };
export const VOICE_RESPONSE_TIMEOUT_MS = 70_000;
export const MAX_VOICE_AUDIO_BYTES = 16 * 1024 * 1024;
const MAX_VOICE_ERROR_BYTES = 64 * 1024;

/** Must match packages/adapters KokoroVoiceProvider placeholder. */
export const KOKORO_LOCAL_TOKEN = "local-kokoro";
export const LOCAL_VOICE_PROVIDER = "kokoro";

/** Keep the saved voice in the UI language, so German speaks German and English speaks English. */
export async function alignVoiceWithLocale(locale: string): Promise<void> {
  const status = await rpc<{ provider: string | null; voiceId: string }>("voice/status");
  if (status.provider !== LOCAL_VOICE_PROVIDER || !status.voiceId) return;
  const voices = await rpc<VoiceInfo[]>("voice/voices", { provider: LOCAL_VOICE_PROVIDER });
  const next = voiceIdForLocale(voices, locale, status.voiceId);
  if (next && next !== status.voiceId) {
    await rpc("voice/setVoice", { voiceId: next, provider: LOCAL_VOICE_PROVIDER });
  }
}

export async function speakText(text: string, opts: SpeechOptions = {}): Promise<boolean> {
  const requestContext = await captureApiRequestContext();
  const prepared = await rpc<{ ready: boolean; utterances: string[] }>(
    "voice/prepare",
    { text, voiceId: opts.voiceId, botId: opts.botId },
    { requestContext },
  );
  if (!prepared.ready) return false;
  for (const utterance of prepared.utterances) {
    await playSpeech(await renderUtterance(utterance, opts, requestContext));
  }
  return true;
}

export async function speakUtterance(
  text: string,
  opts: SpeechOptions & { requestContext?: ApiRequestContext } = {},
): Promise<Uint8Array> {
  const requestContext = opts.requestContext ?? (await captureApiRequestContext());
  await ensureAiDataConsent({
    uses: ["voice"],
    status: () => rpc("aiConsent/status", { uses: ["voice"] }, { requestContext }),
    prompt: promptAiConsent,
    allow: (input) => rpc("aiConsent/allow", input, { requestContext }),
  });
  return renderUtterance(text, opts, requestContext);
}

async function renderUtterance(
  text: string,
  opts: SpeechOptions,
  requestContext: ApiRequestContext,
): Promise<Uint8Array> {
  const deadline = requestDeadline(VOICE_RESPONSE_TIMEOUT_MS);
  try {
    const res = await withAbort(
      fetch(`${requestContext.apiBase}/api/voice/speak`, {
        method: "POST",
        headers: {
          "content-type": "application/json",
          origin: "rakazo://",
          ...requestContext.headers,
        },
        body: JSON.stringify({ text, voiceId: opts.voiceId, botId: opts.botId }),
        signal: deadline.signal,
      }),
      deadline.signal,
    );
    if (!res.ok) {
      const body = await readVoiceError(res, deadline.signal);
      throw new Error(body.error ?? `Voice failed (${res.status})`);
    }
    return await readResponseBytes(res, MAX_VOICE_AUDIO_BYTES, deadline.signal);
  } finally {
    deadline.dispose();
  }
}

/** Kokoro answers with MP3, Piper (German voices) with WAV. */
function clipFormat(bytes: Uint8Array): { type: string; extension: string } {
  const riff = bytes[0] === 0x52 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x46;
  return riff ? { type: "audio/wav", extension: "wav" } : { type: "audio/mpeg", extension: "mp3" };
}

export async function playSpeech(bytes: Uint8Array): Promise<void> {
  const AudioCtor = (globalThis as { Audio?: typeof Audio }).Audio;
  if (typeof AudioCtor === "function") {
    await playWithHtmlAudio(AudioCtor, bytes);
    return;
  }
  await playWithNativeAudio(bytes);
}

async function playWithHtmlAudio(AudioCtor: typeof Audio, bytes: Uint8Array): Promise<void> {
  const blob = new Blob([new Uint8Array(bytes)], { type: clipFormat(bytes).type });
  const url = URL.createObjectURL(blob);
  try {
    const audio = new AudioCtor(url);
    await new Promise<void>((resolve, reject) => {
      let settled = false;
      const finish = (error?: Error) => {
        if (settled) return;
        settled = true;
        audio.onended = null;
        audio.onerror = null;
        if (error) reject(error);
        else resolve();
      };
      audio.onended = () => finish();
      audio.onerror = () => finish(new Error(t("Could not play that clip.")));
      try {
        void audio
          .play()
          .catch((error: unknown) =>
            finish(error instanceof Error ? error : new Error(t("Could not play that clip."))),
          );
      } catch (error) {
        finish(error instanceof Error ? error : new Error(t("Could not play that clip.")));
      }
    });
  } finally {
    URL.revokeObjectURL(url);
  }
}

async function playWithNativeAudio(bytes: Uint8Array): Promise<void> {
  const { createAudioPlayer, setAudioModeAsync } = await import("expo-audio");
  await setAudioModeAsync({
    playsInSilentMode: true,
    interruptionMode: "mixWithOthers",
    shouldPlayInBackground: false,
  });
  const file = new File(Paths.cache, `rakazo-voice-${Date.now()}.${clipFormat(bytes).extension}`);
  file.create({ overwrite: true });
  file.write(bytesToBase64(bytes), { encoding: "base64" });
  const player = createAudioPlayer({ uri: file.uri });
  try {
    await new Promise<void>((resolve, reject) => {
      let settled = false;
      let timer = setTimeout(() => finish(new Error(t("Could not play that clip."))), 15_000);
      const finish = (error?: Error) => {
        if (settled) return;
        settled = true;
        clearTimeout(timer);
        sub.remove();
        if (error) reject(error);
        else resolve();
      };
      const sub = player.addListener("playbackStatusUpdate", (status) => {
        if (status.error) {
          finish(new Error(status.error));
          return;
        }
        if (status.playbackState === "failed") {
          finish(new Error(t("Could not play that clip.")));
          return;
        }
        if (status.didJustFinish) {
          finish();
          return;
        }
        if (status.playing && status.duration > 0) {
          clearTimeout(timer);
          timer = setTimeout(
            () => finish(new Error(t("Could not play that clip."))),
            Math.min(120_000, Math.ceil(status.duration * 1000) + 8_000),
          );
        }
      });
      try {
        player.play();
      } catch (error) {
        finish(error instanceof Error ? error : new Error(t("Could not play that clip.")));
      }
    });
  } finally {
    player.release();
    try {
      file.delete();
    } catch {
      // already gone
    }
  }
}

function bytesToBase64(bytes: Uint8Array): string {
  if (typeof Buffer !== "undefined") return Buffer.from(bytes).toString("base64");
  let binary = "";
  const chunk = 0x8000;
  for (let i = 0; i < bytes.length; i += chunk) {
    binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
  }
  return btoa(binary);
}

function requestDeadline(timeoutMs: number) {
  const controller = new AbortController();
  const timer = setTimeout(
    () => controller.abort(new Error("Voice request timed out.")),
    timeoutMs,
  );
  return {
    signal: controller.signal,
    dispose() {
      clearTimeout(timer);
    },
  };
}

async function readVoiceError(
  response: Response,
  signal: AbortSignal,
): Promise<{ error?: string }> {
  const bytes = await readResponseBytes(response, MAX_VOICE_ERROR_BYTES, signal);
  try {
    const parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown;
    return parsed && typeof parsed === "object" ? (parsed as { error?: string }) : {};
  } catch {
    return {};
  }
}

async function readResponseBytes(
  response: Response,
  maxBytes: number,
  signal: AbortSignal,
): Promise<Uint8Array> {
  const declared = Number(response.headers.get("content-length") ?? 0);
  if (Number.isFinite(declared) && declared > maxBytes) {
    cancelResponse(response);
    throw new Error("Voice response is too large.");
  }
  return readBoundedResponseBytes(response, {
    maxBytes,
    tooLargeMessage: "Voice response is too large.",
    read: (operation) => withAbort(operation(), signal),
  });
}

function withAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
  if (signal.aborted) return Promise.reject(signal.reason ?? new Error("Voice request aborted."));
  return new Promise<T>((resolve, reject) => {
    const onAbort = () => reject(signal.reason ?? new Error("Voice request aborted."));
    signal.addEventListener("abort", onAbort, { once: true });
    promise.then(
      (value) => {
        signal.removeEventListener("abort", onAbort);
        resolve(value);
      },
      (error: unknown) => {
        signal.removeEventListener("abort", onAbort);
        reject(error);
      },
    );
  });
}

function cancelResponse(response: Response): void {
  try {
    void Promise.resolve(response.body?.cancel()).catch(() => undefined);
  } catch {
    // Response cleanup is best-effort.
  }
}