evOo-local/packages/adapters/src/openai-compatible-transcription.ts

import { randomUUID } from "node:crypto";
import type {
  AdapterDescriptor,
  Transcript,
  TranscriptionCapabilities,
  TranscriptionProvider,
  TranscriptionRequest,
  TranscriptSegment,
} from "@rakazo/adapter-kit";
import {
  assertAllowedOpenAiCompatibleUrl,
  assertHttpsForKeyedOpenAiCompatibleUrl,
  OPENAI_COMPATIBLE_PROVIDER_ID,
} from "./openai-compatible-url.js";
import { createOpenAiCompatibleFetch } from "./pi-openai-compatible-provider.js";
import { readBodyCapped } from "./web-ssrf.js";

/** The OpenAI upload limit; whisper.cpp and faster-whisper servers accept at least this much. */
export const OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES = 25 * 1024 * 1024;
/** Used when no model is configured; whisper.cpp ignores it and most servers alias it. */
export const OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL = "whisper-1";
// CPU-only whisper servers need minutes for a long upload; the job's signal still cancels sooner.
const TRANSCRIPTION_TIMEOUT_MS = 15 * 60_000;
const MAX_TRANSCRIPTION_RESPONSE_BYTES = 8 * 1024 * 1024;

export interface OpenAiCompatibleTranscriptionConfig {
  baseUrl: string;
  model?: string;
  apiKey?: string;
  fetch?: typeof globalThis.fetch;
}

type ResponseFormat = "verbose_json" | "json";

/**
 * Speech to text from any server that speaks the OpenAI `POST /v1/audio/transcriptions`
 * protocol: whisper.cpp's server (with `--inference-path /v1/audio/transcriptions`),
 * faster-whisper based servers, vLLM, or a hosted API. Asks for timed segments and falls back
 * to plain text when the server only returns that.
 */
export function createOpenAiCompatibleTranscriptionProvider(
  config: OpenAiCompatibleTranscriptionConfig,
): TranscriptionProvider {
  const baseUrl = assertAllowedOpenAiCompatibleUrl(config.baseUrl);
  const apiKey = config.apiKey?.trim() || undefined;
  assertHttpsForKeyedOpenAiCompatibleUrl(baseUrl, apiKey);
  const model = config.model?.trim() || OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL;
  const endpoint = new URL("audio/transcriptions", `${baseUrl.href}/`).href;
  const fetchImpl = createOpenAiCompatibleFetch(config.fetch);

  function post(
    request: TranscriptionRequest,
    format: ResponseFormat,
    signal: AbortSignal,
  ): Promise<Response> {
    const fields: Record<string, string> = { model, response_format: format };
    const language = languageHint(request.language);
    if (language) fields.language = language;
    const body = multipartBody(fields, {
      name: uploadName(request.fileName, request.mimeType),
      mimeType: request.mimeType || "application/octet-stream",
      bytes: request.audio,
    });
    const headers: Record<string, string> = {
      Accept: "application/json, text/plain",
      "Content-Type": body.contentType,
    };
    if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
    return fetchImpl(endpoint, {
      method: "POST",
      headers,
      body: body.bytes,
      redirect: "error",
      signal,
    });
  }

  async function transcribeAs(
    request: TranscriptionRequest,
    format: ResponseFormat,
  ): Promise<Transcript | { rejectedFormat: true }> {
    const timeout = AbortSignal.timeout(TRANSCRIPTION_TIMEOUT_MS);
    const signal = request.signal ? AbortSignal.any([request.signal, timeout]) : timeout;
    try {
      const response = await post(request, format, signal);
      const text = new TextDecoder().decode(await readResponse(response, signal));
      if (!response.ok) {
        const message = errorMessage(text);
        // Servers that only implement `json`/`text` reject the timed format; ask again plainly.
        if (
          format === "verbose_json" &&
          (response.status === 400 || response.status === 422) &&
          /format|verbose/i.test(message)
        ) {
          return { rejectedFormat: true };
        }
        throw new Error(httpErrorMessage(response.status, message));
      }
      return parseTranscriptionResponse(text, response.headers.get("content-type"));
    } catch (error) {
      if (request.signal?.aborted) throw error;
      if (timeout.aborted) throw new Error("Transcription server timed out");
      // undici reports refused or reset connections as a bare "fetch failed".
      if (error instanceof TypeError && error.message === "fetch failed") {
        throw new Error("Transcription server is unreachable");
      }
      throw error;
    }
  }

  return {
    describe(): AdapterDescriptor<TranscriptionCapabilities> {
      return {
        id: OPENAI_COMPATIBLE_PROVIDER_ID,
        contractVersion: "1",
        adapterVersion: "0.1.0",
        capabilities: {
          segments: true,
          languageDetection: true,
          maxBytes: OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES,
        },
      };
    },

    async transcribe(request: TranscriptionRequest): Promise<Transcript> {
      request.signal?.throwIfAborted();
      if (request.audio.byteLength === 0) throw new Error("The audio is empty");
      if (request.audio.byteLength > OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES) {
        throw new Error("The audio is too large for the transcription server");
      }
      const timed = await transcribeAs(request, "verbose_json");
      if (!("rejectedFormat" in timed)) return timed;
      const plain = await transcribeAs(request, "json");
      if ("rejectedFormat" in plain) throw new Error("Transcription server rejected the request");
      return plain;
    },
  };
}

async function readResponse(response: Response, signal: AbortSignal): Promise<Uint8Array> {
  try {
    return await readBodyCapped(response, MAX_TRANSCRIPTION_RESPONSE_BYTES, signal);
  } catch (error) {
    if (error instanceof Error && error.message === "Response is too large") {
      throw new Error("Transcription server returned too much data");
    }
    throw error;
  }
}

function httpErrorMessage(status: number, detail: string): string {
  if (status === 401 || status === 403) return "Transcription server rejected the API key";
  if (status === 413) return "The audio is too large for the transcription server";
  if (status === 404) return "Transcription server has no /audio/transcriptions endpoint";
  return detail
    ? `Transcription server returned ${status}: ${detail}`
    : `Transcription server returned ${status}`;
}

/** The server's own message from an OpenAI-style `{error: {message}}` or a short plain body. */
function errorMessage(body: string): string {
  let message = body;
  try {
    const parsed: unknown = JSON.parse(body);
    if (isRecord(parsed)) {
      const error = parsed.error;
      message =
        isRecord(error) && typeof error.message === "string"
          ? error.message
          : typeof error === "string"
            ? error
            : typeof parsed.detail === "string"
              ? parsed.detail
              : typeof parsed.message === "string"
                ? parsed.message
                : "";
    }
  } catch {
    // A plain-text error body is used as it is.
  }
  return message.replace(/\s+/g, " ").trim().slice(0, 200);
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}

/**
 * Reads `verbose_json` (`{text, language, segments: [{start, end, text}]}` in seconds), plain
 * `json` (`{text}`) or a `text/plain` body. Malformed segments are dropped rather than trusted.
 */
export function parseTranscriptionResponse(body: string, contentType: string | null): Transcript {
  const trimmed = body.trim();
  const looksJson = /json/i.test(contentType ?? "") || trimmed.startsWith("{");
  if (!looksJson) return { text: trimmed, segments: [] };
  let parsed: unknown;
  try {
    parsed = JSON.parse(trimmed);
  } catch {
    throw new Error("Transcription server returned invalid JSON");
  }
  if (!isRecord(parsed) || typeof parsed.text !== "string") {
    throw new Error("Transcription server returned no text");
  }
  const segments: TranscriptSegment[] = [];
  if (Array.isArray(parsed.segments)) {
    for (const entry of parsed.segments) {
      if (!isRecord(entry) || typeof entry.text !== "string") continue;
      const start = Number(entry.start);
      const end = Number(entry.end);
      const text = entry.text.trim();
      if (!text || !Number.isFinite(start) || !Number.isFinite(end) || start < 0) continue;
      const startMs = Math.round(start * 1000);
      const previousEnd = segments.at(-1)?.endMs ?? 0;
      segments.push({
        startMs: Math.max(startMs, previousEnd),
        endMs: Math.max(Math.round(end * 1000), startMs, previousEnd),
        text,
      });
    }
  }
  const language = languageCode(parsed.language);
  return {
    text: parsed.text.trim() || segments.map((segment) => segment.text).join(" "),
    ...(language ? { language } : {}),
    segments,
  };
}

/** Whisper's language codes; servers report either the code or its English name. */
const WHISPER_LANGUAGES =
  "en zh de es ru ko fr ja pt tr pl ca nl ar sv it id hi fi vi he uk el ms cs ro da hu ta no th ur hr bg lt la mi ml cy sk te fa lv bn sr az sl kn et mk br eu is hy ne mn bs kk sq sw gl mr pa si km sn yo so af oc ka be tg sd gu am yi lo uz fo ht ps tk nn mt sa lb my bo tl mg as tt haw ln ha ba jw su yue".split(
    " ",
  );
let languageNames: Map<string, string> | undefined;

function languageCode(value: unknown): string | undefined {
  if (typeof value !== "string") return undefined;
  const normalized = value.trim().toLowerCase();
  if (WHISPER_LANGUAGES.includes(normalized)) return normalized;
  if (!languageNames) {
    const names = new Intl.DisplayNames(["en"], { type: "language" });
    languageNames = new Map(
      WHISPER_LANGUAGES.map((code) => [(names.of(code) ?? code).toLowerCase(), code]),
    );
  }
  return languageNames.get(normalized);
}

/** OpenAI takes ISO 639-1 codes; a BCP 47 tag such as `de-AT` becomes `de`. */
function languageHint(language: string | undefined): string | undefined {
  const primary = language?.trim().split(/[-_]/)[0]?.toLowerCase();
  return primary && /^[a-z]{2,3}$/.test(primary) ? primary : undefined;
}

const EXTENSIONS: Record<string, string> = {
  "audio/mpeg": "mp3",
  "audio/mp4": "m4a",
  "audio/wav": "wav",
  "audio/x-wav": "wav",
  "audio/webm": "webm",
  "audio/ogg": "ogg",
  "audio/flac": "flac",
  "video/mp4": "mp4",
  "video/webm": "webm",
};

/** Servers pick the decoder from the file extension, so the upload name always carries one. */
function uploadName(fileName: string | undefined, mimeType: string): string {
  const safe = (fileName ?? "").replace(/[^\w.-]+/g, "_").slice(-100);
  if (/\.[a-z0-9]{2,4}$/i.test(safe)) return safe;
  const extension = EXTENSIONS[mimeType.split(";")[0]?.trim().toLowerCase() ?? ""] ?? "wav";
  return `${safe || "audio"}.${extension}`;
}

/**
 * A multipart/form-data body built by hand: the undici fetch used for address-checked requests
 * does not serialize the global FormData class, and a byte body works with every fetch.
 */
function multipartBody(
  fields: Record<string, string>,
  file: { name: string; mimeType: string; bytes: Uint8Array },
): { contentType: string; bytes: Uint8Array<ArrayBuffer> } {
  const boundary = `----rakazo${randomUUID().replaceAll("-", "")}`;
  const encoder = new TextEncoder();
  const parts: Uint8Array[] = [];
  for (const [name, value] of Object.entries(fields)) {
    parts.push(
      encoder.encode(
        `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`,
      ),
    );
  }
  parts.push(
    encoder.encode(
      `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${file.name}"\r\nContent-Type: ${file.mimeType.replace(/[\r\n]/g, "")}\r\n\r\n`,
    ),
    file.bytes,
    encoder.encode(`\r\n--${boundary}--\r\n`),
  );
  const bytes = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
  let offset = 0;
  for (const part of parts) {
    bytes.set(part, offset);
    offset += part.byteLength;
  }
  return { contentType: `multipart/form-data; boundary=${boundary}`, bytes };
}