import { randomUUID } from "node:crypto"; import type { AdapterDescriptor, Transcript, TranscriptionCapabilities, TranscriptionProvider, TranscriptionRequest, TranscriptSegment, } from "@rakazo/adapter-kit"; import { assertAllowedOpenAiCompatibleUrl, assertHttpsForKeyedOpenAiCompatibleUrl, OPENAI_COMPATIBLE_PROVIDER_ID, } from "./openai-compatible-url.js"; import { createOpenAiCompatibleFetch } from "./pi-openai-compatible-provider.js"; import { readBodyCapped } from "./web-ssrf.js"; /** The OpenAI upload limit; whisper.cpp and faster-whisper servers accept at least this much. */ export const OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES = 25 * 1024 * 1024; /** Used when no model is configured; whisper.cpp ignores it and most servers alias it. */ export const OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL = "whisper-1"; // CPU-only whisper servers need minutes for a long upload; the job's signal still cancels sooner. const TRANSCRIPTION_TIMEOUT_MS = 15 * 60_000; const MAX_TRANSCRIPTION_RESPONSE_BYTES = 8 * 1024 * 1024; export interface OpenAiCompatibleTranscriptionConfig { baseUrl: string; model?: string; apiKey?: string; fetch?: typeof globalThis.fetch; } type ResponseFormat = "verbose_json" | "json"; /** * Speech to text from any server that speaks the OpenAI `POST /v1/audio/transcriptions` * protocol: whisper.cpp's server (with `--inference-path /v1/audio/transcriptions`), * faster-whisper based servers, vLLM, or a hosted API. Asks for timed segments and falls back * to plain text when the server only returns that. */ export function createOpenAiCompatibleTranscriptionProvider( config: OpenAiCompatibleTranscriptionConfig, ): TranscriptionProvider { const baseUrl = assertAllowedOpenAiCompatibleUrl(config.baseUrl); const apiKey = config.apiKey?.trim() || undefined; assertHttpsForKeyedOpenAiCompatibleUrl(baseUrl, apiKey); const model = config.model?.trim() || OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL; const endpoint = new URL("audio/transcriptions", `${baseUrl.href}/`).href; const fetchImpl = createOpenAiCompatibleFetch(config.fetch); function post( request: TranscriptionRequest, format: ResponseFormat, signal: AbortSignal, ): Promise { const fields: Record = { model, response_format: format }; const language = languageHint(request.language); if (language) fields.language = language; const body = multipartBody(fields, { name: uploadName(request.fileName, request.mimeType), mimeType: request.mimeType || "application/octet-stream", bytes: request.audio, }); const headers: Record = { Accept: "application/json, text/plain", "Content-Type": body.contentType, }; if (apiKey) headers.Authorization = `Bearer ${apiKey}`; return fetchImpl(endpoint, { method: "POST", headers, body: body.bytes, redirect: "error", signal, }); } async function transcribeAs( request: TranscriptionRequest, format: ResponseFormat, ): Promise { const timeout = AbortSignal.timeout(TRANSCRIPTION_TIMEOUT_MS); const signal = request.signal ? AbortSignal.any([request.signal, timeout]) : timeout; try { const response = await post(request, format, signal); const text = new TextDecoder().decode(await readResponse(response, signal)); if (!response.ok) { const message = errorMessage(text); // Servers that only implement `json`/`text` reject the timed format; ask again plainly. if ( format === "verbose_json" && (response.status === 400 || response.status === 422) && /format|verbose/i.test(message) ) { return { rejectedFormat: true }; } throw new Error(httpErrorMessage(response.status, message)); } return parseTranscriptionResponse(text, response.headers.get("content-type")); } catch (error) { if (request.signal?.aborted) throw error; if (timeout.aborted) throw new Error("Transcription server timed out"); // undici reports refused or reset connections as a bare "fetch failed". if (error instanceof TypeError && error.message === "fetch failed") { throw new Error("Transcription server is unreachable"); } throw error; } } return { describe(): AdapterDescriptor { return { id: OPENAI_COMPATIBLE_PROVIDER_ID, contractVersion: "1", adapterVersion: "0.1.0", capabilities: { segments: true, languageDetection: true, maxBytes: OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES, }, }; }, async transcribe(request: TranscriptionRequest): Promise { request.signal?.throwIfAborted(); if (request.audio.byteLength === 0) throw new Error("The audio is empty"); if (request.audio.byteLength > OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES) { throw new Error("The audio is too large for the transcription server"); } const timed = await transcribeAs(request, "verbose_json"); if (!("rejectedFormat" in timed)) return timed; const plain = await transcribeAs(request, "json"); if ("rejectedFormat" in plain) throw new Error("Transcription server rejected the request"); return plain; }, }; } async function readResponse(response: Response, signal: AbortSignal): Promise { try { return await readBodyCapped(response, MAX_TRANSCRIPTION_RESPONSE_BYTES, signal); } catch (error) { if (error instanceof Error && error.message === "Response is too large") { throw new Error("Transcription server returned too much data"); } throw error; } } function httpErrorMessage(status: number, detail: string): string { if (status === 401 || status === 403) return "Transcription server rejected the API key"; if (status === 413) return "The audio is too large for the transcription server"; if (status === 404) return "Transcription server has no /audio/transcriptions endpoint"; return detail ? `Transcription server returned ${status}: ${detail}` : `Transcription server returned ${status}`; } /** The server's own message from an OpenAI-style `{error: {message}}` or a short plain body. */ function errorMessage(body: string): string { let message = body; try { const parsed: unknown = JSON.parse(body); if (isRecord(parsed)) { const error = parsed.error; message = isRecord(error) && typeof error.message === "string" ? error.message : typeof error === "string" ? error : typeof parsed.detail === "string" ? parsed.detail : typeof parsed.message === "string" ? parsed.message : ""; } } catch { // A plain-text error body is used as it is. } return message.replace(/\s+/g, " ").trim().slice(0, 200); } function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } /** * Reads `verbose_json` (`{text, language, segments: [{start, end, text}]}` in seconds), plain * `json` (`{text}`) or a `text/plain` body. Malformed segments are dropped rather than trusted. */ export function parseTranscriptionResponse(body: string, contentType: string | null): Transcript { const trimmed = body.trim(); const looksJson = /json/i.test(contentType ?? "") || trimmed.startsWith("{"); if (!looksJson) return { text: trimmed, segments: [] }; let parsed: unknown; try { parsed = JSON.parse(trimmed); } catch { throw new Error("Transcription server returned invalid JSON"); } if (!isRecord(parsed) || typeof parsed.text !== "string") { throw new Error("Transcription server returned no text"); } const segments: TranscriptSegment[] = []; if (Array.isArray(parsed.segments)) { for (const entry of parsed.segments) { if (!isRecord(entry) || typeof entry.text !== "string") continue; const start = Number(entry.start); const end = Number(entry.end); const text = entry.text.trim(); if (!text || !Number.isFinite(start) || !Number.isFinite(end) || start < 0) continue; const startMs = Math.round(start * 1000); const previousEnd = segments.at(-1)?.endMs ?? 0; segments.push({ startMs: Math.max(startMs, previousEnd), endMs: Math.max(Math.round(end * 1000), startMs, previousEnd), text, }); } } const language = languageCode(parsed.language); return { text: parsed.text.trim() || segments.map((segment) => segment.text).join(" "), ...(language ? { language } : {}), segments, }; } /** Whisper's language codes; servers report either the code or its English name. */ const WHISPER_LANGUAGES = "en zh de es ru ko fr ja pt tr pl ca nl ar sv it id hi fi vi he uk el ms cs ro da hu ta no th ur hr bg lt la mi ml cy sk te fa lv bn sr az sl kn et mk br eu is hy ne mn bs kk sq sw gl mr pa si km sn yo so af oc ka be tg sd gu am yi lo uz fo ht ps tk nn mt sa lb my bo tl mg as tt haw ln ha ba jw su yue".split( " ", ); let languageNames: Map | undefined; function languageCode(value: unknown): string | undefined { if (typeof value !== "string") return undefined; const normalized = value.trim().toLowerCase(); if (WHISPER_LANGUAGES.includes(normalized)) return normalized; if (!languageNames) { const names = new Intl.DisplayNames(["en"], { type: "language" }); languageNames = new Map( WHISPER_LANGUAGES.map((code) => [(names.of(code) ?? code).toLowerCase(), code]), ); } return languageNames.get(normalized); } /** OpenAI takes ISO 639-1 codes; a BCP 47 tag such as `de-AT` becomes `de`. */ function languageHint(language: string | undefined): string | undefined { const primary = language?.trim().split(/[-_]/)[0]?.toLowerCase(); return primary && /^[a-z]{2,3}$/.test(primary) ? primary : undefined; } const EXTENSIONS: Record = { "audio/mpeg": "mp3", "audio/mp4": "m4a", "audio/wav": "wav", "audio/x-wav": "wav", "audio/webm": "webm", "audio/ogg": "ogg", "audio/flac": "flac", "video/mp4": "mp4", "video/webm": "webm", }; /** Servers pick the decoder from the file extension, so the upload name always carries one. */ function uploadName(fileName: string | undefined, mimeType: string): string { const safe = (fileName ?? "").replace(/[^\w.-]+/g, "_").slice(-100); if (/\.[a-z0-9]{2,4}$/i.test(safe)) return safe; const extension = EXTENSIONS[mimeType.split(";")[0]?.trim().toLowerCase() ?? ""] ?? "wav"; return `${safe || "audio"}.${extension}`; } /** * A multipart/form-data body built by hand: the undici fetch used for address-checked requests * does not serialize the global FormData class, and a byte body works with every fetch. */ function multipartBody( fields: Record, file: { name: string; mimeType: string; bytes: Uint8Array }, ): { contentType: string; bytes: Uint8Array } { const boundary = `----rakazo${randomUUID().replaceAll("-", "")}`; const encoder = new TextEncoder(); const parts: Uint8Array[] = []; for (const [name, value] of Object.entries(fields)) { parts.push( encoder.encode( `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`, ), ); } parts.push( encoder.encode( `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${file.name}"\r\nContent-Type: ${file.mimeType.replace(/[\r\n]/g, "")}\r\n\r\n`, ), file.bytes, encoder.encode(`\r\n--${boundary}--\r\n`), ); const bytes = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0)); let offset = 0; for (const part of parts) { bytes.set(part, offset); offset += part.byteLength; } return { contentType: `multipart/form-data; boundary=${boundary}`, bytes }; }