evOo-local/packages/adapters/src/transcription-emulator.ts
Claude 3f85cb0420
feat(media): transcribe speech on any OpenAI-compatible server and write subtitles
Transcription has to run on the user's own hardware, and the only speech-to-text
path so far was the OpenAI voice key. Self-hosted whisper servers (whisper.cpp,
faster-whisper based servers, vLLM) all speak the OpenAI transcriptions
protocol, so one provider covers them.

- createOpenAiCompatibleTranscriptionProvider posts a multipart upload to
  {base}/audio/transcriptions asking for timed segments, falls back to plain
  JSON for servers that reject verbose_json, reads text/plain bodies, and
  reports failures in words. It reuses the address-checked fetch and URL
  guards of the other OpenAI-compatible adapters, caps uploads at 25 MiB and
  responses at 8 MiB, and builds the multipart body by hand because the
  undici fetch behind the address checks does not serialize the global
  FormData class.
- An in-process server emulator (OpenAI/whisper.cpp, plain-JSON-only and
  text-only flavours) and an in-process provider back an offline
  conformance suite, plus one loopback test through the real fetch.
- toSrt/toVtt in core turn segments into SubRip and WebVTT, wrapping long
  segments into cues of at most two 42-character lines that share the
  segment's time, and escaping WebVTT cue text.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
2026-10-03 13:17:17 +00:00

146 lines
5.2 KiB
TypeScript

import type {
Transcript,
TranscriptionCapabilities,
TranscriptionProvider,
TranscriptionRequest,
} from "@rakazo/adapter-kit";
/** What the emulators "hear" in any audio: two timed sentences. */
export const EMULATED_TRANSCRIPT: Transcript = {
text: "Welcome to the studio. Today we cut a vertical clip.",
language: "en",
segments: [
{ startMs: 0, endMs: 2400, text: "Welcome to the studio." },
{ startMs: 2400, endMs: 5100, text: "Today we cut a vertical clip." },
],
};
const EMULATED_MAX_BYTES = 25 * 1024 * 1024;
/** Offline transcription provider for tests and conformance; no speech server involved. */
export class EmulatorTranscriptionProvider implements TranscriptionProvider {
describe() {
return {
id: "emulator",
contractVersion: "1",
adapterVersion: "0.1.0",
capabilities: {
segments: true,
languageDetection: true,
maxBytes: EMULATED_MAX_BYTES,
} satisfies TranscriptionCapabilities,
};
}
async transcribe(request: TranscriptionRequest): Promise<Transcript> {
request.signal?.throwIfAborted();
if (request.audio.byteLength === 0) throw new Error("The audio is empty");
if (request.audio.byteLength > EMULATED_MAX_BYTES) {
throw new Error("The audio is too large for the transcription server");
}
return {
...EMULATED_TRANSCRIPT,
...(request.language ? { language: request.language } : {}),
segments: EMULATED_TRANSCRIPT.segments.map((segment) => ({ ...segment })),
};
}
}
export interface EmulatedTranscriptionRequest {
url: string;
authorization: string | null;
model: string | null;
responseFormat: string | null;
language: string | null;
fileName: string | null;
fileType: string | null;
fileSize: number;
}
/**
* In-process substitute for an OpenAI-compatible `POST /v1/audio/transcriptions` server, so
* the real HTTP adapter runs in offline tests. `openai` answers every response format like
* OpenAI and whisper.cpp; `plain` mimics servers without timed output that reject
* `verbose_json`; `text` answers with a bare `text/plain` body whatever was asked.
*/
export class OpenAiCompatibleTranscriptionServerEmulator {
readonly requests: EmulatedTranscriptionRequest[] = [];
/** Answer the next request with this HTTP status instead of a transcript. */
failNextStatus: number | undefined;
constructor(
private readonly options: { apiKey?: string; flavor?: "openai" | "plain" | "text" } = {},
) {}
readonly fetch: typeof fetch = async (input, init) => {
init?.signal?.throwIfAborted();
const url = new URL(input instanceof Request ? input.url : String(input));
if (init?.method !== "POST" || !url.pathname.endsWith("/audio/transcriptions")) {
return Response.json({ error: { message: "Not found" } }, { status: 404 });
}
const headers = new Headers(init.headers);
const authorization = headers.get("authorization");
let form: FormData;
try {
form = await new Request(url, { method: "POST", headers, body: init.body }).formData();
} catch {
return Response.json({ error: { message: "Expected multipart/form-data" } }, { status: 400 });
}
const file = form.get("file");
const field = (name: string) => {
const value = form.get(name);
return typeof value === "string" ? value : null;
};
const responseFormat = field("response_format");
this.requests.push({
url: url.href,
authorization,
model: field("model"),
responseFormat,
language: field("language"),
fileName: file instanceof File ? file.name : null,
fileType: file instanceof File ? file.type : null,
fileSize: file instanceof File ? file.size : 0,
});
if (this.failNextStatus) {
const status = this.failNextStatus;
this.failNextStatus = undefined;
return Response.json({ error: { message: "Upstream failure" } }, { status });
}
if (this.options.apiKey && authorization !== `Bearer ${this.options.apiKey}`) {
return Response.json({ error: { message: "Invalid API key" } }, { status: 401 });
}
if (!(file instanceof File) || file.size === 0) {
return Response.json({ error: { message: "file is required" } }, { status: 400 });
}
const flavor = this.options.flavor ?? "openai";
if (flavor === "text" || responseFormat === "text") {
return new Response(EMULATED_TRANSCRIPT.text, {
headers: { "content-type": "text/plain; charset=utf-8" },
});
}
if (responseFormat === "verbose_json") {
if (flavor === "plain") {
return Response.json(
{ error: { message: "Unsupported response_format: verbose_json" } },
{ status: 400 },
);
}
return Response.json({
task: "transcribe",
language: "english",
duration: EMULATED_TRANSCRIPT.segments.at(-1)!.endMs / 1000,
text: EMULATED_TRANSCRIPT.text,
segments: EMULATED_TRANSCRIPT.segments.map((segment, id) => ({
id,
start: segment.startMs / 1000,
end: segment.endMs / 1000,
// Whisper keeps the leading space of each segment.
text: ` ${segment.text}`,
})),
});
}
return Response.json({ text: EMULATED_TRANSCRIPT.text });
};
}