feat(media): transcribe speech on any OpenAI-compatible server and write subtitles
Transcription has to run on the user's own hardware, and the only speech-to-text
path so far was the OpenAI voice key. Self-hosted whisper servers (whisper.cpp,
faster-whisper based servers, vLLM) all speak the OpenAI transcriptions
protocol, so one provider covers them.
- createOpenAiCompatibleTranscriptionProvider posts a multipart upload to
{base}/audio/transcriptions asking for timed segments, falls back to plain
JSON for servers that reject verbose_json, reads text/plain bodies, and
reports failures in words. It reuses the address-checked fetch and URL
guards of the other OpenAI-compatible adapters, caps uploads at 25 MiB and
responses at 8 MiB, and builds the multipart body by hand because the
undici fetch behind the address checks does not serialize the global
FormData class.
- An in-process server emulator (OpenAI/whisper.cpp, plain-JSON-only and
text-only flavours) and an in-process provider back an offline
conformance suite, plus one loopback test through the real fetch.
- toSrt/toVtt in core turn segments into SubRip and WebVTT, wrapping long
segments into cues of at most two 42-character lines that share the
segment's time, and escaping WebVTT cue text.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
parent
7ff93f08bb
commit
3f85cb0420
@ -86,6 +86,7 @@ export * from "./model-selection.js";
|
||||
export * from "./model-vision.js";
|
||||
export * from "./none-sandbox.js";
|
||||
export * from "./openai-compatible-embeddings.js";
|
||||
export * from "./openai-compatible-transcription.js";
|
||||
export * from "./openai-compatible-url.js";
|
||||
export * from "./openai-voice.js";
|
||||
export * from "./page-browser-session.js";
|
||||
@ -120,6 +121,7 @@ export { SupermemoryMemoryProvider } from "./supermemory-memory-provider.js";
|
||||
export * from "./teaching-session.js";
|
||||
export * from "./team-chat-messaging.js";
|
||||
export * from "./third-party-connector-emulator.js";
|
||||
export * from "./transcription-emulator.js";
|
||||
export * from "./voice-factory.js";
|
||||
export * from "./wakeup.js";
|
||||
export * from "./web-limits.js";
|
||||
|
||||
308
packages/adapters/src/openai-compatible-transcription.ts
Normal file
308
packages/adapters/src/openai-compatible-transcription.ts
Normal file
@ -0,0 +1,308 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type {
|
||||
AdapterDescriptor,
|
||||
Transcript,
|
||||
TranscriptionCapabilities,
|
||||
TranscriptionProvider,
|
||||
TranscriptionRequest,
|
||||
TranscriptSegment,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import {
|
||||
assertAllowedOpenAiCompatibleUrl,
|
||||
assertHttpsForKeyedOpenAiCompatibleUrl,
|
||||
OPENAI_COMPATIBLE_PROVIDER_ID,
|
||||
} from "./openai-compatible-url.js";
|
||||
import { createOpenAiCompatibleFetch } from "./pi-openai-compatible-provider.js";
|
||||
import { readBodyCapped } from "./web-ssrf.js";
|
||||
|
||||
/** The OpenAI upload limit; whisper.cpp and faster-whisper servers accept at least this much. */
|
||||
export const OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES = 25 * 1024 * 1024;
|
||||
/** Used when no model is configured; whisper.cpp ignores it and most servers alias it. */
|
||||
export const OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL = "whisper-1";
|
||||
// CPU-only whisper servers need minutes for a long upload; the job's signal still cancels sooner.
|
||||
const TRANSCRIPTION_TIMEOUT_MS = 15 * 60_000;
|
||||
const MAX_TRANSCRIPTION_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
export interface OpenAiCompatibleTranscriptionConfig {
|
||||
baseUrl: string;
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
type ResponseFormat = "verbose_json" | "json";
|
||||
|
||||
/**
|
||||
* Speech to text from any server that speaks the OpenAI `POST /v1/audio/transcriptions`
|
||||
* protocol: whisper.cpp's server (with `--inference-path /v1/audio/transcriptions`),
|
||||
* faster-whisper based servers, vLLM, or a hosted API. Asks for timed segments and falls back
|
||||
* to plain text when the server only returns that.
|
||||
*/
|
||||
export function createOpenAiCompatibleTranscriptionProvider(
|
||||
config: OpenAiCompatibleTranscriptionConfig,
|
||||
): TranscriptionProvider {
|
||||
const baseUrl = assertAllowedOpenAiCompatibleUrl(config.baseUrl);
|
||||
const apiKey = config.apiKey?.trim() || undefined;
|
||||
assertHttpsForKeyedOpenAiCompatibleUrl(baseUrl, apiKey);
|
||||
const model = config.model?.trim() || OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL;
|
||||
const endpoint = new URL("audio/transcriptions", `${baseUrl.href}/`).href;
|
||||
const fetchImpl = createOpenAiCompatibleFetch(config.fetch);
|
||||
|
||||
function post(
|
||||
request: TranscriptionRequest,
|
||||
format: ResponseFormat,
|
||||
signal: AbortSignal,
|
||||
): Promise<Response> {
|
||||
const fields: Record<string, string> = { model, response_format: format };
|
||||
const language = languageHint(request.language);
|
||||
if (language) fields.language = language;
|
||||
const body = multipartBody(fields, {
|
||||
name: uploadName(request.fileName, request.mimeType),
|
||||
mimeType: request.mimeType || "application/octet-stream",
|
||||
bytes: request.audio,
|
||||
});
|
||||
const headers: Record<string, string> = {
|
||||
Accept: "application/json, text/plain",
|
||||
"Content-Type": body.contentType,
|
||||
};
|
||||
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
||||
return fetchImpl(endpoint, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: body.bytes,
|
||||
redirect: "error",
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
async function transcribeAs(
|
||||
request: TranscriptionRequest,
|
||||
format: ResponseFormat,
|
||||
): Promise<Transcript | { rejectedFormat: true }> {
|
||||
const timeout = AbortSignal.timeout(TRANSCRIPTION_TIMEOUT_MS);
|
||||
const signal = request.signal ? AbortSignal.any([request.signal, timeout]) : timeout;
|
||||
try {
|
||||
const response = await post(request, format, signal);
|
||||
const text = new TextDecoder().decode(await readResponse(response, signal));
|
||||
if (!response.ok) {
|
||||
const message = errorMessage(text);
|
||||
// Servers that only implement `json`/`text` reject the timed format; ask again plainly.
|
||||
if (
|
||||
format === "verbose_json" &&
|
||||
(response.status === 400 || response.status === 422) &&
|
||||
/format|verbose/i.test(message)
|
||||
) {
|
||||
return { rejectedFormat: true };
|
||||
}
|
||||
throw new Error(httpErrorMessage(response.status, message));
|
||||
}
|
||||
return parseTranscriptionResponse(text, response.headers.get("content-type"));
|
||||
} catch (error) {
|
||||
if (request.signal?.aborted) throw error;
|
||||
if (timeout.aborted) throw new Error("Transcription server timed out");
|
||||
// undici reports refused or reset connections as a bare "fetch failed".
|
||||
if (error instanceof TypeError && error.message === "fetch failed") {
|
||||
throw new Error("Transcription server is unreachable");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
describe(): AdapterDescriptor<TranscriptionCapabilities> {
|
||||
return {
|
||||
id: OPENAI_COMPATIBLE_PROVIDER_ID,
|
||||
contractVersion: "1",
|
||||
adapterVersion: "0.1.0",
|
||||
capabilities: {
|
||||
segments: true,
|
||||
languageDetection: true,
|
||||
maxBytes: OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES,
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
async transcribe(request: TranscriptionRequest): Promise<Transcript> {
|
||||
request.signal?.throwIfAborted();
|
||||
if (request.audio.byteLength === 0) throw new Error("The audio is empty");
|
||||
if (request.audio.byteLength > OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES) {
|
||||
throw new Error("The audio is too large for the transcription server");
|
||||
}
|
||||
const timed = await transcribeAs(request, "verbose_json");
|
||||
if (!("rejectedFormat" in timed)) return timed;
|
||||
const plain = await transcribeAs(request, "json");
|
||||
if ("rejectedFormat" in plain) throw new Error("Transcription server rejected the request");
|
||||
return plain;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function readResponse(response: Response, signal: AbortSignal): Promise<Uint8Array> {
|
||||
try {
|
||||
return await readBodyCapped(response, MAX_TRANSCRIPTION_RESPONSE_BYTES, signal);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === "Response is too large") {
|
||||
throw new Error("Transcription server returned too much data");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function httpErrorMessage(status: number, detail: string): string {
|
||||
if (status === 401 || status === 403) return "Transcription server rejected the API key";
|
||||
if (status === 413) return "The audio is too large for the transcription server";
|
||||
if (status === 404) return "Transcription server has no /audio/transcriptions endpoint";
|
||||
return detail
|
||||
? `Transcription server returned ${status}: ${detail}`
|
||||
: `Transcription server returned ${status}`;
|
||||
}
|
||||
|
||||
/** The server's own message from an OpenAI-style `{error: {message}}` or a short plain body. */
|
||||
function errorMessage(body: string): string {
|
||||
let message = body;
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(body);
|
||||
if (isRecord(parsed)) {
|
||||
const error = parsed.error;
|
||||
message =
|
||||
isRecord(error) && typeof error.message === "string"
|
||||
? error.message
|
||||
: typeof error === "string"
|
||||
? error
|
||||
: typeof parsed.detail === "string"
|
||||
? parsed.detail
|
||||
: typeof parsed.message === "string"
|
||||
? parsed.message
|
||||
: "";
|
||||
}
|
||||
} catch {
|
||||
// A plain-text error body is used as it is.
|
||||
}
|
||||
return message.replace(/\s+/g, " ").trim().slice(0, 200);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads `verbose_json` (`{text, language, segments: [{start, end, text}]}` in seconds), plain
|
||||
* `json` (`{text}`) or a `text/plain` body. Malformed segments are dropped rather than trusted.
|
||||
*/
|
||||
export function parseTranscriptionResponse(body: string, contentType: string | null): Transcript {
|
||||
const trimmed = body.trim();
|
||||
const looksJson = /json/i.test(contentType ?? "") || trimmed.startsWith("{");
|
||||
if (!looksJson) return { text: trimmed, segments: [] };
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(trimmed);
|
||||
} catch {
|
||||
throw new Error("Transcription server returned invalid JSON");
|
||||
}
|
||||
if (!isRecord(parsed) || typeof parsed.text !== "string") {
|
||||
throw new Error("Transcription server returned no text");
|
||||
}
|
||||
const segments: TranscriptSegment[] = [];
|
||||
if (Array.isArray(parsed.segments)) {
|
||||
for (const entry of parsed.segments) {
|
||||
if (!isRecord(entry) || typeof entry.text !== "string") continue;
|
||||
const start = Number(entry.start);
|
||||
const end = Number(entry.end);
|
||||
const text = entry.text.trim();
|
||||
if (!text || !Number.isFinite(start) || !Number.isFinite(end) || start < 0) continue;
|
||||
const startMs = Math.round(start * 1000);
|
||||
const previousEnd = segments.at(-1)?.endMs ?? 0;
|
||||
segments.push({
|
||||
startMs: Math.max(startMs, previousEnd),
|
||||
endMs: Math.max(Math.round(end * 1000), startMs, previousEnd),
|
||||
text,
|
||||
});
|
||||
}
|
||||
}
|
||||
const language = languageCode(parsed.language);
|
||||
return {
|
||||
text: parsed.text.trim() || segments.map((segment) => segment.text).join(" "),
|
||||
...(language ? { language } : {}),
|
||||
segments,
|
||||
};
|
||||
}
|
||||
|
||||
/** Whisper's language codes; servers report either the code or its English name. */
|
||||
const WHISPER_LANGUAGES =
|
||||
"en zh de es ru ko fr ja pt tr pl ca nl ar sv it id hi fi vi he uk el ms cs ro da hu ta no th ur hr bg lt la mi ml cy sk te fa lv bn sr az sl kn et mk br eu is hy ne mn bs kk sq sw gl mr pa si km sn yo so af oc ka be tg sd gu am yi lo uz fo ht ps tk nn mt sa lb my bo tl mg as tt haw ln ha ba jw su yue".split(
|
||||
" ",
|
||||
);
|
||||
let languageNames: Map<string, string> | undefined;
|
||||
|
||||
function languageCode(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (WHISPER_LANGUAGES.includes(normalized)) return normalized;
|
||||
if (!languageNames) {
|
||||
const names = new Intl.DisplayNames(["en"], { type: "language" });
|
||||
languageNames = new Map(
|
||||
WHISPER_LANGUAGES.map((code) => [(names.of(code) ?? code).toLowerCase(), code]),
|
||||
);
|
||||
}
|
||||
return languageNames.get(normalized);
|
||||
}
|
||||
|
||||
/** OpenAI takes ISO 639-1 codes; a BCP 47 tag such as `de-AT` becomes `de`. */
|
||||
function languageHint(language: string | undefined): string | undefined {
|
||||
const primary = language?.trim().split(/[-_]/)[0]?.toLowerCase();
|
||||
return primary && /^[a-z]{2,3}$/.test(primary) ? primary : undefined;
|
||||
}
|
||||
|
||||
const EXTENSIONS: Record<string, string> = {
|
||||
"audio/mpeg": "mp3",
|
||||
"audio/mp4": "m4a",
|
||||
"audio/wav": "wav",
|
||||
"audio/x-wav": "wav",
|
||||
"audio/webm": "webm",
|
||||
"audio/ogg": "ogg",
|
||||
"audio/flac": "flac",
|
||||
"video/mp4": "mp4",
|
||||
"video/webm": "webm",
|
||||
};
|
||||
|
||||
/** Servers pick the decoder from the file extension, so the upload name always carries one. */
|
||||
function uploadName(fileName: string | undefined, mimeType: string): string {
|
||||
const safe = (fileName ?? "").replace(/[^\w.-]+/g, "_").slice(-100);
|
||||
if (/\.[a-z0-9]{2,4}$/i.test(safe)) return safe;
|
||||
const extension = EXTENSIONS[mimeType.split(";")[0]?.trim().toLowerCase() ?? ""] ?? "wav";
|
||||
return `${safe || "audio"}.${extension}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* A multipart/form-data body built by hand: the undici fetch used for address-checked requests
|
||||
* does not serialize the global FormData class, and a byte body works with every fetch.
|
||||
*/
|
||||
function multipartBody(
|
||||
fields: Record<string, string>,
|
||||
file: { name: string; mimeType: string; bytes: Uint8Array },
|
||||
): { contentType: string; bytes: Uint8Array<ArrayBuffer> } {
|
||||
const boundary = `----rakazo${randomUUID().replaceAll("-", "")}`;
|
||||
const encoder = new TextEncoder();
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [name, value] of Object.entries(fields)) {
|
||||
parts.push(
|
||||
encoder.encode(
|
||||
`--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`,
|
||||
),
|
||||
);
|
||||
}
|
||||
parts.push(
|
||||
encoder.encode(
|
||||
`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${file.name}"\r\nContent-Type: ${file.mimeType.replace(/[\r\n]/g, "")}\r\n\r\n`,
|
||||
),
|
||||
file.bytes,
|
||||
encoder.encode(`\r\n--${boundary}--\r\n`),
|
||||
);
|
||||
const bytes = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
bytes.set(part, offset);
|
||||
offset += part.byteLength;
|
||||
}
|
||||
return { contentType: `multipart/form-data; boundary=${boundary}`, bytes };
|
||||
}
|
||||
286
packages/adapters/src/transcription-conformance.test.ts
Normal file
286
packages/adapters/src/transcription-conformance.test.ts
Normal file
@ -0,0 +1,286 @@
|
||||
import { createServer } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { TranscriptionProvider } from "@rakazo/adapter-kit";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createOpenAiCompatibleTranscriptionProvider,
|
||||
OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL,
|
||||
OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES,
|
||||
parseTranscriptionResponse,
|
||||
} from "./openai-compatible-transcription.js";
|
||||
import {
|
||||
EMULATED_TRANSCRIPT,
|
||||
EmulatorTranscriptionProvider,
|
||||
OpenAiCompatibleTranscriptionServerEmulator,
|
||||
} from "./transcription-emulator.js";
|
||||
|
||||
const AUDIO = new Uint8Array([82, 73, 70, 70, 1, 2, 3, 4]);
|
||||
|
||||
function httpProvider(
|
||||
flavor: "openai" | "plain" | "text",
|
||||
options: { model?: string; apiKey?: string } = {},
|
||||
) {
|
||||
const server = new OpenAiCompatibleTranscriptionServerEmulator({
|
||||
flavor,
|
||||
apiKey: options.apiKey,
|
||||
});
|
||||
const provider = createOpenAiCompatibleTranscriptionProvider({
|
||||
baseUrl: "http://127.0.0.1:8080",
|
||||
...options,
|
||||
fetch: server.fetch,
|
||||
});
|
||||
return { server, provider };
|
||||
}
|
||||
|
||||
const factories: Record<string, () => TranscriptionProvider> = {
|
||||
emulator: () => new EmulatorTranscriptionProvider(),
|
||||
"openai-compatible": () => httpProvider("openai").provider,
|
||||
"openai-compatible without timed output": () => httpProvider("plain").provider,
|
||||
"openai-compatible plain text": () => httpProvider("text").provider,
|
||||
};
|
||||
|
||||
/**
|
||||
* Offline conformance: every TranscriptionProvider declares its limits, returns the spoken
|
||||
* text with ordered, non-overlapping segments in milliseconds (or none), refuses empty or
|
||||
* oversized audio before uploading, and honors cancellation.
|
||||
*/
|
||||
for (const [name, create] of Object.entries(factories)) {
|
||||
describe(`${name} transcription conformance (offline)`, () => {
|
||||
it("describes the contract and its limits", () => {
|
||||
const descriptor = create().describe();
|
||||
expect(descriptor.contractVersion).toBe("1");
|
||||
expect(descriptor.capabilities.maxBytes).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("returns the text and ordered segments", async () => {
|
||||
const transcript = await create().transcribe({ audio: AUDIO, mimeType: "audio/wav" });
|
||||
expect(transcript.text).toBe(EMULATED_TRANSCRIPT.text);
|
||||
let previousEnd = 0;
|
||||
for (const segment of transcript.segments) {
|
||||
expect(segment.text.trim()).toBe(segment.text);
|
||||
expect(segment.text.length).toBeGreaterThan(0);
|
||||
expect(segment.startMs).toBeGreaterThanOrEqual(previousEnd);
|
||||
expect(segment.endMs).toBeGreaterThanOrEqual(segment.startMs);
|
||||
previousEnd = segment.endMs;
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses empty and oversized audio", async () => {
|
||||
const provider = create();
|
||||
await expect(
|
||||
provider.transcribe({ audio: new Uint8Array(), mimeType: "audio/wav" }),
|
||||
).rejects.toThrow(/empty/);
|
||||
const tooLarge = new Uint8Array(provider.describe().capabilities.maxBytes + 1);
|
||||
await expect(provider.transcribe({ audio: tooLarge, mimeType: "audio/wav" })).rejects.toThrow(
|
||||
/too large/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects when the caller cancels", async () => {
|
||||
await expect(
|
||||
create().transcribe({ audio: AUDIO, mimeType: "audio/wav", signal: AbortSignal.abort() }),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe("OpenAI-compatible transcription provider", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("posts a multipart upload to /v1/audio/transcriptions asking for timed segments", async () => {
|
||||
const { server, provider } = httpProvider("openai", { model: " " });
|
||||
const transcript = await provider.transcribe({
|
||||
audio: AUDIO,
|
||||
mimeType: "audio/wav",
|
||||
fileName: "interview take 1.wav",
|
||||
language: "de-AT",
|
||||
});
|
||||
expect(server.requests).toEqual([
|
||||
{
|
||||
url: "http://127.0.0.1:8080/v1/audio/transcriptions",
|
||||
authorization: null,
|
||||
model: OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL,
|
||||
responseFormat: "verbose_json",
|
||||
language: "de",
|
||||
fileName: "interview_take_1.wav",
|
||||
fileType: "audio/wav",
|
||||
fileSize: AUDIO.byteLength,
|
||||
},
|
||||
]);
|
||||
expect(transcript).toEqual({ ...EMULATED_TRANSCRIPT, language: "en" });
|
||||
});
|
||||
|
||||
it("sends a real multipart body through the default address-checked fetch", async () => {
|
||||
const received: { contentType?: string; body?: Buffer } = {};
|
||||
const server = createServer((request, response) => {
|
||||
const chunks: Buffer[] = [];
|
||||
request.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
request.on("end", () => {
|
||||
received.contentType = request.headers["content-type"];
|
||||
received.body = Buffer.concat(chunks);
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.end(JSON.stringify({ text: "loopback" }));
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const { port } = server.address() as AddressInfo;
|
||||
const provider = createOpenAiCompatibleTranscriptionProvider({
|
||||
baseUrl: `http://127.0.0.1:${port}`,
|
||||
});
|
||||
await expect(
|
||||
provider.transcribe({ audio: AUDIO, mimeType: "audio/wav", fileName: "a.wav" }),
|
||||
).resolves.toEqual({ text: "loopback", segments: [] });
|
||||
const form = await new Request("http://127.0.0.1/", {
|
||||
method: "POST",
|
||||
headers: { "content-type": received.contentType ?? "" },
|
||||
body: new Uint8Array(received.body ?? []),
|
||||
}).formData();
|
||||
expect(form.get("response_format")).toBe("verbose_json");
|
||||
const file = form.get("file");
|
||||
expect(file instanceof File && file.name).toBe("a.wav");
|
||||
expect(file instanceof File && new Uint8Array(await file.arrayBuffer())).toEqual(AUDIO);
|
||||
} finally {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
}
|
||||
});
|
||||
|
||||
it("names an upload without an extension after its type and sends the configured model", async () => {
|
||||
const { server, provider } = httpProvider("openai", { model: "Systran/faster-whisper-small" });
|
||||
await provider.transcribe({ audio: AUDIO, mimeType: "audio/mpeg", fileName: "voice note" });
|
||||
expect(server.requests[0]).toMatchObject({
|
||||
model: "Systran/faster-whisper-small",
|
||||
fileName: "voice_note.mp3",
|
||||
});
|
||||
});
|
||||
|
||||
it("asks again for plain JSON when the server rejects timed output", async () => {
|
||||
const { server, provider } = httpProvider("plain");
|
||||
const transcript = await provider.transcribe({ audio: AUDIO, mimeType: "audio/wav" });
|
||||
expect(server.requests.map((request) => request.responseFormat)).toEqual([
|
||||
"verbose_json",
|
||||
"json",
|
||||
]);
|
||||
expect(transcript).toEqual({ text: EMULATED_TRANSCRIPT.text, segments: [] });
|
||||
});
|
||||
|
||||
it("sends the API key as a bearer token over https", async () => {
|
||||
vi.stubEnv("RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC", "1");
|
||||
const server = new OpenAiCompatibleTranscriptionServerEmulator({ apiKey: "test-key" });
|
||||
const provider = createOpenAiCompatibleTranscriptionProvider({
|
||||
baseUrl: "https://speech.example.test/v1",
|
||||
apiKey: "test-key",
|
||||
fetch: server.fetch,
|
||||
});
|
||||
await expect(
|
||||
provider.transcribe({ audio: AUDIO, mimeType: "audio/wav" }),
|
||||
).resolves.toMatchObject({ text: EMULATED_TRANSCRIPT.text });
|
||||
expect(server.requests[0]?.authorization).toBe("Bearer test-key");
|
||||
});
|
||||
|
||||
it("reports server failures in words", async () => {
|
||||
const { server, provider } = httpProvider("openai");
|
||||
server.failNextStatus = 500;
|
||||
await expect(provider.transcribe({ audio: AUDIO, mimeType: "audio/wav" })).rejects.toThrow(
|
||||
"Transcription server returned 500: Upstream failure",
|
||||
);
|
||||
const keyed = httpProvider("openai", { apiKey: "right" });
|
||||
const wrong = createOpenAiCompatibleTranscriptionProvider({
|
||||
baseUrl: "http://127.0.0.1:8080",
|
||||
apiKey: "wrong",
|
||||
fetch: keyed.server.fetch,
|
||||
});
|
||||
await expect(wrong.transcribe({ audio: AUDIO, mimeType: "audio/wav" })).rejects.toThrow(
|
||||
"Transcription server rejected the API key",
|
||||
);
|
||||
const missing = createOpenAiCompatibleTranscriptionProvider({
|
||||
baseUrl: "http://127.0.0.1:8080",
|
||||
fetch: async () => Response.json({ error: { message: "Not found" } }, { status: 404 }),
|
||||
});
|
||||
await expect(missing.transcribe({ audio: AUDIO, mimeType: "audio/wav" })).rejects.toThrow(
|
||||
"Transcription server has no /audio/transcriptions endpoint",
|
||||
);
|
||||
const unreachable = createOpenAiCompatibleTranscriptionProvider({
|
||||
baseUrl: "http://127.0.0.1:8080",
|
||||
fetch: async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
},
|
||||
});
|
||||
await expect(unreachable.transcribe({ audio: AUDIO, mimeType: "audio/wav" })).rejects.toThrow(
|
||||
"Transcription server is unreachable",
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses unsafe endpoints before sending anything", () => {
|
||||
expect(() =>
|
||||
createOpenAiCompatibleTranscriptionProvider({ baseUrl: "http://169.254.169.254" }),
|
||||
).toThrow(/blocked/);
|
||||
expect(() =>
|
||||
createOpenAiCompatibleTranscriptionProvider({ baseUrl: "https://speech.example.test" }),
|
||||
).toThrow(/Public model endpoints are blocked/);
|
||||
vi.stubEnv("RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC", "1");
|
||||
expect(() =>
|
||||
createOpenAiCompatibleTranscriptionProvider({
|
||||
baseUrl: "http://speech.example.test",
|
||||
apiKey: "secret",
|
||||
}),
|
||||
).toThrow(/must use HTTPS/);
|
||||
expect(() =>
|
||||
createOpenAiCompatibleTranscriptionProvider({ baseUrl: "file:///etc/passwd" }),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("caps the upload size at the protocol limit", () => {
|
||||
const { provider } = httpProvider("openai");
|
||||
expect(provider.describe().capabilities.maxBytes).toBe(
|
||||
OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseTranscriptionResponse", () => {
|
||||
it("reads verbose JSON segments in seconds, dropping malformed ones and overlaps", () => {
|
||||
const transcript = parseTranscriptionResponse(
|
||||
JSON.stringify({
|
||||
text: " Hallo. Wie geht's? ",
|
||||
language: "german",
|
||||
segments: [
|
||||
{ start: 0, end: 1.25, text: " Hallo." },
|
||||
{ start: "x", end: 2, text: "bad timing" },
|
||||
{ start: 1, end: 2.5, text: " Wie geht's?" },
|
||||
{ start: 3, end: 4, text: " " },
|
||||
{ start: 3, end: 4 },
|
||||
],
|
||||
}),
|
||||
"application/json",
|
||||
);
|
||||
expect(transcript).toEqual({
|
||||
text: "Hallo. Wie geht's?",
|
||||
language: "de",
|
||||
segments: [
|
||||
{ startMs: 0, endMs: 1250, text: "Hallo." },
|
||||
{ startMs: 1250, endMs: 2500, text: "Wie geht's?" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a language code and reads plain JSON and text bodies", () => {
|
||||
expect(parseTranscriptionResponse('{"text":"hi","language":"en"}', null)).toEqual({
|
||||
text: "hi",
|
||||
language: "en",
|
||||
segments: [],
|
||||
});
|
||||
expect(parseTranscriptionResponse(" just words \n", "text/plain")).toEqual({
|
||||
text: "just words",
|
||||
segments: [],
|
||||
});
|
||||
expect(() => parseTranscriptionResponse('{"segments":[]}', "application/json")).toThrow(
|
||||
"Transcription server returned no text",
|
||||
);
|
||||
expect(() => parseTranscriptionResponse("{oops", "application/json")).toThrow(
|
||||
"Transcription server returned invalid JSON",
|
||||
);
|
||||
});
|
||||
});
|
||||
145
packages/adapters/src/transcription-emulator.ts
Normal file
145
packages/adapters/src/transcription-emulator.ts
Normal file
@ -0,0 +1,145 @@
|
||||
import type {
|
||||
Transcript,
|
||||
TranscriptionCapabilities,
|
||||
TranscriptionProvider,
|
||||
TranscriptionRequest,
|
||||
} from "@rakazo/adapter-kit";
|
||||
|
||||
/** What the emulators "hear" in any audio: two timed sentences. */
|
||||
export const EMULATED_TRANSCRIPT: Transcript = {
|
||||
text: "Welcome to the studio. Today we cut a vertical clip.",
|
||||
language: "en",
|
||||
segments: [
|
||||
{ startMs: 0, endMs: 2400, text: "Welcome to the studio." },
|
||||
{ startMs: 2400, endMs: 5100, text: "Today we cut a vertical clip." },
|
||||
],
|
||||
};
|
||||
|
||||
const EMULATED_MAX_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
/** Offline transcription provider for tests and conformance; no speech server involved. */
|
||||
export class EmulatorTranscriptionProvider implements TranscriptionProvider {
|
||||
describe() {
|
||||
return {
|
||||
id: "emulator",
|
||||
contractVersion: "1",
|
||||
adapterVersion: "0.1.0",
|
||||
capabilities: {
|
||||
segments: true,
|
||||
languageDetection: true,
|
||||
maxBytes: EMULATED_MAX_BYTES,
|
||||
} satisfies TranscriptionCapabilities,
|
||||
};
|
||||
}
|
||||
|
||||
async transcribe(request: TranscriptionRequest): Promise<Transcript> {
|
||||
request.signal?.throwIfAborted();
|
||||
if (request.audio.byteLength === 0) throw new Error("The audio is empty");
|
||||
if (request.audio.byteLength > EMULATED_MAX_BYTES) {
|
||||
throw new Error("The audio is too large for the transcription server");
|
||||
}
|
||||
return {
|
||||
...EMULATED_TRANSCRIPT,
|
||||
...(request.language ? { language: request.language } : {}),
|
||||
segments: EMULATED_TRANSCRIPT.segments.map((segment) => ({ ...segment })),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export interface EmulatedTranscriptionRequest {
|
||||
url: string;
|
||||
authorization: string | null;
|
||||
model: string | null;
|
||||
responseFormat: string | null;
|
||||
language: string | null;
|
||||
fileName: string | null;
|
||||
fileType: string | null;
|
||||
fileSize: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* In-process substitute for an OpenAI-compatible `POST /v1/audio/transcriptions` server, so
|
||||
* the real HTTP adapter runs in offline tests. `openai` answers every response format like
|
||||
* OpenAI and whisper.cpp; `plain` mimics servers without timed output that reject
|
||||
* `verbose_json`; `text` answers with a bare `text/plain` body whatever was asked.
|
||||
*/
|
||||
export class OpenAiCompatibleTranscriptionServerEmulator {
|
||||
readonly requests: EmulatedTranscriptionRequest[] = [];
|
||||
/** Answer the next request with this HTTP status instead of a transcript. */
|
||||
failNextStatus: number | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly options: { apiKey?: string; flavor?: "openai" | "plain" | "text" } = {},
|
||||
) {}
|
||||
|
||||
readonly fetch: typeof fetch = async (input, init) => {
|
||||
init?.signal?.throwIfAborted();
|
||||
const url = new URL(input instanceof Request ? input.url : String(input));
|
||||
if (init?.method !== "POST" || !url.pathname.endsWith("/audio/transcriptions")) {
|
||||
return Response.json({ error: { message: "Not found" } }, { status: 404 });
|
||||
}
|
||||
const headers = new Headers(init.headers);
|
||||
const authorization = headers.get("authorization");
|
||||
let form: FormData;
|
||||
try {
|
||||
form = await new Request(url, { method: "POST", headers, body: init.body }).formData();
|
||||
} catch {
|
||||
return Response.json({ error: { message: "Expected multipart/form-data" } }, { status: 400 });
|
||||
}
|
||||
const file = form.get("file");
|
||||
const field = (name: string) => {
|
||||
const value = form.get(name);
|
||||
return typeof value === "string" ? value : null;
|
||||
};
|
||||
const responseFormat = field("response_format");
|
||||
this.requests.push({
|
||||
url: url.href,
|
||||
authorization,
|
||||
model: field("model"),
|
||||
responseFormat,
|
||||
language: field("language"),
|
||||
fileName: file instanceof File ? file.name : null,
|
||||
fileType: file instanceof File ? file.type : null,
|
||||
fileSize: file instanceof File ? file.size : 0,
|
||||
});
|
||||
if (this.failNextStatus) {
|
||||
const status = this.failNextStatus;
|
||||
this.failNextStatus = undefined;
|
||||
return Response.json({ error: { message: "Upstream failure" } }, { status });
|
||||
}
|
||||
if (this.options.apiKey && authorization !== `Bearer ${this.options.apiKey}`) {
|
||||
return Response.json({ error: { message: "Invalid API key" } }, { status: 401 });
|
||||
}
|
||||
if (!(file instanceof File) || file.size === 0) {
|
||||
return Response.json({ error: { message: "file is required" } }, { status: 400 });
|
||||
}
|
||||
const flavor = this.options.flavor ?? "openai";
|
||||
if (flavor === "text" || responseFormat === "text") {
|
||||
return new Response(EMULATED_TRANSCRIPT.text, {
|
||||
headers: { "content-type": "text/plain; charset=utf-8" },
|
||||
});
|
||||
}
|
||||
if (responseFormat === "verbose_json") {
|
||||
if (flavor === "plain") {
|
||||
return Response.json(
|
||||
{ error: { message: "Unsupported response_format: verbose_json" } },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
return Response.json({
|
||||
task: "transcribe",
|
||||
language: "english",
|
||||
duration: EMULATED_TRANSCRIPT.segments.at(-1)!.endMs / 1000,
|
||||
text: EMULATED_TRANSCRIPT.text,
|
||||
segments: EMULATED_TRANSCRIPT.segments.map((segment, id) => ({
|
||||
id,
|
||||
start: segment.startMs / 1000,
|
||||
end: segment.endMs / 1000,
|
||||
// Whisper keeps the leading space of each segment.
|
||||
text: ` ${segment.text}`,
|
||||
})),
|
||||
});
|
||||
}
|
||||
return Response.json({ text: EMULATED_TRANSCRIPT.text });
|
||||
};
|
||||
}
|
||||
@ -22,6 +22,7 @@ export * from "./featured-connectors.js";
|
||||
export * from "./group-mentions.js";
|
||||
export * from "./http-response.js";
|
||||
export * from "./mcp.js";
|
||||
export * from "./media/subtitles.js";
|
||||
export * from "./message-pages.js";
|
||||
export * from "./message-reactions.js";
|
||||
export * from "./message-visibility.js";
|
||||
|
||||
76
packages/core/src/media/subtitles.test.ts
Normal file
76
packages/core/src/media/subtitles.test.ts
Normal file
@ -0,0 +1,76 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { SUBTITLE_MAX_LINE_CHARS, toSrt, toVtt } from "./subtitles.js";
|
||||
|
||||
const SEGMENTS = [
|
||||
{ startMs: 0, endMs: 2400, text: " Welcome to the studio." },
|
||||
{ startMs: 2400, endMs: 5100, text: "Today we cut a vertical clip." },
|
||||
];
|
||||
|
||||
describe("subtitles", () => {
|
||||
it("writes numbered SubRip cues with comma milliseconds", () => {
|
||||
expect(toSrt(SEGMENTS)).toBe(
|
||||
[
|
||||
"1",
|
||||
"00:00:00,000 --> 00:00:02,400",
|
||||
"Welcome to the studio.",
|
||||
"",
|
||||
"2",
|
||||
"00:00:02,400 --> 00:00:05,100",
|
||||
"Today we cut a vertical clip.",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
|
||||
it("writes WebVTT with a header, dot milliseconds and escaped cue text", () => {
|
||||
expect(
|
||||
toVtt([...SEGMENTS, { startMs: 3_723_004, endMs: 3_725_000, text: "Tom & <Jerry> --> go" }]),
|
||||
).toBe(
|
||||
[
|
||||
"WEBVTT",
|
||||
"",
|
||||
"00:00:00.000 --> 00:00:02.400",
|
||||
"Welcome to the studio.",
|
||||
"",
|
||||
"00:00:02.400 --> 00:00:05.100",
|
||||
"Today we cut a vertical clip.",
|
||||
"",
|
||||
"01:02:03.004 --> 01:02:05.000",
|
||||
"Tom & <Jerry> → go",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns an empty file body for no speech", () => {
|
||||
expect(toSrt([])).toBe("");
|
||||
expect(toVtt([{ startMs: 0, endMs: 10, text: " \n " }])).toBe("WEBVTT\n");
|
||||
});
|
||||
|
||||
it("wraps long segments into two-line cues that share the segment's time", () => {
|
||||
const text =
|
||||
"This is a long sentence that keeps going well past what fits on a phone screen, so it has to wrap across several readable cues.";
|
||||
const srt = toSrt([{ startMs: 1000, endMs: 9000, text }]);
|
||||
const cues = srt.trim().split("\n\n");
|
||||
expect(cues.length).toBeGreaterThan(1);
|
||||
const times = cues.map((cue) => cue.split("\n")[1]);
|
||||
expect(times[0]?.startsWith("00:00:01,000 --> ")).toBe(true);
|
||||
expect(times.at(-1)?.endsWith(" --> 00:00:09,000")).toBe(true);
|
||||
for (let index = 1; index < times.length; index += 1) {
|
||||
expect(times[index]?.split(" --> ")[0]).toBe(times[index - 1]?.split(" --> ")[1]);
|
||||
}
|
||||
for (const cue of cues) {
|
||||
const lines = cue.split("\n").slice(2);
|
||||
expect(lines.length).toBeLessThanOrEqual(2);
|
||||
for (const line of lines) expect(line.length).toBeLessThanOrEqual(SUBTITLE_MAX_LINE_CHARS);
|
||||
}
|
||||
expect(cues.flatMap((cue) => cue.split("\n").slice(2)).join(" ")).toBe(text);
|
||||
});
|
||||
|
||||
it("keeps a word longer than a line whole and never runs time backwards", () => {
|
||||
const word = "a".repeat(60);
|
||||
expect(toSrt([{ startMs: 500, endMs: 100, text: word }])).toBe(
|
||||
`1\n00:00:00,500 --> 00:00:00,500\n${word}\n`,
|
||||
);
|
||||
});
|
||||
});
|
||||
98
packages/core/src/media/subtitles.ts
Normal file
98
packages/core/src/media/subtitles.ts
Normal file
@ -0,0 +1,98 @@
|
||||
/** A timed piece of a transcript, in milliseconds from the start of the media. */
|
||||
export interface SubtitleSegment {
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** Readable on a phone held upright and the common broadcast limit for one subtitle line. */
|
||||
export const SUBTITLE_MAX_LINE_CHARS = 42;
|
||||
const SUBTITLE_MAX_LINES = 2;
|
||||
|
||||
/** SubRip (`.srt`): numbered cues with `HH:MM:SS,mmm` times. */
|
||||
export function toSrt(segments: readonly SubtitleSegment[]): string {
|
||||
return subtitleCues(segments)
|
||||
.map(
|
||||
(cue, index) =>
|
||||
`${index + 1}\n${timestamp(cue.startMs, ",")} --> ${timestamp(cue.endMs, ",")}\n${cue.lines.join("\n")}\n`,
|
||||
)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
/** WebVTT (`.vtt`): the format browsers play in `<track>`, with cue text escaped. */
|
||||
export function toVtt(segments: readonly SubtitleSegment[]): string {
|
||||
const cues = subtitleCues(segments).map(
|
||||
(cue) =>
|
||||
`${timestamp(cue.startMs, ".")} --> ${timestamp(cue.endMs, ".")}\n${cue.lines.map(escapeVtt).join("\n")}\n`,
|
||||
);
|
||||
return ["WEBVTT\n", ...cues].join("\n");
|
||||
}
|
||||
|
||||
interface SubtitleCue {
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
lines: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns transcript segments into cues of at most two short lines. A long segment becomes
|
||||
* several cues that share its time span in proportion to their length, so subtitles stay
|
||||
* readable instead of covering the frame.
|
||||
*/
|
||||
function subtitleCues(segments: readonly SubtitleSegment[]): SubtitleCue[] {
|
||||
const cues: SubtitleCue[] = [];
|
||||
for (const segment of segments) {
|
||||
// A blank line ends a cue in both formats, and "-->" would read as a timing line.
|
||||
const text = segment.text.replaceAll("-->", "→").replace(/\s+/g, " ").trim();
|
||||
if (!text) continue;
|
||||
const startMs = Math.max(0, Math.round(segment.startMs));
|
||||
const endMs = Math.max(startMs, Math.round(segment.endMs));
|
||||
const lines = wrapLines(text, SUBTITLE_MAX_LINE_CHARS);
|
||||
const groups: string[][] = [];
|
||||
for (let index = 0; index < lines.length; index += SUBTITLE_MAX_LINES) {
|
||||
groups.push(lines.slice(index, index + SUBTITLE_MAX_LINES));
|
||||
}
|
||||
const total = groups.reduce((sum, group) => sum + group.join(" ").length, 0);
|
||||
let cursor = startMs;
|
||||
let consumed = 0;
|
||||
for (const [index, group] of groups.entries()) {
|
||||
consumed += group.join(" ").length;
|
||||
const groupEnd =
|
||||
index === groups.length - 1
|
||||
? endMs
|
||||
: startMs + Math.round(((endMs - startMs) * consumed) / total);
|
||||
cues.push({ startMs: cursor, endMs: groupEnd, lines: group });
|
||||
cursor = groupEnd;
|
||||
}
|
||||
}
|
||||
return cues;
|
||||
}
|
||||
|
||||
function wrapLines(text: string, maxChars: number): string[] {
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of text.split(" ")) {
|
||||
if (!line) {
|
||||
line = word;
|
||||
} else if (line.length + 1 + word.length <= maxChars) {
|
||||
line = `${line} ${word}`;
|
||||
} else {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
}
|
||||
}
|
||||
if (line) lines.push(line);
|
||||
return lines;
|
||||
}
|
||||
|
||||
function timestamp(ms: number, separator: "," | "."): string {
|
||||
const hours = Math.floor(ms / 3_600_000);
|
||||
const minutes = Math.floor((ms % 3_600_000) / 60_000);
|
||||
const seconds = Math.floor((ms % 60_000) / 1000);
|
||||
const millis = ms % 1000;
|
||||
return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}${separator}${String(millis).padStart(3, "0")}`;
|
||||
}
|
||||
|
||||
function escapeVtt(line: string): string {
|
||||
return line.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user