diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 7b4ecd0..aa1ca1c 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -86,6 +86,7 @@ export * from "./model-selection.js"; export * from "./model-vision.js"; export * from "./none-sandbox.js"; export * from "./openai-compatible-embeddings.js"; +export * from "./openai-compatible-transcription.js"; export * from "./openai-compatible-url.js"; export * from "./openai-voice.js"; export * from "./page-browser-session.js"; @@ -120,6 +121,7 @@ export { SupermemoryMemoryProvider } from "./supermemory-memory-provider.js"; export * from "./teaching-session.js"; export * from "./team-chat-messaging.js"; export * from "./third-party-connector-emulator.js"; +export * from "./transcription-emulator.js"; export * from "./voice-factory.js"; export * from "./wakeup.js"; export * from "./web-limits.js"; diff --git a/packages/adapters/src/openai-compatible-transcription.ts b/packages/adapters/src/openai-compatible-transcription.ts new file mode 100644 index 0000000..1b71c27 --- /dev/null +++ b/packages/adapters/src/openai-compatible-transcription.ts @@ -0,0 +1,308 @@ +import { randomUUID } from "node:crypto"; +import type { + AdapterDescriptor, + Transcript, + TranscriptionCapabilities, + TranscriptionProvider, + TranscriptionRequest, + TranscriptSegment, +} from "@rakazo/adapter-kit"; +import { + assertAllowedOpenAiCompatibleUrl, + assertHttpsForKeyedOpenAiCompatibleUrl, + OPENAI_COMPATIBLE_PROVIDER_ID, +} from "./openai-compatible-url.js"; +import { createOpenAiCompatibleFetch } from "./pi-openai-compatible-provider.js"; +import { readBodyCapped } from "./web-ssrf.js"; + +/** The OpenAI upload limit; whisper.cpp and faster-whisper servers accept at least this much. */ +export const OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES = 25 * 1024 * 1024; +/** Used when no model is configured; whisper.cpp ignores it and most servers alias it. */ +export const OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL = "whisper-1"; +// CPU-only whisper servers need minutes for a long upload; the job's signal still cancels sooner. +const TRANSCRIPTION_TIMEOUT_MS = 15 * 60_000; +const MAX_TRANSCRIPTION_RESPONSE_BYTES = 8 * 1024 * 1024; + +export interface OpenAiCompatibleTranscriptionConfig { + baseUrl: string; + model?: string; + apiKey?: string; + fetch?: typeof globalThis.fetch; +} + +type ResponseFormat = "verbose_json" | "json"; + +/** + * Speech to text from any server that speaks the OpenAI `POST /v1/audio/transcriptions` + * protocol: whisper.cpp's server (with `--inference-path /v1/audio/transcriptions`), + * faster-whisper based servers, vLLM, or a hosted API. Asks for timed segments and falls back + * to plain text when the server only returns that. + */ +export function createOpenAiCompatibleTranscriptionProvider( + config: OpenAiCompatibleTranscriptionConfig, +): TranscriptionProvider { + const baseUrl = assertAllowedOpenAiCompatibleUrl(config.baseUrl); + const apiKey = config.apiKey?.trim() || undefined; + assertHttpsForKeyedOpenAiCompatibleUrl(baseUrl, apiKey); + const model = config.model?.trim() || OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL; + const endpoint = new URL("audio/transcriptions", `${baseUrl.href}/`).href; + const fetchImpl = createOpenAiCompatibleFetch(config.fetch); + + function post( + request: TranscriptionRequest, + format: ResponseFormat, + signal: AbortSignal, + ): Promise { + const fields: Record = { model, response_format: format }; + const language = languageHint(request.language); + if (language) fields.language = language; + const body = multipartBody(fields, { + name: uploadName(request.fileName, request.mimeType), + mimeType: request.mimeType || "application/octet-stream", + bytes: request.audio, + }); + const headers: Record = { + Accept: "application/json, text/plain", + "Content-Type": body.contentType, + }; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + return fetchImpl(endpoint, { + method: "POST", + headers, + body: body.bytes, + redirect: "error", + signal, + }); + } + + async function transcribeAs( + request: TranscriptionRequest, + format: ResponseFormat, + ): Promise { + const timeout = AbortSignal.timeout(TRANSCRIPTION_TIMEOUT_MS); + const signal = request.signal ? AbortSignal.any([request.signal, timeout]) : timeout; + try { + const response = await post(request, format, signal); + const text = new TextDecoder().decode(await readResponse(response, signal)); + if (!response.ok) { + const message = errorMessage(text); + // Servers that only implement `json`/`text` reject the timed format; ask again plainly. + if ( + format === "verbose_json" && + (response.status === 400 || response.status === 422) && + /format|verbose/i.test(message) + ) { + return { rejectedFormat: true }; + } + throw new Error(httpErrorMessage(response.status, message)); + } + return parseTranscriptionResponse(text, response.headers.get("content-type")); + } catch (error) { + if (request.signal?.aborted) throw error; + if (timeout.aborted) throw new Error("Transcription server timed out"); + // undici reports refused or reset connections as a bare "fetch failed". + if (error instanceof TypeError && error.message === "fetch failed") { + throw new Error("Transcription server is unreachable"); + } + throw error; + } + } + + return { + describe(): AdapterDescriptor { + return { + id: OPENAI_COMPATIBLE_PROVIDER_ID, + contractVersion: "1", + adapterVersion: "0.1.0", + capabilities: { + segments: true, + languageDetection: true, + maxBytes: OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES, + }, + }; + }, + + async transcribe(request: TranscriptionRequest): Promise { + request.signal?.throwIfAborted(); + if (request.audio.byteLength === 0) throw new Error("The audio is empty"); + if (request.audio.byteLength > OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES) { + throw new Error("The audio is too large for the transcription server"); + } + const timed = await transcribeAs(request, "verbose_json"); + if (!("rejectedFormat" in timed)) return timed; + const plain = await transcribeAs(request, "json"); + if ("rejectedFormat" in plain) throw new Error("Transcription server rejected the request"); + return plain; + }, + }; +} + +async function readResponse(response: Response, signal: AbortSignal): Promise { + try { + return await readBodyCapped(response, MAX_TRANSCRIPTION_RESPONSE_BYTES, signal); + } catch (error) { + if (error instanceof Error && error.message === "Response is too large") { + throw new Error("Transcription server returned too much data"); + } + throw error; + } +} + +function httpErrorMessage(status: number, detail: string): string { + if (status === 401 || status === 403) return "Transcription server rejected the API key"; + if (status === 413) return "The audio is too large for the transcription server"; + if (status === 404) return "Transcription server has no /audio/transcriptions endpoint"; + return detail + ? `Transcription server returned ${status}: ${detail}` + : `Transcription server returned ${status}`; +} + +/** The server's own message from an OpenAI-style `{error: {message}}` or a short plain body. */ +function errorMessage(body: string): string { + let message = body; + try { + const parsed: unknown = JSON.parse(body); + if (isRecord(parsed)) { + const error = parsed.error; + message = + isRecord(error) && typeof error.message === "string" + ? error.message + : typeof error === "string" + ? error + : typeof parsed.detail === "string" + ? parsed.detail + : typeof parsed.message === "string" + ? parsed.message + : ""; + } + } catch { + // A plain-text error body is used as it is. + } + return message.replace(/\s+/g, " ").trim().slice(0, 200); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +/** + * Reads `verbose_json` (`{text, language, segments: [{start, end, text}]}` in seconds), plain + * `json` (`{text}`) or a `text/plain` body. Malformed segments are dropped rather than trusted. + */ +export function parseTranscriptionResponse(body: string, contentType: string | null): Transcript { + const trimmed = body.trim(); + const looksJson = /json/i.test(contentType ?? "") || trimmed.startsWith("{"); + if (!looksJson) return { text: trimmed, segments: [] }; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + throw new Error("Transcription server returned invalid JSON"); + } + if (!isRecord(parsed) || typeof parsed.text !== "string") { + throw new Error("Transcription server returned no text"); + } + const segments: TranscriptSegment[] = []; + if (Array.isArray(parsed.segments)) { + for (const entry of parsed.segments) { + if (!isRecord(entry) || typeof entry.text !== "string") continue; + const start = Number(entry.start); + const end = Number(entry.end); + const text = entry.text.trim(); + if (!text || !Number.isFinite(start) || !Number.isFinite(end) || start < 0) continue; + const startMs = Math.round(start * 1000); + const previousEnd = segments.at(-1)?.endMs ?? 0; + segments.push({ + startMs: Math.max(startMs, previousEnd), + endMs: Math.max(Math.round(end * 1000), startMs, previousEnd), + text, + }); + } + } + const language = languageCode(parsed.language); + return { + text: parsed.text.trim() || segments.map((segment) => segment.text).join(" "), + ...(language ? { language } : {}), + segments, + }; +} + +/** Whisper's language codes; servers report either the code or its English name. */ +const WHISPER_LANGUAGES = + "en zh de es ru ko fr ja pt tr pl ca nl ar sv it id hi fi vi he uk el ms cs ro da hu ta no th ur hr bg lt la mi ml cy sk te fa lv bn sr az sl kn et mk br eu is hy ne mn bs kk sq sw gl mr pa si km sn yo so af oc ka be tg sd gu am yi lo uz fo ht ps tk nn mt sa lb my bo tl mg as tt haw ln ha ba jw su yue".split( + " ", + ); +let languageNames: Map | undefined; + +function languageCode(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const normalized = value.trim().toLowerCase(); + if (WHISPER_LANGUAGES.includes(normalized)) return normalized; + if (!languageNames) { + const names = new Intl.DisplayNames(["en"], { type: "language" }); + languageNames = new Map( + WHISPER_LANGUAGES.map((code) => [(names.of(code) ?? code).toLowerCase(), code]), + ); + } + return languageNames.get(normalized); +} + +/** OpenAI takes ISO 639-1 codes; a BCP 47 tag such as `de-AT` becomes `de`. */ +function languageHint(language: string | undefined): string | undefined { + const primary = language?.trim().split(/[-_]/)[0]?.toLowerCase(); + return primary && /^[a-z]{2,3}$/.test(primary) ? primary : undefined; +} + +const EXTENSIONS: Record = { + "audio/mpeg": "mp3", + "audio/mp4": "m4a", + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/webm": "webm", + "audio/ogg": "ogg", + "audio/flac": "flac", + "video/mp4": "mp4", + "video/webm": "webm", +}; + +/** Servers pick the decoder from the file extension, so the upload name always carries one. */ +function uploadName(fileName: string | undefined, mimeType: string): string { + const safe = (fileName ?? "").replace(/[^\w.-]+/g, "_").slice(-100); + if (/\.[a-z0-9]{2,4}$/i.test(safe)) return safe; + const extension = EXTENSIONS[mimeType.split(";")[0]?.trim().toLowerCase() ?? ""] ?? "wav"; + return `${safe || "audio"}.${extension}`; +} + +/** + * A multipart/form-data body built by hand: the undici fetch used for address-checked requests + * does not serialize the global FormData class, and a byte body works with every fetch. + */ +function multipartBody( + fields: Record, + file: { name: string; mimeType: string; bytes: Uint8Array }, +): { contentType: string; bytes: Uint8Array } { + const boundary = `----rakazo${randomUUID().replaceAll("-", "")}`; + const encoder = new TextEncoder(); + const parts: Uint8Array[] = []; + for (const [name, value] of Object.entries(fields)) { + parts.push( + encoder.encode( + `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`, + ), + ); + } + parts.push( + encoder.encode( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${file.name}"\r\nContent-Type: ${file.mimeType.replace(/[\r\n]/g, "")}\r\n\r\n`, + ), + file.bytes, + encoder.encode(`\r\n--${boundary}--\r\n`), + ); + const bytes = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0)); + let offset = 0; + for (const part of parts) { + bytes.set(part, offset); + offset += part.byteLength; + } + return { contentType: `multipart/form-data; boundary=${boundary}`, bytes }; +} diff --git a/packages/adapters/src/transcription-conformance.test.ts b/packages/adapters/src/transcription-conformance.test.ts new file mode 100644 index 0000000..9957653 --- /dev/null +++ b/packages/adapters/src/transcription-conformance.test.ts @@ -0,0 +1,286 @@ +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import type { TranscriptionProvider } from "@rakazo/adapter-kit"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createOpenAiCompatibleTranscriptionProvider, + OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL, + OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES, + parseTranscriptionResponse, +} from "./openai-compatible-transcription.js"; +import { + EMULATED_TRANSCRIPT, + EmulatorTranscriptionProvider, + OpenAiCompatibleTranscriptionServerEmulator, +} from "./transcription-emulator.js"; + +const AUDIO = new Uint8Array([82, 73, 70, 70, 1, 2, 3, 4]); + +function httpProvider( + flavor: "openai" | "plain" | "text", + options: { model?: string; apiKey?: string } = {}, +) { + const server = new OpenAiCompatibleTranscriptionServerEmulator({ + flavor, + apiKey: options.apiKey, + }); + const provider = createOpenAiCompatibleTranscriptionProvider({ + baseUrl: "http://127.0.0.1:8080", + ...options, + fetch: server.fetch, + }); + return { server, provider }; +} + +const factories: Record TranscriptionProvider> = { + emulator: () => new EmulatorTranscriptionProvider(), + "openai-compatible": () => httpProvider("openai").provider, + "openai-compatible without timed output": () => httpProvider("plain").provider, + "openai-compatible plain text": () => httpProvider("text").provider, +}; + +/** + * Offline conformance: every TranscriptionProvider declares its limits, returns the spoken + * text with ordered, non-overlapping segments in milliseconds (or none), refuses empty or + * oversized audio before uploading, and honors cancellation. + */ +for (const [name, create] of Object.entries(factories)) { + describe(`${name} transcription conformance (offline)`, () => { + it("describes the contract and its limits", () => { + const descriptor = create().describe(); + expect(descriptor.contractVersion).toBe("1"); + expect(descriptor.capabilities.maxBytes).toBeGreaterThan(0); + }); + + it("returns the text and ordered segments", async () => { + const transcript = await create().transcribe({ audio: AUDIO, mimeType: "audio/wav" }); + expect(transcript.text).toBe(EMULATED_TRANSCRIPT.text); + let previousEnd = 0; + for (const segment of transcript.segments) { + expect(segment.text.trim()).toBe(segment.text); + expect(segment.text.length).toBeGreaterThan(0); + expect(segment.startMs).toBeGreaterThanOrEqual(previousEnd); + expect(segment.endMs).toBeGreaterThanOrEqual(segment.startMs); + previousEnd = segment.endMs; + } + }); + + it("refuses empty and oversized audio", async () => { + const provider = create(); + await expect( + provider.transcribe({ audio: new Uint8Array(), mimeType: "audio/wav" }), + ).rejects.toThrow(/empty/); + const tooLarge = new Uint8Array(provider.describe().capabilities.maxBytes + 1); + await expect(provider.transcribe({ audio: tooLarge, mimeType: "audio/wav" })).rejects.toThrow( + /too large/, + ); + }); + + it("rejects when the caller cancels", async () => { + await expect( + create().transcribe({ audio: AUDIO, mimeType: "audio/wav", signal: AbortSignal.abort() }), + ).rejects.toThrow(); + }); + }); +} + +describe("OpenAI-compatible transcription provider", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("posts a multipart upload to /v1/audio/transcriptions asking for timed segments", async () => { + const { server, provider } = httpProvider("openai", { model: " " }); + const transcript = await provider.transcribe({ + audio: AUDIO, + mimeType: "audio/wav", + fileName: "interview take 1.wav", + language: "de-AT", + }); + expect(server.requests).toEqual([ + { + url: "http://127.0.0.1:8080/v1/audio/transcriptions", + authorization: null, + model: OPENAI_COMPATIBLE_TRANSCRIPTION_DEFAULT_MODEL, + responseFormat: "verbose_json", + language: "de", + fileName: "interview_take_1.wav", + fileType: "audio/wav", + fileSize: AUDIO.byteLength, + }, + ]); + expect(transcript).toEqual({ ...EMULATED_TRANSCRIPT, language: "en" }); + }); + + it("sends a real multipart body through the default address-checked fetch", async () => { + const received: { contentType?: string; body?: Buffer } = {}; + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + received.contentType = request.headers["content-type"]; + received.body = Buffer.concat(chunks); + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify({ text: "loopback" })); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const { port } = server.address() as AddressInfo; + const provider = createOpenAiCompatibleTranscriptionProvider({ + baseUrl: `http://127.0.0.1:${port}`, + }); + await expect( + provider.transcribe({ audio: AUDIO, mimeType: "audio/wav", fileName: "a.wav" }), + ).resolves.toEqual({ text: "loopback", segments: [] }); + const form = await new Request("http://127.0.0.1/", { + method: "POST", + headers: { "content-type": received.contentType ?? "" }, + body: new Uint8Array(received.body ?? []), + }).formData(); + expect(form.get("response_format")).toBe("verbose_json"); + const file = form.get("file"); + expect(file instanceof File && file.name).toBe("a.wav"); + expect(file instanceof File && new Uint8Array(await file.arrayBuffer())).toEqual(AUDIO); + } finally { + await new Promise((resolve) => server.close(resolve)); + } + }); + + it("names an upload without an extension after its type and sends the configured model", async () => { + const { server, provider } = httpProvider("openai", { model: "Systran/faster-whisper-small" }); + await provider.transcribe({ audio: AUDIO, mimeType: "audio/mpeg", fileName: "voice note" }); + expect(server.requests[0]).toMatchObject({ + model: "Systran/faster-whisper-small", + fileName: "voice_note.mp3", + }); + }); + + it("asks again for plain JSON when the server rejects timed output", async () => { + const { server, provider } = httpProvider("plain"); + const transcript = await provider.transcribe({ audio: AUDIO, mimeType: "audio/wav" }); + expect(server.requests.map((request) => request.responseFormat)).toEqual([ + "verbose_json", + "json", + ]); + expect(transcript).toEqual({ text: EMULATED_TRANSCRIPT.text, segments: [] }); + }); + + it("sends the API key as a bearer token over https", async () => { + vi.stubEnv("RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC", "1"); + const server = new OpenAiCompatibleTranscriptionServerEmulator({ apiKey: "test-key" }); + const provider = createOpenAiCompatibleTranscriptionProvider({ + baseUrl: "https://speech.example.test/v1", + apiKey: "test-key", + fetch: server.fetch, + }); + await expect( + provider.transcribe({ audio: AUDIO, mimeType: "audio/wav" }), + ).resolves.toMatchObject({ text: EMULATED_TRANSCRIPT.text }); + expect(server.requests[0]?.authorization).toBe("Bearer test-key"); + }); + + it("reports server failures in words", async () => { + const { server, provider } = httpProvider("openai"); + server.failNextStatus = 500; + await expect(provider.transcribe({ audio: AUDIO, mimeType: "audio/wav" })).rejects.toThrow( + "Transcription server returned 500: Upstream failure", + ); + const keyed = httpProvider("openai", { apiKey: "right" }); + const wrong = createOpenAiCompatibleTranscriptionProvider({ + baseUrl: "http://127.0.0.1:8080", + apiKey: "wrong", + fetch: keyed.server.fetch, + }); + await expect(wrong.transcribe({ audio: AUDIO, mimeType: "audio/wav" })).rejects.toThrow( + "Transcription server rejected the API key", + ); + const missing = createOpenAiCompatibleTranscriptionProvider({ + baseUrl: "http://127.0.0.1:8080", + fetch: async () => Response.json({ error: { message: "Not found" } }, { status: 404 }), + }); + await expect(missing.transcribe({ audio: AUDIO, mimeType: "audio/wav" })).rejects.toThrow( + "Transcription server has no /audio/transcriptions endpoint", + ); + const unreachable = createOpenAiCompatibleTranscriptionProvider({ + baseUrl: "http://127.0.0.1:8080", + fetch: async () => { + throw new TypeError("fetch failed"); + }, + }); + await expect(unreachable.transcribe({ audio: AUDIO, mimeType: "audio/wav" })).rejects.toThrow( + "Transcription server is unreachable", + ); + }); + + it("refuses unsafe endpoints before sending anything", () => { + expect(() => + createOpenAiCompatibleTranscriptionProvider({ baseUrl: "http://169.254.169.254" }), + ).toThrow(/blocked/); + expect(() => + createOpenAiCompatibleTranscriptionProvider({ baseUrl: "https://speech.example.test" }), + ).toThrow(/Public model endpoints are blocked/); + vi.stubEnv("RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC", "1"); + expect(() => + createOpenAiCompatibleTranscriptionProvider({ + baseUrl: "http://speech.example.test", + apiKey: "secret", + }), + ).toThrow(/must use HTTPS/); + expect(() => + createOpenAiCompatibleTranscriptionProvider({ baseUrl: "file:///etc/passwd" }), + ).toThrow(); + }); + + it("caps the upload size at the protocol limit", () => { + const { provider } = httpProvider("openai"); + expect(provider.describe().capabilities.maxBytes).toBe( + OPENAI_COMPATIBLE_TRANSCRIPTION_MAX_BYTES, + ); + }); +}); + +describe("parseTranscriptionResponse", () => { + it("reads verbose JSON segments in seconds, dropping malformed ones and overlaps", () => { + const transcript = parseTranscriptionResponse( + JSON.stringify({ + text: " Hallo. Wie geht's? ", + language: "german", + segments: [ + { start: 0, end: 1.25, text: " Hallo." }, + { start: "x", end: 2, text: "bad timing" }, + { start: 1, end: 2.5, text: " Wie geht's?" }, + { start: 3, end: 4, text: " " }, + { start: 3, end: 4 }, + ], + }), + "application/json", + ); + expect(transcript).toEqual({ + text: "Hallo. Wie geht's?", + language: "de", + segments: [ + { startMs: 0, endMs: 1250, text: "Hallo." }, + { startMs: 1250, endMs: 2500, text: "Wie geht's?" }, + ], + }); + }); + + it("keeps a language code and reads plain JSON and text bodies", () => { + expect(parseTranscriptionResponse('{"text":"hi","language":"en"}', null)).toEqual({ + text: "hi", + language: "en", + segments: [], + }); + expect(parseTranscriptionResponse(" just words \n", "text/plain")).toEqual({ + text: "just words", + segments: [], + }); + expect(() => parseTranscriptionResponse('{"segments":[]}', "application/json")).toThrow( + "Transcription server returned no text", + ); + expect(() => parseTranscriptionResponse("{oops", "application/json")).toThrow( + "Transcription server returned invalid JSON", + ); + }); +}); diff --git a/packages/adapters/src/transcription-emulator.ts b/packages/adapters/src/transcription-emulator.ts new file mode 100644 index 0000000..b93d983 --- /dev/null +++ b/packages/adapters/src/transcription-emulator.ts @@ -0,0 +1,145 @@ +import type { + Transcript, + TranscriptionCapabilities, + TranscriptionProvider, + TranscriptionRequest, +} from "@rakazo/adapter-kit"; + +/** What the emulators "hear" in any audio: two timed sentences. */ +export const EMULATED_TRANSCRIPT: Transcript = { + text: "Welcome to the studio. Today we cut a vertical clip.", + language: "en", + segments: [ + { startMs: 0, endMs: 2400, text: "Welcome to the studio." }, + { startMs: 2400, endMs: 5100, text: "Today we cut a vertical clip." }, + ], +}; + +const EMULATED_MAX_BYTES = 25 * 1024 * 1024; + +/** Offline transcription provider for tests and conformance; no speech server involved. */ +export class EmulatorTranscriptionProvider implements TranscriptionProvider { + describe() { + return { + id: "emulator", + contractVersion: "1", + adapterVersion: "0.1.0", + capabilities: { + segments: true, + languageDetection: true, + maxBytes: EMULATED_MAX_BYTES, + } satisfies TranscriptionCapabilities, + }; + } + + async transcribe(request: TranscriptionRequest): Promise { + request.signal?.throwIfAborted(); + if (request.audio.byteLength === 0) throw new Error("The audio is empty"); + if (request.audio.byteLength > EMULATED_MAX_BYTES) { + throw new Error("The audio is too large for the transcription server"); + } + return { + ...EMULATED_TRANSCRIPT, + ...(request.language ? { language: request.language } : {}), + segments: EMULATED_TRANSCRIPT.segments.map((segment) => ({ ...segment })), + }; + } +} + +export interface EmulatedTranscriptionRequest { + url: string; + authorization: string | null; + model: string | null; + responseFormat: string | null; + language: string | null; + fileName: string | null; + fileType: string | null; + fileSize: number; +} + +/** + * In-process substitute for an OpenAI-compatible `POST /v1/audio/transcriptions` server, so + * the real HTTP adapter runs in offline tests. `openai` answers every response format like + * OpenAI and whisper.cpp; `plain` mimics servers without timed output that reject + * `verbose_json`; `text` answers with a bare `text/plain` body whatever was asked. + */ +export class OpenAiCompatibleTranscriptionServerEmulator { + readonly requests: EmulatedTranscriptionRequest[] = []; + /** Answer the next request with this HTTP status instead of a transcript. */ + failNextStatus: number | undefined; + + constructor( + private readonly options: { apiKey?: string; flavor?: "openai" | "plain" | "text" } = {}, + ) {} + + readonly fetch: typeof fetch = async (input, init) => { + init?.signal?.throwIfAborted(); + const url = new URL(input instanceof Request ? input.url : String(input)); + if (init?.method !== "POST" || !url.pathname.endsWith("/audio/transcriptions")) { + return Response.json({ error: { message: "Not found" } }, { status: 404 }); + } + const headers = new Headers(init.headers); + const authorization = headers.get("authorization"); + let form: FormData; + try { + form = await new Request(url, { method: "POST", headers, body: init.body }).formData(); + } catch { + return Response.json({ error: { message: "Expected multipart/form-data" } }, { status: 400 }); + } + const file = form.get("file"); + const field = (name: string) => { + const value = form.get(name); + return typeof value === "string" ? value : null; + }; + const responseFormat = field("response_format"); + this.requests.push({ + url: url.href, + authorization, + model: field("model"), + responseFormat, + language: field("language"), + fileName: file instanceof File ? file.name : null, + fileType: file instanceof File ? file.type : null, + fileSize: file instanceof File ? file.size : 0, + }); + if (this.failNextStatus) { + const status = this.failNextStatus; + this.failNextStatus = undefined; + return Response.json({ error: { message: "Upstream failure" } }, { status }); + } + if (this.options.apiKey && authorization !== `Bearer ${this.options.apiKey}`) { + return Response.json({ error: { message: "Invalid API key" } }, { status: 401 }); + } + if (!(file instanceof File) || file.size === 0) { + return Response.json({ error: { message: "file is required" } }, { status: 400 }); + } + const flavor = this.options.flavor ?? "openai"; + if (flavor === "text" || responseFormat === "text") { + return new Response(EMULATED_TRANSCRIPT.text, { + headers: { "content-type": "text/plain; charset=utf-8" }, + }); + } + if (responseFormat === "verbose_json") { + if (flavor === "plain") { + return Response.json( + { error: { message: "Unsupported response_format: verbose_json" } }, + { status: 400 }, + ); + } + return Response.json({ + task: "transcribe", + language: "english", + duration: EMULATED_TRANSCRIPT.segments.at(-1)!.endMs / 1000, + text: EMULATED_TRANSCRIPT.text, + segments: EMULATED_TRANSCRIPT.segments.map((segment, id) => ({ + id, + start: segment.startMs / 1000, + end: segment.endMs / 1000, + // Whisper keeps the leading space of each segment. + text: ` ${segment.text}`, + })), + }); + } + return Response.json({ text: EMULATED_TRANSCRIPT.text }); + }; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c870847..7349f34 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -22,6 +22,7 @@ export * from "./featured-connectors.js"; export * from "./group-mentions.js"; export * from "./http-response.js"; export * from "./mcp.js"; +export * from "./media/subtitles.js"; export * from "./message-pages.js"; export * from "./message-reactions.js"; export * from "./message-visibility.js"; diff --git a/packages/core/src/media/subtitles.test.ts b/packages/core/src/media/subtitles.test.ts new file mode 100644 index 0000000..5e48ec8 --- /dev/null +++ b/packages/core/src/media/subtitles.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { SUBTITLE_MAX_LINE_CHARS, toSrt, toVtt } from "./subtitles.js"; + +const SEGMENTS = [ + { startMs: 0, endMs: 2400, text: " Welcome to the studio." }, + { startMs: 2400, endMs: 5100, text: "Today we cut a vertical clip." }, +]; + +describe("subtitles", () => { + it("writes numbered SubRip cues with comma milliseconds", () => { + expect(toSrt(SEGMENTS)).toBe( + [ + "1", + "00:00:00,000 --> 00:00:02,400", + "Welcome to the studio.", + "", + "2", + "00:00:02,400 --> 00:00:05,100", + "Today we cut a vertical clip.", + "", + ].join("\n"), + ); + }); + + it("writes WebVTT with a header, dot milliseconds and escaped cue text", () => { + expect( + toVtt([...SEGMENTS, { startMs: 3_723_004, endMs: 3_725_000, text: "Tom & --> go" }]), + ).toBe( + [ + "WEBVTT", + "", + "00:00:00.000 --> 00:00:02.400", + "Welcome to the studio.", + "", + "00:00:02.400 --> 00:00:05.100", + "Today we cut a vertical clip.", + "", + "01:02:03.004 --> 01:02:05.000", + "Tom & <Jerry> → go", + "", + ].join("\n"), + ); + }); + + it("returns an empty file body for no speech", () => { + expect(toSrt([])).toBe(""); + expect(toVtt([{ startMs: 0, endMs: 10, text: " \n " }])).toBe("WEBVTT\n"); + }); + + it("wraps long segments into two-line cues that share the segment's time", () => { + const text = + "This is a long sentence that keeps going well past what fits on a phone screen, so it has to wrap across several readable cues."; + const srt = toSrt([{ startMs: 1000, endMs: 9000, text }]); + const cues = srt.trim().split("\n\n"); + expect(cues.length).toBeGreaterThan(1); + const times = cues.map((cue) => cue.split("\n")[1]); + expect(times[0]?.startsWith("00:00:01,000 --> ")).toBe(true); + expect(times.at(-1)?.endsWith(" --> 00:00:09,000")).toBe(true); + for (let index = 1; index < times.length; index += 1) { + expect(times[index]?.split(" --> ")[0]).toBe(times[index - 1]?.split(" --> ")[1]); + } + for (const cue of cues) { + const lines = cue.split("\n").slice(2); + expect(lines.length).toBeLessThanOrEqual(2); + for (const line of lines) expect(line.length).toBeLessThanOrEqual(SUBTITLE_MAX_LINE_CHARS); + } + expect(cues.flatMap((cue) => cue.split("\n").slice(2)).join(" ")).toBe(text); + }); + + it("keeps a word longer than a line whole and never runs time backwards", () => { + const word = "a".repeat(60); + expect(toSrt([{ startMs: 500, endMs: 100, text: word }])).toBe( + `1\n00:00:00,500 --> 00:00:00,500\n${word}\n`, + ); + }); +}); diff --git a/packages/core/src/media/subtitles.ts b/packages/core/src/media/subtitles.ts new file mode 100644 index 0000000..c4f703f --- /dev/null +++ b/packages/core/src/media/subtitles.ts @@ -0,0 +1,98 @@ +/** A timed piece of a transcript, in milliseconds from the start of the media. */ +export interface SubtitleSegment { + startMs: number; + endMs: number; + text: string; +} + +/** Readable on a phone held upright and the common broadcast limit for one subtitle line. */ +export const SUBTITLE_MAX_LINE_CHARS = 42; +const SUBTITLE_MAX_LINES = 2; + +/** SubRip (`.srt`): numbered cues with `HH:MM:SS,mmm` times. */ +export function toSrt(segments: readonly SubtitleSegment[]): string { + return subtitleCues(segments) + .map( + (cue, index) => + `${index + 1}\n${timestamp(cue.startMs, ",")} --> ${timestamp(cue.endMs, ",")}\n${cue.lines.join("\n")}\n`, + ) + .join("\n"); +} + +/** WebVTT (`.vtt`): the format browsers play in ``, with cue text escaped. */ +export function toVtt(segments: readonly SubtitleSegment[]): string { + const cues = subtitleCues(segments).map( + (cue) => + `${timestamp(cue.startMs, ".")} --> ${timestamp(cue.endMs, ".")}\n${cue.lines.map(escapeVtt).join("\n")}\n`, + ); + return ["WEBVTT\n", ...cues].join("\n"); +} + +interface SubtitleCue { + startMs: number; + endMs: number; + lines: string[]; +} + +/** + * Turns transcript segments into cues of at most two short lines. A long segment becomes + * several cues that share its time span in proportion to their length, so subtitles stay + * readable instead of covering the frame. + */ +function subtitleCues(segments: readonly SubtitleSegment[]): SubtitleCue[] { + const cues: SubtitleCue[] = []; + for (const segment of segments) { + // A blank line ends a cue in both formats, and "-->" would read as a timing line. + const text = segment.text.replaceAll("-->", "→").replace(/\s+/g, " ").trim(); + if (!text) continue; + const startMs = Math.max(0, Math.round(segment.startMs)); + const endMs = Math.max(startMs, Math.round(segment.endMs)); + const lines = wrapLines(text, SUBTITLE_MAX_LINE_CHARS); + const groups: string[][] = []; + for (let index = 0; index < lines.length; index += SUBTITLE_MAX_LINES) { + groups.push(lines.slice(index, index + SUBTITLE_MAX_LINES)); + } + const total = groups.reduce((sum, group) => sum + group.join(" ").length, 0); + let cursor = startMs; + let consumed = 0; + for (const [index, group] of groups.entries()) { + consumed += group.join(" ").length; + const groupEnd = + index === groups.length - 1 + ? endMs + : startMs + Math.round(((endMs - startMs) * consumed) / total); + cues.push({ startMs: cursor, endMs: groupEnd, lines: group }); + cursor = groupEnd; + } + } + return cues; +} + +function wrapLines(text: string, maxChars: number): string[] { + const lines: string[] = []; + let line = ""; + for (const word of text.split(" ")) { + if (!line) { + line = word; + } else if (line.length + 1 + word.length <= maxChars) { + line = `${line} ${word}`; + } else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + return lines; +} + +function timestamp(ms: number, separator: "," | "."): string { + const hours = Math.floor(ms / 3_600_000); + const minutes = Math.floor((ms % 3_600_000) / 60_000); + const seconds = Math.floor((ms % 60_000) / 1000); + const millis = ms % 1000; + return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}${separator}${String(millis).padStart(3, "0")}`; +} + +function escapeVtt(line: string): string { + return line.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"); +}