diff --git a/packages/adapters/src/media/transcription.test.ts b/packages/adapters/src/media/transcription.test.ts new file mode 100644 index 0000000..bf6c010 --- /dev/null +++ b/packages/adapters/src/media/transcription.test.ts @@ -0,0 +1,102 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { EmulatorTranscriptionProvider } from "../transcription-emulator.js"; +import { + TRANSCRIPT_SUMMARY_MAX_CHARS, + transcribeChunks, + transcriptionChunkSeconds, + transcriptSummary, +} from "./transcription.js"; + +const dirs: string[] = []; +afterEach(async () => { + await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe("transcriptionChunkSeconds", () => { + it("keeps converted chunks below the upload limit, in whole minutes when long", () => { + expect(transcriptionChunkSeconds(25 * 1024 * 1024)).toBe(720); + expect(transcriptionChunkSeconds(25 * 1024 * 1024) * 32_000).toBeLessThan(25 * 1024 * 1024); + expect(transcriptionChunkSeconds(2 * 1024 * 1024)).toBe(62); + expect(transcriptionChunkSeconds(1024)).toBe(10); + }); +}); + +describe("transcribeChunks", () => { + it("stitches chunks into one transcript on the original timeline", async () => { + const dir = await mkdtemp(join(tmpdir(), "transcribe-chunks-")); + dirs.push(dir); + const first = join(dir, "chunk000.wav"); + const second = join(dir, "chunk001.wav"); + await writeFile(first, new Uint8Array([1, 2, 3])); + await writeFile(second, new Uint8Array([4, 5, 6])); + const progress: number[] = []; + const transcript = await transcribeChunks( + new EmulatorTranscriptionProvider(), + [ + { path: first, offsetMs: 0 }, + { path: second, offsetMs: 600_000 }, + ], + { + mimeType: "audio/wav", + fileName: "talk.wav", + signal: new AbortController().signal, + onProgress: (fraction) => { + progress.push(fraction); + }, + }, + ); + expect(progress).toEqual([0.5, 1]); + expect(transcript.language).toBe("en"); + expect(transcript.text).toBe( + "Welcome to the studio. Today we cut a vertical clip. Welcome to the studio. Today we cut a vertical clip.", + ); + expect(transcript.segments.map((segment) => [segment.startMs, segment.endMs])).toEqual([ + [0, 2400], + [2400, 5100], + [600_000, 602_400], + [602_400, 605_100], + ]); + }); + + it("stops when the job is cancelled", async () => { + await expect( + transcribeChunks(new EmulatorTranscriptionProvider(), [{ path: "/missing", offsetMs: 0 }], { + mimeType: "audio/wav", + fileName: "talk.wav", + signal: AbortSignal.abort(), + }), + ).rejects.toThrow(); + }); +}); + +describe("transcriptSummary", () => { + it("hands the bot the text and says what is attached", () => { + expect( + transcriptSummary("talk.mp4", { + text: "Hello there.", + segments: [{ startMs: 0, endMs: 900, text: "Hello there." }], + }), + ).toBe("Transcript of talk.mp4: Hello there.\nSubtitles are attached as SRT and VTT."); + expect(transcriptSummary("talk.mp4", { text: "Hi.", segments: [] })).toContain( + "attached as text", + ); + expect(transcriptSummary("silence.wav", { text: " ", segments: [] })).toBe( + "No speech found in silence.wav.", + ); + }); + + it("cuts a long transcript at a word", () => { + const summary = transcriptSummary("talk.mp4", { + text: "word ".repeat(1000), + segments: [], + }); + const body = summary.split("\n")[0]!; + expect(body.length).toBeLessThanOrEqual( + "Transcript of talk.mp4: ".length + TRANSCRIPT_SUMMARY_MAX_CHARS + 2, + ); + expect(body.endsWith("word …")).toBe(true); + }); +}); diff --git a/packages/adapters/src/media/transcription.ts b/packages/adapters/src/media/transcription.ts new file mode 100644 index 0000000..0b081e9 --- /dev/null +++ b/packages/adapters/src/media/transcription.ts @@ -0,0 +1,79 @@ +import { readFile } from "node:fs/promises"; +import type { Transcript, TranscriptionProvider, TranscriptSegment } from "@rakazo/adapter-kit"; + +/** Bytes per second of the 16 kHz mono 16-bit WAV that audio is converted to before upload. */ +export const TRANSCRIPTION_WAV_BYTES_PER_SECOND = 16_000 * 2; +/** How much of the transcript the bot that started the job gets back. */ +export const TRANSCRIPT_SUMMARY_MAX_CHARS = 1200; + +/** + * Seconds of converted audio per upload, so each chunk stays below the server's size limit + * with room for the WAV header. Rounded down to whole minutes when that is possible. + */ +export function transcriptionChunkSeconds(maxBytes: number): number { + const seconds = Math.floor((maxBytes * 0.95) / TRANSCRIPTION_WAV_BYTES_PER_SECOND); + return seconds >= 120 ? Math.floor(seconds / 60) * 60 : Math.max(10, seconds); +} + +export interface AudioChunk { + path: string; + /** Where the chunk starts in the original media. */ + offsetMs: number; +} + +/** + * Transcribes audio chunks one after another and stitches them into one transcript with + * segment times relative to the original media. + */ +export async function transcribeChunks( + provider: TranscriptionProvider, + chunks: readonly AudioChunk[], + options: { + mimeType: string; + fileName: string; + language?: string; + signal: AbortSignal; + onProgress?: (fraction: number) => Promise | void; + }, +): Promise { + const texts: string[] = []; + const segments: TranscriptSegment[] = []; + let language: string | undefined; + for (const [index, chunk] of chunks.entries()) { + options.signal.throwIfAborted(); + const audio = new Uint8Array(await readFile(chunk.path)); + const part = await provider.transcribe({ + audio, + mimeType: options.mimeType, + fileName: options.fileName, + ...(options.language ? { language: options.language } : {}), + signal: options.signal, + }); + if (part.text) texts.push(part.text); + language ??= part.language; + for (const segment of part.segments) { + segments.push({ + startMs: segment.startMs + chunk.offsetMs, + endMs: segment.endMs + chunk.offsetMs, + text: segment.text, + }); + } + await options.onProgress?.((index + 1) / chunks.length); + } + return { text: texts.join(" "), ...(language ? { language } : {}), segments }; +} + +/** The transcript for the bot that started the job, cut at a word when it is long. */ +export function transcriptSummary(name: string, transcript: Transcript): string { + const text = transcript.text.replace(/\s+/g, " ").trim(); + if (!text) return `No speech found in ${name}.`; + const cut = + text.length > TRANSCRIPT_SUMMARY_MAX_CHARS + ? `${text.slice(0, TRANSCRIPT_SUMMARY_MAX_CHARS).replace(/\s+\S*$/, "")} …` + : text; + const timing = + transcript.segments.length > 0 + ? "Subtitles are attached as SRT and VTT." + : "The server returned no timings, so the transcript is attached as text."; + return `Transcript of ${name}: ${cut}\n${timing}`; +} diff --git a/packages/adapters/src/studio-specialist.test.ts b/packages/adapters/src/studio-specialist.test.ts new file mode 100644 index 0000000..472d981 --- /dev/null +++ b/packages/adapters/src/studio-specialist.test.ts @@ -0,0 +1,57 @@ +import type { MediaCapabilities } from "@rakazo/adapter-kit"; +import type { PrismaClient } from "@rakazo/db"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const ensureStudioBot = vi.fn(); +vi.mock("@rakazo/db", () => ({ + ensureStudioBot: (...args: unknown[]) => ensureStudioBot(...args), +})); + +import { ensureStudioWhenMediaConfigured, mediaBackendsConfigured } from "./studio-specialist.js"; + +const none: MediaCapabilities = { + generate_image: false, + generate_video: false, + edit_image: false, + edit_video: false, + export_video: false, + transcribe: false, +}; +const prisma = {} as PrismaClient; +const run = { userId: "user-1", spaceId: "space-1" }; + +describe("Studio specialist", () => { + beforeEach(() => { + ensureStudioBot.mockReset(); + }); + + it("counts only media servers, not local edits, as media being set up", () => { + expect(mediaBackendsConfigured({ ...none, edit_image: true, edit_video: true })).toBe(false); + expect(mediaBackendsConfigured({ ...none, transcribe: true })).toBe(true); + expect(mediaBackendsConfigured({ ...none, generate_image: true })).toBe(true); + }); + + it("creates Studio once a media server is configured", async () => { + await ensureStudioWhenMediaConfigured( + { prisma, media: { capabilities: async () => ({ ...none, edit_video: true }) } }, + run, + ); + await ensureStudioWhenMediaConfigured({ prisma }, run); + expect(ensureStudioBot).not.toHaveBeenCalled(); + await ensureStudioWhenMediaConfigured( + { prisma, media: { capabilities: async () => ({ ...none, transcribe: true }) } }, + run, + ); + expect(ensureStudioBot).toHaveBeenCalledWith(prisma, "user-1"); + }); + + it("never fails the run", async () => { + ensureStudioBot.mockRejectedValue(new Error("database unavailable")); + await expect( + ensureStudioWhenMediaConfigured( + { prisma, media: { capabilities: async () => ({ ...none, generate_video: true }) } }, + run, + ), + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/adapters/src/studio-specialist.ts b/packages/adapters/src/studio-specialist.ts new file mode 100644 index 0000000..5d3eab6 --- /dev/null +++ b/packages/adapters/src/studio-specialist.ts @@ -0,0 +1,28 @@ +import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit"; +import { ensureStudioBot, type PrismaClient } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; + +/** + * Generation or transcription only run once the deployment owner set up a media server; local + * edits (ffmpeg, image tools) are always there and alone do not call for a specialist. + */ +export function mediaBackendsConfigured(capabilities: MediaCapabilities): boolean { + return capabilities.generate_image || capabilities.generate_video || capabilities.transcribe; +} + +/** + * Creates the user's Studio specialist the first time their main bot runs with a media server + * configured, so the specialist directory lists it from that run on. Never fails the run. + */ +export async function ensureStudioWhenMediaConfigured( + deps: { prisma: PrismaClient; media?: Pick }, + run: { userId: string; spaceId: string }, +): Promise { + if (!deps.media) return; + try { + if (!mediaBackendsConfigured(await deps.media.capabilities(run.spaceId))) return; + await ensureStudioBot(deps.prisma, run.userId); + } catch (error) { + getLogger().error("studio specialist", error); + } +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index ba2c70b..9a36f8b 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -10,6 +10,7 @@ export * from "./integration-settings.js"; export * from "./local-settings.js"; export * from "./mcp.js"; export * from "./media.js"; +export * from "./media-transcription.js"; export * from "./model-tiers.js"; export * from "./openai-compatible-ui.js"; export * from "./project-goals.js"; diff --git a/packages/contracts/src/media-transcription.ts b/packages/contracts/src/media-transcription.ts new file mode 100644 index 0000000..3d0373a --- /dev/null +++ b/packages/contracts/src/media-transcription.ts @@ -0,0 +1,13 @@ +import * as z from "zod"; +import { Id } from "./ids.js"; + +export const TranscribeInputSchema = z.object({ + artifactId: Id.describe("Audio or video artifact to transcribe."), + language: z + .string() + .trim() + .regex(/^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/, "Use a language code such as en or de") + .optional() + .describe("Spoken language as a code such as en or de; detected when omitted."), +}); +export type TranscribeInput = z.infer;