feat(media): transcription input, chunked transcripts and the Studio trigger

Groundwork for the transcribe operation and for creating Studio when it is
first needed.

- Contracts: TranscribeInputSchema (artifact id, optional language code).
- transcribeChunks uploads audio chunks in order and stitches them into one
  transcript on the original timeline; transcriptionChunkSeconds sizes
  16 kHz mono WAV chunks below the server's upload limit; transcriptSummary
  hands the bot the transcript, cut at a word.
- ensureStudioWhenMediaConfigured creates Studio once a media server
  (generation or transcription) is configured. Local edits alone do not
  count, because ffmpeg and image tools are present on most installs and a
  specialist nobody asked for would appear for everyone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u

packages/adapters/src/media/transcription.test.ts

@@ -0,0 +1,102 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { EmulatorTranscriptionProvider } from "../transcription-emulator.js";
+import {
+  TRANSCRIPT_SUMMARY_MAX_CHARS,
+  transcribeChunks,
+  transcriptionChunkSeconds,
+  transcriptSummary,
+} from "./transcription.js";
+
+const dirs: string[] = [];
+afterEach(async () => {
+  await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
+});
+
+describe("transcriptionChunkSeconds", () => {
+  it("keeps converted chunks below the upload limit, in whole minutes when long", () => {
+    expect(transcriptionChunkSeconds(25 * 1024 * 1024)).toBe(720);
+    expect(transcriptionChunkSeconds(25 * 1024 * 1024) * 32_000).toBeLessThan(25 * 1024 * 1024);
+    expect(transcriptionChunkSeconds(2 * 1024 * 1024)).toBe(62);
+    expect(transcriptionChunkSeconds(1024)).toBe(10);
+  });
+});
+
+describe("transcribeChunks", () => {
+  it("stitches chunks into one transcript on the original timeline", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "transcribe-chunks-"));
+    dirs.push(dir);
+    const first = join(dir, "chunk000.wav");
+    const second = join(dir, "chunk001.wav");
+    await writeFile(first, new Uint8Array([1, 2, 3]));
+    await writeFile(second, new Uint8Array([4, 5, 6]));
+    const progress: number[] = [];
+    const transcript = await transcribeChunks(
+      new EmulatorTranscriptionProvider(),
+      [
+        { path: first, offsetMs: 0 },
+        { path: second, offsetMs: 600_000 },
+      ],
+      {
+        mimeType: "audio/wav",
+        fileName: "talk.wav",
+        signal: new AbortController().signal,
+        onProgress: (fraction) => {
+          progress.push(fraction);
+        },
+      },
+    );
+    expect(progress).toEqual([0.5, 1]);
+    expect(transcript.language).toBe("en");
+    expect(transcript.text).toBe(
+      "Welcome to the studio. Today we cut a vertical clip. Welcome to the studio. Today we cut a vertical clip.",
+    );
+    expect(transcript.segments.map((segment) => [segment.startMs, segment.endMs])).toEqual([
+      [0, 2400],
+      [2400, 5100],
+      [600_000, 602_400],
+      [602_400, 605_100],
+    ]);
+  });
+
+  it("stops when the job is cancelled", async () => {
+    await expect(
+      transcribeChunks(new EmulatorTranscriptionProvider(), [{ path: "/missing", offsetMs: 0 }], {
+        mimeType: "audio/wav",
+        fileName: "talk.wav",
+        signal: AbortSignal.abort(),
+      }),
+    ).rejects.toThrow();
+  });
+});
+
+describe("transcriptSummary", () => {
+  it("hands the bot the text and says what is attached", () => {
+    expect(
+      transcriptSummary("talk.mp4", {
+        text: "Hello there.",
+        segments: [{ startMs: 0, endMs: 900, text: "Hello there." }],
+      }),
+    ).toBe("Transcript of talk.mp4: Hello there.\nSubtitles are attached as SRT and VTT.");
+    expect(transcriptSummary("talk.mp4", { text: "Hi.", segments: [] })).toContain(
+      "attached as text",
+    );
+    expect(transcriptSummary("silence.wav", { text: "  ", segments: [] })).toBe(
+      "No speech found in silence.wav.",
+    );
+  });
+
+  it("cuts a long transcript at a word", () => {
+    const summary = transcriptSummary("talk.mp4", {
+      text: "word ".repeat(1000),
+      segments: [],
+    });
+    const body = summary.split("\n")[0]!;
+    expect(body.length).toBeLessThanOrEqual(
+      "Transcript of talk.mp4: ".length + TRANSCRIPT_SUMMARY_MAX_CHARS + 2,
+    );
+    expect(body.endsWith("word …")).toBe(true);
+  });
+});

packages/adapters/src/media/transcription.ts

@@ -0,0 +1,79 @@
+import { readFile } from "node:fs/promises";
+import type { Transcript, TranscriptionProvider, TranscriptSegment } from "@rakazo/adapter-kit";
+
+/** Bytes per second of the 16 kHz mono 16-bit WAV that audio is converted to before upload. */
+export const TRANSCRIPTION_WAV_BYTES_PER_SECOND = 16_000 * 2;
+/** How much of the transcript the bot that started the job gets back. */
+export const TRANSCRIPT_SUMMARY_MAX_CHARS = 1200;
+
+/**
+ * Seconds of converted audio per upload, so each chunk stays below the server's size limit
+ * with room for the WAV header. Rounded down to whole minutes when that is possible.
+ */
+export function transcriptionChunkSeconds(maxBytes: number): number {
+  const seconds = Math.floor((maxBytes * 0.95) / TRANSCRIPTION_WAV_BYTES_PER_SECOND);
+  return seconds >= 120 ? Math.floor(seconds / 60) * 60 : Math.max(10, seconds);
+}
+
+export interface AudioChunk {
+  path: string;
+  /** Where the chunk starts in the original media. */
+  offsetMs: number;
+}
+
+/**
+ * Transcribes audio chunks one after another and stitches them into one transcript with
+ * segment times relative to the original media.
+ */
+export async function transcribeChunks(
+  provider: TranscriptionProvider,
+  chunks: readonly AudioChunk[],
+  options: {
+    mimeType: string;
+    fileName: string;
+    language?: string;
+    signal: AbortSignal;
+    onProgress?: (fraction: number) => Promise<void> | void;
+  },
+): Promise<Transcript> {
+  const texts: string[] = [];
+  const segments: TranscriptSegment[] = [];
+  let language: string | undefined;
+  for (const [index, chunk] of chunks.entries()) {
+    options.signal.throwIfAborted();
+    const audio = new Uint8Array(await readFile(chunk.path));
+    const part = await provider.transcribe({
+      audio,
+      mimeType: options.mimeType,
+      fileName: options.fileName,
+      ...(options.language ? { language: options.language } : {}),
+      signal: options.signal,
+    });
+    if (part.text) texts.push(part.text);
+    language ??= part.language;
+    for (const segment of part.segments) {
+      segments.push({
+        startMs: segment.startMs + chunk.offsetMs,
+        endMs: segment.endMs + chunk.offsetMs,
+        text: segment.text,
+      });
+    }
+    await options.onProgress?.((index + 1) / chunks.length);
+  }
+  return { text: texts.join(" "), ...(language ? { language } : {}), segments };
+}
+
+/** The transcript for the bot that started the job, cut at a word when it is long. */
+export function transcriptSummary(name: string, transcript: Transcript): string {
+  const text = transcript.text.replace(/\s+/g, " ").trim();
+  if (!text) return `No speech found in ${name}.`;
+  const cut =
+    text.length > TRANSCRIPT_SUMMARY_MAX_CHARS
+      ? `${text.slice(0, TRANSCRIPT_SUMMARY_MAX_CHARS).replace(/\s+\S*$/, "")} …`
+      : text;
+  const timing =
+    transcript.segments.length > 0
+      ? "Subtitles are attached as SRT and VTT."
+      : "The server returned no timings, so the transcript is attached as text.";
+  return `Transcript of ${name}: ${cut}\n${timing}`;
+}

packages/adapters/src/studio-specialist.test.ts

@@ -0,0 +1,57 @@
+import type { MediaCapabilities } from "@rakazo/adapter-kit";
+import type { PrismaClient } from "@rakazo/db";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const ensureStudioBot = vi.fn();
+vi.mock("@rakazo/db", () => ({
+  ensureStudioBot: (...args: unknown[]) => ensureStudioBot(...args),
+}));
+
+import { ensureStudioWhenMediaConfigured, mediaBackendsConfigured } from "./studio-specialist.js";
+
+const none: MediaCapabilities = {
+  generate_image: false,
+  generate_video: false,
+  edit_image: false,
+  edit_video: false,
+  export_video: false,
+  transcribe: false,
+};
+const prisma = {} as PrismaClient;
+const run = { userId: "user-1", spaceId: "space-1" };
+
+describe("Studio specialist", () => {
+  beforeEach(() => {
+    ensureStudioBot.mockReset();
+  });
+
+  it("counts only media servers, not local edits, as media being set up", () => {
+    expect(mediaBackendsConfigured({ ...none, edit_image: true, edit_video: true })).toBe(false);
+    expect(mediaBackendsConfigured({ ...none, transcribe: true })).toBe(true);
+    expect(mediaBackendsConfigured({ ...none, generate_image: true })).toBe(true);
+  });
+
+  it("creates Studio once a media server is configured", async () => {
+    await ensureStudioWhenMediaConfigured(
+      { prisma, media: { capabilities: async () => ({ ...none, edit_video: true }) } },
+      run,
+    );
+    await ensureStudioWhenMediaConfigured({ prisma }, run);
+    expect(ensureStudioBot).not.toHaveBeenCalled();
+    await ensureStudioWhenMediaConfigured(
+      { prisma, media: { capabilities: async () => ({ ...none, transcribe: true }) } },
+      run,
+    );
+    expect(ensureStudioBot).toHaveBeenCalledWith(prisma, "user-1");
+  });
+
+  it("never fails the run", async () => {
+    ensureStudioBot.mockRejectedValue(new Error("database unavailable"));
+    await expect(
+      ensureStudioWhenMediaConfigured(
+        { prisma, media: { capabilities: async () => ({ ...none, generate_video: true }) } },
+        run,
+      ),
+    ).resolves.toBeUndefined();
+  });
+});

packages/adapters/src/studio-specialist.ts

@@ -0,0 +1,28 @@
+import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit";
+import { ensureStudioBot, type PrismaClient } from "@rakazo/db";
+import { getLogger } from "@rakazo/logging";
+
+/**
+ * Generation or transcription only run once the deployment owner set up a media server; local
+ * edits (ffmpeg, image tools) are always there and alone do not call for a specialist.
+ */
+export function mediaBackendsConfigured(capabilities: MediaCapabilities): boolean {
+  return capabilities.generate_image || capabilities.generate_video || capabilities.transcribe;
+}
+
+/**
+ * Creates the user's Studio specialist the first time their main bot runs with a media server
+ * configured, so the specialist directory lists it from that run on. Never fails the run.
+ */
+export async function ensureStudioWhenMediaConfigured(
+  deps: { prisma: PrismaClient; media?: Pick<MediaRuntime, "capabilities"> },
+  run: { userId: string; spaceId: string },
+): Promise<void> {
+  if (!deps.media) return;
+  try {
+    if (!mediaBackendsConfigured(await deps.media.capabilities(run.spaceId))) return;
+    await ensureStudioBot(deps.prisma, run.userId);
+  } catch (error) {
+    getLogger().error("studio specialist", error);
+  }
+}

packages/contracts/src/index.ts

@@ -10,6 +10,7 @@ export * from "./integration-settings.js";
 export * from "./local-settings.js";
 export * from "./mcp.js";
 export * from "./media.js";
+export * from "./media-transcription.js";
 export * from "./model-tiers.js";
 export * from "./openai-compatible-ui.js";
 export * from "./project-goals.js";

packages/contracts/src/media-transcription.ts

@@ -0,0 +1,13 @@
+import * as z from "zod";
+import { Id } from "./ids.js";
+
+export const TranscribeInputSchema = z.object({
+  artifactId: Id.describe("Audio or video artifact to transcribe."),
+  language: z
+    .string()
+    .trim()
+    .regex(/^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/, "Use a language code such as en or de")
+    .optional()
+    .describe("Spoken language as a code such as en or de; detected when omitted."),
+});
+export type TranscribeInput = z.infer<typeof TranscribeInputSchema>;