From 6ba9cef03f29beb3a0b463f46c4cad5645d69cc1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 09:52:53 +0000 Subject: [PATCH] Add remove_silence step to edit_video Talking-head clips are mostly cut by hand to drop pauses, and the bot had no way to find them: it could only keep segments it guessed. The new remove_silence step finds every pause quieter than threshold_db (default -35 dB) and at least min_silence long (default 0.6 s), keeps padding (default 0.12 s) on each side so word onsets survive, and cuts the rest through the existing keep-segments graph, so picture and sound stay in sync. Steps run in order, so silence is detected on the timeline the steps before it leave: the planner gets an analysis mode that sends that audio through silencedetect into a null output, and ametadata prints the marks to a file in the job directory. The detected speech is handed back to the planner per step. Nothing silent makes the step a no-op; nothing left fails with a clear error. At most 200 pieces are kept, bridging the shortest pauses first, so the filter graph stays a sensible size. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- packages/adapters/src/builtin-tools.ts | 2 +- .../adapters/src/media/media-test-support.ts | 21 ++ packages/adapters/src/media/silence.test.ts | 320 ++++++++++++++++++ packages/adapters/src/media/silence.ts | 211 ++++++++++++ .../adapters/src/media/video-edit-plan.ts | 37 +- .../adapters/src/media/video-operations.ts | 14 +- packages/contracts/src/index.ts | 1 + packages/contracts/src/media-analysis.test.ts | 38 +++ packages/contracts/src/media-analysis.ts | 22 ++ packages/contracts/src/media-video.ts | 2 + 10 files changed, 665 insertions(+), 3 deletions(-) create mode 100644 packages/adapters/src/media/silence.test.ts create mode 100644 packages/adapters/src/media/silence.ts create mode 100644 packages/contracts/src/media-analysis.test.ts create mode 100644 packages/contracts/src/media-analysis.ts diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index fb87c63..730c563 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -559,7 +559,7 @@ export const builtinAgentTools: ConnectorTool[] = [ }, { name: "edit_video", - description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`, + description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image; remove_silence {threshold_db -35, min_silence 0.6, padding 0.12} cuts pauses quieter than threshold_db dB and at least min_silence s long, keeping padding s on each side. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`, inputSchema: mediaParameters(EditVideoInputSchema), }, { diff --git a/packages/adapters/src/media/media-test-support.ts b/packages/adapters/src/media/media-test-support.ts index 940419a..8870e7f 100644 --- a/packages/adapters/src/media/media-test-support.ts +++ b/packages/adapters/src/media/media-test-support.ts @@ -104,3 +104,24 @@ export async function fakeMediaJobContext(input: unknown, artifacts: Record { + const path = join(dir, "tone-gap-tone.mp4"); + await runFfmpeg([ + ...["-f", "lavfi", "-i", "color=c=red:size=320x180:rate=25:duration=2"], + ...["-f", "lavfi", "-i", "color=c=blue:size=320x180:rate=25:duration=2"], + ...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=1.5"], + ...["-f", "lavfi", "-i", "anullsrc=channel_layout=mono:sample_rate=48000"], + ...["-f", "lavfi", "-i", "sine=frequency=660:sample_rate=48000:duration=1.5"], + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1:a=0[v];[3:a]atrim=duration=1[gap];[2:a][gap][4:a]concat=n=3:v=0:a=1[a]", + ...["-map", "[v]", "-map", "[a]"], + ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-c:a", "aac"], + `file:${path}`, + ]); + return path; +} diff --git a/packages/adapters/src/media/silence.test.ts b/packages/adapters/src/media/silence.test.ts new file mode 100644 index 0000000..64380e3 --- /dev/null +++ b/packages/adapters/src/media/silence.test.ts @@ -0,0 +1,320 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import type { EditVideoStep } from "@rakazo/contracts"; +import { EditVideoInputSchema } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { detectFfmpeg } from "./ffmpeg.js"; +import { createCutTestClip, fakeMediaJobContext } from "./media-test-support.js"; +import { + MAX_SPEECH_SEGMENTS, + parseFrameMetadata, + parseSilences, + silenceDetectFilters, + speechSegments, +} from "./silence.js"; +import type { PlanInput } from "./video-edit-plan.js"; +import { planVideoEdit } from "./video-edit-plan.js"; +import { createVideoOperations } from "./video-operations.js"; + +const ffmpeg = await detectFfmpeg(); + +// Printed by `silencedetect=noise=-35dB:duration=0.6,ametadata=mode=print:file=…` (ffmpeg 6.1) +// for 1.5 s tone, 1 s silence, 1.5 s tone; then for silence, tone, silence up to the end. +const TONE_GAP_TONE = `frame:98 pts:100352 pts_time:2.09067 +lavfi.silence_start=1.49996 +frame:117 pts:119808 pts_time:2.496 +lavfi.silence_end=2.50004 +lavfi.silence_duration=1.00008 +`; +const SILENT_EDGES = `frame:14 pts:28672 pts_time:0.597333 +lavfi.silence_start=0 +frame:46 pts:94208 pts_time:1.96267 +lavfi.silence_end=2.00008 +lavfi.silence_duration=2.00008 +frame:84 pts:172032 pts_time:3.584 +lavfi.silence_start=2.99996 +`; + +const defaults = { padding: 0.12, thresholdDb: -35 }; + +describe("silence detection output", () => { + it("reads printed frame metadata", () => { + expect(parseFrameMetadata(TONE_GAP_TONE)).toEqual([ + { time: 2.09067, values: { "lavfi.silence_start": "1.49996" } }, + { + time: 2.496, + values: { "lavfi.silence_end": "2.50004", "lavfi.silence_duration": "1.00008" }, + }, + ]); + expect(parseFrameMetadata("frame:1 pts:N/A pts_time:N/A\nlavfi.x=1\n")).toEqual([]); + }); + + it("pairs starts with ends and runs a silence without end to the end of the audio", () => { + expect(parseSilences(TONE_GAP_TONE, 4)).toEqual([{ start: 1.49996, end: 2.50004 }]); + expect(parseSilences(SILENT_EDGES, 4.5)).toEqual([ + { start: 0, end: 2.00008 }, + { start: 2.99996, end: 4.5 }, + ]); + // A probe a little shorter than the decoded audio clamps the ranges to the timeline. + expect(parseSilences(SILENT_EDGES, 2)).toEqual([{ start: 0, end: 2 }]); + expect(parseSilences("", 4)).toEqual([]); + }); + + it("builds the filters from validated numbers and a generated file name", () => { + expect(silenceDetectFilters({ thresholdDb: -35, minSilence: 0.6 }, "silence-2.txt")).toEqual([ + "silencedetect=noise=-35dB:duration=0.6", + "ametadata=mode=print:file=silence-2.txt", + ]); + }); +}); + +describe("speech segments", () => { + it("keeps what is not silent, padded on both sides", () => { + expect(speechSegments([{ start: 1.5, end: 2.5 }], 4, defaults)).toEqual([ + { start: 0, end: 1.62 }, + { start: 2.38, end: 4 }, + ]); + }); + + it("clamps padding to the timeline and drops silent edges", () => { + const segments = speechSegments( + [ + { start: 0, end: 2 }, + { start: 3, end: 4.5 }, + ], + 4.5, + defaults, + ); + expect(segments).toHaveLength(1); + expect(segments?.[0]?.start).toBeCloseTo(1.88); + expect(segments?.[0]?.end).toBeCloseTo(3.12); + expect(speechSegments([{ start: 3.95, end: 4 }], 4, { ...defaults, padding: 1 })).toBeNull(); + }); + + it("merges speech where the padding closes a pause, and overlapping silences", () => { + expect( + speechSegments( + [ + { start: 1, end: 1.2 }, + { start: 2, end: 3 }, + { start: 2.5, end: 3.5 }, + ], + 5, + defaults, + ), + ).toEqual([ + { start: 0, end: 2.12 }, + { start: 3.38, end: 5 }, + ]); + }); + + it("is null when nothing is silent and fails when nothing would be left", () => { + expect(speechSegments([], 4, defaults)).toBeNull(); + expect(() => speechSegments([{ start: 0, end: 4 }], 4, defaults)).toThrow( + "remove_silence found nothing louder than -35 dB, so nothing would be left.", + ); + // Blips shorter than 0.1 s without padding are noise, not speech. + expect(() => + speechSegments( + [ + { start: 0, end: 2 }, + { start: 2.05, end: 4 }, + ], + 4, + { ...defaults, padding: 0 }, + ), + ).toThrow("nothing would be left"); + }); + + it("keeps the longest pauses cut when there would be too many pieces", () => { + const silences = Array.from({ length: 5 }, (_, index) => ({ + start: index * 2 + 1, + end: index * 2 + 1 + (index === 2 ? 0.9 : 0.5), + })); + expect(speechSegments(silences, 11, { padding: 0, thresholdDb: -35, maxSegments: 2 })).toEqual([ + { start: 0, end: 5 }, + { start: 5.9, end: 11 }, + ]); + expect(MAX_SPEECH_SEGMENTS).toBe(200); + }); +}); + +const source: PlanInput = { + path: "/jobs/1/input-source", + mimeType: "video/mp4", + name: "talk.mp4", + probe: { + durationMs: 10_000, + video: { + codec: "h264", + width: 1920, + height: 1080, + rotation: 0, + fps: 25, + sampleAspectRatio: 1, + }, + audio: { codec: "aac", sampleRate: 48_000, channels: 2 }, + hasAudio: true, + }, +}; + +function steps(list: unknown[]): EditVideoStep[] { + return EditVideoInputSchema.parse({ artifact_id: "a", title: "Talk", steps: list }).steps; +} + +function graphOf(args: string[]): string { + return args[args.indexOf("-filter_complex") + 1]!; +} + +describe("remove_silence planning", () => { + it("fills the step's defaults", () => { + expect(steps([{ op: "remove_silence" }])).toEqual([ + { op: "remove_silence", threshold_db: -35, min_silence: 0.6, padding: 0.12 }, + ]); + }); + + it("detects silence on the audio the steps before it leave, into a null output", () => { + const [trim] = steps([{ op: "trim", start: 2, end: 8 }]); + const plan = planVideoEdit({ + source, + steps: [trim!, { op: "speed", factor: 2 }], + analyzeAudio: silenceDetectFilters({ thresholdDb: -35, minSilence: 0.6 }, "silence-2.txt"), + outputPath: "", + }); + expect(plan.args).toEqual([ + ...["-ss", "2", "-t", "6", "-protocol_whitelist", "file", "-f", "mov"], + ...["-i", "file:/jobs/1/input-source"], + "-filter_complex", + "[0:a:0]atempo=2,silencedetect=noise=-35dB:duration=0.6,ametadata=mode=print:file=silence-2.txt[a1]", + ...["-map", "[a1]", "-f", "null", "-"], + ]); + expect(plan).toMatchObject({ kind: "audio", durationMs: 3000 }); + }); + + it("cuts the detected pauses from picture and sound like a keep step", () => { + const list = steps([{ op: "remove_silence" }, { op: "reframe", preset: "vertical" }]); + const plan = planVideoEdit({ + source, + steps: list, + speech: new Map([ + [ + list[0]!, + [ + { start: 0, end: 1.62 }, + { start: 2.38, end: 10 }, + ], + ], + ]), + outputPath: "/jobs/1/output.mp4", + }); + const graph = graphOf(plan.args); + expect(graph).toContain("[0:v:0]split=2[v1][v2]"); + expect(graph).toContain("[v1]trim=start=0:end=1.62,setpts=PTS-STARTPTS[v3]"); + expect(graph).toContain("[v2]trim=start=2.38:end=10,setpts=PTS-STARTPTS[v4]"); + expect(graph).toContain("[0:a:0]asplit=2[a5][a6]"); + expect(graph).toContain("[a5]atrim=start=0:end=1.62,asetpts=PTS-STARTPTS[a7]"); + expect(graph).toContain("concat=n=2:v=1:a=1"); + expect(plan).toMatchObject({ kind: "video", durationMs: 9240, width: 1080, height: 1920 }); + }); + + it("leaves the timeline alone when nothing is silent and needs detection first", () => { + const list = steps([{ op: "remove_silence" }]); + const untouched = planVideoEdit({ + source, + steps: list, + speech: new Map([[list[0]!, null]]), + outputPath: "/jobs/1/output.mp4", + }); + expect(graphOf(untouched.args)).not.toContain("trim"); + expect(untouched.durationMs).toBe(10_000); + expect(() => planVideoEdit({ source, steps: list, outputPath: "/jobs/1/output.mp4" })).toThrow( + "Silence was not detected before planning.", + ); + }); + + it("needs audio to detect silence in", () => { + expect(() => + planVideoEdit({ + source, + steps: [{ op: "mute" }], + analyzeAudio: ["anullsink"], + outputPath: "", + }), + ).toThrow("talk.mp4 has no audio."); + }); +}); + +/** Seconds of each stream of a file, by codec type. */ +async function streamSeconds(ffprobePath: string, path: string) { + const { stdout } = await promisify(execFile)(ffprobePath, [ + ...["-v", "error", "-show_entries", "stream=codec_type,duration", "-of", "json", path], + ]); + const { streams } = JSON.parse(stdout) as { + streams: { codec_type: string; duration: string }[]; + }; + return Object.fromEntries(streams.map((stream) => [stream.codec_type, Number(stream.duration)])); +} + +// CI runners may not have ffmpeg; these run wherever it is installed. +describe.skipIf(!ffmpeg)("remove_silence with ffmpeg", () => { + let dir: string; + let clip: string; + + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "evoo-silence-test-")); + clip = await createCutTestClip(dir); + }); + + afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }); + }); + + async function edit(list: unknown[]) { + const [registration] = createVideoOperations(); + const job = await fakeMediaJobContext( + registration!.inputSchema.parse({ artifact_id: "talk", title: "Ohne Pausen", steps: list }), + { talk: { path: clip, name: "talk.mp4", mimeType: "video/mp4" } }, + ); + try { + await registration!.handler.run(job.context); + const [output] = job.outputs; + // The fake context hands the file over in place; measure it before the temp dir goes. + return { + output: output!, + streams: await streamSeconds(ffmpeg!.ffprobePath, output!.path), + }; + } finally { + await rm(job.tempDir, { recursive: true, force: true }); + } + } + + it("cuts the pause and keeps picture and sound the same length", async () => { + const { output, streams } = await edit([{ op: "remove_silence", padding: 0.1 }]); + // 4 s minus the 1 s pause, of which 0.1 s stays on each side. + expect(Math.abs(output.durationMs! - 3200)).toBeLessThan(100); + expect(Math.abs(streams.video! - 3.2)).toBeLessThan(0.1); + expect(Math.abs(streams.video! - streams.audio!)).toBeLessThan(0.08); + }); + + it("detects silence on the timeline the steps before it leave", async () => { + const { output, streams } = await edit([ + { op: "trim", start: 1, end: 4 }, + { op: "remove_silence", padding: 0.1 }, + ]); + // In the trimmed 3 s the pause runs from 0.5 s to 1.5 s; 0.2 s of it stays. + expect(Math.abs(output.durationMs! - 2200)).toBeLessThan(100); + expect(Math.abs(streams.video! - streams.audio!)).toBeLessThan(0.08); + }); + + it("fails clearly when everything is silent", async () => { + await expect(edit([{ op: "mute" }, { op: "remove_silence" }])).rejects.toThrow( + "talk.mp4 has no audio.", + ); + await expect(edit([{ op: "remove_silence", threshold_db: -10 }])).rejects.toThrow( + "remove_silence found nothing louder than -10 dB", + ); + }); +}); diff --git a/packages/adapters/src/media/silence.ts b/packages/adapters/src/media/silence.ts new file mode 100644 index 0000000..4f9435a --- /dev/null +++ b/packages/adapters/src/media/silence.ts @@ -0,0 +1,211 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { EditVideoStep } from "@rakazo/contracts"; +import { runFfmpeg } from "./ffmpeg.js"; +import type { VideoEditPlanRequest } from "./video-edit-plan.js"; +import { planVideoEdit } from "./video-edit-plan.js"; + +/** + * Silence detection for `remove_silence` and `analyze_video`. ffmpeg's silencedetect marks + * silences in frame metadata, which `ametadata` prints to a file in the job's directory; the + * file holds nothing but frame times and numbers, unlike stderr with its interleaved logs. + */ + +export interface TimeRange { + start: number; + end: number; +} + +/** Shortest piece of speech kept on its own; shorter blips between two pauses are noise. */ +const MIN_SPEECH_SECONDS = 0.1; +/** Most pieces one remove_silence step keeps, so the filter graph stays a sensible size. */ +export const MAX_SPEECH_SEGMENTS = 200; +/** Closer to the end of the timeline than this counts as reaching it. */ +const EDGE_SECONDS = 0.01; + +/** One frame of `metadata=mode=print` output: its time and the values set on it. */ +export interface FrameMetadata { + time: number; + values: Record; +} + +/** + * Parses the output of the `metadata`/`ametadata` filters in print mode: + * `frame:N pts:P pts_time:T` lines, each followed by `key=value` lines. + */ +export function parseFrameMetadata(text: string): FrameMetadata[] { + const frames: FrameMetadata[] = []; + let current: FrameMetadata | undefined; + for (const raw of text.split("\n")) { + const line = raw.trim(); + const frame = /^frame:\s*\d+\s+pts:\s*\S+\s+pts_time:\s*(\S+)/.exec(line); + if (frame) { + const time = Number(frame[1]); + current = Number.isFinite(time) ? { time, values: {} } : undefined; + if (current) frames.push(current); + continue; + } + const separator = line.indexOf("="); + if (current && separator > 0) { + current.values[line.slice(0, separator)] = line.slice(separator + 1); + } + } + return frames; +} + +/** Filters that detect silence and print it to `file`, a generated name in ffmpeg's cwd. */ +export function silenceDetectFilters( + options: { thresholdDb: number; minSilence: number }, + file: string, +): string[] { + return [ + `silencedetect=noise=${number(options.thresholdDb)}dB:duration=${number(options.minSilence)}`, + `ametadata=mode=print:file=${file}`, + ]; +} + +/** + * Silences from silencedetect's printed metadata, sorted and clamped to `[0, duration]`. A + * silence still running when the audio ends has no end mark and lasts to the end. + */ +export function parseSilences(text: string, duration: number): TimeRange[] { + const silences: TimeRange[] = []; + let start: number | undefined; + const add = (from: number, to: number) => { + const range = { start: clamp(from, 0, duration), end: clamp(to, 0, duration) }; + if (range.end > range.start) silences.push(range); + }; + for (const frame of parseFrameMetadata(text)) { + const begins = Number(frame.values["lavfi.silence_start"]); + const ends = Number(frame.values["lavfi.silence_end"]); + // One frame can end a silence and start the next one; the end belongs to the earlier one. + if (frame.values["lavfi.silence_end"] !== undefined && Number.isFinite(ends)) { + add(start ?? 0, ends); + start = undefined; + } + if (frame.values["lavfi.silence_start"] !== undefined && Number.isFinite(begins)) { + start = begins; + } + } + if (start !== undefined) add(start, duration); + return mergeRanges(silences); +} + +/** + * The parts of `[0, duration]` that are not silent, each widened by `padding` on both sides, + * clamped to the timeline and merged where the padding closes a gap. Null when nothing would + * be cut; throws when nothing would be left. + */ +export function speechSegments( + silences: readonly TimeRange[], + duration: number, + options: { padding: number; thresholdDb: number; maxSegments?: number }, +): TimeRange[] | null { + const speech: TimeRange[] = []; + let cursor = 0; + for (const silence of mergeRanges(silences)) { + if (silence.start > cursor) speech.push({ start: cursor, end: silence.start }); + cursor = Math.max(cursor, silence.end); + } + if (cursor < duration) speech.push({ start: cursor, end: duration }); + const padded = mergeRanges( + speech.map((range) => ({ + start: clamp(range.start - options.padding, 0, duration), + end: clamp(range.end + options.padding, 0, duration), + })), + ).filter((range) => range.end - range.start >= MIN_SPEECH_SECONDS); + if (padded.length === 0) { + throw new Error( + `remove_silence found nothing louder than ${number(options.thresholdDb)} dB, so nothing would be left.`, + ); + } + const kept = bridgeShortestGaps(padded, options.maxSegments ?? MAX_SPEECH_SEGMENTS); + const [only] = kept; + if ( + kept.length === 1 && + only && + only.start <= EDGE_SECONDS && + only.end >= duration - EDGE_SECONDS + ) { + return null; + } + return kept; +} + +/** + * Detects the silence each remove_silence step sees, in step order: one audio-only ffmpeg pass + * over the timeline the steps before it leave, so trims, appends and speed changes before it + * are taken into account. The result is the `speech` map `planVideoEdit` cuts with. + */ +export async function detectSpeech( + request: Omit, + options: { workDir: string; signal: AbortSignal }, +): Promise> { + const speech = new Map(); + for (const [index, step] of request.steps.entries()) { + if (step.op !== "remove_silence") continue; + const file = `silence-${index}.txt`; + const plan = planVideoEdit({ + ...request, + steps: request.steps.slice(0, index), + speech, + analyzeAudio: silenceDetectFilters( + { thresholdDb: step.threshold_db, minSilence: step.min_silence }, + file, + ), + outputPath: "", + }); + await runFfmpeg(plan.args, { + signal: options.signal, + cwd: options.workDir, + durationMs: plan.durationMs, + }); + const duration = plan.durationMs / 1000; + const silences = parseSilences(await readFile(join(options.workDir, file), "utf8"), duration); + speech.set( + step, + speechSegments(silences, duration, { + padding: step.padding, + thresholdDb: step.threshold_db, + }), + ); + } + return speech; +} + +/** Sorted, with ranges that overlap or touch merged. */ +function mergeRanges(ranges: readonly TimeRange[]): TimeRange[] { + const sorted = [...ranges].sort((a, b) => a.start - b.start); + const merged: TimeRange[] = []; + for (const range of sorted) { + const last = merged.at(-1); + if (last && range.start <= last.end) last.end = Math.max(last.end, range.end); + else merged.push({ ...range }); + } + return merged; +} + +/** At most `max` segments: the shortest gaps between them stay in until few enough remain. */ +function bridgeShortestGaps(segments: readonly TimeRange[], max: number): TimeRange[] { + if (segments.length <= max) return [...segments]; + const gaps = segments + .slice(1) + .map((segment, index) => ({ index, length: segment.start - segments[index]!.end })) + .sort((a, b) => a.length - b.length || a.index - b.index); + const bridged = new Set(gaps.slice(0, segments.length - max).map((gap) => gap.index)); + const kept: TimeRange[] = []; + for (const [index, segment] of segments.entries()) { + const last = kept.at(-1); + if (last && bridged.has(index - 1)) last.end = segment.end; + else kept.push({ ...segment }); + } + return kept; +} + +function clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); +} + +function number(value: number): string { + return String(Number(value.toFixed(6))); +} diff --git a/packages/adapters/src/media/video-edit-plan.ts b/packages/adapters/src/media/video-edit-plan.ts index f39f8d5..4603c1e 100644 --- a/packages/adapters/src/media/video-edit-plan.ts +++ b/packages/adapters/src/media/video-edit-plan.ts @@ -22,6 +22,16 @@ export interface VideoEditPlanRequest { clips?: ReadonlyMap; /** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */ subtitleFiles?: ReadonlyMap; + /** + * What each remove_silence step keeps, on its own input timeline: speech segments, or null + * when nothing is silent. Detected beforehand by an analysis plan (see silence.ts). + */ + speech?: ReadonlyMap; + /** + * Analysis instead of an encode: the audio left by the steps runs through these filters + * into a null output, so they can report on it (silencedetect writing a file, say). + */ + analyzeAudio?: readonly string[]; outputPath: string; } @@ -106,7 +116,8 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { const video = source.probe.video; if (!video) throw new Error(`${source.name} has no video.`); const sourceSeconds = source.probe.durationMs / 1000; - const audioOnly = request.steps.at(-1)?.op === "extract_audio"; + const audioOnly = + request.steps.at(-1)?.op === "extract_audio" || request.analyzeAudio !== undefined; const graph = new FilterGraph(); const inputs: string[][] = []; const addInput = (input: PlanInput, seek?: { start: number; end: number }) => { @@ -221,6 +232,12 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { case "poster": posterAt = step.at; break; + case "remove_silence": { + const speech = request.speech?.get(step); + if (speech === undefined) throw new Error("Silence was not detected before planning."); + if (speech) keepSegments(graph, timeline, speech); + break; + } } } @@ -233,6 +250,24 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { const output = `file:${request.outputPath}`; if (audioOnly || !timeline.video) { if (!timeline.audio) throw new Error(`${source.name} has no audio.`); + if (request.analyzeAudio) { + timeline.audio.filters.push(...request.analyzeAudio); + const audio = graph.flush(timeline.audio, "a"); + return { + args: [ + ...inputs.flat(), + "-filter_complex", + graph.toString(), + "-map", + `[${audio}]`, + "-f", + "null", + "-", + ], + kind: "audio", + durationMs, + }; + } timeline.audio.filters.push(AUDIO_FORMAT); const audio = graph.flush(timeline.audio, "a"); return { diff --git a/packages/adapters/src/media/video-operations.ts b/packages/adapters/src/media/video-operations.ts index e4e1e6d..eabb5ad 100644 --- a/packages/adapters/src/media/video-operations.ts +++ b/packages/adapters/src/media/video-operations.ts @@ -17,6 +17,7 @@ import { renderPoster, runFfmpeg, } from "./ffmpeg.js"; +import { detectSpeech } from "./silence.js"; import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js"; import { planVideoEdit } from "./video-edit-plan.js"; @@ -129,7 +130,18 @@ async function runEditVideo(context: MediaJobContext): Promise + EditVideoInputSchema.safeParse({ artifact_id: "a", title: "Reel", steps }); + +describe("remove_silence step", () => { + it("fills its defaults and parses again unchanged", () => { + const parsed = EditVideoInputSchema.parse({ + artifact_id: "a", + title: "Reel", + steps: [ + { op: "trim", start: 0, end: 30 }, + { op: "remove_silence", padding: 0 }, + ], + }); + expect(parsed.steps[1]).toEqual({ + op: "remove_silence", + threshold_db: SILENCE_DEFAULTS.thresholdDb, + min_silence: SILENCE_DEFAULTS.minSilence, + padding: 0, + }); + expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed); + }); + + it("keeps its settings in a useful range", () => { + expect(edit([{ op: "remove_silence", threshold_db: -61 }]).success).toBe(false); + expect(edit([{ op: "remove_silence", threshold_db: -5 }]).success).toBe(false); + expect(edit([{ op: "remove_silence", min_silence: 0.1 }]).success).toBe(false); + expect(edit([{ op: "remove_silence", min_silence: 6 }]).success).toBe(false); + expect(edit([{ op: "remove_silence", padding: -0.1 }]).success).toBe(false); + expect(edit([{ op: "remove_silence", padding: 1.5 }]).success).toBe(false); + expect( + edit([{ op: "remove_silence", threshold_db: -60, min_silence: 5, padding: 1 }]).success, + ).toBe(true); + }); +}); diff --git a/packages/contracts/src/media-analysis.ts b/packages/contracts/src/media-analysis.ts new file mode 100644 index 0000000..2c743e9 --- /dev/null +++ b/packages/contracts/src/media-analysis.ts @@ -0,0 +1,22 @@ +import * as z from "zod"; + +/** Cut-point finding for talking-head and social video: the `remove_silence` step of `edit_video`. */ + +/** Defaults of `remove_silence`. */ +export const SILENCE_DEFAULTS = { + /** Quieter than this (dBFS) counts as silence: below speech, above a typical noise floor. */ + thresholdDb: -35, + /** Shorter pauses are the rhythm of speech and stay. */ + minSilence: 0.6, + /** Kept on both sides of what is cut, so breaths and word onsets are not clipped. */ + padding: 0.12, +} as const; + +/** Cuts every pause and keeps sound and picture in sync, like a keep step over the speech. */ +export const RemoveSilenceStepSchema = z.object({ + op: z.literal("remove_silence"), + threshold_db: z.number().min(-60).max(-10).default(SILENCE_DEFAULTS.thresholdDb), + min_silence: z.number().min(0.2).max(5).default(SILENCE_DEFAULTS.minSilence), + padding: z.number().min(0).max(1).default(SILENCE_DEFAULTS.padding), +}); +export type RemoveSilenceStep = z.infer; diff --git a/packages/contracts/src/media-video.ts b/packages/contracts/src/media-video.ts index a12bf51..70f3c86 100644 --- a/packages/contracts/src/media-video.ts +++ b/packages/contracts/src/media-video.ts @@ -1,6 +1,7 @@ import * as z from "zod"; import { Id } from "./ids.js"; import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; +import { RemoveSilenceStepSchema } from "./media-analysis.js"; /** * Inputs of the `edit_video` and `export_video` media operations, which are also the arguments @@ -39,6 +40,7 @@ export const EditVideoStepSchema = z.discriminatedUnion("op", [ z.object({ op: z.literal("extract_audio") }), /** Cover frame at this second of the result; as the only kind of step, the result is that image. */ z.object({ op: z.literal("poster"), at: Seconds }), + RemoveSilenceStepSchema, ]); export type EditVideoStep = z.infer;