Add remove_silence step to edit_video
Talking-head clips are mostly cut by hand to drop pauses, and the bot had no way to find them: it could only keep segments it guessed. The new remove_silence step finds every pause quieter than threshold_db (default -35 dB) and at least min_silence long (default 0.6 s), keeps padding (default 0.12 s) on each side so word onsets survive, and cuts the rest through the existing keep-segments graph, so picture and sound stay in sync. Steps run in order, so silence is detected on the timeline the steps before it leave: the planner gets an analysis mode that sends that audio through silencedetect into a null output, and ametadata prints the marks to a file in the job directory. The detected speech is handed back to the planner per step. Nothing silent makes the step a no-op; nothing left fails with a clear error. At most 200 pieces are kept, bridging the shortest pauses first, so the filter graph stays a sensible size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
parent
404a1d1782
commit
6ba9cef03f
@ -559,7 +559,7 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
},
|
||||
{
|
||||
name: "edit_video",
|
||||
description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`,
|
||||
description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image; remove_silence {threshold_db -35, min_silence 0.6, padding 0.12} cuts pauses quieter than threshold_db dB and at least min_silence s long, keeping padding s on each side. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`,
|
||||
inputSchema: mediaParameters(EditVideoInputSchema),
|
||||
},
|
||||
{
|
||||
|
||||
@ -104,3 +104,24 @@ export async function fakeMediaJobContext(input: unknown, artifacts: Record<stri
|
||||
};
|
||||
return { context, controller, progress, outputs, tempDir };
|
||||
}
|
||||
|
||||
/**
|
||||
* A 4 s talking-head stand-in at 320×180, 25 fps: red for 2 s then a hard cut to blue, over a
|
||||
* 48 kHz tone that pauses from 1.5 s to 2.5 s.
|
||||
*/
|
||||
export async function createCutTestClip(dir: string): Promise<string> {
|
||||
const path = join(dir, "tone-gap-tone.mp4");
|
||||
await runFfmpeg([
|
||||
...["-f", "lavfi", "-i", "color=c=red:size=320x180:rate=25:duration=2"],
|
||||
...["-f", "lavfi", "-i", "color=c=blue:size=320x180:rate=25:duration=2"],
|
||||
...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=1.5"],
|
||||
...["-f", "lavfi", "-i", "anullsrc=channel_layout=mono:sample_rate=48000"],
|
||||
...["-f", "lavfi", "-i", "sine=frequency=660:sample_rate=48000:duration=1.5"],
|
||||
"-filter_complex",
|
||||
"[0:v][1:v]concat=n=2:v=1:a=0[v];[3:a]atrim=duration=1[gap];[2:a][gap][4:a]concat=n=3:v=0:a=1[a]",
|
||||
...["-map", "[v]", "-map", "[a]"],
|
||||
...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-c:a", "aac"],
|
||||
`file:${path}`,
|
||||
]);
|
||||
return path;
|
||||
}
|
||||
|
||||
320
packages/adapters/src/media/silence.test.ts
Normal file
320
packages/adapters/src/media/silence.test.ts
Normal file
@ -0,0 +1,320 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import type { EditVideoStep } from "@rakazo/contracts";
|
||||
import { EditVideoInputSchema } from "@rakazo/contracts";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { detectFfmpeg } from "./ffmpeg.js";
|
||||
import { createCutTestClip, fakeMediaJobContext } from "./media-test-support.js";
|
||||
import {
|
||||
MAX_SPEECH_SEGMENTS,
|
||||
parseFrameMetadata,
|
||||
parseSilences,
|
||||
silenceDetectFilters,
|
||||
speechSegments,
|
||||
} from "./silence.js";
|
||||
import type { PlanInput } from "./video-edit-plan.js";
|
||||
import { planVideoEdit } from "./video-edit-plan.js";
|
||||
import { createVideoOperations } from "./video-operations.js";
|
||||
|
||||
const ffmpeg = await detectFfmpeg();
|
||||
|
||||
// Printed by `silencedetect=noise=-35dB:duration=0.6,ametadata=mode=print:file=…` (ffmpeg 6.1)
|
||||
// for 1.5 s tone, 1 s silence, 1.5 s tone; then for silence, tone, silence up to the end.
|
||||
const TONE_GAP_TONE = `frame:98 pts:100352 pts_time:2.09067
|
||||
lavfi.silence_start=1.49996
|
||||
frame:117 pts:119808 pts_time:2.496
|
||||
lavfi.silence_end=2.50004
|
||||
lavfi.silence_duration=1.00008
|
||||
`;
|
||||
const SILENT_EDGES = `frame:14 pts:28672 pts_time:0.597333
|
||||
lavfi.silence_start=0
|
||||
frame:46 pts:94208 pts_time:1.96267
|
||||
lavfi.silence_end=2.00008
|
||||
lavfi.silence_duration=2.00008
|
||||
frame:84 pts:172032 pts_time:3.584
|
||||
lavfi.silence_start=2.99996
|
||||
`;
|
||||
|
||||
const defaults = { padding: 0.12, thresholdDb: -35 };
|
||||
|
||||
describe("silence detection output", () => {
|
||||
it("reads printed frame metadata", () => {
|
||||
expect(parseFrameMetadata(TONE_GAP_TONE)).toEqual([
|
||||
{ time: 2.09067, values: { "lavfi.silence_start": "1.49996" } },
|
||||
{
|
||||
time: 2.496,
|
||||
values: { "lavfi.silence_end": "2.50004", "lavfi.silence_duration": "1.00008" },
|
||||
},
|
||||
]);
|
||||
expect(parseFrameMetadata("frame:1 pts:N/A pts_time:N/A\nlavfi.x=1\n")).toEqual([]);
|
||||
});
|
||||
|
||||
it("pairs starts with ends and runs a silence without end to the end of the audio", () => {
|
||||
expect(parseSilences(TONE_GAP_TONE, 4)).toEqual([{ start: 1.49996, end: 2.50004 }]);
|
||||
expect(parseSilences(SILENT_EDGES, 4.5)).toEqual([
|
||||
{ start: 0, end: 2.00008 },
|
||||
{ start: 2.99996, end: 4.5 },
|
||||
]);
|
||||
// A probe a little shorter than the decoded audio clamps the ranges to the timeline.
|
||||
expect(parseSilences(SILENT_EDGES, 2)).toEqual([{ start: 0, end: 2 }]);
|
||||
expect(parseSilences("", 4)).toEqual([]);
|
||||
});
|
||||
|
||||
it("builds the filters from validated numbers and a generated file name", () => {
|
||||
expect(silenceDetectFilters({ thresholdDb: -35, minSilence: 0.6 }, "silence-2.txt")).toEqual([
|
||||
"silencedetect=noise=-35dB:duration=0.6",
|
||||
"ametadata=mode=print:file=silence-2.txt",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("speech segments", () => {
|
||||
it("keeps what is not silent, padded on both sides", () => {
|
||||
expect(speechSegments([{ start: 1.5, end: 2.5 }], 4, defaults)).toEqual([
|
||||
{ start: 0, end: 1.62 },
|
||||
{ start: 2.38, end: 4 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("clamps padding to the timeline and drops silent edges", () => {
|
||||
const segments = speechSegments(
|
||||
[
|
||||
{ start: 0, end: 2 },
|
||||
{ start: 3, end: 4.5 },
|
||||
],
|
||||
4.5,
|
||||
defaults,
|
||||
);
|
||||
expect(segments).toHaveLength(1);
|
||||
expect(segments?.[0]?.start).toBeCloseTo(1.88);
|
||||
expect(segments?.[0]?.end).toBeCloseTo(3.12);
|
||||
expect(speechSegments([{ start: 3.95, end: 4 }], 4, { ...defaults, padding: 1 })).toBeNull();
|
||||
});
|
||||
|
||||
it("merges speech where the padding closes a pause, and overlapping silences", () => {
|
||||
expect(
|
||||
speechSegments(
|
||||
[
|
||||
{ start: 1, end: 1.2 },
|
||||
{ start: 2, end: 3 },
|
||||
{ start: 2.5, end: 3.5 },
|
||||
],
|
||||
5,
|
||||
defaults,
|
||||
),
|
||||
).toEqual([
|
||||
{ start: 0, end: 2.12 },
|
||||
{ start: 3.38, end: 5 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("is null when nothing is silent and fails when nothing would be left", () => {
|
||||
expect(speechSegments([], 4, defaults)).toBeNull();
|
||||
expect(() => speechSegments([{ start: 0, end: 4 }], 4, defaults)).toThrow(
|
||||
"remove_silence found nothing louder than -35 dB, so nothing would be left.",
|
||||
);
|
||||
// Blips shorter than 0.1 s without padding are noise, not speech.
|
||||
expect(() =>
|
||||
speechSegments(
|
||||
[
|
||||
{ start: 0, end: 2 },
|
||||
{ start: 2.05, end: 4 },
|
||||
],
|
||||
4,
|
||||
{ ...defaults, padding: 0 },
|
||||
),
|
||||
).toThrow("nothing would be left");
|
||||
});
|
||||
|
||||
it("keeps the longest pauses cut when there would be too many pieces", () => {
|
||||
const silences = Array.from({ length: 5 }, (_, index) => ({
|
||||
start: index * 2 + 1,
|
||||
end: index * 2 + 1 + (index === 2 ? 0.9 : 0.5),
|
||||
}));
|
||||
expect(speechSegments(silences, 11, { padding: 0, thresholdDb: -35, maxSegments: 2 })).toEqual([
|
||||
{ start: 0, end: 5 },
|
||||
{ start: 5.9, end: 11 },
|
||||
]);
|
||||
expect(MAX_SPEECH_SEGMENTS).toBe(200);
|
||||
});
|
||||
});
|
||||
|
||||
const source: PlanInput = {
|
||||
path: "/jobs/1/input-source",
|
||||
mimeType: "video/mp4",
|
||||
name: "talk.mp4",
|
||||
probe: {
|
||||
durationMs: 10_000,
|
||||
video: {
|
||||
codec: "h264",
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
rotation: 0,
|
||||
fps: 25,
|
||||
sampleAspectRatio: 1,
|
||||
},
|
||||
audio: { codec: "aac", sampleRate: 48_000, channels: 2 },
|
||||
hasAudio: true,
|
||||
},
|
||||
};
|
||||
|
||||
function steps(list: unknown[]): EditVideoStep[] {
|
||||
return EditVideoInputSchema.parse({ artifact_id: "a", title: "Talk", steps: list }).steps;
|
||||
}
|
||||
|
||||
function graphOf(args: string[]): string {
|
||||
return args[args.indexOf("-filter_complex") + 1]!;
|
||||
}
|
||||
|
||||
describe("remove_silence planning", () => {
|
||||
it("fills the step's defaults", () => {
|
||||
expect(steps([{ op: "remove_silence" }])).toEqual([
|
||||
{ op: "remove_silence", threshold_db: -35, min_silence: 0.6, padding: 0.12 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("detects silence on the audio the steps before it leave, into a null output", () => {
|
||||
const [trim] = steps([{ op: "trim", start: 2, end: 8 }]);
|
||||
const plan = planVideoEdit({
|
||||
source,
|
||||
steps: [trim!, { op: "speed", factor: 2 }],
|
||||
analyzeAudio: silenceDetectFilters({ thresholdDb: -35, minSilence: 0.6 }, "silence-2.txt"),
|
||||
outputPath: "",
|
||||
});
|
||||
expect(plan.args).toEqual([
|
||||
...["-ss", "2", "-t", "6", "-protocol_whitelist", "file", "-f", "mov"],
|
||||
...["-i", "file:/jobs/1/input-source"],
|
||||
"-filter_complex",
|
||||
"[0:a:0]atempo=2,silencedetect=noise=-35dB:duration=0.6,ametadata=mode=print:file=silence-2.txt[a1]",
|
||||
...["-map", "[a1]", "-f", "null", "-"],
|
||||
]);
|
||||
expect(plan).toMatchObject({ kind: "audio", durationMs: 3000 });
|
||||
});
|
||||
|
||||
it("cuts the detected pauses from picture and sound like a keep step", () => {
|
||||
const list = steps([{ op: "remove_silence" }, { op: "reframe", preset: "vertical" }]);
|
||||
const plan = planVideoEdit({
|
||||
source,
|
||||
steps: list,
|
||||
speech: new Map([
|
||||
[
|
||||
list[0]!,
|
||||
[
|
||||
{ start: 0, end: 1.62 },
|
||||
{ start: 2.38, end: 10 },
|
||||
],
|
||||
],
|
||||
]),
|
||||
outputPath: "/jobs/1/output.mp4",
|
||||
});
|
||||
const graph = graphOf(plan.args);
|
||||
expect(graph).toContain("[0:v:0]split=2[v1][v2]");
|
||||
expect(graph).toContain("[v1]trim=start=0:end=1.62,setpts=PTS-STARTPTS[v3]");
|
||||
expect(graph).toContain("[v2]trim=start=2.38:end=10,setpts=PTS-STARTPTS[v4]");
|
||||
expect(graph).toContain("[0:a:0]asplit=2[a5][a6]");
|
||||
expect(graph).toContain("[a5]atrim=start=0:end=1.62,asetpts=PTS-STARTPTS[a7]");
|
||||
expect(graph).toContain("concat=n=2:v=1:a=1");
|
||||
expect(plan).toMatchObject({ kind: "video", durationMs: 9240, width: 1080, height: 1920 });
|
||||
});
|
||||
|
||||
it("leaves the timeline alone when nothing is silent and needs detection first", () => {
|
||||
const list = steps([{ op: "remove_silence" }]);
|
||||
const untouched = planVideoEdit({
|
||||
source,
|
||||
steps: list,
|
||||
speech: new Map([[list[0]!, null]]),
|
||||
outputPath: "/jobs/1/output.mp4",
|
||||
});
|
||||
expect(graphOf(untouched.args)).not.toContain("trim");
|
||||
expect(untouched.durationMs).toBe(10_000);
|
||||
expect(() => planVideoEdit({ source, steps: list, outputPath: "/jobs/1/output.mp4" })).toThrow(
|
||||
"Silence was not detected before planning.",
|
||||
);
|
||||
});
|
||||
|
||||
it("needs audio to detect silence in", () => {
|
||||
expect(() =>
|
||||
planVideoEdit({
|
||||
source,
|
||||
steps: [{ op: "mute" }],
|
||||
analyzeAudio: ["anullsink"],
|
||||
outputPath: "",
|
||||
}),
|
||||
).toThrow("talk.mp4 has no audio.");
|
||||
});
|
||||
});
|
||||
|
||||
/** Seconds of each stream of a file, by codec type. */
|
||||
async function streamSeconds(ffprobePath: string, path: string) {
|
||||
const { stdout } = await promisify(execFile)(ffprobePath, [
|
||||
...["-v", "error", "-show_entries", "stream=codec_type,duration", "-of", "json", path],
|
||||
]);
|
||||
const { streams } = JSON.parse(stdout) as {
|
||||
streams: { codec_type: string; duration: string }[];
|
||||
};
|
||||
return Object.fromEntries(streams.map((stream) => [stream.codec_type, Number(stream.duration)]));
|
||||
}
|
||||
|
||||
// CI runners may not have ffmpeg; these run wherever it is installed.
|
||||
describe.skipIf(!ffmpeg)("remove_silence with ffmpeg", () => {
|
||||
let dir: string;
|
||||
let clip: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), "evoo-silence-test-"));
|
||||
clip = await createCutTestClip(dir);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (dir) await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function edit(list: unknown[]) {
|
||||
const [registration] = createVideoOperations();
|
||||
const job = await fakeMediaJobContext(
|
||||
registration!.inputSchema.parse({ artifact_id: "talk", title: "Ohne Pausen", steps: list }),
|
||||
{ talk: { path: clip, name: "talk.mp4", mimeType: "video/mp4" } },
|
||||
);
|
||||
try {
|
||||
await registration!.handler.run(job.context);
|
||||
const [output] = job.outputs;
|
||||
// The fake context hands the file over in place; measure it before the temp dir goes.
|
||||
return {
|
||||
output: output!,
|
||||
streams: await streamSeconds(ffmpeg!.ffprobePath, output!.path),
|
||||
};
|
||||
} finally {
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("cuts the pause and keeps picture and sound the same length", async () => {
|
||||
const { output, streams } = await edit([{ op: "remove_silence", padding: 0.1 }]);
|
||||
// 4 s minus the 1 s pause, of which 0.1 s stays on each side.
|
||||
expect(Math.abs(output.durationMs! - 3200)).toBeLessThan(100);
|
||||
expect(Math.abs(streams.video! - 3.2)).toBeLessThan(0.1);
|
||||
expect(Math.abs(streams.video! - streams.audio!)).toBeLessThan(0.08);
|
||||
});
|
||||
|
||||
it("detects silence on the timeline the steps before it leave", async () => {
|
||||
const { output, streams } = await edit([
|
||||
{ op: "trim", start: 1, end: 4 },
|
||||
{ op: "remove_silence", padding: 0.1 },
|
||||
]);
|
||||
// In the trimmed 3 s the pause runs from 0.5 s to 1.5 s; 0.2 s of it stays.
|
||||
expect(Math.abs(output.durationMs! - 2200)).toBeLessThan(100);
|
||||
expect(Math.abs(streams.video! - streams.audio!)).toBeLessThan(0.08);
|
||||
});
|
||||
|
||||
it("fails clearly when everything is silent", async () => {
|
||||
await expect(edit([{ op: "mute" }, { op: "remove_silence" }])).rejects.toThrow(
|
||||
"talk.mp4 has no audio.",
|
||||
);
|
||||
await expect(edit([{ op: "remove_silence", threshold_db: -10 }])).rejects.toThrow(
|
||||
"remove_silence found nothing louder than -10 dB",
|
||||
);
|
||||
});
|
||||
});
|
||||
211
packages/adapters/src/media/silence.ts
Normal file
211
packages/adapters/src/media/silence.ts
Normal file
@ -0,0 +1,211 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { EditVideoStep } from "@rakazo/contracts";
|
||||
import { runFfmpeg } from "./ffmpeg.js";
|
||||
import type { VideoEditPlanRequest } from "./video-edit-plan.js";
|
||||
import { planVideoEdit } from "./video-edit-plan.js";
|
||||
|
||||
/**
|
||||
* Silence detection for `remove_silence` and `analyze_video`. ffmpeg's silencedetect marks
|
||||
* silences in frame metadata, which `ametadata` prints to a file in the job's directory; the
|
||||
* file holds nothing but frame times and numbers, unlike stderr with its interleaved logs.
|
||||
*/
|
||||
|
||||
export interface TimeRange {
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
|
||||
/** Shortest piece of speech kept on its own; shorter blips between two pauses are noise. */
|
||||
const MIN_SPEECH_SECONDS = 0.1;
|
||||
/** Most pieces one remove_silence step keeps, so the filter graph stays a sensible size. */
|
||||
export const MAX_SPEECH_SEGMENTS = 200;
|
||||
/** Closer to the end of the timeline than this counts as reaching it. */
|
||||
const EDGE_SECONDS = 0.01;
|
||||
|
||||
/** One frame of `metadata=mode=print` output: its time and the values set on it. */
|
||||
export interface FrameMetadata {
|
||||
time: number;
|
||||
values: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the output of the `metadata`/`ametadata` filters in print mode:
|
||||
* `frame:N pts:P pts_time:T` lines, each followed by `key=value` lines.
|
||||
*/
|
||||
export function parseFrameMetadata(text: string): FrameMetadata[] {
|
||||
const frames: FrameMetadata[] = [];
|
||||
let current: FrameMetadata | undefined;
|
||||
for (const raw of text.split("\n")) {
|
||||
const line = raw.trim();
|
||||
const frame = /^frame:\s*\d+\s+pts:\s*\S+\s+pts_time:\s*(\S+)/.exec(line);
|
||||
if (frame) {
|
||||
const time = Number(frame[1]);
|
||||
current = Number.isFinite(time) ? { time, values: {} } : undefined;
|
||||
if (current) frames.push(current);
|
||||
continue;
|
||||
}
|
||||
const separator = line.indexOf("=");
|
||||
if (current && separator > 0) {
|
||||
current.values[line.slice(0, separator)] = line.slice(separator + 1);
|
||||
}
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
|
||||
/** Filters that detect silence and print it to `file`, a generated name in ffmpeg's cwd. */
|
||||
export function silenceDetectFilters(
|
||||
options: { thresholdDb: number; minSilence: number },
|
||||
file: string,
|
||||
): string[] {
|
||||
return [
|
||||
`silencedetect=noise=${number(options.thresholdDb)}dB:duration=${number(options.minSilence)}`,
|
||||
`ametadata=mode=print:file=${file}`,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Silences from silencedetect's printed metadata, sorted and clamped to `[0, duration]`. A
|
||||
* silence still running when the audio ends has no end mark and lasts to the end.
|
||||
*/
|
||||
export function parseSilences(text: string, duration: number): TimeRange[] {
|
||||
const silences: TimeRange[] = [];
|
||||
let start: number | undefined;
|
||||
const add = (from: number, to: number) => {
|
||||
const range = { start: clamp(from, 0, duration), end: clamp(to, 0, duration) };
|
||||
if (range.end > range.start) silences.push(range);
|
||||
};
|
||||
for (const frame of parseFrameMetadata(text)) {
|
||||
const begins = Number(frame.values["lavfi.silence_start"]);
|
||||
const ends = Number(frame.values["lavfi.silence_end"]);
|
||||
// One frame can end a silence and start the next one; the end belongs to the earlier one.
|
||||
if (frame.values["lavfi.silence_end"] !== undefined && Number.isFinite(ends)) {
|
||||
add(start ?? 0, ends);
|
||||
start = undefined;
|
||||
}
|
||||
if (frame.values["lavfi.silence_start"] !== undefined && Number.isFinite(begins)) {
|
||||
start = begins;
|
||||
}
|
||||
}
|
||||
if (start !== undefined) add(start, duration);
|
||||
return mergeRanges(silences);
|
||||
}
|
||||
|
||||
/**
|
||||
* The parts of `[0, duration]` that are not silent, each widened by `padding` on both sides,
|
||||
* clamped to the timeline and merged where the padding closes a gap. Null when nothing would
|
||||
* be cut; throws when nothing would be left.
|
||||
*/
|
||||
export function speechSegments(
|
||||
silences: readonly TimeRange[],
|
||||
duration: number,
|
||||
options: { padding: number; thresholdDb: number; maxSegments?: number },
|
||||
): TimeRange[] | null {
|
||||
const speech: TimeRange[] = [];
|
||||
let cursor = 0;
|
||||
for (const silence of mergeRanges(silences)) {
|
||||
if (silence.start > cursor) speech.push({ start: cursor, end: silence.start });
|
||||
cursor = Math.max(cursor, silence.end);
|
||||
}
|
||||
if (cursor < duration) speech.push({ start: cursor, end: duration });
|
||||
const padded = mergeRanges(
|
||||
speech.map((range) => ({
|
||||
start: clamp(range.start - options.padding, 0, duration),
|
||||
end: clamp(range.end + options.padding, 0, duration),
|
||||
})),
|
||||
).filter((range) => range.end - range.start >= MIN_SPEECH_SECONDS);
|
||||
if (padded.length === 0) {
|
||||
throw new Error(
|
||||
`remove_silence found nothing louder than ${number(options.thresholdDb)} dB, so nothing would be left.`,
|
||||
);
|
||||
}
|
||||
const kept = bridgeShortestGaps(padded, options.maxSegments ?? MAX_SPEECH_SEGMENTS);
|
||||
const [only] = kept;
|
||||
if (
|
||||
kept.length === 1 &&
|
||||
only &&
|
||||
only.start <= EDGE_SECONDS &&
|
||||
only.end >= duration - EDGE_SECONDS
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects the silence each remove_silence step sees, in step order: one audio-only ffmpeg pass
|
||||
* over the timeline the steps before it leave, so trims, appends and speed changes before it
|
||||
* are taken into account. The result is the `speech` map `planVideoEdit` cuts with.
|
||||
*/
|
||||
export async function detectSpeech(
|
||||
request: Omit<VideoEditPlanRequest, "speech" | "analyzeAudio" | "outputPath">,
|
||||
options: { workDir: string; signal: AbortSignal },
|
||||
): Promise<Map<EditVideoStep, TimeRange[] | null>> {
|
||||
const speech = new Map<EditVideoStep, TimeRange[] | null>();
|
||||
for (const [index, step] of request.steps.entries()) {
|
||||
if (step.op !== "remove_silence") continue;
|
||||
const file = `silence-${index}.txt`;
|
||||
const plan = planVideoEdit({
|
||||
...request,
|
||||
steps: request.steps.slice(0, index),
|
||||
speech,
|
||||
analyzeAudio: silenceDetectFilters(
|
||||
{ thresholdDb: step.threshold_db, minSilence: step.min_silence },
|
||||
file,
|
||||
),
|
||||
outputPath: "",
|
||||
});
|
||||
await runFfmpeg(plan.args, {
|
||||
signal: options.signal,
|
||||
cwd: options.workDir,
|
||||
durationMs: plan.durationMs,
|
||||
});
|
||||
const duration = plan.durationMs / 1000;
|
||||
const silences = parseSilences(await readFile(join(options.workDir, file), "utf8"), duration);
|
||||
speech.set(
|
||||
step,
|
||||
speechSegments(silences, duration, {
|
||||
padding: step.padding,
|
||||
thresholdDb: step.threshold_db,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return speech;
|
||||
}
|
||||
|
||||
/** Sorted, with ranges that overlap or touch merged. */
|
||||
function mergeRanges(ranges: readonly TimeRange[]): TimeRange[] {
|
||||
const sorted = [...ranges].sort((a, b) => a.start - b.start);
|
||||
const merged: TimeRange[] = [];
|
||||
for (const range of sorted) {
|
||||
const last = merged.at(-1);
|
||||
if (last && range.start <= last.end) last.end = Math.max(last.end, range.end);
|
||||
else merged.push({ ...range });
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/** At most `max` segments: the shortest gaps between them stay in until few enough remain. */
|
||||
function bridgeShortestGaps(segments: readonly TimeRange[], max: number): TimeRange[] {
|
||||
if (segments.length <= max) return [...segments];
|
||||
const gaps = segments
|
||||
.slice(1)
|
||||
.map((segment, index) => ({ index, length: segment.start - segments[index]!.end }))
|
||||
.sort((a, b) => a.length - b.length || a.index - b.index);
|
||||
const bridged = new Set(gaps.slice(0, segments.length - max).map((gap) => gap.index));
|
||||
const kept: TimeRange[] = [];
|
||||
for (const [index, segment] of segments.entries()) {
|
||||
const last = kept.at(-1);
|
||||
if (last && bridged.has(index - 1)) last.end = segment.end;
|
||||
else kept.push({ ...segment });
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function number(value: number): string {
|
||||
return String(Number(value.toFixed(6)));
|
||||
}
|
||||
@ -22,6 +22,16 @@ export interface VideoEditPlanRequest {
|
||||
clips?: ReadonlyMap<string, PlanInput>;
|
||||
/** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */
|
||||
subtitleFiles?: ReadonlyMap<string, string>;
|
||||
/**
|
||||
* What each remove_silence step keeps, on its own input timeline: speech segments, or null
|
||||
* when nothing is silent. Detected beforehand by an analysis plan (see silence.ts).
|
||||
*/
|
||||
speech?: ReadonlyMap<EditVideoStep, readonly { start: number; end: number }[] | null>;
|
||||
/**
|
||||
* Analysis instead of an encode: the audio left by the steps runs through these filters
|
||||
* into a null output, so they can report on it (silencedetect writing a file, say).
|
||||
*/
|
||||
analyzeAudio?: readonly string[];
|
||||
outputPath: string;
|
||||
}
|
||||
|
||||
@ -106,7 +116,8 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||
const video = source.probe.video;
|
||||
if (!video) throw new Error(`${source.name} has no video.`);
|
||||
const sourceSeconds = source.probe.durationMs / 1000;
|
||||
const audioOnly = request.steps.at(-1)?.op === "extract_audio";
|
||||
const audioOnly =
|
||||
request.steps.at(-1)?.op === "extract_audio" || request.analyzeAudio !== undefined;
|
||||
const graph = new FilterGraph();
|
||||
const inputs: string[][] = [];
|
||||
const addInput = (input: PlanInput, seek?: { start: number; end: number }) => {
|
||||
@ -221,6 +232,12 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||
case "poster":
|
||||
posterAt = step.at;
|
||||
break;
|
||||
case "remove_silence": {
|
||||
const speech = request.speech?.get(step);
|
||||
if (speech === undefined) throw new Error("Silence was not detected before planning.");
|
||||
if (speech) keepSegments(graph, timeline, speech);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -233,6 +250,24 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||
const output = `file:${request.outputPath}`;
|
||||
if (audioOnly || !timeline.video) {
|
||||
if (!timeline.audio) throw new Error(`${source.name} has no audio.`);
|
||||
if (request.analyzeAudio) {
|
||||
timeline.audio.filters.push(...request.analyzeAudio);
|
||||
const audio = graph.flush(timeline.audio, "a");
|
||||
return {
|
||||
args: [
|
||||
...inputs.flat(),
|
||||
"-filter_complex",
|
||||
graph.toString(),
|
||||
"-map",
|
||||
`[${audio}]`,
|
||||
"-f",
|
||||
"null",
|
||||
"-",
|
||||
],
|
||||
kind: "audio",
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
timeline.audio.filters.push(AUDIO_FORMAT);
|
||||
const audio = graph.flush(timeline.audio, "a");
|
||||
return {
|
||||
|
||||
@ -17,6 +17,7 @@ import {
|
||||
renderPoster,
|
||||
runFfmpeg,
|
||||
} from "./ffmpeg.js";
|
||||
import { detectSpeech } from "./silence.js";
|
||||
import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js";
|
||||
import { planVideoEdit } from "./video-edit-plan.js";
|
||||
|
||||
@ -129,7 +130,18 @@ async function runEditVideo(context: MediaJobContext): Promise<MediaOperationRes
|
||||
}
|
||||
const audioOnly = input.steps.at(-1)?.op === "extract_audio";
|
||||
const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4");
|
||||
const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath });
|
||||
const speech = await detectSpeech(
|
||||
{ source, steps: input.steps, clips, subtitleFiles },
|
||||
{ workDir, signal: context.signal },
|
||||
);
|
||||
const plan = planVideoEdit({
|
||||
source,
|
||||
steps: input.steps,
|
||||
clips,
|
||||
subtitleFiles,
|
||||
speech,
|
||||
outputPath,
|
||||
});
|
||||
const output = await encode(context, plan, outputPath, workDir, progress.range(0, 1), {
|
||||
name: `${name}${audioOnly ? ".m4a" : ".mp4"}`,
|
||||
parentArtifactId: source.artifactId,
|
||||
|
||||
@ -15,6 +15,7 @@ export * from "./integration-settings.js";
|
||||
export * from "./local-settings.js";
|
||||
export * from "./mcp.js";
|
||||
export * from "./media.js";
|
||||
export * from "./media-analysis.js";
|
||||
export * from "./media-generation.js";
|
||||
export * from "./media-jobs.js";
|
||||
export * from "./media-settings.js";
|
||||
|
||||
38
packages/contracts/src/media-analysis.test.ts
Normal file
38
packages/contracts/src/media-analysis.test.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { SILENCE_DEFAULTS } from "./media-analysis.js";
|
||||
import { EditVideoInputSchema } from "./media-video.js";
|
||||
|
||||
const edit = (steps: unknown[]) =>
|
||||
EditVideoInputSchema.safeParse({ artifact_id: "a", title: "Reel", steps });
|
||||
|
||||
describe("remove_silence step", () => {
|
||||
it("fills its defaults and parses again unchanged", () => {
|
||||
const parsed = EditVideoInputSchema.parse({
|
||||
artifact_id: "a",
|
||||
title: "Reel",
|
||||
steps: [
|
||||
{ op: "trim", start: 0, end: 30 },
|
||||
{ op: "remove_silence", padding: 0 },
|
||||
],
|
||||
});
|
||||
expect(parsed.steps[1]).toEqual({
|
||||
op: "remove_silence",
|
||||
threshold_db: SILENCE_DEFAULTS.thresholdDb,
|
||||
min_silence: SILENCE_DEFAULTS.minSilence,
|
||||
padding: 0,
|
||||
});
|
||||
expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed);
|
||||
});
|
||||
|
||||
it("keeps its settings in a useful range", () => {
|
||||
expect(edit([{ op: "remove_silence", threshold_db: -61 }]).success).toBe(false);
|
||||
expect(edit([{ op: "remove_silence", threshold_db: -5 }]).success).toBe(false);
|
||||
expect(edit([{ op: "remove_silence", min_silence: 0.1 }]).success).toBe(false);
|
||||
expect(edit([{ op: "remove_silence", min_silence: 6 }]).success).toBe(false);
|
||||
expect(edit([{ op: "remove_silence", padding: -0.1 }]).success).toBe(false);
|
||||
expect(edit([{ op: "remove_silence", padding: 1.5 }]).success).toBe(false);
|
||||
expect(
|
||||
edit([{ op: "remove_silence", threshold_db: -60, min_silence: 5, padding: 1 }]).success,
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
22
packages/contracts/src/media-analysis.ts
Normal file
22
packages/contracts/src/media-analysis.ts
Normal file
@ -0,0 +1,22 @@
|
||||
import * as z from "zod";
|
||||
|
||||
/** Cut-point finding for talking-head and social video: the `remove_silence` step of `edit_video`. */
|
||||
|
||||
/** Defaults of `remove_silence`. */
|
||||
export const SILENCE_DEFAULTS = {
|
||||
/** Quieter than this (dBFS) counts as silence: below speech, above a typical noise floor. */
|
||||
thresholdDb: -35,
|
||||
/** Shorter pauses are the rhythm of speech and stay. */
|
||||
minSilence: 0.6,
|
||||
/** Kept on both sides of what is cut, so breaths and word onsets are not clipped. */
|
||||
padding: 0.12,
|
||||
} as const;
|
||||
|
||||
/** Cuts every pause and keeps sound and picture in sync, like a keep step over the speech. */
|
||||
export const RemoveSilenceStepSchema = z.object({
|
||||
op: z.literal("remove_silence"),
|
||||
threshold_db: z.number().min(-60).max(-10).default(SILENCE_DEFAULTS.thresholdDb),
|
||||
min_silence: z.number().min(0.2).max(5).default(SILENCE_DEFAULTS.minSilence),
|
||||
padding: z.number().min(0).max(1).default(SILENCE_DEFAULTS.padding),
|
||||
});
|
||||
export type RemoveSilenceStep = z.infer<typeof RemoveSilenceStepSchema>;
|
||||
@ -1,6 +1,7 @@
|
||||
import * as z from "zod";
|
||||
import { Id } from "./ids.js";
|
||||
import { MediaFitSchema, MediaPresetIdSchema } from "./media.js";
|
||||
import { RemoveSilenceStepSchema } from "./media-analysis.js";
|
||||
|
||||
/**
|
||||
* Inputs of the `edit_video` and `export_video` media operations, which are also the arguments
|
||||
@ -39,6 +40,7 @@ export const EditVideoStepSchema = z.discriminatedUnion("op", [
|
||||
z.object({ op: z.literal("extract_audio") }),
|
||||
/** Cover frame at this second of the result; as the only kind of step, the result is that image. */
|
||||
z.object({ op: z.literal("poster"), at: Seconds }),
|
||||
RemoveSilenceStepSchema,
|
||||
]);
|
||||
export type EditVideoStep = z.infer<typeof EditVideoStepSchema>;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user