import { readFile } from "node:fs/promises"; import { join } from "node:path"; import type { EditVideoStep } from "@rakazo/contracts"; import { runFfmpeg } from "./ffmpeg.js"; import type { VideoEditPlanRequest } from "./video-edit-plan.js"; import { planVideoEdit } from "./video-edit-plan.js"; /** * Silence detection for `remove_silence` and `analyze_video`. ffmpeg's silencedetect marks * silences in frame metadata, which `ametadata` prints to a file in the job's directory; the * file holds nothing but frame times and numbers, unlike stderr with its interleaved logs. */ export interface TimeRange { start: number; end: number; } /** Shortest piece of speech kept on its own; shorter blips between two pauses are noise. */ const MIN_SPEECH_SECONDS = 0.1; /** Most pieces one remove_silence step keeps, so the filter graph stays a sensible size. */ export const MAX_SPEECH_SEGMENTS = 200; /** Closer to the end of the timeline than this counts as reaching it. */ const EDGE_SECONDS = 0.01; /** One frame of `metadata=mode=print` output: its time and the values set on it. */ export interface FrameMetadata { time: number; values: Record; } /** * Parses the output of the `metadata`/`ametadata` filters in print mode: * `frame:N pts:P pts_time:T` lines, each followed by `key=value` lines. */ export function parseFrameMetadata(text: string): FrameMetadata[] { const frames: FrameMetadata[] = []; let current: FrameMetadata | undefined; for (const raw of text.split("\n")) { const line = raw.trim(); const frame = /^frame:\s*\d+\s+pts:\s*\S+\s+pts_time:\s*(\S+)/.exec(line); if (frame) { const time = Number(frame[1]); current = Number.isFinite(time) ? { time, values: {} } : undefined; if (current) frames.push(current); continue; } const separator = line.indexOf("="); if (current && separator > 0) { current.values[line.slice(0, separator)] = line.slice(separator + 1); } } return frames; } /** Filters that detect silence and print it to `file`, a generated name in ffmpeg's cwd. */ export function silenceDetectFilters( options: { thresholdDb: number; minSilence: number }, file: string, ): string[] { return [ `silencedetect=noise=${number(options.thresholdDb)}dB:duration=${number(options.minSilence)}`, `ametadata=mode=print:file=${file}`, ]; } /** * Silences from silencedetect's printed metadata, sorted and clamped to `[0, duration]`. A * silence still running when the audio ends has no end mark and lasts to the end. */ export function parseSilences(text: string, duration: number): TimeRange[] { const silences: TimeRange[] = []; let start: number | undefined; const add = (from: number, to: number) => { const range = { start: clamp(from, 0, duration), end: clamp(to, 0, duration) }; if (range.end > range.start) silences.push(range); }; for (const frame of parseFrameMetadata(text)) { const begins = Number(frame.values["lavfi.silence_start"]); const ends = Number(frame.values["lavfi.silence_end"]); // One frame can end a silence and start the next one; the end belongs to the earlier one. if (frame.values["lavfi.silence_end"] !== undefined && Number.isFinite(ends)) { add(start ?? 0, ends); start = undefined; } if (frame.values["lavfi.silence_start"] !== undefined && Number.isFinite(begins)) { start = begins; } } if (start !== undefined) add(start, duration); return mergeRanges(silences); } /** * The parts of `[0, duration]` that are not silent, each widened by `padding` on both sides, * clamped to the timeline and merged where the padding closes a gap. Null when nothing would * be cut; throws when nothing would be left. */ export function speechSegments( silences: readonly TimeRange[], duration: number, options: { padding: number; thresholdDb: number; maxSegments?: number }, ): TimeRange[] | null { const speech: TimeRange[] = []; let cursor = 0; for (const silence of mergeRanges(silences)) { if (silence.start > cursor) speech.push({ start: cursor, end: silence.start }); cursor = Math.max(cursor, silence.end); } if (cursor < duration) speech.push({ start: cursor, end: duration }); const padded = mergeRanges( speech.map((range) => ({ start: clamp(range.start - options.padding, 0, duration), end: clamp(range.end + options.padding, 0, duration), })), ).filter((range) => range.end - range.start >= MIN_SPEECH_SECONDS); if (padded.length === 0) { throw new Error( `remove_silence found nothing louder than ${number(options.thresholdDb)} dB, so nothing would be left.`, ); } const kept = bridgeShortestGaps(padded, options.maxSegments ?? MAX_SPEECH_SEGMENTS); const [only] = kept; if ( kept.length === 1 && only && only.start <= EDGE_SECONDS && only.end >= duration - EDGE_SECONDS ) { return null; } return kept; } /** * Detects the silence each remove_silence step sees, in step order: one audio-only ffmpeg pass * over the timeline the steps before it leave, so trims, appends and speed changes before it * are taken into account. The result is the `speech` map `planVideoEdit` cuts with. */ export async function detectSpeech( request: Omit, options: { workDir: string; signal: AbortSignal }, ): Promise> { const speech = new Map(); for (const [index, step] of request.steps.entries()) { if (step.op !== "remove_silence") continue; const file = `silence-${index}.txt`; const plan = planVideoEdit({ ...request, steps: request.steps.slice(0, index), speech, analyzeAudio: silenceDetectFilters( { thresholdDb: step.threshold_db, minSilence: step.min_silence }, file, ), outputPath: "", }); await runFfmpeg(plan.args, { signal: options.signal, cwd: options.workDir, durationMs: plan.durationMs, }); const duration = plan.durationMs / 1000; const silences = parseSilences(await readFile(join(options.workDir, file), "utf8"), duration); speech.set( step, speechSegments(silences, duration, { padding: step.padding, thresholdDb: step.threshold_db, }), ); } return speech; } /** Sorted, with ranges that overlap or touch merged. */ function mergeRanges(ranges: readonly TimeRange[]): TimeRange[] { const sorted = [...ranges].sort((a, b) => a.start - b.start); const merged: TimeRange[] = []; for (const range of sorted) { const last = merged.at(-1); if (last && range.start <= last.end) last.end = Math.max(last.end, range.end); else merged.push({ ...range }); } return merged; } /** At most `max` segments: the shortest gaps between them stay in until few enough remain. */ function bridgeShortestGaps(segments: readonly TimeRange[], max: number): TimeRange[] { if (segments.length <= max) return [...segments]; const gaps = segments .slice(1) .map((segment, index) => ({ index, length: segment.start - segments[index]!.end })) .sort((a, b) => a.length - b.length || a.index - b.index); const bridged = new Set(gaps.slice(0, segments.length - max).map((gap) => gap.index)); const kept: TimeRange[] = []; for (const [index, segment] of segments.entries()) { const last = kept.at(-1); if (last && bridged.has(index - 1)) last.end = segment.end; else kept.push({ ...segment }); } return kept; } function clamp(value: number, min: number, max: number): number { return Math.min(max, Math.max(min, value)); } function number(value: number): string { return String(Number(value.toFixed(6))); }