evOo-local/packages/adapters/src/media/silence.ts

import { readFile } from "node:fs/promises";
import { join } from "node:path";
import type { EditVideoStep } from "@rakazo/contracts";
import { runFfmpeg } from "./ffmpeg.js";
import type { VideoEditPlanRequest } from "./video-edit-plan.js";
import { planVideoEdit } from "./video-edit-plan.js";

/**
 * Silence detection for `remove_silence` and `analyze_video`. ffmpeg's silencedetect marks
 * silences in frame metadata, which `ametadata` prints to a file in the job's directory; the
 * file holds nothing but frame times and numbers, unlike stderr with its interleaved logs.
 */

export interface TimeRange {
  start: number;
  end: number;
}

/** Shortest piece of speech kept on its own; shorter blips between two pauses are noise. */
const MIN_SPEECH_SECONDS = 0.1;
/** Most pieces one remove_silence step keeps, so the filter graph stays a sensible size. */
export const MAX_SPEECH_SEGMENTS = 200;
/** Closer to the end of the timeline than this counts as reaching it. */
const EDGE_SECONDS = 0.01;

/** One frame of `metadata=mode=print` output: its time and the values set on it. */
export interface FrameMetadata {
  time: number;
  values: Record<string, string>;
}

/**
 * Parses the output of the `metadata`/`ametadata` filters in print mode:
 * `frame:N pts:P pts_time:T` lines, each followed by `key=value` lines.
 */
export function parseFrameMetadata(text: string): FrameMetadata[] {
  const frames: FrameMetadata[] = [];
  let current: FrameMetadata | undefined;
  for (const raw of text.split("\n")) {
    const line = raw.trim();
    const frame = /^frame:\s*\d+\s+pts:\s*\S+\s+pts_time:\s*(\S+)/.exec(line);
    if (frame) {
      const time = Number(frame[1]);
      current = Number.isFinite(time) ? { time, values: {} } : undefined;
      if (current) frames.push(current);
      continue;
    }
    const separator = line.indexOf("=");
    if (current && separator > 0) {
      current.values[line.slice(0, separator)] = line.slice(separator + 1);
    }
  }
  return frames;
}

/** Filters that detect silence and print it to `file`, a generated name in ffmpeg's cwd. */
export function silenceDetectFilters(
  options: { thresholdDb: number; minSilence: number },
  file: string,
): string[] {
  return [
    `silencedetect=noise=${number(options.thresholdDb)}dB:duration=${number(options.minSilence)}`,
    `ametadata=mode=print:file=${file}`,
  ];
}

/**
 * Silences from silencedetect's printed metadata, sorted and clamped to `[0, duration]`. A
 * silence still running when the audio ends has no end mark and lasts to the end.
 */
export function parseSilences(text: string, duration: number): TimeRange[] {
  const silences: TimeRange[] = [];
  let start: number | undefined;
  const add = (from: number, to: number) => {
    const range = { start: clamp(from, 0, duration), end: clamp(to, 0, duration) };
    if (range.end > range.start) silences.push(range);
  };
  for (const frame of parseFrameMetadata(text)) {
    const begins = Number(frame.values["lavfi.silence_start"]);
    const ends = Number(frame.values["lavfi.silence_end"]);
    // One frame can end a silence and start the next one; the end belongs to the earlier one.
    if (frame.values["lavfi.silence_end"] !== undefined && Number.isFinite(ends)) {
      add(start ?? 0, ends);
      start = undefined;
    }
    if (frame.values["lavfi.silence_start"] !== undefined && Number.isFinite(begins)) {
      start = begins;
    }
  }
  if (start !== undefined) add(start, duration);
  return mergeRanges(silences);
}

/**
 * The parts of `[0, duration]` that are not silent, each widened by `padding` on both sides,
 * clamped to the timeline and merged where the padding closes a gap. Null when nothing would
 * be cut; throws when nothing would be left.
 */
export function speechSegments(
  silences: readonly TimeRange[],
  duration: number,
  options: { padding: number; thresholdDb: number; maxSegments?: number },
): TimeRange[] | null {
  const speech: TimeRange[] = [];
  let cursor = 0;
  for (const silence of mergeRanges(silences)) {
    if (silence.start > cursor) speech.push({ start: cursor, end: silence.start });
    cursor = Math.max(cursor, silence.end);
  }
  if (cursor < duration) speech.push({ start: cursor, end: duration });
  const padded = mergeRanges(
    speech.map((range) => ({
      start: clamp(range.start - options.padding, 0, duration),
      end: clamp(range.end + options.padding, 0, duration),
    })),
  ).filter((range) => range.end - range.start >= MIN_SPEECH_SECONDS);
  if (padded.length === 0) {
    throw new Error(
      `remove_silence found nothing louder than ${number(options.thresholdDb)} dB, so nothing would be left.`,
    );
  }
  const kept = bridgeShortestGaps(padded, options.maxSegments ?? MAX_SPEECH_SEGMENTS);
  const [only] = kept;
  if (
    kept.length === 1 &&
    only &&
    only.start <= EDGE_SECONDS &&
    only.end >= duration - EDGE_SECONDS
  ) {
    return null;
  }
  return kept;
}

/**
 * Detects the silence each remove_silence step sees, in step order: one audio-only ffmpeg pass
 * over the timeline the steps before it leave, so trims, appends and speed changes before it
 * are taken into account. The result is the `speech` map `planVideoEdit` cuts with.
 */
export async function detectSpeech(
  request: Omit<VideoEditPlanRequest, "speech" | "analyzeAudio" | "outputPath">,
  options: { workDir: string; signal: AbortSignal },
): Promise<Map<EditVideoStep, TimeRange[] | null>> {
  const speech = new Map<EditVideoStep, TimeRange[] | null>();
  for (const [index, step] of request.steps.entries()) {
    if (step.op !== "remove_silence") continue;
    const file = `silence-${index}.txt`;
    const plan = planVideoEdit({
      ...request,
      steps: request.steps.slice(0, index),
      speech,
      analyzeAudio: silenceDetectFilters(
        { thresholdDb: step.threshold_db, minSilence: step.min_silence },
        file,
      ),
      outputPath: "",
    });
    await runFfmpeg(plan.args, {
      signal: options.signal,
      cwd: options.workDir,
      durationMs: plan.durationMs,
    });
    const duration = plan.durationMs / 1000;
    const silences = parseSilences(await readFile(join(options.workDir, file), "utf8"), duration);
    speech.set(
      step,
      speechSegments(silences, duration, {
        padding: step.padding,
        thresholdDb: step.threshold_db,
      }),
    );
  }
  return speech;
}

/** Sorted, with ranges that overlap or touch merged. */
function mergeRanges(ranges: readonly TimeRange[]): TimeRange[] {
  const sorted = [...ranges].sort((a, b) => a.start - b.start);
  const merged: TimeRange[] = [];
  for (const range of sorted) {
    const last = merged.at(-1);
    if (last && range.start <= last.end) last.end = Math.max(last.end, range.end);
    else merged.push({ ...range });
  }
  return merged;
}

/** At most `max` segments: the shortest gaps between them stay in until few enough remain. */
function bridgeShortestGaps(segments: readonly TimeRange[], max: number): TimeRange[] {
  if (segments.length <= max) return [...segments];
  const gaps = segments
    .slice(1)
    .map((segment, index) => ({ index, length: segment.start - segments[index]!.end }))
    .sort((a, b) => a.length - b.length || a.index - b.index);
  const bridged = new Set(gaps.slice(0, segments.length - max).map((gap) => gap.index));
  const kept: TimeRange[] = [];
  for (const [index, segment] of segments.entries()) {
    const last = kept.at(-1);
    if (last && bridged.has(index - 1)) last.end = segment.end;
    else kept.push({ ...segment });
  }
  return kept;
}

function clamp(value: number, min: number, max: number): number {
  return Math.min(max, Math.max(min, value));
}

function number(value: number): string {
  return String(Number(value.toFixed(6)));
}