videopython 0.26.6__tar.gz → 0.26.8__tar.gz

{videopython-0.26.6 → videopython-0.26.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: videopython
-Version: 0.26.6
+Version: 0.26.8
 Summary: Minimal video generation and processing library.
 Project-URL: Homepage, https://videopython.com
 Project-URL: Repository, https://github.com/bartwojtowicz/videopython/

{videopython-0.26.6 → videopython-0.26.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "videopython"
-version = "0.26.6"
+version = "0.26.8"
 description = "Minimal video generation and processing library."
 authors = [
     { name = "Bartosz Wójtowicz", email = "bartoszwojtowicz@outlook.com" },

{videopython-0.26.6 → videopython-0.26.8}/src/videopython/ai/dubbing/dubber.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import logging
-import tempfile
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable
 
@@ -170,7 +169,7 @@ class VideoDubber:
             ``DubbingResult`` with the dubbed audio, translated segments, and
             source transcription. The output video is written to ``output_path``.
         """
-        from videopython.ai.dubbing.remux import replace_audio_stream
+        from videopython.ai.dubbing.remux import replace_audio_stream_from_audio
         from videopython.base.audio import Audio
 
         input_path = Path(input_path)
@@ -196,17 +195,14 @@ class VideoDubber:
             transcription=transcription,
         )
 
-        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
-            dubbed_audio_path = Path(tmp.name)
-        try:
-            result.dubbed_audio.save(dubbed_audio_path)
-            replace_audio_stream(
-                video_path=input_path,
-                audio_path=dubbed_audio_path,
-                output_path=output_path,
-            )
-        finally:
-            dubbed_audio_path.unlink(missing_ok=True)
+        # Stream the dubbed Audio directly into ffmpeg via stdin instead of
+        # going through a temp WAV on disk. For a 2h dub the temp file would
+        # be ~10 GB written-then-read; the streaming path drops both copies.
+        replace_audio_stream_from_audio(
+            video_path=input_path,
+            audio=result.dubbed_audio,
+            output_path=output_path,
+        )
 
         return result
 

{videopython-0.26.6 → videopython-0.26.8}/src/videopython/ai/dubbing/pipeline.py

@@ -3,14 +3,45 @@
 from __future__ import annotations
 
 import logging
+import tempfile
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Literal
 
+import numpy as np
+
 from videopython.ai.dubbing.models import DubbingResult, RevoiceResult, SeparatedAudio
 from videopython.ai.dubbing.timing import TimingSynchronizer
 
 if TYPE_CHECKING:
     from videopython.base.audio import Audio
 
+
+def _peak_match(target: Audio, reference: Audio) -> Audio:
+    """Scale ``target`` so its peak amplitude matches ``reference``.
+
+    Demucs background normalization and the timing-assembler peak guard
+    each clamp at 1.0 instead of restoring headroom, so a dubbed mix
+    typically lands quieter than the source — perceptually "thinner."
+    A single peak match recovers most of that drift without LUFS deps.
+
+    No-op when either side has zero peak (silent input or all-silent dub).
+    The new ``Audio`` shares no buffer with ``target``.
+    """
+    from videopython.base.audio import Audio as _Audio
+
+    target_peak = float(np.max(np.abs(target.data))) if target.data.size else 0.0
+    reference_peak = float(np.max(np.abs(reference.data))) if reference.data.size else 0.0
+
+    if target_peak <= 0.0 or reference_peak <= 0.0:
+        return target
+
+    scale = reference_peak / target_peak
+    if abs(scale - 1.0) < 1e-3:
+        return target
+
+    return _Audio(target.data * scale, target.metadata)
+
+
 WhisperModel = Literal["tiny", "base", "small", "medium", "large", "turbo"]
 
 logger = logging.getLogger(__name__)
@@ -237,7 +268,19 @@ class LocalDubbingPipeline:
             if self._separator is None:
                 self._init_separator()
 
-            separated_audio = self._separator.separate(source_audio)
+            # Limit Demucs to the speech-bearing portion of the audio. The
+            # transcription has already located every speech region; running
+            # source separation outside those is pure overhead (no vocals to
+            # isolate). On talk-heavy sources with silence/music gaps this
+            # roughly halves separation time. When speech covers most of the
+            # track separate_regions falls back to a full-track separate().
+            from videopython.ai.understanding.separation import _merge_regions
+
+            speech_regions = _merge_regions(
+                [(s.start, s.end) for s in transcription.segments],
+                audio_duration=source_audio.metadata.duration_seconds,
+            )
+            separated_audio = self._separator.separate_regions(source_audio, speech_regions)
             self._maybe_unload("_separator")
             vocal_audio = separated_audio.vocals
             background_audio = separated_audio.background
@@ -278,24 +321,46 @@ class LocalDubbingPipeline:
         target_durations: list[float] = []
         start_times: list[float] = []
 
-        for i, segment in enumerate(translated_segments):
-            if segment.duration < 0.1:
-                continue
-
-            progress = 0.50 + (0.30 * (i / len(translated_segments)))
-            report_progress(f"Generating speech ({i + 1}/{len(translated_segments)})", progress)
-
-            speaker = segment.speaker or "speaker_0"
-            voice_sample = voice_samples.get(speaker)
-
-            if voice_clone and voice_sample is not None:
-                dubbed_audio = self._tts.generate_audio(segment.translated_text, voice_sample=voice_sample)
-            else:
-                dubbed_audio = self._tts.generate_audio(segment.translated_text)
-
-            dubbed_segments.append(dubbed_audio)
-            target_durations.append(segment.duration)
-            start_times.append(segment.start)
+        # Encode each speaker's voice sample to a temp WAV exactly once and
+        # reuse the path across every segment for that speaker. Without this
+        # cache, TextToSpeech.generate_audio re-encodes the same voice sample
+        # on every call (one temp WAV write + delete per segment), which is
+        # pure overhead for long dubs with many segments per speaker.
+        speaker_wav_paths: dict[str, Path] = {}
+        try:
+            if voice_clone:
+                for speaker, sample in voice_samples.items():
+                    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+                        sample.save(f.name)
+                        speaker_wav_paths[speaker] = Path(f.name)
+
+            for i, segment in enumerate(translated_segments):
+                if segment.duration < 0.1:
+                    continue
+                # Translation filter (translation.py:_is_translatable_text)
+                # leaves translated_text="" for punctuation-only or empty
+                # segments. Don't TTS those — saves a model call and avoids
+                # injecting hallucinated speech into the dubbed track.
+                if not segment.translated_text.strip():
+                    continue
+
+                progress = 0.50 + (0.30 * (i / len(translated_segments)))
+                report_progress(f"Generating speech ({i + 1}/{len(translated_segments)})", progress)
+
+                speaker = segment.speaker or "speaker_0"
+                cached_path = speaker_wav_paths.get(speaker) if voice_clone else None
+
+                if cached_path is not None:
+                    dubbed_audio = self._tts.generate_audio(segment.translated_text, voice_sample_path=cached_path)
+                else:
+                    dubbed_audio = self._tts.generate_audio(segment.translated_text)
+
+                dubbed_segments.append(dubbed_audio)
+                target_durations.append(segment.duration)
+                start_times.append(segment.start)
+        finally:
+            for path in speaker_wav_paths.values():
+                path.unlink(missing_ok=True)
 
         self._maybe_unload("_tts")
 
@@ -325,6 +390,11 @@ class LocalDubbingPipeline:
         else:
             final_audio = dubbed_speech
 
+        # Peak-match against the source so the dub doesn't land quieter
+        # than the original. Done last so it captures both vocals+background
+        # mixes and speech-only outputs uniformly.
+        final_audio = _peak_match(final_audio, source_audio)
+
         report_progress("Complete", 1.0)
 
         return DubbingResult(
@@ -375,7 +445,13 @@ class LocalDubbingPipeline:
             if self._separator is None:
                 self._init_separator()
 
-            separated_audio = self._separator.separate(source_audio)
+            from videopython.ai.understanding.separation import _merge_regions
+
+            speech_regions = _merge_regions(
+                [(s.start, s.end) for s in transcription.segments],
+                audio_duration=source_audio.metadata.duration_seconds,
+            )
+            separated_audio = self._separator.separate_regions(source_audio, speech_regions)
             self._maybe_unload("_separator")
             vocal_audio = separated_audio.vocals
             background_audio = separated_audio.background
@@ -430,6 +506,8 @@ class LocalDubbingPipeline:
         else:
             final_audio = generated_speech
 
+        final_audio = _peak_match(final_audio, source_audio)
+
         report_progress("Complete", 1.0)
 
         return RevoiceResult(

videopython-0.26.8/src/videopython/ai/dubbing/remux.py

@@ -0,0 +1,159 @@
+"""ffmpeg helper for replacing a video file's audio track without re-encoding video."""
+
+from __future__ import annotations
+
+import io
+import logging
+import subprocess
+import wave
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from videopython.base.audio import Audio
+
+logger = logging.getLogger(__name__)
+
+
+class RemuxError(RuntimeError):
+    """ffmpeg failed while replacing an audio stream."""
+
+
+def replace_audio_stream(
+    video_path: str | Path,
+    audio_path: str | Path,
+    output_path: str | Path,
+    audio_codec: str = "aac",
+    audio_bitrate: str = "192k",
+) -> None:
+    """Copy ``video_path``'s video stream and mux in ``audio_path`` as the audio track.
+
+    Uses ffmpeg stream-copy for video (no re-encode) and encodes audio to AAC.
+    ``-shortest`` trims to the shorter of the two streams so the output duration
+    matches the source video when the dubbed audio is slightly longer.
+
+    Args:
+        video_path: Source video file (video stream is copied unchanged).
+        audio_path: Audio file to use as the new audio track.
+        output_path: Destination file. Overwritten if it exists.
+        audio_codec: ffmpeg audio codec name. Defaults to ``aac`` (MP4-compatible).
+        audio_bitrate: Audio bitrate passed to ffmpeg (``-b:a``).
+
+    Raises:
+        FileNotFoundError: If ``video_path`` or ``audio_path`` does not exist.
+        RemuxError: If ffmpeg returns a non-zero exit code.
+    """
+    video_path = Path(video_path)
+    audio_path = Path(audio_path)
+    output_path = Path(output_path)
+
+    if not video_path.exists():
+        raise FileNotFoundError(f"Video file not found: {video_path}")
+    if not audio_path.exists():
+        raise FileNotFoundError(f"Audio file not found: {audio_path}")
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-i",
+        str(video_path),
+        "-i",
+        str(audio_path),
+        "-map",
+        "0:v:0",
+        "-map",
+        "1:a:0",
+        "-c:v",
+        "copy",
+        "-c:a",
+        audio_codec,
+        "-b:a",
+        audio_bitrate,
+        "-shortest",
+        str(output_path),
+    ]
+
+    logger.info("replace_audio_stream: %s + %s -> %s", video_path, audio_path, output_path)
+    result = subprocess.run(cmd, capture_output=True)
+    if result.returncode != 0:
+        raise RemuxError(f"ffmpeg failed (exit {result.returncode}): {result.stderr.decode(errors='replace')}")
+
+
+def replace_audio_stream_from_audio(
+    video_path: str | Path,
+    audio: Audio,
+    output_path: str | Path,
+    audio_codec: str = "aac",
+    audio_bitrate: str = "192k",
+) -> None:
+    """Like ``replace_audio_stream`` but takes an in-memory ``Audio`` and pipes WAV to ffmpeg.
+
+    Avoids the ``Audio.save -> read-from-disk -> ffmpeg`` round-trip used by
+    the path-based variant: we serialize the WAV in memory and feed it to
+    ffmpeg via stdin. For long dubs this saves a full WAV write+read of the
+    output audio (~10 GB for a 2h source).
+
+    Args:
+        video_path: Source video file (video stream is copied unchanged).
+        audio: ``Audio`` instance to mux in as the new audio track.
+        output_path: Destination file. Overwritten if it exists.
+        audio_codec: ffmpeg audio codec name. Defaults to ``aac``.
+        audio_bitrate: Audio bitrate passed to ffmpeg (``-b:a``).
+
+    Raises:
+        FileNotFoundError: If ``video_path`` does not exist.
+        RemuxError: If ffmpeg returns a non-zero exit code.
+    """
+    video_path = Path(video_path)
+    output_path = Path(output_path)
+
+    if not video_path.exists():
+        raise FileNotFoundError(f"Video file not found: {video_path}")
+
+    # Serialize Audio to WAV bytes in memory. Mirrors Audio.save's WAV writer:
+    # int16 samples, header from metadata. We stream these bytes to ffmpeg's
+    # stdin as the second input (the first is the video file on disk).
+    int_data = (audio.data * np.iinfo(np.int16).max).astype(np.int16)
+    wav_io = io.BytesIO()
+    with wave.open(wav_io, "wb") as wav_file:
+        wav_file.setnchannels(audio.metadata.channels)
+        wav_file.setsampwidth(audio.metadata.sample_width)
+        wav_file.setframerate(audio.metadata.sample_rate)
+        wav_file.writeframes(int_data.tobytes())
+    wav_bytes = wav_io.getvalue()
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-i",
+        str(video_path),
+        "-f",
+        "wav",
+        "-i",
+        "-",
+        "-map",
+        "0:v:0",
+        "-map",
+        "1:a:0",
+        "-c:v",
+        "copy",
+        "-c:a",
+        audio_codec,
+        "-b:a",
+        audio_bitrate,
+        "-shortest",
+        str(output_path),
+    ]
+
+    logger.info(
+        "replace_audio_stream_from_audio: %s + <stdin wav %d bytes> -> %s",
+        video_path,
+        len(wav_bytes),
+        output_path,
+    )
+    process = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+    _, stderr = process.communicate(wav_bytes)
+    if process.returncode != 0:
+        raise RemuxError(f"ffmpeg failed (exit {process.returncode}): {stderr.decode(errors='replace')}")

{videopython-0.26.6 → videopython-0.26.8}/src/videopython/ai/dubbing/timing.py

@@ -4,7 +4,9 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 
-from videopython.base.audio import Audio
+import numpy as np
+
+from videopython.base.audio import Audio, AudioMetadata
 
 
 @dataclass
@@ -181,32 +183,58 @@ class TimingSynchronizer:
         if len(audio_segments) != len(start_times):
             raise ValueError(f"Length mismatch: {len(audio_segments)} segments vs {len(start_times)} start times")
 
+        for start_time in start_times:
+            if start_time < 0:
+                raise ValueError(f"Invalid start time: {start_time}")
+
         if not audio_segments:
             return Audio.create_silent(total_duration, stereo=False)
 
-        # Determine sample rate from first segment
+        # Single-pass assembler: allocate one mono float32 buffer and add each
+        # segment in place at its start sample. The previous implementation
+        # called Audio.overlay() per segment, which allocates np.zeros and
+        # copies the full track on every call — O(N * total_samples) memory
+        # traffic. For long dubs (thousands of segments) this loop dominated
+        # wall time and peak RAM.
         sample_rate = audio_segments[0].metadata.sample_rate
-
-        # Create base silent track
-        output = Audio.create_silent(total_duration, stereo=False, sample_rate=sample_rate)
-
-        # Overlay each segment at its start time
+        base_samples = max(int(total_duration * sample_rate), 0)
+
+        # Pre-normalize each segment to (mono, target sample rate) and compute
+        # placement bounds so the output buffer is sized to fit any segment
+        # that runs past total_duration (mirrors Audio.overlay's extend-on-OOB
+        # behavior so we don't silently truncate speech).
+        normalized: list[tuple[int, np.ndarray]] = []
+        end_sample = base_samples
         for audio, start_time in zip(audio_segments, start_times):
-            if start_time < 0:
-                raise ValueError(f"Invalid start time: {start_time}")
-
-            # Resample if needed
             if audio.metadata.sample_rate != sample_rate:
                 audio = audio.resample(sample_rate)
-
-            # Convert to mono if needed
             if audio.metadata.channels > 1:
                 audio = audio.to_mono()
-
-            # Overlay at position
-            output = output.overlay(audio, position=start_time)
-
-        return output
+            start_sample = int(np.ceil(start_time * sample_rate))
+            seg_data = audio.data
+            normalized.append((start_sample, seg_data))
+            end_sample = max(end_sample, start_sample + len(seg_data))
+
+        output = np.zeros(end_sample, dtype=np.float32)
+        for start_sample, seg_data in normalized:
+            stop = start_sample + len(seg_data)
+            output[start_sample:stop] += seg_data
+
+        # Single post-mix peak guard, equivalent to Audio.overlay's per-call
+        # rescale collapsed into one pass. For non-overlapping dub segments
+        # this is a no-op; only the rare overlap case touches it.
+        max_amplitude = float(np.max(np.abs(output))) if output.size else 0.0
+        if max_amplitude > 1.0:
+            output /= max_amplitude
+
+        metadata = AudioMetadata(
+            sample_rate=sample_rate,
+            channels=1,
+            sample_width=audio_segments[0].metadata.sample_width,
+            duration_seconds=end_sample / sample_rate,
+            frame_count=end_sample,
+        )
+        return Audio(output, metadata)
 
     def check_overlaps(
         self,

{videopython-0.26.6 → videopython-0.26.8}/src/videopython/ai/generation/audio.py

@@ -2,11 +2,14 @@
 
 from __future__ import annotations
 
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from videopython.ai._device import log_device_initialization, release_device_memory, select_device
 from videopython.base.audio import Audio, AudioMetadata
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 
 class TextToSpeech:
     """Generates speech audio from text using Chatterbox Multilingual.
@@ -47,6 +50,7 @@ class TextToSpeech:
         self,
         text: str,
         voice_sample: Audio | None = None,
+        voice_sample_path: str | Path | None = None,
     ) -> Audio:
         """Generate speech audio from text.
 
@@ -54,6 +58,12 @@ class TextToSpeech:
             text: Text to synthesize.
             voice_sample: Optional voice sample to clone. Falls back to the
                 instance's ``voice`` and then to Chatterbox's default speaker.
+            voice_sample_path: Optional pre-encoded WAV path to use directly as
+                the speaker prompt. Skips the per-call temp-WAV encode that
+                ``voice_sample`` would otherwise trigger. When set, takes
+                precedence over ``voice_sample`` and ``self.voice``. Used by
+                the dubbing pipeline to encode each speaker's sample once and
+                reuse it across all of that speaker's segments.
         """
         import tempfile
         from pathlib import Path
@@ -63,13 +73,18 @@ class TextToSpeech:
         if self._model is None:
             self._init_model()
 
-        effective_sample = voice_sample or self.voice
         speaker_wav_path: Path | None = None
-
-        if effective_sample is not None:
-            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
-                effective_sample.save(f.name)
-                speaker_wav_path = Path(f.name)
+        cleanup_path = False
+
+        if voice_sample_path is not None:
+            speaker_wav_path = Path(voice_sample_path)
+        else:
+            effective_sample = voice_sample or self.voice
+            if effective_sample is not None:
+                with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+                    effective_sample.save(f.name)
+                    speaker_wav_path = Path(f.name)
+                    cleanup_path = True
 
         try:
             wav = self._model.generate(
@@ -91,8 +106,8 @@ class TextToSpeech:
             )
             return Audio(audio_data, metadata)
         finally:
-            if speaker_wav_path is not None:
-                speaker_wav_path.unlink()
+            if cleanup_path and speaker_wav_path is not None:
+                speaker_wav_path.unlink(missing_ok=True)
 
     def unload(self) -> None:
         """Release the TTS model so the next generate_audio() re-initializes.

{videopython-0.26.6 → videopython-0.26.8}/src/videopython/ai/generation/translation.py

@@ -8,6 +8,17 @@ from videopython.ai._device import log_device_initialization, release_device_mem
 from videopython.ai.dubbing.models import TranslatedSegment
 from videopython.base.text.transcription import TranscriptionSegment
 
+
+def _is_translatable_text(text: str) -> bool:
+    """Return True if text has enough content to be worth translating.
+
+    Whisper routinely emits punctuation-only or single-character segments
+    (" .", "...", "?", "♪") that MarianMT can hallucinate full sentences
+    from. Require at least 2 alphanumeric characters to filter these out.
+    """
+    return sum(1 for c in text if c.isalnum()) >= 2
+
+
 LANGUAGE_NAMES = {
     "en": "English",
     "es": "Spanish",
@@ -159,17 +170,28 @@ class TextTranslator:
         target_lang: str,
         source_lang: str | None = None,
     ) -> list[TranslatedSegment]:
-        """Translate transcription segments while preserving timing/speaker info."""
+        """Translate transcription segments while preserving timing/speaker info.
+
+        Segments whose text is empty or contains fewer than 2 alphanumeric
+        characters are not sent to the model — they receive
+        ``translated_text=""`` instead. This avoids MarianMT hallucinating
+        full sentences from " .", "...", or single-token Whisper segments,
+        which would otherwise be TTS'd into the dubbed track.
+        """
         effective_source = source_lang or "en"
-        texts = [segment.text for segment in segments]
-        translated_texts = self.translate_batch(texts, target_lang, source_lang)
+
+        translatable_indices = [i for i, segment in enumerate(segments) if _is_translatable_text(segment.text)]
+        translatable_texts = [segments[i].text for i in translatable_indices]
+        translated_texts = self.translate_batch(translatable_texts, target_lang, source_lang)
+
+        translation_map: dict[int, str] = dict(zip(translatable_indices, translated_texts))
 
         translated_segments = []
-        for segment, translated_text in zip(segments, translated_texts):
+        for i, segment in enumerate(segments):
             translated_segments.append(
                 TranslatedSegment(
                     original_segment=segment,
-                    translated_text=translated_text,
+                    translated_text=translation_map.get(i, ""),
                     source_lang=effective_source,
                     target_lang=target_lang,
                     speaker=segment.speaker,

videopython-0.26.8/src/videopython/ai/understanding/separation.py

@@ -0,0 +1,304 @@
+"""Audio source separation using local Demucs models."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from videopython.ai._device import log_device_initialization, release_device_memory, select_device
+from videopython.ai.dubbing.models import SeparatedAudio
+from videopython.base.audio import Audio, AudioMetadata
+
+logger = logging.getLogger(__name__)
+
+
+def _merge_regions(
+    regions: list[tuple[float, float]],
+    audio_duration: float,
+    pad: float = 0.5,
+    merge_gap: float = 1.0,
+) -> list[tuple[float, float]]:
+    """Merge overlapping/adjacent (start, end) ranges and pad each side.
+
+    Args:
+        regions: Speech regions in seconds. Order does not matter.
+        audio_duration: Total audio duration; output is clamped to ``[0, audio_duration]``.
+        pad: Seconds added to each side. Demucs needs context to separate
+            cleanly at boundaries; 0.5s avoids clipped onsets/decays.
+        merge_gap: Adjacent regions whose padded edges are within this
+            many seconds are merged. Avoids running Demucs on very short
+            slices (where its temporal context isn't there).
+
+    Returns:
+        Sorted list of non-overlapping (start, end) regions covering the
+        speech-bearing portion of the audio.
+    """
+    if not regions:
+        return []
+
+    sorted_regions = sorted(regions)
+
+    merged: list[tuple[float, float]] = []
+    for start, end in sorted_regions:
+        if end <= start:
+            continue
+        padded_start = max(0.0, start - pad)
+        padded_end = min(audio_duration, end + pad)
+        if padded_start >= audio_duration or padded_end <= 0.0:
+            continue
+
+        if merged and padded_start - merged[-1][1] <= merge_gap:
+            merged[-1] = (merged[-1][0], max(merged[-1][1], padded_end))
+        else:
+            merged.append((padded_start, padded_end))
+
+    return merged
+
+
+class AudioSeparator:
+    """Separates audio into vocals and background components using Demucs."""
+
+    SUPPORTED_MODELS: list[str] = ["htdemucs", "htdemucs_ft", "htdemucs_6s", "mdx_extra"]
+    STEM_NAMES = ["drums", "bass", "other", "vocals"]
+    STEM_NAMES_6S = ["drums", "bass", "other", "vocals", "guitar", "piano"]
+
+    def __init__(self, model_name: str = "htdemucs", device: str | None = None):
+        if model_name not in self.SUPPORTED_MODELS:
+            raise ValueError(f"Model '{model_name}' not supported. Supported: {self.SUPPORTED_MODELS}")
+
+        self.model_name = model_name
+        self.device = device
+        self._model: Any = None
+
+    def _init_local(self) -> None:
+        """Initialize local Demucs model."""
+        from demucs.pretrained import get_model
+
+        requested_device = self.device
+        device = select_device(self.device, mps_allowed=False)
+
+        self._model = get_model(self.model_name)
+        self._model.to(device)
+        self._model.eval()
+        self.device = device
+        log_device_initialization(
+            "AudioSeparator",
+            requested_device=requested_device,
+            resolved_device=device,
+        )
+
+    def _separate_local(self, audio: Audio) -> SeparatedAudio:
+        """Separate audio using local Demucs model.
+
+        Keeps the input tensor on CPU and passes ``device=self.device`` to
+        ``apply_model`` so per-chunk compute runs on GPU while the full
+        ``(stems, channels, samples)`` output is stored in CPU RAM. For long
+        sources this is the difference between OOM-on-GPU and running cleanly:
+        a 2h stereo @ 44.1kHz output is ~10 GB — too big for an 8 GB card but
+        comfortable on a 32 GB host.
+        """
+        import numpy as np
+        import torch
+        from demucs.apply import apply_model
+
+        if self._model is None:
+            self._init_local()
+
+        target_sr = self._model.samplerate
+
+        if audio.metadata.channels == 1:
+            audio = audio._to_stereo()
+
+        if audio.metadata.sample_rate != target_sr:
+            audio = audio.resample(target_sr)
+
+        audio_data = audio.data
+        if audio_data.ndim == 1:
+            audio_data = np.stack([audio_data, audio_data])
+        elif audio_data.ndim == 2:
+            audio_data = audio_data.T
+
+        wav = torch.tensor(audio_data, dtype=torch.float32).unsqueeze(0)
+
+        with torch.no_grad():
+            sources = apply_model(self._model, wav, device=self.device)
+
+        sources_np = sources[0].cpu().numpy()
+        del sources
+
+        stem_names = self.STEM_NAMES_6S if self.model_name == "htdemucs_6s" else self.STEM_NAMES
+        vocals_idx = stem_names.index("vocals")
+        non_vocal_indices = [i for i in range(len(stem_names)) if i != vocals_idx]
+
+        vocals_data = sources_np[vocals_idx].T
+        background_data = sources_np[non_vocal_indices].sum(axis=0).T
+        del sources_np
+
+        max_val = np.max(np.abs(background_data))
+        if max_val > 1.0:
+            background_data /= max_val
+
+        metadata = AudioMetadata(
+            sample_rate=target_sr,
+            channels=2,
+            sample_width=2,
+            duration_seconds=vocals_data.shape[0] / target_sr,
+            frame_count=vocals_data.shape[0],
+        )
+        vocals = Audio(np.ascontiguousarray(vocals_data, dtype=np.float32), metadata)
+        background = Audio(np.ascontiguousarray(background_data, dtype=np.float32), metadata)
+
+        return SeparatedAudio(
+            vocals=vocals,
+            background=background,
+            original=audio,
+            music=None,
+            effects=None,
+        )
+
+    def separate(self, audio: Audio) -> SeparatedAudio:
+        """Separate audio into vocals and background components."""
+        return self._separate_local(audio)
+
+    def separate_regions(
+        self,
+        audio: Audio,
+        regions: list[tuple[float, float]],
+        full_separation_threshold: float = 0.9,
+    ) -> SeparatedAudio:
+        """Separate only the given (start, end) regions; pass the rest through.
+
+        Demucs is the slowest stage of the dubbing pipeline. On talk-heavy
+        sources (podcasts, interviews) most of the track is speech, but
+        long pauses, silence, or music-only stretches don't need vocal
+        isolation — there's nothing to isolate. We run Demucs only on the
+        speech-bearing regions and treat the rest as pure background.
+
+        Output is full-length: vocals are silent outside the given
+        regions; background is the original audio outside the given
+        regions and the Demucs-separated background inside.
+
+        Args:
+            audio: Source audio (typically the full track).
+            regions: List of ``(start, end)`` second pairs marking
+                speech-bearing portions. Caller is responsible for
+                merging/padding (use ``_merge_regions``).
+            full_separation_threshold: If the regions cover more than
+                this fraction of the audio, fall back to full-track
+                ``separate()`` since per-region slicing+stitching
+                overhead would exceed the savings. Default 0.9.
+
+        Returns:
+            ``SeparatedAudio`` with full-length vocals and background.
+        """
+        import numpy as np
+
+        if not regions:
+            logger.info("separate_regions: no regions, returning silent vocals over original audio")
+            return self._passthrough_separation(audio)
+
+        total_duration = audio.metadata.duration_seconds
+        speech_duration = sum(end - start for start, end in regions)
+        if total_duration > 0 and speech_duration / total_duration >= full_separation_threshold:
+            logger.info(
+                "separate_regions: speech covers %.0f%% of audio (>=%.0f%%), using full-track separation",
+                speech_duration / total_duration * 100,
+                full_separation_threshold * 100,
+            )
+            return self._separate_local(audio)
+
+        logger.info(
+            "separate_regions: separating %.1fs of speech across %d region(s) (full duration: %.1fs)",
+            speech_duration,
+            len(regions),
+            total_duration,
+        )
+
+        # Build full-length output buffers. Background defaults to the
+        # original audio (so non-speech gaps pass through unchanged); vocals
+        # default to silence (no speech to isolate outside the regions).
+        # Both are stereo to match the full-track separation contract.
+        sr = audio.metadata.sample_rate
+        stereo_audio = audio if audio.metadata.channels == 2 else audio._to_stereo()
+
+        total_samples = len(stereo_audio.data)
+        vocals_full = np.zeros((total_samples, 2), dtype=np.float32)
+        background_full = stereo_audio.data.astype(np.float32, copy=True)
+
+        for start, end in regions:
+            chunk = audio.slice(start, end)
+            separated_chunk = self._separate_local(chunk)
+            chunk_vocals = separated_chunk.vocals.data
+            chunk_background = separated_chunk.background.data
+
+            # Demucs operates at its model sample rate (typically 44.1 kHz)
+            # and returns stereo. The slice of `audio` we passed in may have
+            # been resampled inside _separate_local, so resample the chunk
+            # outputs back to the source sample rate before splicing.
+            chunk_sr = separated_chunk.vocals.metadata.sample_rate
+            if chunk_sr != sr:
+                chunk_vocals = separated_chunk.vocals.resample(sr).data
+                chunk_background = separated_chunk.background.resample(sr).data
+
+            start_sample = int(start * sr)
+            end_sample = min(start_sample + len(chunk_vocals), total_samples)
+            length = end_sample - start_sample
+            if length <= 0:
+                continue
+
+            vocals_full[start_sample:end_sample] = chunk_vocals[:length]
+            background_full[start_sample:end_sample] = chunk_background[:length]
+
+        metadata = AudioMetadata(
+            sample_rate=sr,
+            channels=2,
+            sample_width=audio.metadata.sample_width,
+            duration_seconds=total_samples / sr,
+            frame_count=total_samples,
+        )
+        vocals = Audio(np.ascontiguousarray(vocals_full, dtype=np.float32), metadata)
+        background = Audio(np.ascontiguousarray(background_full, dtype=np.float32), metadata)
+
+        return SeparatedAudio(
+            vocals=vocals,
+            background=background,
+            original=stereo_audio,
+            music=None,
+            effects=None,
+        )
+
+    def _passthrough_separation(self, audio: Audio) -> SeparatedAudio:
+        """Return the original audio as background with silent vocals.
+
+        Used when no speech regions are present — there's nothing to
+        separate, so the entire signal is background by definition.
+        """
+        import numpy as np
+
+        stereo_audio = audio if audio.metadata.channels == 2 else audio._to_stereo()
+        silent_vocals_data = np.zeros_like(stereo_audio.data, dtype=np.float32)
+        vocals = Audio(silent_vocals_data, stereo_audio.metadata)
+
+        return SeparatedAudio(
+            vocals=vocals,
+            background=stereo_audio,
+            original=stereo_audio,
+            music=None,
+            effects=None,
+        )
+
+    def extract_vocals(self, audio: Audio) -> Audio:
+        """Convenience method to extract only vocals from audio."""
+        return self.separate(audio).vocals
+
+    def extract_background(self, audio: Audio) -> Audio:
+        """Convenience method to extract only background from audio."""
+        return self.separate(audio).background
+
+    def unload(self) -> None:
+        """Release the Demucs model so the next separate() re-initializes.
+
+        Used by low-memory dubbing to free VRAM between pipeline stages.
+        """
+        self._model = None
+        release_device_memory(self.device)

videopython-0.26.6/src/videopython/ai/dubbing/remux.py

@@ -1,73 +0,0 @@
-"""ffmpeg helper for replacing a video file's audio track without re-encoding video."""
-
-from __future__ import annotations
-
-import logging
-import subprocess
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-
-class RemuxError(RuntimeError):
-    """ffmpeg failed while replacing an audio stream."""
-
-
-def replace_audio_stream(
-    video_path: str | Path,
-    audio_path: str | Path,
-    output_path: str | Path,
-    audio_codec: str = "aac",
-    audio_bitrate: str = "192k",
-) -> None:
-    """Copy ``video_path``'s video stream and mux in ``audio_path`` as the audio track.
-
-    Uses ffmpeg stream-copy for video (no re-encode) and encodes audio to AAC.
-    ``-shortest`` trims to the shorter of the two streams so the output duration
-    matches the source video when the dubbed audio is slightly longer.
-
-    Args:
-        video_path: Source video file (video stream is copied unchanged).
-        audio_path: Audio file to use as the new audio track.
-        output_path: Destination file. Overwritten if it exists.
-        audio_codec: ffmpeg audio codec name. Defaults to ``aac`` (MP4-compatible).
-        audio_bitrate: Audio bitrate passed to ffmpeg (``-b:a``).
-
-    Raises:
-        FileNotFoundError: If ``video_path`` or ``audio_path`` does not exist.
-        RemuxError: If ffmpeg returns a non-zero exit code.
-    """
-    video_path = Path(video_path)
-    audio_path = Path(audio_path)
-    output_path = Path(output_path)
-
-    if not video_path.exists():
-        raise FileNotFoundError(f"Video file not found: {video_path}")
-    if not audio_path.exists():
-        raise FileNotFoundError(f"Audio file not found: {audio_path}")
-
-    cmd = [
-        "ffmpeg",
-        "-y",
-        "-i",
-        str(video_path),
-        "-i",
-        str(audio_path),
-        "-map",
-        "0:v:0",
-        "-map",
-        "1:a:0",
-        "-c:v",
-        "copy",
-        "-c:a",
-        audio_codec,
-        "-b:a",
-        audio_bitrate,
-        "-shortest",
-        str(output_path),
-    ]
-
-    logger.info("replace_audio_stream: %s + %s -> %s", video_path, audio_path, output_path)
-    result = subprocess.run(cmd, capture_output=True)
-    if result.returncode != 0:
-        raise RemuxError(f"ffmpeg failed (exit {result.returncode}): {result.stderr.decode(errors='replace')}")

videopython-0.26.6/src/videopython/ai/understanding/separation.py

@@ -1,131 +0,0 @@
-"""Audio source separation using local Demucs models."""
-
-from __future__ import annotations
-
-from typing import Any
-
-from videopython.ai._device import log_device_initialization, release_device_memory, select_device
-from videopython.ai.dubbing.models import SeparatedAudio
-from videopython.base.audio import Audio, AudioMetadata
-
-
-class AudioSeparator:
-    """Separates audio into vocals and background components using Demucs."""
-
-    SUPPORTED_MODELS: list[str] = ["htdemucs", "htdemucs_ft", "htdemucs_6s", "mdx_extra"]
-    STEM_NAMES = ["drums", "bass", "other", "vocals"]
-    STEM_NAMES_6S = ["drums", "bass", "other", "vocals", "guitar", "piano"]
-
-    def __init__(self, model_name: str = "htdemucs", device: str | None = None):
-        if model_name not in self.SUPPORTED_MODELS:
-            raise ValueError(f"Model '{model_name}' not supported. Supported: {self.SUPPORTED_MODELS}")
-
-        self.model_name = model_name
-        self.device = device
-        self._model: Any = None
-
-    def _init_local(self) -> None:
-        """Initialize local Demucs model."""
-        from demucs.pretrained import get_model
-
-        requested_device = self.device
-        device = select_device(self.device, mps_allowed=False)
-
-        self._model = get_model(self.model_name)
-        self._model.to(device)
-        self._model.eval()
-        self.device = device
-        log_device_initialization(
-            "AudioSeparator",
-            requested_device=requested_device,
-            resolved_device=device,
-        )
-
-    def _separate_local(self, audio: Audio) -> SeparatedAudio:
-        """Separate audio using local Demucs model.
-
-        Keeps the input tensor on CPU and passes ``device=self.device`` to
-        ``apply_model`` so per-chunk compute runs on GPU while the full
-        ``(stems, channels, samples)`` output is stored in CPU RAM. For long
-        sources this is the difference between OOM-on-GPU and running cleanly:
-        a 2h stereo @ 44.1kHz output is ~10 GB — too big for an 8 GB card but
-        comfortable on a 32 GB host.
-        """
-        import numpy as np
-        import torch
-        from demucs.apply import apply_model
-
-        if self._model is None:
-            self._init_local()
-
-        target_sr = self._model.samplerate
-
-        if audio.metadata.channels == 1:
-            audio = audio._to_stereo()
-
-        if audio.metadata.sample_rate != target_sr:
-            audio = audio.resample(target_sr)
-
-        audio_data = audio.data
-        if audio_data.ndim == 1:
-            audio_data = np.stack([audio_data, audio_data])
-        elif audio_data.ndim == 2:
-            audio_data = audio_data.T
-
-        wav = torch.tensor(audio_data, dtype=torch.float32).unsqueeze(0)
-
-        with torch.no_grad():
-            sources = apply_model(self._model, wav, device=self.device)
-
-        sources_np = sources[0].cpu().numpy()
-        del sources
-
-        stem_names = self.STEM_NAMES_6S if self.model_name == "htdemucs_6s" else self.STEM_NAMES
-        vocals_idx = stem_names.index("vocals")
-        non_vocal_indices = [i for i in range(len(stem_names)) if i != vocals_idx]
-
-        vocals_data = sources_np[vocals_idx].T
-        background_data = sources_np[non_vocal_indices].sum(axis=0).T
-        del sources_np
-
-        max_val = np.max(np.abs(background_data))
-        if max_val > 1.0:
-            background_data /= max_val
-
-        metadata = AudioMetadata(
-            sample_rate=target_sr,
-            channels=2,
-            sample_width=2,
-            duration_seconds=vocals_data.shape[0] / target_sr,
-            frame_count=vocals_data.shape[0],
-        )
-        vocals = Audio(np.ascontiguousarray(vocals_data, dtype=np.float32), metadata)
-        background = Audio(np.ascontiguousarray(background_data, dtype=np.float32), metadata)
-
-        return SeparatedAudio(
-            vocals=vocals,
-            background=background,
-            original=audio,
-            music=None,
-            effects=None,
-        )
-
-    def separate(self, audio: Audio) -> SeparatedAudio:
-        """Separate audio into vocals and background components."""
-        return self._separate_local(audio)
-
-    def extract_vocals(self, audio: Audio) -> Audio:
-        """Convenience method to extract only vocals from audio."""
-        return self.separate(audio).vocals
-
-    def extract_background(self, audio: Audio) -> Audio:
-        """Convenience method to extract only background from audio."""
-        return self.separate(audio).background
-
-    def unload(self) -> None:
-        """Release the Demucs model so the next separate() re-initializes.
-
-        Used by low-memory dubbing to free VRAM between pipeline stages.
-        """
-        self._model = None
-        release_device_memory(self.device)