videopython 0.25.4__tar.gz → 0.25.5__tar.gz

{videopython-0.25.4 → videopython-0.25.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: videopython
-Version: 0.25.4
+Version: 0.25.5
 Summary: Minimal video generation and processing library.
 Project-URL: Homepage, https://videopython.com
 Project-URL: Repository, https://github.com/bartwojtowicz/videopython/
@@ -25,7 +25,7 @@ Requires-Dist: torchcodec>=0.9.1
 Requires-Dist: tqdm>=4.66.3
 Provides-Extra: ai
 Requires-Dist: accelerate>=0.29.2; extra == 'ai'
-Requires-Dist: coqui-tts>=0.24.0; extra == 'ai'
+Requires-Dist: chatterbox-tts>=0.1.7; extra == 'ai'
 Requires-Dist: demucs>=4.0.0; extra == 'ai'
 Requires-Dist: diffusers>=0.26.3; extra == 'ai'
 Requires-Dist: easyocr>=1.7.0; extra == 'ai'
@@ -36,6 +36,7 @@ Requires-Dist: openai-whisper>=20240930; extra == 'ai'
 Requires-Dist: pyannote-audio>=4.0.0; extra == 'ai'
 Requires-Dist: scikit-learn>=1.3.0; extra == 'ai'
 Requires-Dist: scipy>=1.10.0; extra == 'ai'
+Requires-Dist: sentencepiece>=0.1.99; extra == 'ai'
 Requires-Dist: torch>=2.1.0; extra == 'ai'
 Requires-Dist: transformers>=5.2.0; extra == 'ai'
 Requires-Dist: transnetv2-pytorch>=1.0.5; extra == 'ai'

{videopython-0.25.4 → videopython-0.25.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "videopython"
-version = "0.25.4"
+version = "0.25.5"
 description = "Minimal video generation and processing library."
 authors = [
     { name = "Bartosz Wójtowicz", email = "bartoszwojtowicz@outlook.com" },
@@ -72,8 +72,10 @@ ai = [
     # Audio classification (AST via transformers - no separate dep needed)
     # Scene detection
     "transnetv2-pytorch>=1.0.5",
-    # Voice cloning TTS (coqui-tts is the maintained fork of TTS)
-    "coqui-tts>=0.24.0",
+    # Voice cloning TTS (Chatterbox Multilingual by Resemble AI)
+    "chatterbox-tts>=0.1.7",
+    # Translation (Marian MT tokenizer requires sentencepiece)
+    "sentencepiece>=0.1.99",
     # Audio source separation
     "demucs>=4.0.0",
 ]
@@ -107,8 +109,10 @@ ai = [
     # Audio classification (AST via transformers - no separate dep needed)
     # Scene detection
     "transnetv2-pytorch>=1.0.5",
-    # Voice cloning TTS (coqui-tts is the maintained fork of TTS)
-    "coqui-tts>=0.24.0",
+    # Voice cloning TTS (Chatterbox Multilingual by Resemble AI)
+    "chatterbox-tts>=0.1.7",
+    # Translation (Marian MT tokenizer requires sentencepiece)
+    "sentencepiece>=0.1.99",
     # Audio source separation
     "demucs>=4.0.0",
 ]
@@ -130,13 +134,19 @@ module = [
     "easyocr", "easyocr.*",
     "transformers", "transformers.*",
     "transnetv2_pytorch", "transnetv2_pytorch.*",
-    "TTS", "TTS.*", "coqui", "coqui.*",
+    "chatterbox", "chatterbox.*",
     "demucs", "demucs.*",
     "pyannote", "pyannote.*",
     "cv2", "cv2.*",
 ]
 ignore_missing_imports = true
 
+[tool.uv]
+# chatterbox-tts 0.1.7 pins strict versions of torch, torchaudio, numpy, and
+# diffusers that conflict with pyannote-audio (torch>=2.8) and CogVideoX
+# (diffusers>=0.30). Override to let the resolver pick compatible versions.
+override-dependencies = ["torch>=2.8.0", "torchaudio>=2.8.0", "numpy>=2.0.0", "diffusers>=0.30.0"]
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

{videopython-0.25.4 → videopython-0.25.5}/src/videopython/ai/dubbing/dubber.py

@@ -34,9 +34,15 @@ class VideoDubber:
         source_lang: str | None = None,
         preserve_background: bool = True,
         voice_clone: bool = True,
+        enable_diarization: bool = False,
         progress_callback: Callable[[str, float], None] | None = None,
     ) -> DubbingResult:
-        """Dub a video into a target language."""
+        """Dub a video into a target language.
+
+        Args:
+            enable_diarization: Enable speaker diarization to clone each speaker's
+                voice separately. Requires additional VRAM for the diarization model.
+        """
         if self._local_pipeline is None:
             self._init_local_pipeline()
 
@@ -46,6 +52,7 @@ class VideoDubber:
             source_lang=source_lang,
             preserve_background=preserve_background,
             voice_clone=voice_clone,
+            enable_diarization=enable_diarization,
             progress_callback=progress_callback,
         )
 
@@ -56,6 +63,7 @@ class VideoDubber:
         source_lang: str | None = None,
         preserve_background: bool = True,
         voice_clone: bool = True,
+        enable_diarization: bool = False,
         progress_callback: Callable[[str, float], None] | None = None,
     ) -> Video:
         """Dub a video and return a new video with the dubbed audio."""
@@ -65,6 +73,7 @@ class VideoDubber:
             source_lang=source_lang,
             preserve_background=preserve_background,
             voice_clone=voice_clone,
+            enable_diarization=enable_diarization,
             progress_callback=progress_callback,
         )
         return video.add_audio(result.dubbed_audio, overlay=False)

{videopython-0.25.4 → videopython-0.25.5}/src/videopython/ai/dubbing/pipeline.py

@@ -28,11 +28,11 @@ class LocalDubbingPipeline:
         self._separator: Any = None
         self._synchronizer: TimingSynchronizer | None = None
 
-    def _init_transcriber(self) -> None:
+    def _init_transcriber(self, enable_diarization: bool = False) -> None:
         """Initialize the transcription model."""
         from videopython.ai.understanding.audio import AudioToText
 
-        self._transcriber = AudioToText(device=self.device)
+        self._transcriber = AudioToText(device=self.device, enable_diarization=enable_diarization)
 
     def _init_translator(self) -> None:
         """Initialize the translation model."""
@@ -40,17 +40,18 @@ class LocalDubbingPipeline:
 
         self._translator = TextTranslator(device=self.device)
 
-    def _init_tts(self, voice_clone: bool = False) -> None:
+    def _init_tts(self, voice_clone: bool = False, language: str = "en") -> None:
         """Initialize the text-to-speech model."""
         from videopython.ai.generation.audio import TextToSpeech
 
         if voice_clone:
             self._tts = TextToSpeech(
-                model_size="xtts",
+                model_size="chatterbox",
                 device=self.device,
+                language=language,
             )
         else:
-            self._tts = TextToSpeech(device=self.device)
+            self._tts = TextToSpeech(device=self.device, language=language)
 
     def _init_separator(self) -> None:
         """Initialize the audio separator."""
@@ -108,6 +109,7 @@ class LocalDubbingPipeline:
         source_lang: str | None = None,
         preserve_background: bool = True,
         voice_clone: bool = True,
+        enable_diarization: bool = False,
         progress_callback: Callable[[str, float], None] | None = None,
     ) -> DubbingResult:
         """Process a video through the local dubbing pipeline."""
@@ -119,7 +121,7 @@ class LocalDubbingPipeline:
 
         report_progress("Transcribing audio", 0.05)
         if self._transcriber is None:
-            self._init_transcriber()
+            self._init_transcriber(enable_diarization=enable_diarization)
 
         source_audio = video.audio
         transcription = self._transcriber.transcribe(source_audio)
@@ -133,7 +135,7 @@ class LocalDubbingPipeline:
                 target_lang=target_lang,
             )
 
-        detected_lang = source_lang or "en"
+        detected_lang = source_lang or transcription.language or "en"
 
         separated_audio: SeparatedAudio | None = None
         vocal_audio = source_audio
@@ -163,13 +165,16 @@ class LocalDubbingPipeline:
 
         report_progress("Generating dubbed speech", 0.50)
         if self._tts is None:
-            self._init_tts(voice_clone=voice_clone)
+            self._init_tts(voice_clone=voice_clone, language=target_lang)
 
         dubbed_segments: list[Audio] = []
         target_durations: list[float] = []
         start_times: list[float] = []
 
         for i, segment in enumerate(translated_segments):
+            if segment.duration < 0.1:
+                continue
+
             progress = 0.50 + (0.30 * (i / len(translated_segments)))
             report_progress(f"Generating speech ({i + 1}/{len(translated_segments)})", progress)
 

{videopython-0.25.4 → videopython-0.25.5}/src/videopython/ai/dubbing/timing.py

@@ -79,8 +79,8 @@ class TimingSynchronizer:
         """
         original_duration = audio.metadata.duration_seconds
 
-        if original_duration <= 0:
-            # Empty audio, return as-is
+        if original_duration <= 0 or target_duration <= 0:
+            # Empty audio or zero-length target, return as-is
             return audio, TimingAdjustment(
                 segment_index=segment_index,
                 original_duration=original_duration,

{videopython-0.25.4 → videopython-0.25.5}/src/videopython/ai/generation/audio.py

@@ -11,10 +11,13 @@ from videopython.base.audio import Audio, AudioMetadata
 class TextToSpeech:
     """Generates speech audio from text using local models.
 
-    Supports Bark (`base`, `small`) and XTTS voice cloning (`xtts`).
+    Supports Bark (`base`, `small`) for general TTS and Chatterbox Multilingual
+    (`chatterbox`) for multilingual voice cloning.
     """
 
-    SUPPORTED_LOCAL_MODELS: list[str] = ["base", "small", "xtts"]
+    SUPPORTED_LOCAL_MODELS: list[str] = ["base", "small", "chatterbox"]
+
+    CHATTERBOX_SAMPLE_RATE: int = 24000
 
     def __init__(
         self,
@@ -32,7 +35,7 @@ class TextToSpeech:
         self.language = language
         self._model: Any = None
         self._processor: Any = None
-        self._xtts_model: Any = None
+        self._chatterbox_model: Any = None
 
     def _init_local(self) -> None:
         """Initialize local Bark model."""
@@ -51,43 +54,14 @@ class TextToSpeech:
             resolved_device=device,
         )
 
-    def _patch_xtts_load_audio(self) -> None:
-        """Patch XTTS load_audio to avoid torchcodec dependency issues."""
-        import TTS.tts.models.xtts as xtts_module
-
-        def load_audio_soundfile(audiopath: str, sampling_rate: int):
-            import soundfile as sf  # type: ignore[import-untyped]
-            import torch
-            import torchaudio.functional as F  # type: ignore[import-untyped]
-
-            audio_np, sr = sf.read(audiopath, dtype="float32")
-
-            audio = torch.from_numpy(audio_np)
-            if audio.dim() == 1:
-                audio = audio.unsqueeze(0)
-            else:
-                audio = audio.T
-
-            if audio.size(0) != 1:
-                audio = torch.mean(audio, dim=0, keepdim=True)
-
-            if sr != sampling_rate:
-                audio = F.resample(audio, sr, sampling_rate)
-
-            return audio
-
-        xtts_module.load_audio = load_audio_soundfile
-
-    def _init_xtts(self) -> None:
-        """Initialize XTTS-v2 model for voice cloning."""
-        from TTS.api import TTS
-
-        self._patch_xtts_load_audio()
+    def _init_chatterbox(self) -> None:
+        """Initialize Chatterbox Multilingual model for voice cloning."""
+        from chatterbox.mtl_tts import ChatterboxMultilingualTTS  # type: ignore[import-untyped]
 
         requested_device = self.device
         device = select_device(self.device, mps_allowed=False)
 
-        self._xtts_model = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(device)
+        self._chatterbox_model = ChatterboxMultilingualTTS.from_pretrained(device=device)
         self.device = device
         log_device_initialization(
             "TextToSpeech",
@@ -120,28 +94,32 @@ class TextToSpeech:
         )
         return Audio(audio_data, metadata)
 
-    def _generate_xtts(self, text: str, voice_sample: Audio) -> Audio:
-        """Generate speech using XTTS-v2 with voice cloning."""
+    def _generate_chatterbox(self, text: str, voice_sample: Audio) -> Audio:
+        """Generate speech using Chatterbox Multilingual with voice cloning."""
         import tempfile
         from pathlib import Path
 
         import numpy as np
 
-        if self._xtts_model is None:
-            self._init_xtts()
+        if self._chatterbox_model is None:
+            self._init_chatterbox()
 
         with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
             voice_sample.save(f.name)
             speaker_wav_path = Path(f.name)
 
         try:
-            audio_list = self._xtts_model.tts(
+            wav = self._chatterbox_model.generate(
                 text=text,
-                speaker_wav=str(speaker_wav_path),
-                language=self.language,
+                language_id=self.language,
+                audio_prompt_path=str(speaker_wav_path),
             )
-            audio_data = np.array(audio_list, dtype=np.float32)
-            sample_rate = 24000
+
+            audio_data = wav.cpu().float().numpy().squeeze()
+            if audio_data.ndim == 0:
+                audio_data = np.array([audio_data], dtype=np.float32)
+
+            sample_rate = self.CHATTERBOX_SAMPLE_RATE
 
             metadata = AudioMetadata(
                 sample_rate=sample_rate,
@@ -163,12 +141,13 @@ class TextToSpeech:
         """Generate speech audio from text."""
         effective_voice = voice_preset or self.voice
 
-        if self.model_size == "xtts" or voice_sample is not None:
+        if self.model_size == "chatterbox" or voice_sample is not None:
             if voice_sample is None:
                 raise ValueError(
-                    "voice_sample is required for XTTS voice cloning. Provide an Audio sample of the voice to clone."
+                    "voice_sample is required for Chatterbox voice cloning. "
+                    "Provide an Audio sample of the voice to clone."
                 )
-            return self._generate_xtts(text, voice_sample)
+            return self._generate_chatterbox(text, voice_sample)
 
         return self._generate_local(text, effective_voice)
 

{videopython-0.25.4 → videopython-0.25.5}/src/videopython/ai/generation/translation.py

@@ -61,15 +61,15 @@ class TextTranslator:
         return f"Helsinki-NLP/opus-mt-{source_lang}-{target_lang}"
 
     def _init_local(self, source_lang: str, target_lang: str) -> None:
-        from transformers import AutoModelForSeq2SeqLM, AutoTokenizer  # type: ignore[attr-defined]
+        from transformers import MarianMTModel, MarianTokenizer  # type: ignore[attr-defined]
 
         model_name = self._get_local_model_name(source_lang, target_lang)
 
         requested_device = self.device
         device = select_device(self.device, mps_allowed=True)
 
-        self._tokenizer = AutoTokenizer.from_pretrained(model_name)
-        self._model = AutoModelForSeq2SeqLM.from_pretrained(model_name).to(device)
+        self._tokenizer = MarianTokenizer.from_pretrained(model_name)
+        self._model = MarianMTModel.from_pretrained(model_name).to(device)
         self.device = device
         log_device_initialization(
             "TextTranslator",
@@ -103,6 +103,8 @@ class TextTranslator:
             return text
 
         effective_source = source_lang or "en"
+        if effective_source == target_lang:
+            return text
         return self._translate_local(text, target_lang, effective_source)
 
     def translate_batch(
@@ -118,6 +120,8 @@ class TextTranslator:
             return []
 
         effective_source = source_lang or "en"
+        if effective_source == target_lang:
+            return list(texts)
         if self._model is None or self._current_lang_pair != (effective_source, target_lang):
             self._init_local(effective_source, target_lang)