videopython 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

videopython/{generation → ai/generation}/audio.py

@@ -1,6 +1,5 @@
-import numpy as np
 import torch
-from pydub import AudioSegment
+from soundpython import Audio, AudioMetadata
 from transformers import (
     AutoProcessor,
     AutoTokenizer,
@@ -17,15 +16,24 @@ class TextToSpeech:
         self.pipeline = VitsModel.from_pretrained(TEXT_TO_SPEECH_MODEL)
         self.tokenizer = AutoTokenizer.from_pretrained(TEXT_TO_SPEECH_MODEL)
 
-    def generate_audio(self, text: str) -> AudioSegment:
+    def generate_audio(self, text: str) -> Audio:
         tokenized = self.tokenizer(text, return_tensors="pt")
 
         with torch.no_grad():
             output = self.pipeline(**tokenized).waveform
 
-        output = (output.T.float().numpy() * (2**31 - 1)).astype(np.int32)
-        audio = AudioSegment(data=output, frame_rate=self.pipeline.config.sampling_rate, sample_width=4, channels=1)
-        return audio
+        # Convert to float32 and normalize to [-1, 1]
+        audio_data = output.T.float().numpy()
+
+        metadata = AudioMetadata(
+            sample_rate=self.pipeline.config.sampling_rate,
+            channels=1,
+            sample_width=4,
+            duration_seconds=len(audio_data) / self.pipeline.config.sampling_rate,
+            frame_count=len(audio_data),
+        )
+
+        return Audio(audio_data, metadata)
 
 
 class TextToMusic:
@@ -37,7 +45,7 @@ class TextToMusic:
         self.processor = AutoProcessor.from_pretrained(MUSIC_GENERATION_MODEL_SMALL)
         self.model = MusicgenForConditionalGeneration.from_pretrained(MUSIC_GENERATION_MODEL_SMALL)
 
-    def generate_audio(self, text: str, max_new_tokens: int) -> AudioSegment:
+    def generate_audio(self, text: str, max_new_tokens: int) -> Audio:
         inputs = self.processor(
             text=[text],
             padding=True,
@@ -45,12 +53,16 @@ class TextToMusic:
         )
         audio_values = self.model.generate(**inputs, max_new_tokens=max_new_tokens)
         sampling_rate = self.model.config.audio_encoder.sampling_rate
-        output = (audio_values[0, 0].float().numpy() * (2**31 - 1)).astype(np.int32)
 
-        audio = AudioSegment(
-            data=output.tobytes(),
-            frame_rate=sampling_rate,
-            sample_width=4,
+        # Convert to float32 and normalize to [-1, 1]
+        audio_data = audio_values[0, 0].float().numpy()
+
+        metadata = AudioMetadata(
+            sample_rate=sampling_rate,
             channels=1,
+            sample_width=4,
+            duration_seconds=len(audio_data) / sampling_rate,
+            frame_count=len(audio_data),
         )
-        return audio
+
+        return Audio(audio_data, metadata)

videopython/{generation → ai/generation}/image.py

@@ -1,6 +1,3 @@
-import io
-import os
-
 import torch
 from diffusers import DiffusionPipeline
 from PIL import Image

videopython/ai/understanding/transcribe.py

@@ -0,0 +1,37 @@
+from typing import Literal
+
+import whisper
+
+from videopython.base.transcription import Transcription, TranscriptionSegment
+from videopython.base.video import Video
+
+
+class VideoTranscription:
+    def __init__(self, model_name: Literal["tiny", "base", "small", "medium", "large", "turbo"] = "small") -> None:
+        self.model = whisper.load_model(name=model_name)
+
+    def transcribe_video(self, video: Video) -> Transcription:
+        """Transcribes video to text.
+
+        Args:
+            video: Video to transcribe.
+
+        Returns:
+            List of dictionaries with segments of text and their start and end times.
+        """
+        if video.audio.is_silent:
+            return Transcription(segments=[])
+
+        audio = video.audio.to_mono()
+        audio = audio.resample(whisper.audio.SAMPLE_RATE)
+        audio_data = audio.data
+
+        transcription = self.model.transcribe(audio=audio_data, word_timestamps=True)
+
+        transcription_segments = [
+            TranscriptionSegment(start=segment["start"], end=segment["end"], text=segment["text"])
+            for segment in transcription["segments"]
+        ]
+        result = Transcription(segments=transcription_segments)
+
+        return result

videopython/base/effects.py

@@ -156,13 +156,13 @@ class Zoom(Effect):
 
         width = video.metadata.width
         height = video.metadata.height
-        crop_sizes_w, crop_sizes_h = np.linspace(width // self.zoom_factor, width, n_frames), np.linspace(
-            height // self.zoom_factor, height, n_frames
+        crop_sizes_w, crop_sizes_h = (
+            np.linspace(width // self.zoom_factor, width, n_frames),
+            np.linspace(height // self.zoom_factor, height, n_frames),
         )
 
         if self.mode == "in":
             for frame, w, h in tqdm(zip(video.frames, reversed(crop_sizes_w), reversed(crop_sizes_h))):
-
                 x = width / 2 - w / 2
                 y = height / 2 - h / 2
 

videopython/base/transcription.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class TranscriptionSegment:
+    start: float
+    end: float
+    text: str
+
+
+@dataclass
+class Transcription:
+    segments: list[TranscriptionSegment]

videopython/base/transforms.py

@@ -1,7 +1,6 @@
 from abc import ABC, abstractmethod
 from enum import Enum
 from multiprocessing import Pool
-from typing import Literal
 
 import cv2
 import numpy as np
@@ -154,7 +153,6 @@ class CropMode(Enum):
 
 
 class Crop(Transformation):
-
     def __init__(self, width: int, height: int, mode: CropMode = CropMode.CENTER):
         self.width = width
         self.height = height

videopython/base/transitions.py

@@ -67,7 +67,7 @@ class FadeTransition(Transition):
             ],
             fps=video_fps,
         )
-        faded_videos.audio = videos[0].audio.append(videos[1].audio, crossfade=(effect_time_fps / video_fps) * 1000)
+        faded_videos.audio = videos[0].audio.concat(videos[1].audio, crossfade=(effect_time_fps / video_fps))
         return faded_videos
 
 
@@ -102,5 +102,5 @@ class BlurTransition(Transition):
             ],
             fps=video_fps,
         )
-        blurred_videos.audio = videos[0].audio.append(videos[1].audio)
+        blurred_videos.audio = videos[0].audio.concat(videos[1].audio)
         return blurred_videos