videopython 0.9.1__py3-none-any.whl

videopython/__init__.py

@@ -0,0 +1,3 @@
+from videopython.base import Video, VideoMetadata
+
+__all__ = ["Video", "VideoMetadata"]

videopython/ai/__init__.py

@@ -0,0 +1,39 @@
+from .generation import ImageToVideo, TextToImage, TextToMusic, TextToSpeech, TextToVideo
+from .understanding import (
+    AudioClassifier,
+    AudioToText,
+    CameraMotionDetector,
+    CombinedFrameAnalyzer,
+    FaceDetector,
+    ImageToText,
+    LLMSummarizer,
+    MotionAnalyzer,
+    ObjectDetector,
+    ShotTypeClassifier,
+    TextDetector,
+    VideoAnalyzer,
+)
+
+__all__ = [
+    # Generation
+    "TextToVideo",
+    "ImageToVideo",
+    "TextToImage",
+    "TextToSpeech",
+    "TextToMusic",
+    # Understanding
+    "AudioToText",
+    "AudioClassifier",
+    "ImageToText",
+    "LLMSummarizer",
+    "VideoAnalyzer",
+    # Detection
+    "ObjectDetector",
+    "FaceDetector",
+    "TextDetector",
+    "ShotTypeClassifier",
+    "CameraMotionDetector",
+    "CombinedFrameAnalyzer",
+    # Motion
+    "MotionAnalyzer",
+]

videopython/ai/backends.py

@@ -0,0 +1,77 @@
+"""Backend utilities for videopython.ai module."""
+
+from __future__ import annotations
+
+import os
+from typing import Literal
+
+# Backend type definitions per task
+TextToVideoBackend = Literal["local", "luma"]
+ImageToVideoBackend = Literal["local", "luma", "runway"]
+TextToSpeechBackend = Literal["local", "openai", "elevenlabs"]
+TextToMusicBackend = Literal["local"]
+TextToImageBackend = Literal["local", "openai"]
+ImageToTextBackend = Literal["local", "openai", "gemini"]
+AudioToTextBackend = Literal["local", "openai", "gemini"]
+AudioClassifierBackend = Literal["local"]
+LLMBackend = Literal["local", "openai", "gemini"]
+
+# Environment variable names per provider
+API_KEY_ENV_VARS: dict[str, str] = {
+    "openai": "OPENAI_API_KEY",
+    "gemini": "GOOGLE_API_KEY",
+    "elevenlabs": "ELEVENLABS_API_KEY",
+    "runway": "RUNWAYML_API_KEY",
+    "luma": "LUMAAI_API_KEY",
+}
+
+
+class BackendError(Exception):
+    """Base exception for backend-related errors."""
+
+    pass
+
+
+class MissingAPIKeyError(BackendError):
+    """Raised when a required API key is not found."""
+
+    def __init__(self, provider: str):
+        env_var = API_KEY_ENV_VARS.get(provider, f"{provider.upper()}_API_KEY")
+        super().__init__(
+            f"API key for '{provider}' not found. Set the {env_var} environment variable or pass api_key parameter."
+        )
+        self.provider = provider
+
+
+class UnsupportedBackendError(BackendError):
+    """Raised when an unsupported backend is requested."""
+
+    def __init__(self, backend: str, supported: list[str]):
+        super().__init__(f"Backend '{backend}' is not supported. Supported backends: {', '.join(supported)}")
+        self.backend = backend
+        self.supported = supported
+
+
+def get_api_key(provider: str, api_key: str | None = None) -> str:
+    """Get API key for a provider.
+
+    Args:
+        provider: Provider name (e.g., 'openai', 'runway', 'luma')
+        api_key: Optional explicit API key. If provided, returns this directly.
+
+    Returns:
+        The API key string.
+
+    Raises:
+        MissingAPIKeyError: If no API key is found.
+    """
+    if api_key:
+        return api_key
+
+    env_var = API_KEY_ENV_VARS.get(provider)
+    if env_var:
+        key = os.environ.get(env_var)
+        if key:
+            return key
+
+    raise MissingAPIKeyError(provider)

videopython/ai/config.py

@@ -0,0 +1,125 @@
+"""Configuration loader for videopython.ai module."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import tomllib
+
+# Default backends per task (used when no config file is found)
+DEFAULT_BACKENDS: dict[str, str] = {
+    "text_to_video": "local",
+    "image_to_video": "local",
+    "video_upscaler": "local",
+    "text_to_speech": "local",
+    "text_to_music": "local",
+    "text_to_image": "local",
+    "image_to_text": "local",
+    "audio_to_text": "local",
+    "audio_classifier": "local",
+    "llm_summarizer": "local",
+}
+
+
+def _find_config_file() -> Path | None:
+    """Find the configuration file in current directory or parents.
+
+    Looks for:
+    1. videopython.toml in current directory
+    2. pyproject.toml in current directory
+
+    Returns:
+        Path to config file if found, None otherwise.
+    """
+    cwd = Path.cwd()
+
+    # Check for videopython.toml
+    videopython_toml = cwd / "videopython.toml"
+    if videopython_toml.exists():
+        return videopython_toml
+
+    # Check for pyproject.toml
+    pyproject_toml = cwd / "pyproject.toml"
+    if pyproject_toml.exists():
+        return pyproject_toml
+
+    return None
+
+
+def _load_toml(path: Path) -> dict[str, Any]:
+    """Load and parse a TOML file."""
+    with open(path, "rb") as f:
+        return tomllib.load(f)
+
+
+def _extract_config(data: dict[str, Any], filename: str) -> dict[str, Any]:
+    """Extract videopython config from parsed TOML data.
+
+    Args:
+        data: Parsed TOML data
+        filename: Name of the file (to determine extraction method)
+
+    Returns:
+        The videopython configuration section, or empty dict if not found.
+    """
+    if filename == "videopython.toml":
+        return data
+    elif filename == "pyproject.toml":
+        return data.get("tool", {}).get("videopython", {})
+    return {}
+
+
+@lru_cache(maxsize=1)
+def _get_cached_config() -> dict[str, Any]:
+    """Load and cache the configuration.
+
+    Returns:
+        The loaded configuration, or empty dict if no config file found.
+    """
+    config_path = _find_config_file()
+    if config_path is None:
+        return {}
+
+    try:
+        data = _load_toml(config_path)
+        return _extract_config(data, config_path.name)
+    except Exception:
+        return {}
+
+
+def get_config() -> dict[str, Any]:
+    """Get the current configuration.
+
+    Returns:
+        The configuration dictionary.
+    """
+    return _get_cached_config()
+
+
+def get_default_backend(task: str) -> str:
+    """Get the default backend for a task.
+
+    Priority:
+    1. Config file setting
+    2. Hardcoded default ("local")
+
+    Args:
+        task: Task name (e.g., 'text_to_video', 'text_to_speech')
+
+    Returns:
+        The backend name to use.
+    """
+    config = get_config()
+    ai_defaults = config.get("ai", {}).get("defaults", {})
+
+    if task in ai_defaults:
+        return ai_defaults[task]
+
+    return DEFAULT_BACKENDS.get(task, "local")
+
+
+def clear_config_cache() -> None:
+    """Clear the configuration cache. Useful for testing."""
+    _get_cached_config.cache_clear()

videopython/ai/generation/__init__.py

@@ -0,0 +1,11 @@
+from .audio import TextToMusic, TextToSpeech
+from .image import TextToImage
+from .video import ImageToVideo, TextToVideo
+
+__all__ = [
+    "ImageToVideo",
+    "TextToSpeech",
+    "TextToImage",
+    "TextToVideo",
+    "TextToMusic",
+]

videopython/ai/generation/audio.py

@@ -0,0 +1,285 @@
+"""Audio generation with multi-backend support."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from videopython.ai.backends import (
+    TextToMusicBackend,
+    TextToSpeechBackend,
+    UnsupportedBackendError,
+    get_api_key,
+)
+from videopython.ai.config import get_default_backend
+from videopython.base.audio import Audio, AudioMetadata
+
+
+class TextToSpeech:
+    """Generates speech audio from text."""
+
+    SUPPORTED_BACKENDS: list[str] = ["local", "openai", "elevenlabs"]
+
+    def __init__(
+        self,
+        backend: TextToSpeechBackend | None = None,
+        model_size: str = "base",
+        voice: str | None = None,
+        api_key: str | None = None,
+        device: str | None = None,
+    ):
+        """Initialize text-to-speech generator.
+
+        Args:
+            backend: Backend to use. If None, uses config default or 'local'.
+            model_size: Model size for local backend ('base' or 'small').
+            voice: Voice to use (backend-specific).
+            api_key: API key for cloud backends. If None, reads from environment.
+            device: Device for local backend ('cuda' or 'cpu').
+        """
+        resolved_backend: str = backend if backend is not None else get_default_backend("text_to_speech")
+        if resolved_backend not in self.SUPPORTED_BACKENDS:
+            raise UnsupportedBackendError(resolved_backend, self.SUPPORTED_BACKENDS)
+
+        self.backend: TextToSpeechBackend = resolved_backend  # type: ignore[assignment]
+        self.model_size = model_size
+        self.voice = voice
+        self.api_key = api_key
+        self.device = device
+        self._model: Any = None
+        self._processor: Any = None
+
+    def _init_local(self) -> None:
+        """Initialize local Bark model."""
+        import torch
+        from transformers import AutoModel, AutoProcessor
+
+        if self.model_size not in ["base", "small"]:
+            raise ValueError(f"model_size must be 'base' or 'small', got '{self.model_size}'")
+
+        device = self.device
+        if device is None:
+            if torch.cuda.is_available():
+                device = "cuda"
+            elif torch.backends.mps.is_available():
+                device = "mps"
+            else:
+                device = "cpu"
+
+        model_name = "suno/bark" if self.model_size == "base" else "suno/bark-small"
+        self._processor = AutoProcessor.from_pretrained(model_name)
+        self._model = AutoModel.from_pretrained(model_name).to(device)
+        self.device = device
+
+    def _generate_local(
+        self,
+        text: str,
+        voice_preset: str | None,
+    ) -> Audio:
+        """Generate speech using local Bark model."""
+        import torch
+
+        if self._model is None:
+            self._init_local()
+
+        inputs = self._processor(text=[text], return_tensors="pt", voice_preset=voice_preset)
+        inputs = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()}
+
+        with torch.no_grad():
+            speech_values = self._model.generate(**inputs, do_sample=True)
+
+        audio_data = speech_values.cpu().float().numpy().squeeze()
+        sample_rate = self._model.generation_config.sample_rate
+
+        metadata = AudioMetadata(
+            sample_rate=sample_rate,
+            channels=1,
+            sample_width=2,
+            duration_seconds=len(audio_data) / sample_rate,
+            frame_count=len(audio_data),
+        )
+        return Audio(audio_data, metadata)
+
+    def _generate_openai(self, text: str) -> Audio:
+        """Generate speech using OpenAI TTS."""
+        import numpy as np
+        from openai import OpenAI
+
+        api_key = get_api_key("openai", self.api_key)
+        client = OpenAI(api_key=api_key)
+
+        voice = self.voice or "alloy"
+        response = client.audio.speech.create(
+            model="tts-1-hd",
+            voice=voice,  # type: ignore
+            input=text,
+            response_format="pcm",
+        )
+
+        # OpenAI returns raw PCM at 24kHz, 16-bit, mono
+        audio_bytes = response.read()
+        audio_data = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+        sample_rate = 24000
+
+        metadata = AudioMetadata(
+            sample_rate=sample_rate,
+            channels=1,
+            sample_width=2,
+            duration_seconds=len(audio_data) / sample_rate,
+            frame_count=len(audio_data),
+        )
+        return Audio(audio_data, metadata)
+
+    def _generate_elevenlabs(self, text: str) -> Audio:
+        """Generate speech using ElevenLabs."""
+        import numpy as np
+        from elevenlabs import ElevenLabs
+
+        api_key = get_api_key("elevenlabs", self.api_key)
+        client = ElevenLabs(api_key=api_key)
+
+        voice = self.voice or "Sarah"
+
+        # Resolve voice name to ID if needed (voice IDs are 20+ chars)
+        if len(voice) < 20:
+            voices = client.voices.get_all()
+            voice_id = None
+            for v in voices.voices:
+                if v.name and voice.lower() in v.name.lower():
+                    voice_id = v.voice_id
+                    break
+            if voice_id is None:
+                raise ValueError(f"Voice '{voice}' not found. Use a voice ID or valid name.")
+            voice = voice_id
+
+        # Generate audio - returns generator
+        audio_chunks = []
+        for chunk in client.text_to_speech.convert(
+            voice_id=voice,
+            text=text,
+            model_id="eleven_multilingual_v2",
+            output_format="pcm_24000",
+        ):
+            audio_chunks.append(chunk)
+
+        audio_bytes = b"".join(audio_chunks)
+        audio_data = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+        sample_rate = 24000
+
+        metadata = AudioMetadata(
+            sample_rate=sample_rate,
+            channels=1,
+            sample_width=2,
+            duration_seconds=len(audio_data) / sample_rate,
+            frame_count=len(audio_data),
+        )
+        return Audio(audio_data, metadata)
+
+    def generate_audio(
+        self,
+        text: str,
+        voice_preset: str | None = None,
+    ) -> Audio:
+        """Generate speech audio from text.
+
+        Args:
+            text: Text to synthesize. For local backend, can include emotion markers
+                  like [laughs], [sighs].
+            voice_preset: Voice preset (backend-specific). For local backend, use
+                          IDs like "v2/en_speaker_0".
+
+        Returns:
+            Generated speech audio.
+        """
+        effective_voice = voice_preset or self.voice
+
+        if self.backend == "local":
+            return self._generate_local(text, effective_voice)
+        elif self.backend == "openai":
+            return self._generate_openai(text)
+        elif self.backend == "elevenlabs":
+            return self._generate_elevenlabs(text)
+        else:
+            raise UnsupportedBackendError(self.backend, self.SUPPORTED_BACKENDS)
+
+
+class TextToMusic:
+    """Generates music from text descriptions."""
+
+    SUPPORTED_BACKENDS: list[str] = ["local"]
+
+    def __init__(
+        self,
+        backend: TextToMusicBackend | None = None,
+    ):
+        """Initialize text-to-music generator.
+
+        Args:
+            backend: Backend to use. If None, uses config default or 'local'.
+        """
+        resolved_backend: str = backend if backend is not None else get_default_backend("text_to_music")
+        if resolved_backend not in self.SUPPORTED_BACKENDS:
+            raise UnsupportedBackendError(resolved_backend, self.SUPPORTED_BACKENDS)
+
+        self.backend: TextToMusicBackend = resolved_backend  # type: ignore[assignment]
+        self._processor: Any = None
+        self._model: Any = None
+        self._device: str = "cpu"
+
+    def _init_local(self) -> None:
+        """Initialize local MusicGen model."""
+        import os
+
+        import torch
+        from transformers import AutoProcessor, MusicgenForConditionalGeneration
+
+        # Enable MPS fallback for unsupported operations
+        os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+
+        if torch.cuda.is_available():
+            self._device = "cuda"
+        elif torch.backends.mps.is_available():
+            self._device = "mps"
+        else:
+            self._device = "cpu"
+
+        model_name = "facebook/musicgen-small"
+        self._processor = AutoProcessor.from_pretrained(model_name)
+        self._model = MusicgenForConditionalGeneration.from_pretrained(model_name)
+        self._model.to(self._device)
+
+    def _generate_local(self, text: str, max_new_tokens: int) -> Audio:
+        """Generate music using local MusicGen model."""
+        if self._model is None:
+            self._init_local()
+
+        inputs = self._processor(text=[text], padding=True, return_tensors="pt")
+        # Move inputs to the same device as the model
+        inputs = {k: v.to(self._device) if hasattr(v, "to") else v for k, v in inputs.items()}
+        audio_values = self._model.generate(**inputs, max_new_tokens=max_new_tokens)
+        sampling_rate = self._model.config.audio_encoder.sampling_rate
+
+        audio_data = audio_values[0, 0].cpu().float().numpy()
+
+        metadata = AudioMetadata(
+            sample_rate=sampling_rate,
+            channels=1,
+            sample_width=2,
+            duration_seconds=len(audio_data) / sampling_rate,
+            frame_count=len(audio_data),
+        )
+        return Audio(audio_data, metadata)
+
+    def generate_audio(self, text: str, max_new_tokens: int = 256) -> Audio:
+        """Generate music audio from text description.
+
+        Args:
+            text: Text description of desired music.
+            max_new_tokens: Maximum length of generated audio in tokens.
+
+        Returns:
+            Generated music audio.
+        """
+        if self.backend == "local":
+            return self._generate_local(text, max_new_tokens)
+        else:
+            raise UnsupportedBackendError(self.backend, self.SUPPORTED_BACKENDS)

videopython/ai/generation/image.py

@@ -0,0 +1,117 @@
+"""Image generation with multi-backend support."""
+
+from __future__ import annotations
+
+import io
+from typing import Any
+
+from PIL import Image
+
+from videopython.ai.backends import TextToImageBackend, UnsupportedBackendError, get_api_key
+from videopython.ai.config import get_default_backend
+
+
+class TextToImage:
+    """Generates images from text descriptions."""
+
+    SUPPORTED_BACKENDS: list[str] = ["local", "openai"]
+
+    def __init__(
+        self,
+        backend: TextToImageBackend | None = None,
+        api_key: str | None = None,
+    ):
+        """Initialize text-to-image generator.
+
+        Args:
+            backend: Backend to use. If None, uses config default or 'local'.
+            api_key: API key for cloud backends. If None, reads from environment.
+        """
+        resolved_backend: str = backend if backend is not None else get_default_backend("text_to_image")
+        if resolved_backend not in self.SUPPORTED_BACKENDS:
+            raise UnsupportedBackendError(resolved_backend, self.SUPPORTED_BACKENDS)
+
+        self.backend: TextToImageBackend = resolved_backend  # type: ignore[assignment]
+        self.api_key = api_key
+        self._pipeline: Any = None
+
+    def _init_local(self) -> None:
+        """Initialize local diffusion pipeline."""
+        import torch
+        from diffusers import DiffusionPipeline
+
+        if torch.cuda.is_available():
+            device = "cuda"
+            dtype = torch.float16
+            variant = "fp16"
+        elif torch.backends.mps.is_available():
+            device = "mps"
+            dtype = torch.float32  # MPS works better with float32
+            variant = None  # No fp16 variant for MPS
+        else:
+            raise ValueError("No GPU available. Local TextToImage requires CUDA or MPS (Apple Silicon).")
+
+        model_name = "stabilityai/stable-diffusion-xl-base-1.0"
+        self._pipeline = DiffusionPipeline.from_pretrained(
+            model_name, torch_dtype=dtype, variant=variant, use_safetensors=True
+        )
+        self._pipeline.to(device)
+
+        # Enable attention slicing for MPS memory efficiency
+        if device == "mps":
+            self._pipeline.enable_attention_slicing()
+
+    def _generate_local(self, prompt: str) -> Image.Image:
+        """Generate image using local diffusion model."""
+        if self._pipeline is None:
+            self._init_local()
+
+        return self._pipeline(prompt=prompt).images[0]
+
+    def _generate_openai(self, prompt: str, size: str) -> Image.Image:
+        """Generate image using OpenAI DALL-E."""
+        import httpx
+        from openai import OpenAI
+
+        api_key = get_api_key("openai", self.api_key)
+        client = OpenAI(api_key=api_key)
+
+        response = client.images.generate(
+            model="dall-e-3",
+            prompt=prompt,
+            size=size,  # type: ignore
+            quality="hd",
+            n=1,
+        )
+
+        image_url = response.data[0].url
+        if image_url is None:
+            raise RuntimeError("OpenAI returned no image URL")
+
+        # Download the image
+        with httpx.Client() as http_client:
+            image_response = http_client.get(image_url, timeout=60.0)
+            image_response.raise_for_status()
+
+            return Image.open(io.BytesIO(image_response.content))
+
+    def generate_image(
+        self,
+        prompt: str,
+        size: str = "1024x1024",
+    ) -> Image.Image:
+        """Generate image from text prompt.
+
+        Args:
+            prompt: Text description of desired image.
+            size: Image size (OpenAI backend only). Options: "1024x1024", "1792x1024", "1024x1792".
+
+        Returns:
+            Generated PIL Image.
+        """
+        if self.backend == "local":
+            return self._generate_local(prompt)
+        elif self.backend == "openai":
+            return self._generate_openai(prompt, size)
+        else:
+            raise UnsupportedBackendError(self.backend, self.SUPPORTED_BACKENDS)