videopython 0.53.0__tar.gz → 0.54.0__tar.gz

{videopython-0.53.0 → videopython-0.54.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: videopython
-Version: 0.53.0
+Version: 0.54.0
 Summary: Minimal video generation and processing library.
 Project-URL: Homepage, https://videopython.com
 Project-URL: Repository, https://github.com/bartwojtowicz/videopython/
@@ -26,7 +26,8 @@ Provides-Extra: ai
 Requires-Dist: accelerate>=0.29.2; extra == 'ai'
 Requires-Dist: chatterbox-tts>=0.1.7; extra == 'ai'
 Requires-Dist: demucs>=4.0.0; extra == 'ai'
-Requires-Dist: diffusers>=0.30.0; extra == 'ai'
+Requires-Dist: diffusers>=0.35.0; extra == 'ai'
+Requires-Dist: ftfy>=6.1; extra == 'ai'
 Requires-Dist: imagehash>=4.3; extra == 'ai'
 Requires-Dist: ollama>=0.5.0; extra == 'ai'
 Requires-Dist: openai-whisper>=20240930; extra == 'ai'

{videopython-0.53.0 → videopython-0.54.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "videopython"
-version = "0.53.0"
+version = "0.54.0"
 description = "Minimal video generation and processing library."
 authors = [
     { name = "Bartosz Wójtowicz", email = "bartoszwojtowicz@outlook.com" },
@@ -78,8 +78,13 @@ ai = [
     # Voice cloning TTS (generation/audio.py — Chatterbox); its strict pins drive
     # the [tool.uv].override-dependencies block below.
     "chatterbox-tts>=0.1.7",
-    # Local media generation: SDXL/CogVideoX + MusicGen (generation/*)
-    "diffusers>=0.30.0",
+    # Local media generation: Qwen-Image/Wan2.2 + MusicGen (generation/*).
+    # >=0.35 is the floor that ships QwenImagePipeline + Wan2.2 support (tested on 0.37.1).
+    "diffusers>=0.35.0",
+    # Wan2.2 i2v pipeline cleans prompts with ftfy via a hard import (diffusers'
+    # pipeline_wan_i2v); the t2v pipeline guards it with is_ftfy_available(), i2v
+    # does not, so it is a required dep for ImageToVideo.
+    "ftfy>=6.1",
     "accelerate>=0.29.2",
     # Dubbing loudness matching (dubbing/loudness.py)
     "pyloudnorm>=0.1.1",
@@ -136,7 +141,7 @@ ignore_missing_imports = true
 # These overrides let the combined [ai] resolve pick compatible versions; the
 # floors in [ai] are aligned with them so pip and uv resolve similar versions.
 override-dependencies = [
-    "torch>=2.8.0", "torchaudio>=2.8.0", "torchvision>=0.23.0,<0.24.0", "numpy>=2.0.0", "diffusers>=0.30.0",
+    "torch>=2.8.0", "torchaudio>=2.8.0", "torchvision>=0.23.0,<0.24.0", "numpy>=2.0.0", "diffusers>=0.35.0",
     # Some transitive deps pull opencv-python, which conflicts with our
     # opencv-python-headless (both provide cv2). Exclude opencv-python so
     # only the headless variant is installed.

{videopython-0.53.0 → videopython-0.54.0}/src/videopython/ai/_revisions.py

@@ -71,11 +71,14 @@ MODEL_REVISIONS: dict[str, str] = {
     "opencv/face_detection_yunet": "3cc26e7f1014a5ee5d74a42acee58bafc9d0a310",
     # MusicGen (ai/generation/audio.py: TextToMusic)
     "facebook/musicgen-small": "4c8334b02c6ec4e8664a91979669a501ec497792",
-    # SDXL (ai/generation/image.py: TextToImage)
-    "stabilityai/stable-diffusion-xl-base-1.0": "462165984030d82259a11f4367a4eed129e94a7b",
-    # CogVideoX (ai/generation/video.py: TextToVideo / ImageToVideo)
-    "THUDM/CogVideoX1.5-5B": "fdc5267c90b5c06492985b966e43aae984e189e0",
-    "THUDM/CogVideoX1.5-5B-I2V": "46c90528707aebbe69066390b4fe7e7d24c9c2a4",
+    # Text-to-image — Qwen-Image (ai/generation/image.py: TextToImage), Apache-2.0;
+    # replaced the OpenRAIL++-licensed SDXL.
+    "Qwen/Qwen-Image-2512": "25468b98e3276ca6700de15c6628e51b7de54a26",
+    # Text/image-to-video — Wan2.2 (ai/generation/video.py: TextToVideo / ImageToVideo),
+    # Apache-2.0; replaced the custom-licensed CogVideoX. The fp32 VAE is loaded from the
+    # same repo (subfolder="vae"), so revision=pinned(repo) covers both from_pretrained calls.
+    "Wan-AI/Wan2.2-T2V-A14B-Diffusers": "5be7df9619b54f4e2667b2755bc6a756675b5cd7",
+    "Wan-AI/Wan2.2-I2V-A14B-Diffusers": "596658fd9ca6b7b71d5057529bbf319ecbc61d74",
 }
 
 

{videopython-0.53.0 → videopython-0.54.0}/src/videopython/ai/generation/__init__.py

@@ -2,7 +2,7 @@
 
 Each symbol's backing leaf module is imported only on first access, so
 ``from videopython.ai.generation import TextToImage`` does not pull in
-``audio`` (chatterbox/musicgen) or ``video`` (diffusers/CogVideoX). The
+``audio`` (chatterbox/musicgen) or ``video`` (diffusers/Wan2.2). The
 ``TYPE_CHECKING`` block keeps the symbols visible to mypy and IDEs.
 """
 

videopython-0.54.0/src/videopython/ai/generation/image.py

@@ -0,0 +1,93 @@
+"""Image generation using local diffusion models (Qwen-Image)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from PIL import Image
+
+from videopython.ai._device import log_device_initialization, select_device
+from videopython.ai._predictor import ManagedPredictor
+from videopython.ai._revisions import pinned
+
+_MODEL_NAME = "Qwen/Qwen-Image-2512"
+
+# Qwen-Image recommends appending a quality "magic" suffix to the prompt
+# (verbatim from the model card).
+_POSITIVE_MAGIC = ", Ultra HD, 4K, cinematic composition."
+
+
+class TextToImage(ManagedPredictor):
+    """Generates images from text descriptions using local models (Qwen-Image, Apache-2.0)."""
+
+    _model_attrs = ("_pipeline",)
+
+    def __init__(self, device: str | None = None):
+        self.device = device
+        self._pipeline: Any = None
+
+    def _init_local(self) -> None:
+        """Initialize the local Qwen-Image diffusion pipeline (CUDA-only)."""
+        import torch
+
+        from videopython.ai._optional import require
+
+        requested_device = self.device
+        device = select_device(self.device, mps_allowed=False)
+        if device != "cuda":
+            raise RuntimeError("TextToImage requires a CUDA GPU; Qwen-Image (~20B) is impractical on CPU/MPS.")
+
+        QwenImagePipeline = require("diffusers", feature="TextToImage").QwenImagePipeline
+        self._pipeline = QwenImagePipeline.from_pretrained(
+            _MODEL_NAME,
+            revision=pinned(_MODEL_NAME),
+            torch_dtype=torch.bfloat16,
+            use_safetensors=True,
+        )
+        # ~20B params (Qwen2.5-VL text encoder + transformer + VAE). Offload submodules
+        # to the GPU on demand so it fits a single GPU; offload manages device placement,
+        # so we must NOT also call .to("cuda").
+        self._pipeline.enable_model_cpu_offload()
+        self._pipeline.enable_vae_tiling()
+
+        self.device = device
+        log_device_initialization(
+            "TextToImage",
+            requested_device=requested_device,
+            resolved_device=device,
+        )
+
+    def generate_image(
+        self,
+        prompt: str,
+        *,
+        negative_prompt: str = " ",
+        true_cfg_scale: float = 4.0,
+        num_inference_steps: int = 50,
+        width: int = 1328,
+        height: int = 1328,
+        add_magic: bool = True,
+        seed: int = 42,
+    ) -> Image.Image:
+        """Generate an image from a text prompt.
+
+        Qwen-Image uses ``true_cfg_scale`` (not ``guidance_scale``) for
+        classifier-free guidance; a non-empty ``negative_prompt`` (default a single
+        space) is required to enable it. ``add_magic`` appends the model's
+        recommended quality suffix to ``prompt``.
+        """
+        import torch
+
+        if self._pipeline is None:
+            self._init_local()
+
+        full_prompt = prompt + _POSITIVE_MAGIC if add_magic else prompt
+        return self._pipeline(
+            prompt=full_prompt,
+            negative_prompt=negative_prompt,
+            true_cfg_scale=true_cfg_scale,
+            num_inference_steps=num_inference_steps,
+            width=width,
+            height=height,
+            generator=torch.Generator(device=self.device).manual_seed(seed),
+        ).images[0]

videopython-0.54.0/src/videopython/ai/generation/video.py

@@ -0,0 +1,207 @@
+"""Video generation using local diffusion models (Wan2.2)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from videopython.ai._device import log_device_initialization, select_device
+from videopython.ai._predictor import ManagedPredictor
+from videopython.ai._revisions import pinned
+from videopython.base.video import Video
+
+if TYPE_CHECKING:
+    from PIL.Image import Image
+
+# Canonical Wan2.2 negative prompt (from the diffusers WanPipeline docstring example).
+_WAN_NEGATIVE_PROMPT = (
+    "Bright tones, overexposed, static, blurred details, subtitles, style, works, "
+    "paintings, images, static, overall gray, worst quality, low quality, JPEG "
+    "compression residue, ugly, incomplete, extra fingers, poorly drawn hands, "
+    "poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, "
+    "still picture, messy background, three legs, many people in the background, "
+    "walking backwards"
+)
+
+
+def _retie_umt5_embeddings(pipeline: Any) -> None:
+    """Restore Wan's text conditioning under transformers>=5.2.
+
+    Wan's checkpoint ships the UMT5 token embedding only as ``shared.weight``, but
+    transformers 5.2's ``UMT5EncoderModel`` has ``tie_word_embeddings=False`` and reads a
+    *separate* ``encoder.embed_tokens`` that the checkpoint never populates -- so it stays
+    zero-initialized. The encoder then emits an all-zero embedding for every prompt and
+    generation silently ignores the text (verified on GPU: prompt has no effect). Point the
+    embedding lookup back at the loaded ``shared`` weight to make conditioning live again.
+    """
+    text_encoder = pipeline.text_encoder
+    text_encoder.encoder.embed_tokens = text_encoder.shared
+
+
+def _require_cuda_device(device: str | None, component: str) -> str:
+    """Resolve to a CUDA device or raise.
+
+    Wan2.2-A14B (~28B MoE) is too large to run on CPU/MPS, so generation requires
+    a CUDA GPU rather than silently falling back to an unusable device.
+    """
+    selected_device = select_device(device, mps_allowed=False)
+    if selected_device != "cuda":
+        raise RuntimeError(f"{component} requires a CUDA GPU; Wan2.2-A14B (~28B) is impractical on CPU/MPS.")
+    return selected_device
+
+
+class TextToVideo(ManagedPredictor):
+    """Generates videos from text descriptions using Wan2.2-T2V (Apache-2.0)."""
+
+    _model_attrs = ("_pipeline",)
+
+    def __init__(self, device: str | None = None):
+        self.device = device
+        self._pipeline: Any = None
+
+    def _init_local(self) -> None:
+        import torch
+
+        from videopython.ai._optional import require
+
+        diffusers = require("diffusers", feature="TextToVideo")
+        WanPipeline = diffusers.WanPipeline
+        AutoencoderKLWan = diffusers.AutoencoderKLWan
+
+        requested_device = self.device
+        device = _require_cuda_device(self.device, "TextToVideo")
+
+        model_name = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
+        revision = pinned(model_name)
+        # VAE stays float32 for decode quality; the transformer/pipeline use bf16.
+        vae = AutoencoderKLWan.from_pretrained(
+            model_name, subfolder="vae", revision=revision, torch_dtype=torch.float32
+        )
+        self._pipeline = WanPipeline.from_pretrained(model_name, vae=vae, revision=revision, torch_dtype=torch.bfloat16)
+        _retie_umt5_embeddings(self._pipeline)
+        # A14B is a MoE (high+low-noise experts), too large for a single-GPU .to("cuda");
+        # offload submodules on demand (offload manages placement, so do NOT also call .to).
+        self._pipeline.enable_model_cpu_offload()
+
+        self.device = device
+        log_device_initialization(
+            "TextToVideo",
+            requested_device=requested_device,
+            resolved_device=device,
+        )
+
+    def generate_video(
+        self,
+        prompt: str,
+        num_steps: int = 40,
+        num_frames: int = 81,
+        guidance_scale: float = 4.0,
+    ) -> Video:
+        """Generate video from text prompt."""
+        import torch
+
+        if self._pipeline is None:
+            self._init_local()
+
+        # output_type="pil" is required: the "np" default returns float32 in [0, 1],
+        # which the uint8 cast below would floor to an all-black video.
+        video_frames = self._pipeline(
+            prompt=prompt,
+            negative_prompt=_WAN_NEGATIVE_PROMPT,
+            height=720,
+            width=1280,
+            num_frames=num_frames,
+            num_inference_steps=num_steps,
+            guidance_scale=guidance_scale,
+            guidance_scale_2=3.0,  # low-noise MoE expert; None would reuse guidance_scale
+            output_type="pil",
+            generator=torch.Generator(device=self.device).manual_seed(42),
+        ).frames[0]
+        video_frames = np.asarray(video_frames, dtype=np.uint8)
+        return Video.from_frames(video_frames, fps=16.0)
+
+
+class ImageToVideo(ManagedPredictor):
+    """Generates videos from static images using Wan2.2-I2V (Apache-2.0)."""
+
+    _model_attrs = ("_pipeline",)
+
+    def __init__(self, device: str | None = None):
+        self.device = device
+        self._pipeline: Any = None
+
+    def _init_local(self) -> None:
+        import torch
+
+        from videopython.ai._optional import require
+
+        diffusers = require("diffusers", feature="ImageToVideo")
+        WanImageToVideoPipeline = diffusers.WanImageToVideoPipeline
+        AutoencoderKLWan = diffusers.AutoencoderKLWan
+
+        requested_device = self.device
+        device = _require_cuda_device(self.device, "ImageToVideo")
+
+        model_name = "Wan-AI/Wan2.2-I2V-A14B-Diffusers"
+        revision = pinned(model_name)
+        vae = AutoencoderKLWan.from_pretrained(
+            model_name, subfolder="vae", revision=revision, torch_dtype=torch.float32
+        )
+        self._pipeline = WanImageToVideoPipeline.from_pretrained(
+            model_name, vae=vae, revision=revision, torch_dtype=torch.bfloat16
+        )
+        _retie_umt5_embeddings(self._pipeline)
+        # A14B MoE is too large for a single-GPU .to("cuda"); offload submodules on demand.
+        self._pipeline.enable_model_cpu_offload()
+
+        self.device = device
+        log_device_initialization(
+            "ImageToVideo",
+            requested_device=requested_device,
+            resolved_device=device,
+        )
+
+    def _resize_to_model_grid(self, image: Image) -> tuple[Image, int, int]:
+        """Resize ``image`` to Wan's area budget, snapped to the model's spatial grid.
+
+        Returns the resized image plus the ``(height, width)`` to request, derived
+        from the input aspect ratio against Wan's 480x832 area (per the model card).
+        """
+        max_area = 480 * 832
+        aspect_ratio = image.height / image.width
+        mod_value = self._pipeline.vae_scale_factor_spatial * self._pipeline.transformer.config.patch_size[1]
+        height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
+        width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
+        return image.resize((width, height)), height, width
+
+    def generate_video(
+        self,
+        image: Image,
+        prompt: str = "",
+        num_steps: int = 40,
+        num_frames: int = 81,
+        guidance_scale: float = 3.5,
+    ) -> Video:
+        """Generate video animation from a static image."""
+        import torch
+
+        if self._pipeline is None:
+            self._init_local()
+
+        image, height, width = self._resize_to_model_grid(image)
+        video_frames = self._pipeline(
+            image=image,
+            prompt=prompt,
+            negative_prompt=_WAN_NEGATIVE_PROMPT,
+            height=height,
+            width=width,
+            num_frames=num_frames,
+            num_inference_steps=num_steps,
+            guidance_scale=guidance_scale,
+            # No guidance_scale_2: the low-noise MoE expert reuses guidance_scale (its None default).
+            output_type="pil",
+            generator=torch.Generator(device=self.device).manual_seed(42),
+        ).frames[0]
+        video_frames = np.asarray(video_frames, dtype=np.uint8)
+        return Video.from_frames(video_frames, fps=16.0)

videopython-0.53.0/src/videopython/ai/generation/image.py

@@ -1,59 +0,0 @@
-"""Image generation using local diffusion models."""
-
-from __future__ import annotations
-
-from typing import Any
-
-from PIL import Image
-
-from videopython.ai._device import log_device_initialization, select_device
-from videopython.ai._predictor import ManagedPredictor
-from videopython.ai._revisions import pinned
-
-
-class TextToImage(ManagedPredictor):
-    """Generates images from text descriptions using local models."""
-
-    _model_attrs = ("_pipeline",)
-
-    def __init__(self, device: str | None = None):
-        self.device = device
-        self._pipeline: Any = None
-
-    def _init_local(self) -> None:
-        """Initialize local diffusion pipeline."""
-        import torch
-
-        from videopython.ai._optional import require
-
-        DiffusionPipeline = require("diffusers", feature="TextToImage").DiffusionPipeline
-
-        requested_device = self.device
-        device = select_device(self.device, mps_allowed=True)
-        dtype = torch.float16 if device == "cuda" else torch.float32
-        variant = "fp16" if device == "cuda" else None
-
-        model_name = "stabilityai/stable-diffusion-xl-base-1.0"
-        self._pipeline = DiffusionPipeline.from_pretrained(
-            model_name,
-            revision=pinned(model_name),
-            torch_dtype=dtype,
-            variant=variant,
-            use_safetensors=True,
-        )
-        self._pipeline.to(device)
-        self.device = device
-        log_device_initialization(
-            "TextToImage",
-            requested_device=requested_device,
-            resolved_device=device,
-        )
-
-        if device == "mps":
-            self._pipeline.enable_attention_slicing()
-
-    def generate_image(self, prompt: str) -> Image.Image:
-        """Generate an image from a text prompt."""
-        if self._pipeline is None:
-            self._init_local()
-        return self._pipeline(prompt=prompt).images[0]

videopython-0.53.0/src/videopython/ai/generation/video.py

@@ -1,131 +0,0 @@
-"""Video generation using local diffusion models."""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any
-
-import numpy as np
-
-from videopython.ai._device import log_device_initialization, select_device
-from videopython.ai._predictor import ManagedPredictor
-from videopython.ai._revisions import pinned
-from videopython.base.video import Video
-
-if TYPE_CHECKING:
-    from PIL.Image import Image
-
-
-def _get_torch_device_and_dtype(device: str | None) -> tuple[str, Any]:
-    """Get the best available torch device and dtype for CogVideoX."""
-    import torch
-
-    selected_device = select_device(device, mps_allowed=False)
-    if selected_device == "cuda":
-        return selected_device, torch.bfloat16
-    return selected_device, torch.float32
-
-
-class TextToVideo(ManagedPredictor):
-    """Generates videos from text descriptions using local diffusion models."""
-
-    _model_attrs = ("_pipeline",)
-
-    def __init__(self, device: str | None = None):
-        self.device = device
-        self._pipeline: Any = None
-
-    def _init_local(self) -> None:
-        from videopython.ai._optional import require
-
-        CogVideoXPipeline = require("diffusers", feature="TextToVideo").CogVideoXPipeline
-
-        requested_device = self.device
-        device, dtype = _get_torch_device_and_dtype(self.device)
-
-        model_name = "THUDM/CogVideoX1.5-5B"
-        self._pipeline = CogVideoXPipeline.from_pretrained(model_name, revision=pinned(model_name), torch_dtype=dtype)
-        self._pipeline.to(device)
-        self.device = device
-        log_device_initialization(
-            "TextToVideo",
-            requested_device=requested_device,
-            resolved_device=device,
-        )
-
-    def generate_video(
-        self,
-        prompt: str,
-        num_steps: int = 50,
-        num_frames: int = 81,
-        guidance_scale: float = 6.0,
-    ) -> Video:
-        """Generate video from text prompt."""
-        import torch
-
-        if self._pipeline is None:
-            self._init_local()
-
-        video_frames = self._pipeline(
-            prompt=prompt,
-            num_inference_steps=num_steps,
-            num_frames=num_frames,
-            guidance_scale=guidance_scale,
-            generator=torch.Generator(device=self.device).manual_seed(42),
-        ).frames[0]
-        video_frames = np.asarray(video_frames, dtype=np.uint8)
-        return Video.from_frames(video_frames, fps=16.0)
-
-
-class ImageToVideo(ManagedPredictor):
-    """Generates videos from static images using local video diffusion."""
-
-    _model_attrs = ("_pipeline",)
-
-    def __init__(self, device: str | None = None):
-        self.device = device
-        self._pipeline: Any = None
-
-    def _init_local(self) -> None:
-        from videopython.ai._optional import require
-
-        CogVideoXImageToVideoPipeline = require("diffusers", feature="ImageToVideo").CogVideoXImageToVideoPipeline
-
-        requested_device = self.device
-        device, dtype = _get_torch_device_and_dtype(self.device)
-
-        model_name = "THUDM/CogVideoX1.5-5B-I2V"
-        self._pipeline = CogVideoXImageToVideoPipeline.from_pretrained(
-            model_name, revision=pinned(model_name), torch_dtype=dtype
-        )
-        self._pipeline.to(device)
-        self.device = device
-        log_device_initialization(
-            "ImageToVideo",
-            requested_device=requested_device,
-            resolved_device=device,
-        )
-
-    def generate_video(
-        self,
-        image: Image,
-        prompt: str = "",
-        num_steps: int = 50,
-        num_frames: int = 81,
-        guidance_scale: float = 6.0,
-    ) -> Video:
-        """Generate video animation from a static image."""
-        import torch
-
-        if self._pipeline is None:
-            self._init_local()
-
-        video_frames = self._pipeline(
-            prompt=prompt,
-            image=image,
-            num_inference_steps=num_steps,
-            num_frames=num_frames,
-            guidance_scale=guidance_scale,
-            generator=torch.Generator(device=self.device).manual_seed(42),
-        ).frames[0]
-        video_frames = np.asarray(video_frames, dtype=np.uint8)
-        return Video.from_frames(video_frames, fps=16.0)