cache-dit 0.2.25__py3-none-any.whl → 0.2.27__py3-none-any.whl

cache_dit/cache_factory/patch_functors/functor_chroma.py

@@ -0,0 +1,276 @@
+import inspect
+
+import torch
+import numpy as np
+from typing import Tuple, Optional, Dict, Any, Union
+from diffusers import ChromaTransformer2DModel
+from diffusers.models.transformers.transformer_chroma import (
+    ChromaSingleTransformerBlock,
+    Transformer2DModelOutput,
+)
+from diffusers.utils import (
+    USE_PEFT_BACKEND,
+    scale_lora_layers,
+    unscale_lora_layers,
+)
+
+from cache_dit.cache_factory.patch_functors.functor_base import (
+    PatchFunctor,
+)
+from cache_dit.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+class ChromaPatchFunctor(PatchFunctor):
+
+    def apply(
+        self,
+        transformer: ChromaTransformer2DModel,
+        blocks: torch.nn.ModuleList = None,
+        **kwargs,
+    ) -> ChromaTransformer2DModel:
+        if getattr(transformer, "_is_patched", False):
+            return transformer
+
+        if blocks is None:
+            blocks = transformer.single_transformer_blocks
+
+        is_patched = False
+        for block in blocks:
+            if isinstance(block, ChromaSingleTransformerBlock):
+                forward_parameters = inspect.signature(
+                    block.forward
+                ).parameters.keys()
+                if "encoder_hidden_states" not in forward_parameters:
+                    block.forward = __patch_single_forward__.__get__(block)
+                    is_patched = True
+
+        if is_patched:
+            logger.warning("Patched Chroma for cache-dit.")
+            assert not getattr(transformer, "_is_parallelized", False), (
+                "Please call `cache_dit.enable_cache` before Parallelize, "
+                "the __patch_transformer_forward__ will overwrite the "
+                "parallized forward and cause a downgrade of performance."
+            )
+            transformer.forward = __patch_transformer_forward__.__get__(
+                transformer
+            )
+            transformer._is_patched = True
+
+        cls_name = transformer.__class__.__name__
+        logger.info(
+            f"Applied {self.__class__.__name__} for {cls_name}, "
+            f"Patch: {is_patched}."
+        )
+
+        return transformer
+
+
+# adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_chroma.py
+def __patch_single_forward__(
+    self: ChromaSingleTransformerBlock,  # Almost same as FluxSingleTransformerBlock
+    hidden_states: torch.Tensor,
+    encoder_hidden_states: torch.Tensor,
+    temb: torch.Tensor,
+    image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+    joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    text_seq_len = encoder_hidden_states.shape[1]
+    hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1)
+
+    residual = hidden_states
+    norm_hidden_states, gate = self.norm(hidden_states, emb=temb)
+    mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states))
+    joint_attention_kwargs = joint_attention_kwargs or {}
+    attn_output = self.attn(
+        hidden_states=norm_hidden_states,
+        image_rotary_emb=image_rotary_emb,
+        **joint_attention_kwargs,
+    )
+
+    hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2)
+    gate = gate.unsqueeze(1)
+    hidden_states = gate * self.proj_out(hidden_states)
+    hidden_states = residual + hidden_states
+    if hidden_states.dtype == torch.float16:
+        hidden_states = hidden_states.clip(-65504, 65504)
+
+    encoder_hidden_states, hidden_states = (
+        hidden_states[:, :text_seq_len],
+        hidden_states[:, text_seq_len:],
+    )
+    return encoder_hidden_states, hidden_states
+
+
+# Adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_chroma.py
+def __patch_transformer_forward__(
+    self: ChromaTransformer2DModel,
+    hidden_states: torch.Tensor,
+    encoder_hidden_states: torch.Tensor = None,
+    timestep: torch.LongTensor = None,
+    img_ids: torch.Tensor = None,
+    txt_ids: torch.Tensor = None,
+    attention_mask: torch.Tensor = None,
+    joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+    controlnet_block_samples=None,
+    controlnet_single_block_samples=None,
+    return_dict: bool = True,
+    controlnet_blocks_repeat: bool = False,
+) -> Union[torch.Tensor, Transformer2DModelOutput]:
+    if joint_attention_kwargs is not None:
+        joint_attention_kwargs = joint_attention_kwargs.copy()
+        lora_scale = joint_attention_kwargs.pop("scale", 1.0)
+    else:
+        lora_scale = 1.0
+
+    if USE_PEFT_BACKEND:
+        # weight the lora layers by setting `lora_scale` for each PEFT layer
+        scale_lora_layers(self, lora_scale)
+    else:
+        if (
+            joint_attention_kwargs is not None
+            and joint_attention_kwargs.get("scale", None) is not None
+        ):
+            logger.warning(
+                "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective."
+            )
+
+    hidden_states = self.x_embedder(hidden_states)
+
+    timestep = timestep.to(hidden_states.dtype) * 1000
+
+    input_vec = self.time_text_embed(timestep)
+    pooled_temb = self.distilled_guidance_layer(input_vec)
+
+    encoder_hidden_states = self.context_embedder(encoder_hidden_states)
+
+    if txt_ids.ndim == 3:
+        logger.warning(
+            "Passing `txt_ids` 3d torch.Tensor is deprecated."
+            "Please remove the batch dimension and pass it as a 2d torch Tensor"
+        )
+        txt_ids = txt_ids[0]
+    if img_ids.ndim == 3:
+        logger.warning(
+            "Passing `img_ids` 3d torch.Tensor is deprecated."
+            "Please remove the batch dimension and pass it as a 2d torch Tensor"
+        )
+        img_ids = img_ids[0]
+
+    ids = torch.cat((txt_ids, img_ids), dim=0)
+    image_rotary_emb = self.pos_embed(ids)
+
+    if (
+        joint_attention_kwargs is not None
+        and "ip_adapter_image_embeds" in joint_attention_kwargs
+    ):
+        ip_adapter_image_embeds = joint_attention_kwargs.pop(
+            "ip_adapter_image_embeds"
+        )
+        ip_hidden_states = self.encoder_hid_proj(ip_adapter_image_embeds)
+        joint_attention_kwargs.update({"ip_hidden_states": ip_hidden_states})
+
+    for index_block, block in enumerate(self.transformer_blocks):
+        img_offset = 3 * len(self.single_transformer_blocks)
+        txt_offset = img_offset + 6 * len(self.transformer_blocks)
+        img_modulation = img_offset + 6 * index_block
+        text_modulation = txt_offset + 6 * index_block
+        temb = torch.cat(
+            (
+                pooled_temb[:, img_modulation : img_modulation + 6],
+                pooled_temb[:, text_modulation : text_modulation + 6],
+            ),
+            dim=1,
+        )
+        if torch.is_grad_enabled() and self.gradient_checkpointing:
+            encoder_hidden_states, hidden_states = (
+                self._gradient_checkpointing_func(
+                    block,
+                    hidden_states,
+                    encoder_hidden_states,
+                    temb,
+                    image_rotary_emb,
+                    attention_mask,
+                )
+            )
+
+        else:
+            encoder_hidden_states, hidden_states = block(
+                hidden_states=hidden_states,
+                encoder_hidden_states=encoder_hidden_states,
+                temb=temb,
+                image_rotary_emb=image_rotary_emb,
+                attention_mask=attention_mask,
+                joint_attention_kwargs=joint_attention_kwargs,
+            )
+
+        # controlnet residual
+        if controlnet_block_samples is not None:
+            interval_control = len(self.transformer_blocks) / len(
+                controlnet_block_samples
+            )
+            interval_control = int(np.ceil(interval_control))
+            # For Xlabs ControlNet.
+            if controlnet_blocks_repeat:
+                hidden_states = (
+                    hidden_states
+                    + controlnet_block_samples[
+                        index_block % len(controlnet_block_samples)
+                    ]
+                )
+            else:
+                hidden_states = (
+                    hidden_states
+                    + controlnet_block_samples[index_block // interval_control]
+                )
+
+    for index_block, block in enumerate(self.single_transformer_blocks):
+        start_idx = 3 * index_block
+        temb = pooled_temb[:, start_idx : start_idx + 3]
+        if torch.is_grad_enabled() and self.gradient_checkpointing:
+            encoder_hidden_states, hidden_states = (
+                self._gradient_checkpointing_func(
+                    block,
+                    hidden_states,
+                    encoder_hidden_states,
+                    temb,
+                    image_rotary_emb,
+                )
+            )
+
+        else:
+            encoder_hidden_states, hidden_states = block(
+                hidden_states=hidden_states,
+                encoder_hidden_states=encoder_hidden_states,
+                temb=temb,
+                image_rotary_emb=image_rotary_emb,
+                attention_mask=attention_mask,
+                joint_attention_kwargs=joint_attention_kwargs,
+            )
+
+        # controlnet residual
+        if controlnet_single_block_samples is not None:
+            interval_control = len(self.single_transformer_blocks) / len(
+                controlnet_single_block_samples
+            )
+            interval_control = int(np.ceil(interval_control))
+            hidden_states = (
+                hidden_states
+                + controlnet_single_block_samples[
+                    index_block // interval_control
+                ]
+            )
+
+    temb = pooled_temb[:, -2:]
+    hidden_states = self.norm_out(hidden_states, temb)
+    output = self.proj_out(hidden_states)
+
+    if USE_PEFT_BACKEND:
+        # remove `lora_scale` from each PEFT layer
+        unscale_lora_layers(self, lora_scale)
+
+    if not return_dict:
+        return (output,)
+
+    return Transformer2DModelOutput(sample=output)

cache_dit/cache_factory/{patch/flux.py → patch_functors/functor_flux.py}

@@ -14,12 +14,60 @@ from diffusers.utils import (
     unscale_lora_layers,
 )
 
-
+from cache_dit.cache_factory.patch_functors.functor_base import (
+    PatchFunctor,
+)
 from cache_dit.logger import init_logger
 
 logger = init_logger(__name__)
 
 
+class FluxPatchFunctor(PatchFunctor):
+
+    def apply(
+        self,
+        transformer: FluxTransformer2DModel,
+        blocks: torch.nn.ModuleList = None,
+        **kwargs,
+    ) -> FluxTransformer2DModel:
+
+        if getattr(transformer, "_is_patched", False):
+            return transformer
+
+        if blocks is None:
+            blocks = transformer.single_transformer_blocks
+
+        is_patched = False
+        for block in blocks:
+            if isinstance(block, FluxSingleTransformerBlock):
+                forward_parameters = inspect.signature(
+                    block.forward
+                ).parameters.keys()
+                if "encoder_hidden_states" not in forward_parameters:
+                    block.forward = __patch_single_forward__.__get__(block)
+                    is_patched = True
+
+        if is_patched:
+            logger.warning("Patched Flux for cache-dit.")
+            assert not getattr(transformer, "_is_parallelized", False), (
+                "Please call `cache_dit.enable_cache` before Parallelize, "
+                "the __patch_transformer_forward__ will overwrite the "
+                "parallized forward and cause a downgrade of performance."
+            )
+            transformer.forward = __patch_transformer_forward__.__get__(
+                transformer
+            )
+            transformer._is_patched = True
+
+        cls_name = transformer.__class__.__name__
+        logger.info(
+            f"Applied {self.__class__.__name__} for {cls_name}, "
+            f"Patch: {is_patched}."
+        )
+
+        return transformer
+
+
 # copy from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_flux.py#L380
 def __patch_single_forward__(
     self: FluxSingleTransformerBlock,
@@ -217,33 +265,3 @@ def __patch_transformer_forward__(
         return (output,)
 
     return Transformer2DModelOutput(sample=output)
-
-
-def maybe_patch_flux_transformer(
-    transformer: FluxTransformer2DModel,
-    blocks: torch.nn.ModuleList = None,
-) -> FluxTransformer2DModel:
-    if blocks is None:
-        blocks = transformer.single_transformer_blocks
-
-    is_patched = False
-    for block in blocks:
-        if isinstance(block, FluxSingleTransformerBlock):
-            forward_parameters = inspect.signature(
-                block.forward
-            ).parameters.keys()
-            if "encoder_hidden_states" not in forward_parameters:
-                block.forward = __patch_single_forward__.__get__(block)
-                is_patched = True
-
-    if is_patched:
-        logger.warning("Patched Flux for cache-dit.")
-        assert not getattr(transformer, "_is_parallelized", False), (
-            "Please call apply_cache_on_pipe before Parallelize, "
-            "the __patch_transformer_forward__ will overwrite the "
-            "parallized forward and cause a downgrade of performance."
-        )
-        transformer.forward = __patch_transformer_forward__.__get__(transformer)
-        transformer._is_patched = True
-
-    return transformer

cache_dit/quantize/quantize_ao.py

@@ -10,12 +10,13 @@ logger = init_logger(__name__)
 def quantize_ao(
     module: torch.nn.Module,
     quant_type: str = "fp8_w8a8_dq",
-    per_row: bool = True,
     exclude_layers: List[str] = [
         "embedder",
         "embed",
     ],
     filter_fn: Optional[Callable] = None,
+    # paramters for fp8 quantization
+    per_row: bool = True,
     **kwargs,
 ) -> torch.nn.Module:
     # Apply FP8 DQ for module and skip any `embed` modules
@@ -89,17 +90,30 @@ def quantize_ao(
                 )
 
                 quantization_fn = float8_dynamic_activation_float8_weight(
+                    weight_dtype=kwargs.get(
+                        "weight_dtype",
+                        torch.float8_e4m3fn,
+                    ),
+                    activation_dtype=kwargs.get(
+                        "activation_dtype",
+                        torch.float8_e4m3fn,
+                    ),
                     granularity=(
                         ((PerRow(), PerRow()))
                         if per_row
                         else ((PerTensor(), PerTensor()))
-                    )
+                    ),
                 )
 
             elif quant_type == "fp8_w8a16_wo":
                 from torchao.quantization import float8_weight_only
 
-                quantization_fn = float8_weight_only()
+                quantization_fn = float8_weight_only(
+                    weight_dtype=kwargs.get(
+                        "weight_dtype",
+                        torch.float8_e4m3fn,
+                    ),
+                )
 
             elif quant_type == "int8_w8a8_dq":
                 from torchao.quantization import (
@@ -159,12 +173,13 @@ def quantize_ao(
         module,
         _quantization_fn(),
         filter_fn=_filter_fn if filter_fn is None else filter_fn,
-        **kwargs,
+        device=kwargs.get("device", None),
     )
 
     force_empty_cache()
 
     logger.info(
+        f"Quantized        Method: {quant_type:>5}\n"
         f"Quantized Linear Layers: {num_quant_linear:>5}\n"
         f"Skipped   Linear Layers: {num_skip_linear:>5}\n"
         f"Total     Linear Layers: {num_linear_layers:>5}\n"

cache_dit/quantize/quantize_interface.py

@@ -9,13 +9,13 @@ def quantize(
     module: torch.nn.Module,
     quant_type: str = "fp8_w8a8_dq",
     backend: str = "ao",
-    # only for fp8_w8a8_dq
-    per_row: bool = True,
     exclude_layers: List[str] = [
         "embedder",
         "embed",
     ],
     filter_fn: Optional[Callable] = None,
+    # only for fp8_w8a8_dq
+    per_row: bool = True,
     **kwargs,
 ) -> torch.nn.Module:
     assert isinstance(module, torch.nn.Module)

cache_dit/utils.py

@@ -30,26 +30,32 @@ class CacheStats:
 
 
 def summary(
-    pipe_or_transformer: DiffusionPipeline | torch.nn.Module,
+    pipe_or_module: DiffusionPipeline | torch.nn.Module | Any,
     details: bool = False,
     logging: bool = True,
 ) -> CacheStats:
     cache_stats = CacheStats()
-    cls_name = pipe_or_transformer.__class__.__name__
-    if isinstance(pipe_or_transformer, DiffusionPipeline):
-        transformer = pipe_or_transformer.transformer
+
+    if not isinstance(pipe_or_module, torch.nn.Module):
+        assert hasattr(pipe_or_module, "transformer")
+        module = pipe_or_module.transformer
+        cls_name = module.__class__.__name__
     else:
-        transformer = pipe_or_transformer
+        module = pipe_or_module
+
+    cls_name = module.__class__.__name__
+    if isinstance(module, torch.nn.ModuleList):
+        cls_name = module[0].__class__.__name__
 
-    if hasattr(transformer, "_cache_context_kwargs"):
-        cache_options = transformer._cache_context_kwargs
+    if hasattr(module, "_cache_context_kwargs"):
+        cache_options = module._cache_context_kwargs
         cache_stats.cache_options = cache_options
         if logging:
             print(f"\n🤗Cache Options: {cls_name}\n\n{cache_options}")
 
-    if hasattr(transformer, "_cached_steps"):
-        cached_steps: list[int] = transformer._cached_steps
-        residual_diffs: dict[str, float] = dict(transformer._residual_diffs)
+    if hasattr(module, "_cached_steps"):
+        cached_steps: list[int] = module._cached_steps
+        residual_diffs: dict[str, float] = dict(module._residual_diffs)
         cache_stats.cached_steps = cached_steps
         cache_stats.residual_diffs = residual_diffs
 
@@ -90,11 +96,9 @@ def summary(
                     compact=True,
                 )
 
-    if hasattr(transformer, "_cfg_cached_steps"):
-        cfg_cached_steps: list[int] = transformer._cfg_cached_steps
-        cfg_residual_diffs: dict[str, float] = dict(
-            transformer._cfg_residual_diffs
-        )
+    if hasattr(module, "_cfg_cached_steps"):
+        cfg_cached_steps: list[int] = module._cfg_cached_steps
+        cfg_residual_diffs: dict[str, float] = dict(module._cfg_residual_diffs)
         cache_stats.cfg_cached_steps = cfg_cached_steps
         cache_stats.cfg_residual_diffs = cfg_residual_diffs