cache-dit 0.2.24__py3-none-any.whl → 0.2.26__py3-none-any.whl

cache_dit/__init__.py

@@ -11,7 +11,10 @@ from cache_dit.cache_factory import block_range
 from cache_dit.cache_factory import CacheType
 from cache_dit.cache_factory import BlockAdapter
 from cache_dit.cache_factory import ForwardPattern
+from cache_dit.cache_factory import PatchFunctor
+from cache_dit.cache_factory import supported_pipelines
 from cache_dit.compile import set_compile_configs
+from cache_dit.quantize import quantize
 from cache_dit.utils import summary
 from cache_dit.utils import strify
 from cache_dit.logger import init_logger
@@ -23,3 +26,5 @@ Forward_Pattern_0 = ForwardPattern.Pattern_0
 Forward_Pattern_1 = ForwardPattern.Pattern_1
 Forward_Pattern_2 = ForwardPattern.Pattern_2
 Forward_Pattern_3 = ForwardPattern.Pattern_3
+Forward_Pattern_4 = ForwardPattern.Pattern_4
+Forward_Pattern_5 = ForwardPattern.Pattern_5

cache_dit/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.24'
-__version_tuple__ = version_tuple = (0, 2, 24)
+__version__ = version = '0.2.26'
+__version_tuple__ = version_tuple = (0, 2, 26)
 
 __commit_id__ = commit_id = None

cache_dit/cache_factory/__init__.py

@@ -5,4 +5,6 @@ from cache_dit.cache_factory.cache_types import block_range
 from cache_dit.cache_factory.cache_adapters import BlockAdapter
 from cache_dit.cache_factory.cache_adapters import UnifiedCacheAdapter
 from cache_dit.cache_factory.cache_interface import enable_cache
+from cache_dit.cache_factory.cache_interface import supported_pipelines
+from cache_dit.cache_factory.patch_functors import PatchFunctor
 from cache_dit.cache_factory.utils import load_options

cache_dit/cache_factory/cache_adapters.py

@@ -8,15 +8,14 @@ import dataclasses
 from typing import Any, Tuple, List, Optional
 from contextlib import ExitStack
 from diffusers import DiffusionPipeline
-from cache_dit.cache_factory.patch.flux import (
-    maybe_patch_flux_transformer,
-)
 from cache_dit.cache_factory import CacheType
+from cache_dit.cache_factory import cache_context
 from cache_dit.cache_factory import ForwardPattern
+from cache_dit.cache_factory.patch_functors import PatchFunctor
 from cache_dit.cache_factory.cache_blocks import (
-    cache_context,
-    DBCachedTransformerBlocks,
+    DBCachedBlocks,
 )
+
 from cache_dit.logger import init_logger
 
 logger = init_logger(__name__)
@@ -24,12 +23,14 @@ logger = init_logger(__name__)
 
 @dataclasses.dataclass
 class BlockAdapter:
-    pipe: DiffusionPipeline = None
+    pipe: DiffusionPipeline | Any = None
     transformer: torch.nn.Module = None
     blocks: torch.nn.ModuleList = None
     # transformer_blocks, blocks, etc.
     blocks_name: str = None
-    dummy_blocks_names: list[str] = dataclasses.field(default_factory=list)
+    dummy_blocks_names: List[str] = dataclasses.field(default_factory=list)
+    # patch functor: Flux, etc.
+    patch_functor: Optional[PatchFunctor] = None
     # flags to control auto block adapter
     auto: bool = False
     allow_prefixes: List[str] = dataclasses.field(
@@ -38,6 +39,8 @@ class BlockAdapter:
             "single_transformer",
             "blocks",
             "layers",
+            "single_stream_blocks",
+            "double_stream_blocks",
         ]
     )
     check_prefixes: bool = True
@@ -50,17 +53,19 @@ class BlockAdapter:
     )
 
     def __post_init__(self):
-        self.maybe_apply_patch()
+        assert any((self.pipe is not None, self.transformer is not None))
+        self.patchify()
 
-    def maybe_apply_patch(self):
+    def patchify(self, *args, **kwargs):
         # Process some specificial cases, specific for transformers
         # that has different forward patterns between single_transformer_blocks
         # and transformer_blocks , such as Flux (diffusers < 0.35.0).
-        if self.transformer.__class__.__name__.startswith("Flux"):
-            self.transformer = maybe_patch_flux_transformer(
-                self.transformer,
-                blocks=self.blocks,
-            )
+        if self.patch_functor is not None:
+            if self.transformer is not None:
+                self.patch_functor.apply(self.transformer, *args, **kwargs)
+            else:
+                assert hasattr(self.pipe, "transformer")
+                self.patch_functor.apply(self.pipe.transformer, *args, **kwargs)
 
     @staticmethod
     def auto_block_adapter(
@@ -99,7 +104,9 @@ class BlockAdapter:
     @staticmethod
     def check_block_adapter(adapter: "BlockAdapter") -> bool:
         if (
-            isinstance(adapter.pipe, DiffusionPipeline)
+            # NOTE: pipe may not need to be DiffusionPipeline?
+            # isinstance(adapter.pipe, DiffusionPipeline)
+            adapter.pipe is not None
             and adapter.transformer is not None
             and adapter.blocks is not None
             and adapter.blocks_name is not None
@@ -287,11 +294,34 @@ class UnifiedCacheAdapter:
         "EasyAnimate",
         "SkyReelsV2",
         "SD3",
+        "ConsisID",
+        "DiT",
+        "Amused",
+        "Bria",
+        "HunyuanDiT",
+        "HunyuanDiTPAG",
+        "Lumina",
+        "Lumina2",
+        "OmniGen",
+        "PixArt",
+        "Sana",
+        "ShapE",
+        "StableAudio",
+        "VisualCloze",
+        "AuraFlow",
+        "Chroma",
+        "HiDream",
     ]
 
     def __call__(self, *args, **kwargs):
         return self.apply(*args, **kwargs)
 
+    @classmethod
+    def supported_pipelines(cls) -> Tuple[int, List[str]]:
+        return len(cls._supported_pipelines), [
+            p + "*" for p in cls._supported_pipelines
+        ]
+
     @classmethod
     def is_supported(cls, pipe: DiffusionPipeline) -> bool:
         pipe_cls_name: str = pipe.__class__.__name__
@@ -303,8 +333,10 @@ class UnifiedCacheAdapter:
     @classmethod
     def get_params(cls, pipe: DiffusionPipeline) -> UnifiedCacheParams:
         pipe_cls_name: str = pipe.__class__.__name__
+
         if pipe_cls_name.startswith("Flux"):
             from diffusers import FluxTransformer2DModel
+            from cache_dit.cache_factory.patch_functors import FluxPatchFunctor
 
             assert isinstance(pipe.transformer, FluxTransformer2DModel)
             return UnifiedCacheParams(
@@ -317,9 +349,11 @@ class UnifiedCacheAdapter:
                     ),
                     blocks_name="transformer_blocks",
                     dummy_blocks_names=["single_transformer_blocks"],
+                    patch_functor=FluxPatchFunctor(),
                 ),
                 forward_pattern=ForwardPattern.Pattern_1,
             )
+
         elif pipe_cls_name.startswith("Mochi"):
             from diffusers import MochiTransformer3DModel
 
@@ -334,6 +368,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_0,
             )
+
         elif pipe_cls_name.startswith("CogVideoX"):
             from diffusers import CogVideoXTransformer3DModel
 
@@ -348,6 +383,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_0,
             )
+
         elif pipe_cls_name.startswith("Wan"):
             from diffusers import (
                 WanTransformer3DModel,
@@ -358,16 +394,35 @@ class UnifiedCacheAdapter:
                 pipe.transformer,
                 (WanTransformer3DModel, WanVACETransformer3DModel),
             )
-            return UnifiedCacheParams(
-                block_adapter=BlockAdapter(
-                    pipe=pipe,
-                    transformer=pipe.transformer,
-                    blocks=pipe.transformer.blocks,
-                    blocks_name="blocks",
-                    dummy_blocks_names=[],
-                ),
-                forward_pattern=ForwardPattern.Pattern_2,
-            )
+            if getattr(pipe, "transformer_2", None):
+                # Wan 2.2, cache for low-noise transformer
+                assert isinstance(
+                    pipe.transformer_2,
+                    (WanTransformer3DModel, WanVACETransformer3DModel),
+                )
+                return UnifiedCacheParams(
+                    block_adapter=BlockAdapter(
+                        pipe=pipe,
+                        transformer=pipe.transformer_2,
+                        blocks=pipe.transformer_2.blocks,
+                        blocks_name="blocks",
+                        dummy_blocks_names=[],
+                    ),
+                    forward_pattern=ForwardPattern.Pattern_2,
+                )
+            else:
+                # Wan 2.1
+                return UnifiedCacheParams(
+                    block_adapter=BlockAdapter(
+                        pipe=pipe,
+                        transformer=pipe.transformer,
+                        blocks=pipe.transformer.blocks,
+                        blocks_name="blocks",
+                        dummy_blocks_names=[],
+                    ),
+                    forward_pattern=ForwardPattern.Pattern_2,
+                )
+
         elif pipe_cls_name.startswith("HunyuanVideo"):
             from diffusers import HunyuanVideoTransformer3DModel
 
@@ -384,6 +439,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_0,
             )
+
         elif pipe_cls_name.startswith("QwenImage"):
             from diffusers import QwenImageTransformer2DModel
 
@@ -398,6 +454,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_1,
             )
+
         elif pipe_cls_name.startswith("LTXVideo"):
             from diffusers import LTXVideoTransformer3DModel
 
@@ -412,6 +469,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_2,
             )
+
         elif pipe_cls_name.startswith("Allegro"):
             from diffusers import AllegroTransformer3DModel
 
@@ -426,6 +484,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_2,
             )
+
         elif pipe_cls_name.startswith("CogView3Plus"):
             from diffusers import CogView3PlusTransformer2DModel
 
@@ -440,6 +499,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_0,
             )
+
         elif pipe_cls_name.startswith("CogView4"):
             from diffusers import CogView4Transformer2DModel
 
@@ -454,6 +514,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_0,
             )
+
         elif pipe_cls_name.startswith("Cosmos"):
             from diffusers import CosmosTransformer3DModel
 
@@ -468,6 +529,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_2,
             )
+
         elif pipe_cls_name.startswith("EasyAnimate"):
             from diffusers import EasyAnimateTransformer3DModel
 
@@ -482,6 +544,7 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_0,
             )
+
         elif pipe_cls_name.startswith("SkyReelsV2"):
             from diffusers import SkyReelsV2Transformer3DModel
 
@@ -510,6 +573,284 @@ class UnifiedCacheAdapter:
                 ),
                 forward_pattern=ForwardPattern.Pattern_1,
             )
+
+        elif pipe_cls_name.startswith("ConsisID"):
+            from diffusers import ConsisIDTransformer3DModel
+
+            assert isinstance(pipe.transformer, ConsisIDTransformer3DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.transformer_blocks,
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_0,
+            )
+
+        elif pipe_cls_name.startswith("DiT"):
+            from diffusers import DiTTransformer2DModel
+
+            assert isinstance(pipe.transformer, DiTTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.transformer_blocks,
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("Amused"):
+            from diffusers import UVit2DModel
+
+            assert isinstance(pipe.transformer, UVit2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.transformer_layers,
+                    blocks_name="transformer_layers",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("Bria"):
+            from diffusers import BriaTransformer2DModel
+
+            assert isinstance(pipe.transformer, BriaTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=(
+                        pipe.transformer.transformer_blocks
+                        + pipe.transformer.single_transformer_blocks
+                    ),
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=["single_transformer_blocks"],
+                ),
+                forward_pattern=ForwardPattern.Pattern_0,
+            )
+
+        elif pipe_cls_name.startswith("HunyuanDiT"):
+            from diffusers import HunyuanDiT2DModel, HunyuanDiT2DControlNetModel
+
+            assert isinstance(
+                pipe.transformer,
+                (HunyuanDiT2DModel, HunyuanDiT2DControlNetModel),
+            )
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.blocks,
+                    blocks_name="blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("HunyuanDiTPAG"):
+            from diffusers import HunyuanDiT2DModel
+
+            assert isinstance(pipe.transformer, HunyuanDiT2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.blocks,
+                    blocks_name="blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("Lumina"):
+            from diffusers import LuminaNextDiT2DModel
+
+            assert isinstance(pipe.transformer, LuminaNextDiT2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.layers,
+                    blocks_name="layers",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("Lumina2"):
+            from diffusers import Lumina2Transformer2DModel
+
+            assert isinstance(pipe.transformer, Lumina2Transformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.layers,
+                    blocks_name="layers",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("OmniGen"):
+            from diffusers import OmniGenTransformer2DModel
+
+            assert isinstance(pipe.transformer, OmniGenTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.layers,
+                    blocks_name="layers",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("PixArt"):
+            from diffusers import PixArtTransformer2DModel
+
+            assert isinstance(pipe.transformer, PixArtTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.transformer_blocks,
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("Sana"):
+            from diffusers import SanaTransformer2DModel
+
+            assert isinstance(pipe.transformer, SanaTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.transformer_blocks,
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("ShapE"):
+            from diffusers import PriorTransformer
+
+            assert isinstance(pipe.prior, PriorTransformer)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.prior,
+                    blocks=pipe.prior.transformer_blocks,
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("StableAudio"):
+            from diffusers import StableAudioDiTModel
+
+            assert isinstance(pipe.transformer, StableAudioDiTModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=pipe.transformer.transformer_blocks,
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("VisualCloze"):
+            from diffusers import FluxTransformer2DModel
+            from cache_dit.cache_factory.patch_functors import FluxPatchFunctor
+
+            assert isinstance(pipe.transformer, FluxTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=(
+                        pipe.transformer.transformer_blocks
+                        + pipe.transformer.single_transformer_blocks
+                    ),
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=["single_transformer_blocks"],
+                    patch_functor=FluxPatchFunctor(),
+                ),
+                forward_pattern=ForwardPattern.Pattern_1,
+            )
+
+        elif pipe_cls_name.startswith("AuraFlow"):
+            from diffusers import AuraFlowTransformer2DModel
+
+            assert isinstance(pipe.transformer, AuraFlowTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    # Only support caching single_transformer_blocks for AuraFlow now.
+                    # TODO: Support AuraFlowPatchFunctor.
+                    blocks=pipe.transformer.single_transformer_blocks,
+                    blocks_name="single_transformer_blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
+        elif pipe_cls_name.startswith("Chroma"):
+            from diffusers import ChromaTransformer2DModel
+            from cache_dit.cache_factory.patch_functors import (
+                ChromaPatchFunctor,
+            )
+
+            assert isinstance(pipe.transformer, ChromaTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    blocks=(
+                        pipe.transformer.transformer_blocks
+                        + pipe.transformer.single_transformer_blocks
+                    ),
+                    blocks_name="transformer_blocks",
+                    dummy_blocks_names=["single_transformer_blocks"],
+                    patch_functor=ChromaPatchFunctor(),
+                ),
+                forward_pattern=ForwardPattern.Pattern_1,
+            )
+
+        elif pipe_cls_name.startswith("HiDream"):
+            from diffusers import HiDreamImageTransformer2DModel
+
+            assert isinstance(pipe.transformer, HiDreamImageTransformer2DModel)
+            return UnifiedCacheParams(
+                block_adapter=BlockAdapter(
+                    pipe=pipe,
+                    transformer=pipe.transformer,
+                    # Only support caching single_stream_blocks for HiDream now.
+                    # TODO: Support HiDreamPatchFunctor.
+                    blocks=pipe.transformer.single_stream_blocks,
+                    blocks_name="single_stream_blocks",
+                    dummy_blocks_names=[],
+                ),
+                forward_pattern=ForwardPattern.Pattern_3,
+            )
+
         else:
             raise ValueError(f"Unknown pipeline class name: {pipe_cls_name}")
 
@@ -608,6 +949,14 @@ class UnifiedCacheAdapter:
             return True
         elif cls_name.startswith("Wan"):
             return True
+        elif cls_name.startswith("CogView4"):
+            return True
+        elif cls_name.startswith("Cosmos"):
+            return True
+        elif cls_name.startswith("SkyReelsV2"):
+            return True
+        elif cls_name.startswith("Chroma"):
+            return True
         return False
 
     @classmethod
@@ -680,7 +1029,7 @@ class UnifiedCacheAdapter:
         # Apply cache on transformer: mock cached transformer blocks
         cached_blocks = torch.nn.ModuleList(
             [
-                DBCachedTransformerBlocks(
+                DBCachedBlocks(
                     block_adapter.blocks,
                     transformer=block_adapter.transformer,
                     forward_pattern=forward_pattern,

cache_dit/cache_factory/cache_blocks/__init__.py

@@ -0,0 +1,20 @@
+from cache_dit.cache_factory.cache_blocks.pattern_0_1_2 import (
+    DBCachedBlocks_Pattern_0_1_2,
+)
+from cache_dit.cache_factory.cache_blocks.pattern_3_4_5 import (
+    DBCachedBlocks_Pattern_3_4_5,
+)
+
+
+class DBCachedBlocks:
+    def __new__(cls, *args, **kwargs):
+        forward_pattern = kwargs.get("forward_pattern", None)
+        assert forward_pattern is not None, "forward_pattern can't be None."
+        if forward_pattern in DBCachedBlocks_Pattern_0_1_2._supported_patterns:
+            return DBCachedBlocks_Pattern_0_1_2(*args, **kwargs)
+        elif (
+            forward_pattern in DBCachedBlocks_Pattern_3_4_5._supported_patterns
+        ):
+            return DBCachedBlocks_Pattern_3_4_5(*args, **kwargs)
+        else:
+            raise ValueError(f"Pattern {forward_pattern} is not supported now!")

cache_dit/cache_factory/cache_blocks/pattern_0_1_2.py

@@ -0,0 +1,16 @@
+from cache_dit.cache_factory import ForwardPattern
+from cache_dit.cache_factory.cache_blocks.pattern_base import (
+    DBCachedBlocks_Pattern_Base,
+)
+from cache_dit.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+class DBCachedBlocks_Pattern_0_1_2(DBCachedBlocks_Pattern_Base):
+    _supported_patterns = [
+        ForwardPattern.Pattern_0,
+        ForwardPattern.Pattern_1,
+        ForwardPattern.Pattern_2,
+    ]
+    ...