lycoris-lora 3.2.0.dev2__tar.gz → 3.2.0.post2__tar.gz

{lycoris_lora-3.2.0.dev2/lycoris_lora.egg-info → lycoris_lora-3.2.0.post2}/PKG-INFO

@@ -1,17 +1,17 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: lycoris_lora
-Version: 3.2.0.dev2
+Version: 3.2.0.post2
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao
 Author-email: kohaku@kblueleaf.net
+License: Apache-2.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: GPU :: NVIDIA CUDA :: 11.8
 Classifier: Environment :: GPU :: NVIDIA CUDA :: 12
 Classifier: Environment :: GPU :: NVIDIA CUDA :: 12 :: 12.1
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: End Users/Desktop
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -27,6 +27,8 @@ Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
 Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/README.md

@@ -259,21 +259,21 @@ See [docs/Conversion-scripts.md](docs/Conversion-scripts.md) for more informatio
 
 For full log, please see [Change.md](Change.md)
 
-
-### 2024/12/09 update to 3.1.1
+### 2025/04/23 update to 3.2.0
 
 #### New Features
 
-* use `wd_on_output=True` can enable "correct" weight-decomposition implementation which use the output dimension of weight to calc the norm. The original implementation in LyCORIS calculate things on input dimension due to ambiguos annotation in paper.
+* Support lora-plus learning rate scaling
+* Support HunYuanVideo model and Wan2.1 model
+* LyCORIS now have `onfly_merge` and `onfly_restore` method. Which can be used in inference time to merge the weights of LyCORIS into the original model. This will save the memory and speed up the inference time.
 
 #### Improvements
 
-* BOFT now have more efficient implementation which avoid einops.rearrange.
-* `.merge_to()` will automatically match the device and dtype now.
+* [BREAKING CHANGES] Now LyCORIS will use `wd_on_output=True` by default. This will make the weight norm more consistent with the original paper.
 
 #### Bug fixes
 
-* `scale_weight_norm` working correctly now.
+* `bypass_mode=False` will turn off the bypass mode correctly now.
 
 ## Todo list
 

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/lycoris/config.py

@@ -12,6 +12,9 @@ PRESET = {
             "SingleDiTBlock",  # SD3.5
             "MMDoubleStreamBlock",  # HunYuanVideo
             "MMSingleStreamBlock",  # HunYuanVideo
+            "WanAttentionBlock", # Wan
+            "HunyuanVideoTransformerBlock", # FramePack
+            "HunyuanVideoSingleTransformerBlock", # FramePack
         ],
         "unet_target_name": [
             "conv_in",
@@ -39,6 +42,9 @@ PRESET = {
             "SingleDiTBlock",
             "MMDoubleStreamBlock",  # HunYuanVideo
             "MMSingleStreamBlock",  # HunYuanVideo
+            "WanAttentionBlock", # Wan
+            "HunyuanVideoTransformerBlock", # FramePack
+            "HunyuanVideoSingleTransformerBlock", # FramePack
         ],
         "unet_target_name": [
             "time_embedding.linear_1",
@@ -63,6 +69,9 @@ PRESET = {
             "SingleDiTBlock",
             "MMDoubleStreamBlock",  # HunYuanVideo
             "MMSingleStreamBlock",  # HunYuanVideo
+            "WanAttentionBlock", # Wan
+            "HunyuanVideoTransformerBlock", # FramePack
+            "HunyuanVideoSingleTransformerBlock", # FramePack
         ],
         "unet_target_name": [],
         "text_encoder_target_module": [
@@ -102,6 +111,9 @@ PRESET = {
             "SingleDiTBlock",
             "MMDoubleStreamBlock",  # HunYuanVideo
             "MMSingleStreamBlock",  # HunYuanVideo
+            "WanAttentionBlock", # Wan
+            "HunyuanVideoTransformerBlock", # FramePack
+            "HunyuanVideoSingleTransformerBlock", # FramePack
         ],
         "unet_target_name": [
             "conv_in",
@@ -122,6 +134,9 @@ PRESET = {
             "SingleDiTBlock",
             "MMDoubleStreamBlock",  # HunYuanVideo
             "MMSingleStreamBlock",  # HunYuanVideo
+            "WanAttentionBlock", # Wan
+            "HunyuanVideoTransformerBlock", # FramePack
+            "HunyuanVideoSingleTransformerBlock", # FramePack
         ],
         "unet_target_name": [],
         "text_encoder_target_module": [],

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/lycoris/functional/lokr.py

@@ -105,12 +105,12 @@ def weight_gen(
             w2 = torch.empty(shape[0][1], shape[1][1])
 
     if use_w2:
-        torch.nn.init.constant_(w2, 1)
+        torch.nn.init.constant_(w2, 0)
     else:
         if tucker:
             torch.nn.init.kaiming_uniform_(t2, a=math.sqrt(5))
         torch.nn.init.kaiming_uniform_(w2a, a=math.sqrt(5))
-        torch.nn.init.constant_(w2b, 1)
+        torch.nn.init.constant_(w2b, 0)
 
     if use_w1:
         torch.nn.init.kaiming_uniform_(w1, a=math.sqrt(5))

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/lycoris/kohya.py

@@ -57,9 +57,9 @@ def create_network(
     constraint = float(kwargs.get("constraint", None) or 0)
     rescaled = str_bool(kwargs.get("rescaled", False))
     weight_decompose = str_bool(kwargs.get("dora_wd", False))
-    wd_on_output = str_bool(kwargs.get("wd_on_output", False))
+    wd_on_output = str_bool(kwargs.get("wd_on_output", True))
     full_matrix = str_bool(kwargs.get("full_matrix", False))
-    bypass_mode = str_bool(kwargs.get("bypass_mode", None))
+    bypass_mode = str_bool(kwargs.get("bypass_mode", False))
     rs_lora = str_bool(kwargs.get("rs_lora", False))
     unbalanced_factorization = str_bool(kwargs.get("unbalanced_factorization", False))
     train_t5xxl = str_bool(kwargs.get("train_t5xxl", False))
@@ -252,6 +252,9 @@ class LycorisNetworkKohya(LycorisNetwork):
         "SingleDiTBlock",
         "MMDoubleStreamBlock",  # HunYuanVideo
         "MMSingleStreamBlock",  # HunYuanVideo
+        "WanAttentionBlock", # Wan
+        "HunyuanVideoTransformerBlock", # FramePack
+        "HunyuanVideoSingleTransformerBlock", # FramePack
     ]
     UNET_TARGET_REPLACE_NAME = [
         "conv_in",
@@ -699,7 +702,7 @@ class LycorisNetworkKohya(LycorisNetwork):
         if self.text_encoder_loras:
             params, descriptions = assemble_params(
                 self.text_encoder_loras,
-                text_encoder_lr if text_encoder_lr is not None else default_lr,
+                text_encoder_lr if text_encoder_lr is not None else learning_rate,
                 self.loraplus_text_encoder_lr_ratio or self.loraplus_lr_ratio,
             )
             all_params.extend(params)
@@ -710,7 +713,7 @@ class LycorisNetworkKohya(LycorisNetwork):
         if self.unet_loras:
             params, descriptions = assemble_params(
                 self.unet_loras,
-                unet_lr if unet_lr is not None else default_lr,
+                unet_lr if unet_lr is not None else learning_rate,
                 self.loraplus_unet_lr_ratio or self.loraplus_lr_ratio,
             )
             all_params.extend(params)
@@ -727,10 +730,10 @@ class LycorisNetworkKohya(LycorisNetwork):
     def prepare_grad_etc(self, *args):
         self.requires_grad_(True)
 
-    def on_epoch_start(self, unet):
+    def on_epoch_start(self, *args):
         self.train()
 
-    def on_step_start(self):
+    def on_step_start(self, *args):
         pass
 
     def get_trainable_params(self):

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/lycoris/modules/locon.py

@@ -49,7 +49,7 @@ class LoConModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        wd_on_out=False,
+        wd_on_out=True,
         bypass_mode=None,
         rs_lora=False,
         **kwargs,

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/lycoris/modules/loha.py

@@ -41,7 +41,7 @@ class LohaModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        wd_on_out=False,
+        wd_on_out=True,
         bypass_mode=None,
         rs_lora=False,
         **kwargs,

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/lycoris/modules/lokr.py

@@ -58,7 +58,7 @@ class LokrModule(LycorisBaseModule):
         factor: int = -1,  # factorization factor
         rank_dropout_scale=False,
         weight_decompose=False,
-        wd_on_out=False,
+        wd_on_out=True,
         full_matrix=False,
         bypass_mode=None,
         rs_lora=False,

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/lycoris/wrapper.py

@@ -89,9 +89,9 @@ def create_lycoris(module, multiplier=1.0, linear_dim=4, linear_alpha=1, **kwarg
     constraint = float(kwargs.get("constraint", 0) or 0)
     rescaled = str_bool(kwargs.get("rescaled", False))
     weight_decompose = str_bool(kwargs.get("dora_wd", False))
-    wd_on_output = str_bool(kwargs.get("wd_on_output", False))
+    wd_on_output = str_bool(kwargs.get("wd_on_output", True))
     full_matrix = str_bool(kwargs.get("full_matrix", False))
-    bypass_mode = str_bool(kwargs.get("bypass_mode", None))
+    bypass_mode = str_bool(kwargs.get("bypass_mode", False))
     unbalanced_factorization = str_bool(kwargs.get("unbalanced_factorization", False))
 
     if unbalanced_factorization:

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2/lycoris_lora.egg-info}/PKG-INFO

@@ -1,17 +1,17 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: lycoris_lora
-Version: 3.2.0.dev2
+Version: 3.2.0.post2
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao
 Author-email: kohaku@kblueleaf.net
+License: Apache-2.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: GPU :: NVIDIA CUDA :: 11.8
 Classifier: Environment :: GPU :: NVIDIA CUDA :: 12
 Classifier: Environment :: GPU :: NVIDIA CUDA :: 12 :: 12.1
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: End Users/Desktop
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -27,6 +27,8 @@ Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
 Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{lycoris_lora-3.2.0.dev2 → lycoris_lora-3.2.0.post2}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.2.0.dev2",
+    version="3.2.0.post2",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",
@@ -12,6 +12,7 @@ setup(
     zip_safe=False,
     install_requires=["torch", "einops", "toml", "tqdm"],
     python_requires=">=3.10",
+    license="Apache-2.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Environment :: GPU :: NVIDIA CUDA :: 11.8",
@@ -19,7 +20,6 @@ setup(
         "Environment :: GPU :: NVIDIA CUDA :: 12 :: 12.1",
         "Intended Audience :: Developers",
         "Intended Audience :: End Users/Desktop",
-        "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",