lycoris-lora 3.0.1.dev1__tar.gz → 3.0.1.dev3__tar.gz

{lycoris_lora-3.0.1.dev1/lycoris_lora.egg-info → lycoris_lora-3.0.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev1
+Version: 3.0.1.dev3
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/README.md

@@ -27,7 +27,7 @@ Please check [List of Implemented Algorithms](docs/Algo-List.md) and [Guidelines
 
 A simple comparison of some of these methods are provided below (to be taken with a grain of salt)
 
-|                       | Full | LoRA | LoHa | LoKr low factor | LoKr high factor$^+$ |
+|                       | Full | LoRA | LoHa | LoKr low factor | LoKr high factor $^+$ |
 | --------------------- | ---- | ---- | ---- | --------------- | ---------------------- |
 | Fidelity              | ★   | ●   | ▲   | ◉              | ▲                     |
 | Flexibility $^*$     | ★   | ●   | ◉   | ▲              | ● $^†$              |
@@ -39,9 +39,9 @@ A simple comparison of some of these methods are provided below (to be taken wit
 ★ > ◉ > ● > ▲
 [> means better and smaller size is better]
 
-$^+$ Usually we take `factor <= 0.5 * sqrt(dim)` as low factor and `factor >= sqrt(dim` as high factor. For example, factor<=8 for SD1.x/SD2.x/SDXL can be seen as low factor, and, factor>=16 can be seen as high factor. `<br>`
-$^*$ Flexibility means anything related to generating images not similar to those in the training set, and combination of multiple concepts, whether they are trained together or not `<br>`
-$^†$ It may become more difficult to switch base model or combine multiple concepts in this situation `<br>`
+$^+$ Usually we take `factor <= 0.5 * sqrt(dim)` as low factor and `factor >= sqrt(dim` as high factor. For example, factor<=8 for SD1.x/SD2.x/SDXL can be seen as low factor, and, factor>=16 can be seen as high factor. <br>
+$^*$ Flexibility means anything related to generating images not similar to those in the training set, and combination of multiple concepts, whether they are trained together or not <br>
+$^†$ It may become more difficult to switch base model or combine multiple concepts in this situation <br>
 
 **The actual performance may vary depending on the datasets, tasks, and hyperparameters used. It is recommended to experiment with different settings to achieve optimal results.**
 

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/lycoris/config.py

@@ -7,6 +7,8 @@ PRESET = {
             "Downsample2D",
             "Upsample2D",
             "HunYuanDiTBlock",
+            "DoubleStreamBlock",
+            "SingleStreamBlock",
         ],
         "unet_target_name": [
             "conv_in",
@@ -28,6 +30,8 @@ PRESET = {
             "Transformer2DModel",
             "ResnetBlock2D",
             "HunYuanDiTBlock",
+            "DoubleStreamBlock",
+            "SingleStreamBlock",
         ],
         "unet_target_name": [
             "time_embedding.linear_1",
@@ -46,6 +50,8 @@ PRESET = {
         "unet_target_module": [
             "Transformer2DModel",
             "HunYuanDiTBlock",
+            "DoubleStreamBlock",
+            "SingleStreamBlock",
         ],
         "unet_target_name": [],
         "text_encoder_target_module": [
@@ -78,6 +84,8 @@ PRESET = {
             "Downsample2D",
             "Upsample2D",
             "HunYuanDiTBlock",
+            "DoubleStreamBlock",
+            "SingleStreamBlock",
         ],
         "unet_target_name": [
             "conv_in",
@@ -93,6 +101,8 @@ PRESET = {
         "unet_target_module": [
             "Transformer2DModel",
             "HunYuanDiTBlock",
+            "DoubleStreamBlock",
+            "SingleStreamBlock",
         ],
         "unet_target_name": [],
         "text_encoder_target_module": [],

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/lycoris/functional/loha.py

@@ -141,7 +141,7 @@ def diff_weight(w1d, w1u, w2d, w2u, t1=None, t2=None, gamma=1.0):
         w1d = w1d.reshape(w1d.size(0), -1)
         w1u = w1u.reshape(-1, w1u.size(1))
         w2d = w2d.reshape(w2d.size(0), -1)
-        w2u = w1u.reshape(-1, w2u.size(1))
+        w2u = w2u.reshape(-1, w2u.size(1))
         result = make_weight(w1d, w1u, w2d, w2u, gamma)
 
     result = result.reshape(O, I, *k)

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/lycoris/kohya.py

@@ -214,6 +214,8 @@ class LycorisNetworkKohya(LycorisNetwork):
         "Downsample2D",
         "Upsample2D",
         "HunYuanDiTBlock",
+        "DoubleStreamBlock",
+        "SingleStreamBlock",
     ]
     UNET_TARGET_REPLACE_NAME = [
         "conv_in",

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/lycoris/modules/base.py

@@ -159,11 +159,11 @@ class LycorisBaseModule(ModuleCustomSD):
 
         self.register_buffer("dtype_tensor", torch.tensor(0.0), persistent=False)
 
-        self.is_bnb = False
+        self.is_quant = False
         if isinstance(org_module, QuantLinears):
             if not bypass_mode:
                 log_bypass()
-            self.is_bnb = True
+            self.is_quant = True
             bypass_mode = True
         self.bypass_mode = bypass_mode
         self.dropout = dropout

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/lycoris/modules/full.py

@@ -44,7 +44,7 @@ class FullModule(LycorisBaseModule):
         if self.module_type not in self.support_module:
             raise ValueError(f"{self.module_type} is not supported in Full algo.")
 
-        if self.is_bnb:
+        if self.is_quant:
             raise ValueError(
                 "Quant Linear is not supported and meaningless in Full algo."
             )
@@ -57,6 +57,12 @@ class FullModule(LycorisBaseModule):
             self.bias = nn.Parameter(torch.zeros_like(org_module.bias))
         else:
             self.bias = None
+        self.is_diff = True
+        self._org_weight = [self.org_module[0].weight.data.cpu().clone()]
+        if self.org_module[0].bias is not None:
+            self.org_bias = [self.org_module[0].bias.data.cpu().clone()]
+        else:
+            self.org_bias = None
 
     @classmethod
     def make_module_from_state_dict(cls, lora_name, orig_module, diff, diff_b):
@@ -71,6 +77,7 @@ class FullModule(LycorisBaseModule):
                 module.bias.copy_(diff_b)
             else:
                 module.bias = nn.Parameter(diff_b)
+        module.is_diff = True
         return module
 
     @property
@@ -93,21 +100,7 @@ class FullModule(LycorisBaseModule):
             delattr(self.org_module[0], "bias")
         else:
             self.org_bias = None
-
-    def merge_to(self, multiplier=1.0):
-        need_restore = False
-        if not hasattr(self, "_org_weight"):
-            need_restore = True
-            self.apply_to()
-        result = super().merge_to(multiplier)
-        if need_restore:
-            self.weight = nn.Parameter(torch.zeros_like(self.org_module[0].weight))
-            if self.org_module[0].bias is not None:
-                self.bias = nn.Parameter(torch.zeros_like(self.org_module[0].bias))
-            else:
-                self.bias = None
-            self.org_module[0].forward = self.org_forward
-        return result
+        self.is_diff = False
 
     def restore(self):
         self.org_module[0].forward = self.org_forward
@@ -143,16 +136,24 @@ class FullModule(LycorisBaseModule):
             if self.rank_dropout and self.training
             else 1
         )
-        if drop != 1 or scale != 1:
+        if drop != 1 or scale != 1 or self.is_diff:
             diff_w, diff_b = self.get_diff_weight(scale, device=device)
-            weight = self.org_module[0].weight + diff_w * drop
-            bias = self.org_module[0].bias + diff_b * drop
+            weight = self._org_weight + diff_w * drop
+            if self.org_bias is not None:
+                bias = self.org_bias + diff_b * drop
+            else:
+                bias = None
         else:
             weight = self.weight
             bias = self.bias
         return weight, bias
 
     def get_diff_weight(self, multiplier=1, shape=None, device=None):
+        if self.is_diff:
+            diff_b = None
+            if self.bias is not None:
+                diff_b = self.bias * multiplier
+            return self.weight * multiplier, diff_b
         org_weight = self.org_module[0].weight.to(device, dtype=self.weight.dtype)
         diff = self.weight.to(device) - org_weight
         diff_b = None

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/lycoris/modules/norms.py

@@ -37,12 +37,14 @@ class NormModule(LycorisBaseModule):
         if self.module_type == "unknown":
             if not hasattr(org_module, "weight") or not hasattr(org_module, "_norm"):
                 warning_once(f"{type(org_module)} is not supported in Norm algo.")
+                self.not_supported=True
                 return
             else:
                 self.dim = org_module.weight.numel()
                 self.not_supported = False
         elif self.module_type not in self.support_module:
             warning_once(f"{self.module_type} is not supported in Norm algo.")
+            self.not_supported=True
             return
 
         self.w_norm = nn.Parameter(torch.zeros(self.dim))

lycoris_lora-3.0.1.dev3/lycoris/utils/bnb.py

@@ -0,0 +1,77 @@
+from functools import cache
+
+SUPPORT_QUANT = False
+try:
+    from bitsandbytes.nn import LinearNF4, Linear8bitLt, LinearFP4
+
+    SUPPORT_QUANT = True
+except Exception:
+    import torch.nn as nn
+
+    class LinearNF4(nn.Linear):
+        pass
+
+    class Linear8bitLt(nn.Linear):
+        pass
+
+    class LinearFP4(nn.Linear):
+        pass
+
+
+try:
+    from quanto.nn import QLinear, QConv2d, QLayerNorm
+
+    SUPPORT_QUANT = True
+except Exception:
+    import torch.nn as nn
+
+    class QLinear(nn.Linear):
+        pass
+
+    class QConv2d(nn.Conv2d):
+        pass
+
+    class QLayerNorm(nn.LayerNorm):
+        pass
+
+
+try:
+    from optimum.quanto.nn import (
+        QLinear as QLinearOpt,
+        QConv2d as QConv2dOpt,
+        QLayerNorm as QLayerNormOpt,
+    )
+
+    SUPPORT_QUANT = True
+except Exception:
+    import torch.nn as nn
+
+    class QLinearOpt(nn.Linear):
+        pass
+
+    class QConv2dOpt(nn.Conv2d):
+        pass
+
+    class QLayerNormOpt(nn.LayerNorm):
+        pass
+
+
+from ..logging import logger
+
+
+QuantLinears = (
+    Linear8bitLt,
+    LinearFP4,
+    LinearNF4,
+    QLinear,
+    QConv2d,
+    QLayerNorm,
+    QLinearOpt,
+    QConv2dOpt,
+    QLayerNormOpt,
+)
+
+
+@cache
+def log_bypass():
+    return logger.warning("Using bnb/quanto/optimum-quanto with LyCORIS will enable force-bypass mode.")

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/lycoris/wrapper.py

@@ -44,7 +44,7 @@ deprecated_arg_dict = {
 }
 
 
-def create_lycoris(module, multiplier, linear_dim, linear_alpha, **kwargs):
+def create_lycoris(module, multiplier=1.0, linear_dim=4, linear_alpha=1, **kwargs):
     for key, value in kwargs.items():
         if key in deprecated_arg_dict:
             logger.warning(

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev1
+Version: 3.0.1.dev3
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev1 → lycoris_lora-3.0.1.dev3}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.1.dev1",
+    version="3.0.1.dev3",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",

lycoris_lora-3.0.1.dev1/lycoris/utils/bnb.py

@@ -1,29 +0,0 @@
-from functools import cache
-
-try:
-    from bitsandbytes.nn import LinearNF4, Linear8bitLt, LinearFP4
-
-    SUPPORT_BNB = True
-except Exception:
-    SUPPORT_BNB = False
-    import torch.nn as nn
-
-    class LinearNF4(nn.Linear):
-        pass
-
-    class Linear8bitLt(nn.Linear):
-        pass
-
-    class LinearFP4(nn.Linear):
-        pass
-
-
-from ..logging import logger
-
-
-QuantLinears = (Linear8bitLt, LinearFP4, LinearNF4)
-
-
-@cache
-def log_bypass():
-    return logger.warning("Using bnb with LyCORIS will enable force-bypass mode.")