lycoris-lora 3.0.1.dev11__tar.gz → 3.0.1.dev13__tar.gz

{lycoris_lora-3.0.1.dev11/lycoris_lora.egg-info → lycoris_lora-3.0.1.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev11
+Version: 3.0.1.dev13
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/functional/lokr.py

@@ -14,7 +14,10 @@ def make_kron(w1, w2, scale):
     w2 = w2.contiguous()
     rebuild = torch.kron(w1, w2)
 
-    return rebuild * scale
+    if scale != 1:
+        rebuild = rebuild * scale
+
+    return rebuild
 
 
 def weight_gen(

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/kohya.py

@@ -58,7 +58,7 @@ def create_network(
     rescaled = str_bool(kwargs.get("rescaled", False))
     weight_decompose = str_bool(kwargs.get("dora_wd", False))
     full_matrix = str_bool(kwargs.get("full_matrix", False))
-    bypass_mode = str_bool(kwargs.get("bypass_mode", False))
+    bypass_mode = str_bool(kwargs.get("bypass_mode", None))
     rs_lora = str_bool(kwargs.get("rs_lora", False))
     unbalanced_factorization = str_bool(kwargs.get("unbalanced_factorization", False))
     train_t5xxl = str_bool(kwargs.get("train_t5xxl", False))

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/base.py

@@ -5,7 +5,7 @@ import torch.nn as nn
 import torch.nn.functional as F
 import torch.nn.utils.parametrize as parametrize
 
-from ..utils.bnb import QuantLinears, log_bypass
+from ..utils.quant import QuantLinears, log_bypass, log_suspect
 
 
 class ModuleCustomSD(nn.Module):
@@ -77,7 +77,7 @@ class LycorisBaseModule(ModuleCustomSD):
         rank_dropout=0.0,
         module_dropout=0.0,
         rank_dropout_scale=False,
-        bypass_mode=False,
+        bypass_mode=None,
         **kwargs,
     ):
         """if alpha == 0 or None, alpha is rank (no scaling)."""
@@ -165,6 +165,12 @@ class LycorisBaseModule(ModuleCustomSD):
                 log_bypass()
             self.is_quant = True
             bypass_mode = True
+        if isinstance(org_module, nn.Linear) and org_module.__class__.__name__ != "Linear":
+            if bypass_mode is None:
+                log_suspect()
+                bypass_mode = True
+            if bypass_mode == True:
+                self.is_quant = True
         self.bypass_mode = bypass_mode
         self.dropout = dropout
         self.rank_dropout = rank_dropout

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/boft.py

@@ -10,7 +10,6 @@ from einops import rearrange
 from .base import LycorisBaseModule
 from ..functional import power2factorization
 from ..logging import logger
-from ..utils.bnb import LinearNF4
 
 
 @cache
@@ -64,7 +63,7 @@ class ButterflyOFTModule(LycorisBaseModule):
         rank_dropout_scale=False,
         constraint=0,
         rescaled=False,
-        bypass_mode=False,
+        bypass_mode=None,
         **kwargs,
     ):
         super().__init__(
@@ -116,10 +115,13 @@ class ButterflyOFTModule(LycorisBaseModule):
             lora_name,
             orig_module,
             1,
-            lora_dim=n,
-            alpha=float(alpha),
-            rescale=rescale is not None,
+            lora_dim=s,
+            constraint=float(alpha),
+            rescaled=rescale is not None,
         )
+        module.oft_blocks.copy_(oft_blocks)
+        if rescale is not None:
+            module.rescale.copy_(rescale)
         return module
 
     @property

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/diag_oft.py

@@ -7,7 +7,6 @@ import torch.nn.functional as F
 from .base import LycorisBaseModule
 from ..functional import factorization
 from ..logging import logger
-from ..utils.bnb import LinearNF4
 
 
 @cache
@@ -49,7 +48,7 @@ class DiagOFTModule(LycorisBaseModule):
         rank_dropout_scale=False,
         constraint=0,
         rescaled=False,
-        bypass_mode=False,
+        bypass_mode=None,
         **kwargs,
     ):
         super().__init__(
@@ -104,9 +103,12 @@ class DiagOFTModule(LycorisBaseModule):
             orig_module,
             1,
             lora_dim=s,
-            alpha=float(alpha),
-            rescale=rescale is not None,
+            constraint=float(alpha),
+            rescaled=rescale is not None,
         )
+        module.oft_blocks.copy_(oft_blocks)
+        if rescale is not None:
+            module.rescale.copy_(rescale)
         return module
 
     @property

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/dylora.py

@@ -6,7 +6,6 @@ import torch.nn as nn
 
 from .base import LycorisBaseModule
 from ..utils import product
-from ..utils.bnb import LinearNF4
 
 
 class DyLoraModule(LycorisBaseModule):
@@ -32,7 +31,7 @@ class DyLoraModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         train_on_input=False,
         **kwargs,

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/full.py

@@ -28,7 +28,7 @@ class FullModule(LycorisBaseModule):
         use_tucker=False,
         use_scalar=False,
         rank_dropout_scale=False,
-        bypass_mode=False,
+        bypass_mode=None,
         **kwargs,
     ):
         super().__init__(

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/glora.py

@@ -6,7 +6,6 @@ import torch.nn.functional as F
 
 from .base import LycorisBaseModule
 from ..functional import tucker_weight_from_conv
-from ..utils.bnb import LinearNF4
 
 
 class GLoRAModule(LycorisBaseModule):
@@ -41,7 +40,7 @@ class GLoRAModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         **kwargs,
     ):

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/ia3.py

@@ -2,7 +2,6 @@ import torch
 import torch.nn as nn
 
 from .base import LycorisBaseModule
-from ..utils.bnb import LinearNF4
 
 
 class IA3Module(LycorisBaseModule):
@@ -30,7 +29,7 @@ class IA3Module(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         train_on_input=False,
         **kwargs,

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/locon.py

@@ -7,8 +7,7 @@ import torch.nn.functional as F
 
 from .base import LycorisBaseModule
 from ..functional.general import rebuild_tucker
-from ..logging import logger, warning_once
-from ..utils.bnb import LinearNF4
+from ..logging import logger
 
 
 @cache
@@ -50,7 +49,7 @@ class LoConModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         **kwargs,
     ):

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/loha.py

@@ -5,7 +5,6 @@ import torch.nn as nn
 
 from .base import LycorisBaseModule
 from ..functional.loha import diff_weight as loha_diff_weight
-from ..utils.bnb import LinearNF4
 
 
 class LohaModule(LycorisBaseModule):
@@ -42,7 +41,7 @@ class LohaModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         **kwargs,
     ):
@@ -182,6 +181,9 @@ class LohaModule(LycorisBaseModule):
             )
 
     def get_weight(self, shape):
+        scale = torch.tensor(
+            self.scale, dtype=self.hada_w1_b.dtype, device=self.hada_w1_b.device
+        )
         if self.tucker:
             weight = loha_diff_weight(
                 self.hada_w1_b,
@@ -190,7 +192,7 @@ class LohaModule(LycorisBaseModule):
                 self.hada_w2_a,
                 self.hada_t1,
                 self.hada_t2,
-                gamma=torch.tensor(self.scale),
+                gamma=scale,
             )
         else:
             weight = loha_diff_weight(
@@ -200,7 +202,7 @@ class LohaModule(LycorisBaseModule):
                 self.hada_w2_a,
                 None,
                 None,
-                gamma=torch.tensor(self.scale),
+                gamma=scale,
             )
         if shape is not None:
             weight = weight.reshape(shape)

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/modules/lokr.py

@@ -5,13 +5,10 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 
-from einops import rearrange
-
 from .base import LycorisBaseModule
 from ..functional import factorization, rebuild_tucker
 from ..functional.lokr import make_kron
 from ..logging import logger
-from ..utils.bnb import LinearNF4
 
 
 @cache
@@ -62,7 +59,7 @@ class LokrModule(LycorisBaseModule):
         rank_dropout_scale=False,
         weight_decompose=False,
         full_matrix=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         unbalanced_factorization=False,
         **kwargs,
@@ -288,12 +285,22 @@ class LokrModule(LycorisBaseModule):
             w_shape = (w1_shape[0] * w2_shape[0], w1_shape[1] * w2_shape[1])
             factor1 = max(w1.shape) if w1 is not None else max(w1a.size(0), w1b.size(1))
             factor2 = max(w2.shape) if w2 is not None else max(w2a.size(0), w2b.size(1))
-            if w_shape[0] % factor1 == 0 and w_shape[1] % factor1 == 0 and factor1 in shape_group_1 and factor1 in shape_group_2:
+            if (
+                w_shape[0] % factor1 == 0
+                and w_shape[1] % factor1 == 0
+                and factor1 in shape_group_1
+                and factor1 in shape_group_2
+            ):
                 factor = factor1
-            elif w_shape[0] % factor2 == 0 and w_shape[1] % factor2 == 0 and factor2 in shape_group_1 and factor2 in shape_group_2:
+            elif (
+                w_shape[0] % factor2 == 0
+                and w_shape[1] % factor2 == 0
+                and factor2 in shape_group_1
+                and factor2 in shape_group_2
+            ):
                 factor = factor2
             else:
-                factor = max(factor1, factor2)
+                factor = min(factor1, factor2)
 
         module = cls(
             lora_name,
@@ -349,7 +356,7 @@ class LokrModule(LycorisBaseModule):
                     else self.lokr_w2_a @ self.lokr_w2_b
                 )
             ),
-            torch.tensor(self.scale),
+            self.scale,
         )
         dtype = weight.dtype
         if shape is not None:
@@ -528,6 +535,8 @@ class LokrModule(LycorisBaseModule):
                 weight = self.apply_weight_decompose(
                     weight + diff_weight, self.multiplier
                 )
+            elif self.multiplier == 1:
+                weight = weight + diff_weight
             else:
                 weight = weight + diff_weight * self.multiplier
             bias = (

lycoris_lora-3.0.1.dev11/lycoris/utils/bnb.py → lycoris_lora-3.0.1.dev13/lycoris/utils/quant.py

@@ -77,3 +77,12 @@ def log_bypass():
     return logger.warning(
         "Using bnb/quanto/optimum-quanto with LyCORIS will enable force-bypass mode."
     )
+
+
+@cache
+def log_suspect():
+    return logger.warning(
+        "Non-native Linear detected but bypass_mode is not set. "
+        "Automatically using force-bypass mode to avoid possible issues. "
+        "Please set bypass_mode=False explicitly if there are no quantized layers."
+    )

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris/wrapper.py

@@ -90,7 +90,7 @@ def create_lycoris(module, multiplier=1.0, linear_dim=4, linear_alpha=1, **kwarg
     rescaled = str_bool(kwargs.get("rescaled", False))
     weight_decompose = str_bool(kwargs.get("dora_wd", False))
     full_matrix = str_bool(kwargs.get("full_matrix", False))
-    bypass_mode = str_bool(kwargs.get("bypass_mode", False))
+    bypass_mode = str_bool(kwargs.get("bypass_mode", None))
     unbalanced_factorization = str_bool(kwargs.get("unbalanced_factorization", False))
 
     if unbalanced_factorization:

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev11
+Version: 3.0.1.dev13
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/lycoris_lora.egg-info/SOURCES.txt

@@ -26,10 +26,10 @@ lycoris/modules/loha.py
 lycoris/modules/lokr.py
 lycoris/modules/norms.py
 lycoris/utils/__init__.py
-lycoris/utils/bnb.py
 lycoris/utils/general.py
 lycoris/utils/logger.py
 lycoris/utils/preset.py
+lycoris/utils/quant.py
 lycoris/utils/xformers_utils.py
 lycoris_lora.egg-info/PKG-INFO
 lycoris_lora.egg-info/SOURCES.txt

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.1.dev11",
+    version="3.0.1.dev13",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/test/kohya.py

@@ -104,9 +104,11 @@ class LycorisKohyaWrapperTests(unittest.TestCase):
         )
         network.apply_to([sd_te1, sd_te2], sdxl_unet, True, True)
         network.restore()
-        create_network_from_weights(
+        network = create_network_from_weights(
             1, "", vae, [sd_te1, sd_te2], sdxl_unet, weights_sd=network.state_dict()
         )
+        network.merge_to()
+        del network
         torch.cuda.empty_cache()
 
     @parameterized.expand(extract_param_list)

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev13}/test/wrapper.py

@@ -206,6 +206,7 @@ class LycorisWrapperTests(unittest.TestCase):
             test_input = torch.randn(1, 16, 8, 8, 8).to(device, dtype)
             test_output = test_net(test_input)
             test_lycoris.restore()
+            test_lycoris.merge_to()
 
             state_dict = test_lycoris.state_dict()
             test_lycoris_from_weights: LycorisNetwork
@@ -219,9 +220,13 @@ class LycorisWrapperTests(unittest.TestCase):
             test_lycoris_from_weights.load_state_dict(test_lycoris.state_dict())
 
             self.assertTrue(
-                len(test_lycoris.loras) == len(test_lycoris_from_weights.loras)
+                len(test_lycoris.loras) == len(test_lycoris_from_weights.loras),
+                f"{len(test_lycoris.loras)} != {len(test_lycoris_from_weights.loras)}",
+            )
+            self.assertTrue(
+                torch.allclose(test_output, test_output_from_weights),
+                f"diff: {torch.nn.functional.mse_loss(test_output, test_output_from_weights).item()}",
             )
-            self.assertTrue(torch.allclose(test_output, test_output_from_weights))
         finally:
             reset_globals()