lycoris-lora 3.0.1.dev12__tar.gz → 3.0.1.dev14__tar.gz

{lycoris_lora-3.0.1.dev12/lycoris_lora.egg-info → lycoris_lora-3.0.1.dev14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev12
+Version: 3.0.1.dev14
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/kohya.py

@@ -58,7 +58,7 @@ def create_network(
     rescaled = str_bool(kwargs.get("rescaled", False))
     weight_decompose = str_bool(kwargs.get("dora_wd", False))
     full_matrix = str_bool(kwargs.get("full_matrix", False))
-    bypass_mode = str_bool(kwargs.get("bypass_mode", False))
+    bypass_mode = str_bool(kwargs.get("bypass_mode", None))
     rs_lora = str_bool(kwargs.get("rs_lora", False))
     unbalanced_factorization = str_bool(kwargs.get("unbalanced_factorization", False))
     train_t5xxl = str_bool(kwargs.get("train_t5xxl", False))

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/base.py

@@ -5,7 +5,7 @@ import torch.nn as nn
 import torch.nn.functional as F
 import torch.nn.utils.parametrize as parametrize
 
-from ..utils.bnb import QuantLinears, log_bypass
+from ..utils.quant import QuantLinears, log_bypass, log_suspect
 
 
 class ModuleCustomSD(nn.Module):
@@ -77,7 +77,7 @@ class LycorisBaseModule(ModuleCustomSD):
         rank_dropout=0.0,
         module_dropout=0.0,
         rank_dropout_scale=False,
-        bypass_mode=False,
+        bypass_mode=None,
         **kwargs,
     ):
         """if alpha == 0 or None, alpha is rank (no scaling)."""
@@ -165,6 +165,12 @@ class LycorisBaseModule(ModuleCustomSD):
                 log_bypass()
             self.is_quant = True
             bypass_mode = True
+        if isinstance(org_module, nn.Linear) and org_module.__class__.__name__ != "Linear":
+            if bypass_mode is None:
+                log_suspect()
+                bypass_mode = True
+            if bypass_mode == True:
+                self.is_quant = True
         self.bypass_mode = bypass_mode
         self.dropout = dropout
         self.rank_dropout = rank_dropout

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/boft.py

@@ -10,7 +10,6 @@ from einops import rearrange
 from .base import LycorisBaseModule
 from ..functional import power2factorization
 from ..logging import logger
-from ..utils.bnb import LinearNF4
 
 
 @cache
@@ -64,7 +63,7 @@ class ButterflyOFTModule(LycorisBaseModule):
         rank_dropout_scale=False,
         constraint=0,
         rescaled=False,
-        bypass_mode=False,
+        bypass_mode=None,
         **kwargs,
     ):
         super().__init__(
@@ -116,10 +115,13 @@ class ButterflyOFTModule(LycorisBaseModule):
             lora_name,
             orig_module,
             1,
-            lora_dim=n,
-            alpha=float(alpha),
-            rescale=rescale is not None,
+            lora_dim=s,
+            constraint=float(alpha),
+            rescaled=rescale is not None,
         )
+        module.oft_blocks.copy_(oft_blocks)
+        if rescale is not None:
+            module.rescale.copy_(rescale)
         return module
 
     @property

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/diag_oft.py

@@ -7,7 +7,6 @@ import torch.nn.functional as F
 from .base import LycorisBaseModule
 from ..functional import factorization
 from ..logging import logger
-from ..utils.bnb import LinearNF4
 
 
 @cache
@@ -49,7 +48,7 @@ class DiagOFTModule(LycorisBaseModule):
         rank_dropout_scale=False,
         constraint=0,
         rescaled=False,
-        bypass_mode=False,
+        bypass_mode=None,
         **kwargs,
     ):
         super().__init__(
@@ -104,9 +103,12 @@ class DiagOFTModule(LycorisBaseModule):
             orig_module,
             1,
             lora_dim=s,
-            alpha=float(alpha),
-            rescale=rescale is not None,
+            constraint=float(alpha),
+            rescaled=rescale is not None,
         )
+        module.oft_blocks.copy_(oft_blocks)
+        if rescale is not None:
+            module.rescale.copy_(rescale)
         return module
 
     @property

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/dylora.py

@@ -6,7 +6,6 @@ import torch.nn as nn
 
 from .base import LycorisBaseModule
 from ..utils import product
-from ..utils.bnb import LinearNF4
 
 
 class DyLoraModule(LycorisBaseModule):
@@ -32,7 +31,7 @@ class DyLoraModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         train_on_input=False,
         **kwargs,

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/full.py

@@ -1,7 +1,19 @@
+from functools import cache
+
 import torch
 import torch.nn as nn
 
 from .base import LycorisBaseModule
+from ..logging import logger
+
+
+@cache
+def log_bypass_override():
+    return logger.warning(
+        "Automatic Bypass-Mode detected in algo=full, "
+        "override with bypass_mode=False since algo=full not support bypass mode. "
+        "If you are using quantized model which require bypass mode, please don't use algo=full. "
+    )
 
 
 class FullModule(LycorisBaseModule):
@@ -28,9 +40,10 @@ class FullModule(LycorisBaseModule):
         use_tucker=False,
         use_scalar=False,
         rank_dropout_scale=False,
-        bypass_mode=False,
+        bypass_mode=None,
         **kwargs,
     ):
+        org_bypass = bypass_mode
         super().__init__(
             lora_name,
             org_module,
@@ -41,6 +54,10 @@ class FullModule(LycorisBaseModule):
             rank_dropout_scale,
             bypass_mode,
         )
+        if bypass_mode and org_bypass is None:
+            self.bypass_mode = False
+            log_bypass_override()
+
         if self.module_type not in self.support_module:
             raise ValueError(f"{self.module_type} is not supported in Full algo.")
 
@@ -138,7 +155,7 @@ class FullModule(LycorisBaseModule):
         )
         if drop != 1 or scale != 1 or self.is_diff:
             diff_w, diff_b = self.get_diff_weight(scale, device=device)
-            weight = self._org_weight + diff_w * drop
+            weight = self.org_weight + diff_w * drop
             if self.org_bias is not None:
                 bias = self.org_bias + diff_b * drop
             else:

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/glora.py

@@ -6,7 +6,6 @@ import torch.nn.functional as F
 
 from .base import LycorisBaseModule
 from ..functional import tucker_weight_from_conv
-from ..utils.bnb import LinearNF4
 
 
 class GLoRAModule(LycorisBaseModule):
@@ -41,7 +40,7 @@ class GLoRAModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         **kwargs,
     ):

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/ia3.py

@@ -2,7 +2,6 @@ import torch
 import torch.nn as nn
 
 from .base import LycorisBaseModule
-from ..utils.bnb import LinearNF4
 
 
 class IA3Module(LycorisBaseModule):
@@ -30,7 +29,7 @@ class IA3Module(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         train_on_input=False,
         **kwargs,

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/locon.py

@@ -7,8 +7,7 @@ import torch.nn.functional as F
 
 from .base import LycorisBaseModule
 from ..functional.general import rebuild_tucker
-from ..logging import logger, warning_once
-from ..utils.bnb import LinearNF4
+from ..logging import logger
 
 
 @cache
@@ -50,7 +49,7 @@ class LoConModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         **kwargs,
     ):

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/loha.py

@@ -5,7 +5,6 @@ import torch.nn as nn
 
 from .base import LycorisBaseModule
 from ..functional.loha import diff_weight as loha_diff_weight
-from ..utils.bnb import LinearNF4
 
 
 class LohaModule(LycorisBaseModule):
@@ -42,7 +41,7 @@ class LohaModule(LycorisBaseModule):
         use_scalar=False,
         rank_dropout_scale=False,
         weight_decompose=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         **kwargs,
     ):

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/modules/lokr.py

@@ -5,13 +5,10 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 
-from einops import rearrange
-
 from .base import LycorisBaseModule
 from ..functional import factorization, rebuild_tucker
 from ..functional.lokr import make_kron
 from ..logging import logger
-from ..utils.bnb import LinearNF4
 
 
 @cache
@@ -62,7 +59,7 @@ class LokrModule(LycorisBaseModule):
         rank_dropout_scale=False,
         weight_decompose=False,
         full_matrix=False,
-        bypass_mode=False,
+        bypass_mode=None,
         rs_lora=False,
         unbalanced_factorization=False,
         **kwargs,
@@ -303,7 +300,7 @@ class LokrModule(LycorisBaseModule):
             ):
                 factor = factor2
             else:
-                factor = max(factor1, factor2)
+                factor = min(factor1, factor2)
 
         module = cls(
             lora_name,

lycoris_lora-3.0.1.dev12/lycoris/utils/bnb.py → lycoris_lora-3.0.1.dev14/lycoris/utils/quant.py

@@ -77,3 +77,12 @@ def log_bypass():
     return logger.warning(
         "Using bnb/quanto/optimum-quanto with LyCORIS will enable force-bypass mode."
     )
+
+
+@cache
+def log_suspect():
+    return logger.warning(
+        "Non-native Linear detected but bypass_mode is not set. "
+        "Automatically using force-bypass mode to avoid possible issues. "
+        "Please set bypass_mode=False explicitly if there are no quantized layers."
+    )

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris/wrapper.py

@@ -90,7 +90,7 @@ def create_lycoris(module, multiplier=1.0, linear_dim=4, linear_alpha=1, **kwarg
     rescaled = str_bool(kwargs.get("rescaled", False))
     weight_decompose = str_bool(kwargs.get("dora_wd", False))
     full_matrix = str_bool(kwargs.get("full_matrix", False))
-    bypass_mode = str_bool(kwargs.get("bypass_mode", False))
+    bypass_mode = str_bool(kwargs.get("bypass_mode", None))
     unbalanced_factorization = str_bool(kwargs.get("unbalanced_factorization", False))
 
     if unbalanced_factorization:

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev12
+Version: 3.0.1.dev14
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/lycoris_lora.egg-info/SOURCES.txt

@@ -26,10 +26,10 @@ lycoris/modules/loha.py
 lycoris/modules/lokr.py
 lycoris/modules/norms.py
 lycoris/utils/__init__.py
-lycoris/utils/bnb.py
 lycoris/utils/general.py
 lycoris/utils/logger.py
 lycoris/utils/preset.py
+lycoris/utils/quant.py
 lycoris/utils/xformers_utils.py
 lycoris_lora.egg-info/PKG-INFO
 lycoris_lora.egg-info/SOURCES.txt

{lycoris_lora-3.0.1.dev12 → lycoris_lora-3.0.1.dev14}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.1.dev12",
+    version="3.0.1.dev14",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",