lycoris-lora 2.3.0.dev7__tar.gz → 2.3.0.dev9__tar.gz

{lycoris_lora-2.3.0.dev7/lycoris_lora.egg-info → lycoris_lora-2.3.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.3.0.dev7
+Version: 2.3.0.dev9
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/README.md

@@ -81,7 +81,7 @@ In any case, please install this package in the corresponding virtual environmen
   pip install .
   ```
 
-A detilaed description of the network arguments is provided in [docs/Network-Args.md](docs/Network-Args.md).
+A detailed description of the network arguments is provided in [docs/Network-Args.md](docs/Network-Args.md).
 
 #### kohya script
 

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/lycoris/modules/__init__.py

@@ -1,4 +1,5 @@
 import torch
+import torch.nn as nn
 
 from .full import FullModule
 from .norms import NormModule
@@ -122,9 +123,12 @@ def make_module(lyco_type, params, lora_name, orig_module):
             orig_module,
             1,
         )
-        module.diff.copy_(diff)
+        module.weight.copy_(diff + orig_module[0].weight.data)
         if diff_b is not None:
-            module.diff_b.copy_(diff_b)
+            if orig_module[0].bias is not None:
+                module.bias.copy_(diff_b + orig_module[0].bias.data)
+            else:
+                module.bias = nn.Parameter(diff_b)
     elif lyco_type == "ia3":
         pass
     else:

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/lycoris/modules/base.py

@@ -164,7 +164,6 @@ class LycorisBaseModule(ModuleCustomSD):
 
         self.multiplier = multiplier
         self.org_module = [org_module]
-        self.org_forward = self.org_module[0].forward
 
     @property
     def org_weight(self):
@@ -192,6 +191,9 @@ class LycorisBaseModule(ModuleCustomSD):
             else:
                 self.org_module[0].bias = nn.Parameter(bias)
 
+    def get_diff_weight(self, multiplier=1.0, shape=None, device=None):
+        raise NotImplementedError
+
     def get_merged_weight(self, multiplier=1.0, shape=None, device=None):
         raise NotImplementedError
 
@@ -199,6 +201,9 @@ class LycorisBaseModule(ModuleCustomSD):
     def apply_max_norm(self, max_norm, device=None):
         raise NotImplementedError
 
+    def bypass_forward_diff(self, x, scale=1):
+        raise NotImplementedError
+
     def bypass_forward(self, x, scale=1):
         raise NotImplementedError
 

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/lycoris/modules/boft.py

@@ -124,7 +124,7 @@ class ButterflyOFTModule(LycorisBaseModule):
         r = (I + normed_q) @ (I - normed_q).float().inverse()
         return r
 
-    def make_weight(self, scale=1, device=None):
+    def make_weight(self, scale=1, device=None, diff=False):
         m = self.boft_m
         b = self.boft_b
         r_b = b // 2
@@ -135,9 +135,9 @@ class ButterflyOFTModule(LycorisBaseModule):
             bi = r[i]  # b_num, b_size, b_size
             if i == 0:
                 # Apply multiplier/scale and rescale into first weight
-                bi = bi * scale + (1 - scale) * self.I
                 if self.rescaled:
                     bi = bi * self.rescale
+            bi = bi * scale - scale * self.I + (self.I if diff else 0)
             inp = rearrange(inp, "(c g k) ... -> (c k g) ...", g=2, k=2**i * r_b)
             inp = rearrange(inp, "(d b) ... -> d b ...", b=b)
             inp = torch.einsum("b i j, b j ... -> b i ...", bi, inp)
@@ -146,6 +146,12 @@ class ButterflyOFTModule(LycorisBaseModule):
 
         return inp
 
+    def get_diff_weight(self, multiplier=1, shape=None, device=None):
+        diff = self.make_weight(scale=multiplier, device=device, diff=True)
+        if shape is not None:
+            diff = diff.view(shape)
+        return diff
+
     def get_merged_weight(self, multiplier=1, shape=None, device=None):
         diff = self.make_weight(scale=multiplier, device=device)
         if shape is not None:
@@ -165,7 +171,7 @@ class ButterflyOFTModule(LycorisBaseModule):
 
         return scaled, orig_norm * ratio
 
-    def bypass_forward(self, x, scale=1):
+    def _bypass_forward(self, x, scale=1, diff=False):
         m = self.boft_m
         b = self.boft_b
         r_b = b // 2
@@ -176,9 +182,9 @@ class ButterflyOFTModule(LycorisBaseModule):
             bi = r[i]  # b_num, b_size, b_size
             if i == 0:
                 # Apply multiplier/scale and rescale into first weight
-                bi = bi * scale + (1 - scale) * self.I
                 if self.rescaled:
                     bi = bi * self.rescale
+            bi = bi * scale - scale * self.I + (self.I if diff else 0)
             inp = rearrange(inp, "... (c g k) ->... (c k g)", g=2, k=2**i * r_b)
             inp = rearrange(inp, "... (d b) -> ... d b", b=b)
             inp = torch.einsum("b i j, ... b j -> ... b i", bi, inp)
@@ -187,6 +193,12 @@ class ButterflyOFTModule(LycorisBaseModule):
 
         return inp
 
+    def bypass_forward_diff(self, x, scale=1):
+        return self._bypass_forward(x, scale, diff=True)
+
+    def bypass_forward(self, x, scale=1):
+        return self._bypass_forward(x, scale, diff=False)
+
     def forward(self, x, *args, **kwargs):
         if self.module_dropout and self.training:
             if torch.rand(1) < self.module_dropout:

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/lycoris/modules/diag_oft.py

@@ -98,7 +98,7 @@ class DiagOFTModule(LycorisBaseModule):
         r = (I + normed_q) @ (I - normed_q).float().inverse()
         return r
 
-    def make_weight(self, scale=1, device=None):
+    def make_weight(self, scale=1, device=None, diff=False):
         r = self.get_r()
         org_weight = self.org_weight.to(device, dtype=r.dtype)
         org_weight = rearrange(
@@ -110,7 +110,7 @@ class DiagOFTModule(LycorisBaseModule):
         # Init R=0, so add I on it to ensure the output of step0 is original model output
         weight = torch.einsum(
             "k n m, k n ... -> k m ...",
-            r * scale + (1 - scale) * self.I,
+            r * scale - scale * self.I + (self.I if diff else 0),
             org_weight,
         )
         weight = rearrange(weight, "k m ... -> (k m) ...")
@@ -118,6 +118,12 @@ class DiagOFTModule(LycorisBaseModule):
             weight = self.rescale * weight
         return weight
 
+    def get_diff_weight(self, multiplier=1, shape=None, device=None):
+        diff = self.make_weight(scale=multiplier, device=device, diff=True)
+        if shape is not None:
+            diff = diff.view(shape)
+        return diff
+
     def get_merged_weight(self, multiplier=1, shape=None, device=None):
         diff = self.make_weight(scale=multiplier, device=device)
         if shape is not None:
@@ -137,7 +143,7 @@ class DiagOFTModule(LycorisBaseModule):
 
         return scaled, orig_norm * ratio
 
-    def bypass_forward(self, x, scale=1):
+    def _bypass_forward(self, x, scale=1, diff=False):
         r = self.get_r()
         org_out = self.org_forward(x)
         if self.op == F.conv2d:
@@ -149,13 +155,21 @@ class DiagOFTModule(LycorisBaseModule):
             n=self.block_size,
         )
         oft_out = torch.einsum(
-            "k n m, ... k n -> ... k m", r * scale + (1 - scale) * self.I, org_out
+            "k n m, ... k n -> ... k m",
+            r * scale - scale * self.I + (self.I if diff else 0),
+            org_out,
         )
         out = rearrange(oft_out, "... k m -> ... (k m)")
         if self.op == F.conv2d:
             out = out.transpose(1, -1)
         return out
 
+    def bypass_forward_diff(self, x, scale=1):
+        return self._bypass_forward(x, scale, diff=True)
+
+    def bypass_forward(self, x, scale=1):
+        return self._bypass_forward(x, scale, diff=False)
+
     def forward(self, x: torch.Tensor, *args, **kwargs):
         if self.module_dropout and self.training:
             if torch.rand(1) < self.module_dropout:

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/lycoris/modules/locon.py

@@ -174,17 +174,22 @@ class LoConModule(LycorisBaseModule):
 
         return weight * self.scalar.to(device)
 
-    def get_merged_weight(self, multiplier=1, shape=None, device=None):
+    def get_diff_weight(self, multiplier=1, shape=None, device=None):
         scale = self.scale * multiplier
         diff = self.make_weight(device=device) * scale
         if shape is not None:
             diff = diff.view(shape)
         if device is not None:
             diff = diff.to(device)
-        merged = self.org_module[0].weight.data + diff
+        return diff
+
+    def get_merged_weight(self, multiplier=1, shape=None, device=None):
+        merged = self.org_module[0].weight.data + self.get_diff_weight(
+            multiplier=multiplier, shape=shape, device=device
+        )
         if self.wd:
             merged = self.apply_weight_decompose(merged)
-        return merged
+        return merged, None
 
     def apply_weight_decompose(self, weight):
         weight = weight.to(self.dora_scale.dtype)
@@ -222,7 +227,7 @@ class LoConModule(LycorisBaseModule):
 
         return scaled, orig_norm * ratio
 
-    def bypass_forward(self, x, scale=1):
+    def bypass_forward_diff(self, x, scale=1):
         if self.tucker:
             mid = self.lora_mid(self.lora_down(x))
         else:
@@ -240,9 +245,10 @@ class LoConModule(LycorisBaseModule):
                 drop = drop.view(*[1] * (dims - 1), -1)
             mid = mid * drop
 
-        return self.org_forward(x) + self.dropout(
-            self.lora_up(mid) * self.scalar * self.scale * scale
-        )
+        return self.dropout(self.lora_up(mid) * self.scalar * self.scale * scale)
+
+    def bypass_forward(self, x, scale=1):
+        return self.org_forward(x) + self.bypass_forward_diff(x, scale=scale)
 
     def forward(self, x):
         if self.module_dropout and self.training:

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/lycoris/modules/loha.py

@@ -247,12 +247,17 @@ class LohaModule(LycorisBaseModule):
             weight *= drop
         return weight
 
-    def get_merged_weight(self, multiplier=1, shape=None, device=None):
+    def get_diff_weight(self, multiplier=1, shape=None, device=None):
         scale = self.scale * multiplier
         diff = self.get_weight(shape) * scale
         if device is not None:
             diff = diff.to(device)
-        merged = self.org_module[0].weight.data + diff
+        return diff
+
+    def get_merged_weight(self, multiplier=1, shape=None, device=None):
+        merged = self.org_module[0].weight.data + self.get_diff_weight(
+            multiplier=multiplier, shape=shape, device=device
+        )
         if self.wd:
             merged = self.apply_weight_decompose(merged)
         return merged
@@ -296,10 +301,12 @@ class LohaModule(LycorisBaseModule):
 
         return scaled, orig_norm * ratio
 
-    def bypass_forward(self, x, scale=1):
+    def bypass_forward_diff(self, x, scale=1):
         diff_weight = self.get_weight(self.shape) * self.scalar * scale
-        print(diff_weight.shape, x.shape, self.shape)
-        return self.org_forward(x) + self.op(x, diff_weight, **self.kw_dict)
+        return self.op(x, diff_weight, **self.kw_dict)
+
+    def bypass_forward(self, x, scale=1):
+        return self.org_forward(x) + self.bypass_forward_diff(x, scale=scale)
 
     def forward(self, x: torch.Tensor, *args, **kwargs):
         if self.module_dropout and self.training:

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/lycoris/modules/lokr.py

@@ -376,11 +376,10 @@ class LokrModule(LycorisBaseModule):
 
         return scaled, orig_norm * ratio
 
-    def bypass_forward(self, h):
+    def bypass_forward_diff(self, h):
         if len(self.shape) > 2 and self.shape[2] > 1:
             return (
-                self.org_forward(h)
-                + self.op(h, self.get_weight(self.shape), **self.kw_dict)
+                self.op(h, self.get_weight(self.shape), **self.kw_dict)
                 * self.multiplier
             )
         if self.module_type.startswith("conv"):
@@ -413,7 +412,10 @@ class LokrModule(LycorisBaseModule):
         if self.module_type.startswith("conv"):
             h = h.transpose(1, -1)
 
-        return h * self.multiplier + self.org_forward(h)
+        return h * self.multiplier
+
+    def bypass_forward(self, x, scale=1):
+        return self.org_forward(x) + self.bypass_forward_diff(x, scale=scale)
 
     def forward(self, x: torch.Tensor, *args, **kwargs):
         if self.module_dropout and self.training:

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.3.0.dev7
+Version: 2.3.0.dev9
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/lycoris_lora.egg-info/SOURCES.txt

@@ -39,7 +39,6 @@ lycoris/utils/__init__.py
 lycoris/utils/bnb.py
 lycoris/utils/logger.py
 lycoris/utils/preset.py
-lycoris/utils/tensor_norm.py
 lycoris/utils/xformers_utils.py
 lycoris_lora.egg-info/PKG-INFO
 lycoris_lora.egg-info/SOURCES.txt

{lycoris_lora-2.3.0.dev7 → lycoris_lora-2.3.0.dev9}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="2.3.0.dev7",
+    version="2.3.0.dev9",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",

lycoris_lora-2.3.0.dev7/lycoris/utils/tensor_norm.py

@@ -1,7 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-def multi_dim_norm(x: torch.Tensor, dim: int = 1) -> torch.Tensor:
-    return torch.norm(x, dim=dim, p=2, keepdim=True)