lycoris-lora 2.3.0.dev9__tar.gz → 2.3.0.dev11__tar.gz

{lycoris_lora-2.3.0.dev9/lycoris_lora.egg-info → lycoris_lora-2.3.0.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.3.0.dev9
+Version: 2.3.0.dev11
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11}/README.md

@@ -1,4 +1,4 @@
-# LyCORIS - Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion.
+# LyCORIS - Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion. (ICLR'24)
 
 ![banner image](docs/images/banner.png)
 
@@ -10,7 +10,7 @@ This project originated from LoCon (see archive branch).
 
 [![Discord!](https://i.imgur.com/A8tOvFS.jpg)](https://discord.gg/VtTFKrj9gJ)
 
-**If you want to check more in-depth experiment results and discussions for LyCORIS, you can check our [paper](https://arxiv.org/abs/2309.14859)**
+**If you want to check more in-depth experiment results and discussions for LyCORIS, you can check our [paper](https://openreview.net/forum?id=wfzXa8e783)**
 
 ## Algorithm Overview
 
@@ -20,10 +20,10 @@ Please check [List of Implemented Algorithms](docs/Algo-List.md) and [Guidelines
 
 A simple comparison of some of these methods are provided below (to be taken with a grain of salt)
 
-|                       | Full | LoRA | LoHa | LoKr low factor | LoKr high factor |
+|                       | Full | LoRA | LoHa | LoKr low factor | LoKr high factor $^+$ |
 | --------------------- | ---- | ---- | ---- | --------------- | ---------------- |
 | Fidelity              | ★   | ●   | ▲   | ◉              | ▲               |
-| Flexibility$^*$     | ★   | ●   | ◉   | ▲              | ●$^†$        |
+| Flexibility $^*$     | ★   | ●   | ◉   | ▲              | ● $^†$        |
 | Diversity             | ▲   | ◉   | ★   | ●              | ★               |
 | Size                  | ▲   | ●   | ●   | ●              | ★               |
 | Training Speed Linear | ★   | ●   | ●   | ★              | ★               |
@@ -32,8 +32,11 @@ A simple comparison of some of these methods are provided below (to be taken wit
 ★ > ◉ > ● > ▲
 [> means better and smaller size is better]
 
-$^*$ Flexibility means anything related to generating images not similar to those in the training set, and combination of multiple concepts, whether they are trained together or not
-$^†$ It may become more difficult to switch base model or combine multiple concepts in this situation
+$^+$ Usually we take `factor <= 0.5 * sqrt(dim)` as low factor and `factor >= sqrt(dim` as high factor. For example, factor<=8 for SD1.x/SD2.x/SDXL can be seen as low factor, and, factor>=16 can be seen as high factor. <br>
+$^*$ Flexibility means anything related to generating images not similar to those in the training set, and combination of multiple concepts, whether they are trained together or not <br>
+$^†$ It may become more difficult to switch base model or combine multiple concepts in this situation <br>
+
+**The actual performance may vary depending on the datasets, tasks, and hyperparameters used. It is recommended to experiment with different settings to achieve optimal results.**
 
 ## Usage
 

lycoris_lora-2.3.0.dev11/lycoris/functional/__init__.py

@@ -0,0 +1,4 @@
+from .locon import (
+    lora_diff_weight,
+    lora_bypass_forward_diff,
+)

lycoris_lora-2.3.0.dev11/lycoris/functional/general.py

@@ -0,0 +1,8 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+def rebuild_tucker(t, wa, wb):
+    rebuild2 = torch.einsum("i j ..., i p, j r -> p r ...", t, wa, wb)
+    return rebuild2

lycoris_lora-2.3.0.dev11/lycoris/functional/locon.py

@@ -0,0 +1,109 @@
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .general import rebuild_tucker
+
+
+def lora_weight_gen(org_weight, rank, tucker=True):
+    """### lora_weight_gen
+
+    Args:
+        out_dim (int): output dimension
+        in_dim (int): input dimension
+        rank (int): low rank
+
+    Returns:
+        torch.Tensor: down proj weight
+        torch.Tensor: up proj weight
+        torch.Tensor(optional): mid proj weight
+    """
+    out_dim, in_dim, *k = org_weight.shape
+    if k and tucker:
+        down = torch.empty(rank, in_dim, *(1 for _ in k))
+        up = torch.empty(out_dim, rank, *(1 for _ in k))
+        mid = torch.empty(rank, rank, *k)
+        nn.init.kaiming_uniform_(down, a=math.sqrt(5))
+        nn.init.constant_(up, 0)
+        nn.init.kaiming_uniform_(mid, a=math.sqrt(5))
+        return down, up, mid
+    else:
+        down = torch.empty(rank, in_dim)
+        up = torch.empty(out_dim, rank)
+        nn.init.kaiming_uniform_(down, a=math.sqrt(5))
+        nn.init.constant_(up, 0)
+        return down, up
+
+
+def lora_diff_weight(d, u, m=None, gamma=1.0):
+    """### lora_diff_weight
+
+    Get ΔW = BA, where BA is low rank decomposition
+
+    Args:
+        d (torch.Tensor): weight of down proj linear/conv layer
+        u (torch.Tensor): weight of up proj linear/conv layer
+        m (torch.Tensor, optional): weight of mid proj linear/conv layer, \
+            for tucker deomposition
+        gamma (float, optional): scale factor, normally alpha/rank here
+
+    Returns:
+        torch.Tensor: ΔW
+    """
+    assert d.size(0) == u.size(1)
+    R, I, *k = d.shape
+    O, R, *_ = u.shape
+    u = u * gamma
+
+    if m is None:
+        result = u.reshape(-1, u.size(1)) @ d.reshape(d.size(0), -1)
+    else:
+        R, R, *k = m.shape
+        u = u.reshape(u.size(0), -1).transpose(0, 1)
+        d = d.reshape(d.size(0), -1)
+        result = rebuild_tucker(m, u, d)
+    return result.reshape(O, I, *k)
+
+
+FUNC_LIST = [None, None, F.linear, F.conv1d, F.conv2d, F.conv3d]
+
+
+def lora_bypass_forward_diff(x, d, u, m=None, gamma=1.0, extra_args={}):
+    """### lora_bypass_forward_diff
+
+    Args:
+        x (torch.Tensor): input tensor
+        d (torch.Tensor): weight of down proj linear/conv layer
+        u (torch.Tensor): weight of up proj linear/conv layer
+        m (torch.Tensor, optional): weight of mid proj linear/conv layer, \
+            for tucker deomposition
+        gamma (float, optional): scale factor, normally alpha/rank here
+
+    Returns:
+        torch.Tensor: output tensor
+    """
+    if m is not None:
+        down = FUNC_LIST[d.dim()](x, d)
+        mid = FUNC_LIST[d.dim()](down, m, **extra_args)
+        up = FUNC_LIST[d.dim()](mid, u)
+    else:
+        down = FUNC_LIST[d.dim()](x, d, **extra_args)
+        up = FUNC_LIST[d.dim()](down, u)
+    return up * gamma
+
+
+if __name__ == "__main__":
+    w = torch.randn(32, 32, 3, 3, 3)
+    d, u, m = lora_weight_gen(w, 4)
+    u = u+0.01
+    extra_args = {"padding": 1}
+
+    x = torch.randn(1, 32, 8, 8, 8)
+    y = FUNC_LIST[d.dim()](x, w, **extra_args)
+    diff_w = lora_diff_weight(d, u, m, 1)
+    diff_y = lora_bypass_forward_diff(x, d, u, m, 1, extra_args)
+
+    print(F.mse_loss(y, y + diff_y))
+    print(F.mse_loss(w, w + diff_w))

lycoris_lora-2.3.0.dev11/lycoris/functional/loha.py

@@ -0,0 +1,153 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class HadaWeight(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, w1d, w1u, w2d, w2u, scale=torch.tensor(1)):
+        ctx.save_for_backward(w1d, w1u, w2d, w2u, scale)
+        diff_weight = ((w1u @ w1d) * (w2u @ w2d)) * scale
+        return diff_weight
+
+    @staticmethod
+    def backward(ctx, grad_out):
+        (w1d, w1u, w2d, w2u, scale) = ctx.saved_tensors
+        grad_out = grad_out * scale
+        temp = grad_out * (w2u @ w2d)
+        grad_w1d = temp @ w1d.T
+        grad_w1u = w1u.T @ temp
+
+        temp = grad_out * (w1u @ w1d)
+        grad_w2d = temp @ w2d.T
+        grad_w2u = w2u.T @ temp
+
+        del temp
+        return grad_w1d, grad_w1u, grad_w2d, grad_w2u, None
+
+
+class HadaWeightTucker(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, t1, w1d, w1u, t2, w2d, w2u, scale=torch.tensor(1)):
+        ctx.save_for_backward(t1, w1d, w1u, t2, w2d, w2u, scale)
+
+        rebuild1 = torch.einsum("i j ..., j r, i p -> p r ...", t1, w1d, w1u)
+        rebuild2 = torch.einsum("i j ..., j r, i p -> p r ...", t2, w2d, w2u)
+
+        return rebuild1 * rebuild2 * scale
+
+    @staticmethod
+    def backward(ctx, grad_out):
+        (t1, w1d, w1u, t2, w2d, w2u, scale) = ctx.saved_tensors
+        grad_out = grad_out * scale
+
+        temp = torch.einsum("i j ..., j r -> i r ...", t2, w2d)
+        rebuild = torch.einsum("i j ..., i r -> r j ...", temp, w2u)
+
+        grad_w = rebuild * grad_out
+        del rebuild
+
+        grad_w1d = torch.einsum("r j ..., i j ... -> r i", temp, grad_w)
+        grad_temp = torch.einsum("i j ..., i r -> r j ...", grad_w, w1u.T)
+        del grad_w, temp
+
+        grad_w1u = torch.einsum("i r ..., i j ... -> r j", t1, grad_temp)
+        grad_t1 = torch.einsum("i j ..., j r -> i r ...", grad_temp, w1d.T)
+        del grad_temp
+
+        temp = torch.einsum("i j ..., j r -> i r ...", t1, w1d)
+        rebuild = torch.einsum("i j ..., i r -> r j ...", temp, w1u)
+
+        grad_w = rebuild * grad_out
+        del rebuild
+
+        grad_w2d = torch.einsum("r j ..., i j ... -> r i", temp, grad_w)
+        grad_temp = torch.einsum("i j ..., i r -> r j ...", grad_w, w2u.T)
+        del grad_w, temp
+
+        grad_w2u = torch.einsum("i r ..., i j ... -> r j", t2, grad_temp)
+        grad_t2 = torch.einsum("i j ..., j r -> i r ...", grad_temp, w2d.T)
+        del grad_temp
+        return grad_t1, grad_w1d, grad_w1u, grad_t2, grad_w2d, grad_w2u, None
+
+
+def make_weight(w1d, w1u, w2d, w2u, scale):
+    return HadaWeight.apply(w1d, w1u, w2d, w2u, scale)
+
+
+def make_weight_tucker(t1, w1d, w1u, t2, w2d, w2u, scale):
+    return HadaWeightTucker.apply(t1, w1d, w1u, t2, w2d, w2u, scale)
+
+
+def loha_diff_weight(w1d, w1u, w2d, w2u, t1=None, t2=None, gamma=1.0):
+    """### loha_diff_weight
+
+    Get ΔW = BA, where BA is low rank decomposition
+
+    Args:
+        w1d, w2d (torch.Tensor): weight of down proj linear/conv layer
+        w1u, w2u (torch.Tensor): weight of up proj linear/conv layer
+        gamma (float, optional): scale factor, normally alpha/rank here
+
+    Returns:
+        torch.Tensor: ΔW
+    """
+    if t1 is not None and t2 is not None:
+        assert w1d.size(0) == t1.size(1)
+        assert w1u.size(0) == t1.size(0)
+        assert w2d.size(0) == t2.size(1)
+        assert w2u.size(0) == t2.size(0)
+        R, I = w1d.shape
+        R, O = w1u.shape
+        R, R, *k = t1.shape
+        result = make_weight_tucker(t1, w1d, w1u, t2, w2d, w2u, gamma)
+    else:
+        assert w1d.size(0) == w1u.size(1)
+        assert w2d.size(0) == w2u.size(1)
+        R, I, *k = w1d.shape
+        O, R, *_ = w1u.shape
+        w1d = w1d.reshape(w1d.size(0), -1)
+        w1u = w1u.reshape(-1, w1u.size(1))
+        w2d = w2d.reshape(w2d.size(0), -1)
+        w2u = w1u.reshape(-1, w2u.size(1))
+        result = make_weight(w1d, w1u, w2d, w2u, gamma)
+
+    result = result.reshape(O, I, *k)
+    return result
+
+
+FUNC_LIST = [None, None, F.linear, F.conv1d, F.conv2d, F.conv3d]
+
+
+def loha_bypass_forward_diff(
+    x, w1d, w1u, w2d, w2u, t1=None, t2=None, gamma=1.0, extra_args={}
+):
+    """### loha_bypass_forward_diff
+
+    Args:
+        x (torch.Tensor): input tensor
+        w1d, w2d (torch.Tensor): weight of up proj linear/conv layer
+        w1u, w2u (torch.Tensor): weight of down proj linear/conv layer
+        gamma (float, optional): scale factor, normally alpha/rank here
+
+    Returns:
+        torch.Tensor: output tensor
+    """
+    diff_w = loha_diff_weight(w1d, w1u, w2d, w2u, t1, t2, gamma)
+    return FUNC_LIST[w1d.dim() if t1 is None else t1.dim()](x, diff_w, **extra_args)
+
+
+if __name__ == "__main__":
+    w = torch.randn(128, 128, 3, 3, 3)
+    a = torch.randn(16, 128) * 0.01
+    b = torch.randn(16, 128) * 0.01
+    m = torch.randn(16, 16, 3, 3, 3) * 0.01
+    extra_args = {"padding": 1}
+
+    x = torch.randn(1, 128, 8, 8, 8)
+    y = FUNC_LIST[w.dim()](x, w, **extra_args)
+    diff_w = loha_diff_weight(a, b, a, b, m, m, 1)
+    diff_y = loha_bypass_forward_diff(x, a, b, a, b, m, m, 1, extra_args)
+
+    print(F.mse_loss(y, y + diff_y))
+    print(F.mse_loss(w, w + diff_w))

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11}/lycoris/modules/lilora.py

@@ -157,7 +157,7 @@ class LiLoConModule(ModuleCustomSD):
             if "scalar" in key:
                 del missing_keys[missing_keys.index(key)]
         if isinstance(self.scalar, nn.Parameter):
-            self.scalar.copy_(torch.ones_like(self.scalar))
+            self.scalar.data.copy_(torch.ones_like(self.scalar))
         else:
             self.scalar = torch.ones_like(self.scalar)
 

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11}/lycoris/modules/locon.py

@@ -6,6 +6,7 @@ import torch.nn as nn
 import torch.nn.functional as F
 
 from .base import LycorisBaseModule
+from ..functional.general import rebuild_tucker
 from ..logging import logger
 from ..utils.bnb import LinearNF4
 
@@ -149,7 +150,7 @@ class LoConModule(LycorisBaseModule):
             if "scalar" in key:
                 del missing_keys[missing_keys.index(key)]
         if isinstance(self.scalar, nn.Parameter):
-            self.scalar.copy_(torch.ones_like(self.scalar))
+            self.scalar.data.copy_(torch.ones_like(self.scalar))
         else:
             self.scalar = torch.ones_like(self.scalar)
 
@@ -158,9 +159,9 @@ class LoConModule(LycorisBaseModule):
         wb = self.lora_down.weight.to(device)
         if self.tucker:
             t = self.lora_mid.weight.to(device)
-            wa = wa.squeeze(-1, -2)
-            wb = wb.squeeze(-1, -2)
-            weight = torch.einsum("i j k l, p i, j r -> p r k l", t, wa, wb)
+            wa = wa.reshape(wa.size(0), -1).transpose(0, 1)
+            wb = wb.reshape(wb.size(0), -1)
+            weight = rebuild_tucker(t, wa, wb)
         else:
             weight = wa.view(wa.size(0), -1) @ wb.view(wb.size(0), -1)
 

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11}/lycoris/modules/loha.py

@@ -5,85 +5,10 @@ import torch.nn as nn
 import torch.nn.functional as F
 
 from .base import LycorisBaseModule
+from ..functional.loha import loha_diff_weight
 from ..utils.bnb import LinearNF4
 
 
-class HadaWeight(torch.autograd.Function):
-    @staticmethod
-    def forward(ctx, w1a, w1b, w2a, w2b, scale=torch.tensor(1)):
-        ctx.save_for_backward(w1a, w1b, w2a, w2b, scale)
-        diff_weight = ((w1a @ w1b) * (w2a @ w2b)) * scale
-        return diff_weight
-
-    @staticmethod
-    def backward(ctx, grad_out):
-        (w1a, w1b, w2a, w2b, scale) = ctx.saved_tensors
-        grad_out = grad_out * scale
-        temp = grad_out * (w2a @ w2b)
-        grad_w1a = temp @ w1b.T
-        grad_w1b = w1a.T @ temp
-
-        temp = grad_out * (w1a @ w1b)
-        grad_w2a = temp @ w2b.T
-        grad_w2b = w2a.T @ temp
-
-        del temp
-        return grad_w1a, grad_w1b, grad_w2a, grad_w2b, None
-
-
-class HadaWeightCP(torch.autograd.Function):
-    @staticmethod
-    def forward(ctx, t1, w1a, w1b, t2, w2a, w2b, scale=torch.tensor(1)):
-        ctx.save_for_backward(t1, w1a, w1b, t2, w2a, w2b, scale)
-
-        rebuild1 = torch.einsum("i j k l, j r, i p -> p r k l", t1, w1b, w1a)
-        rebuild2 = torch.einsum("i j k l, j r, i p -> p r k l", t2, w2b, w2a)
-
-        return rebuild1 * rebuild2 * scale
-
-    @staticmethod
-    def backward(ctx, grad_out):
-        (t1, w1a, w1b, t2, w2a, w2b, scale) = ctx.saved_tensors
-        grad_out = grad_out * scale
-
-        temp = torch.einsum("i j k l, j r -> i r k l", t2, w2b)
-        rebuild = torch.einsum("i j k l, i r -> r j k l", temp, w2a)
-
-        grad_w = rebuild * grad_out
-        del rebuild
-
-        grad_w1a = torch.einsum("r j k l, i j k l -> r i", temp, grad_w)
-        grad_temp = torch.einsum("i j k l, i r -> r j k l", grad_w, w1a.T)
-        del grad_w, temp
-
-        grad_w1b = torch.einsum("i r k l, i j k l -> r j", t1, grad_temp)
-        grad_t1 = torch.einsum("i j k l, j r -> i r k l", grad_temp, w1b.T)
-        del grad_temp
-
-        temp = torch.einsum("i j k l, j r -> i r k l", t1, w1b)
-        rebuild = torch.einsum("i j k l, i r -> r j k l", temp, w1a)
-
-        grad_w = rebuild * grad_out
-        del rebuild
-
-        grad_w2a = torch.einsum("r j k l, i j k l -> r i", temp, grad_w)
-        grad_temp = torch.einsum("i j k l, i r -> r j k l", grad_w, w2a.T)
-        del grad_w, temp
-
-        grad_w2b = torch.einsum("i r k l, i j k l -> r j", t2, grad_temp)
-        grad_t2 = torch.einsum("i j k l, j r -> i r k l", grad_temp, w2b.T)
-        del grad_temp
-        return grad_t1, grad_w1a, grad_w1b, grad_t2, grad_w2a, grad_w2b, None
-
-
-def make_weight(w1a, w1b, w2a, w2b, scale):
-    return HadaWeight.apply(w1a, w1b, w2a, w2b, scale)
-
-
-def make_weight_tucker(t1, w1a, w1b, t2, w2a, w2b, scale):
-    return HadaWeightCP.apply(t1, w1a, w1b, t2, w2a, w2b, scale)
-
-
 class LohaModule(LycorisBaseModule):
     support_module = {
         "linear",
@@ -214,28 +139,30 @@ class LohaModule(LycorisBaseModule):
             if "scalar" in key:
                 del missing_keys[missing_keys.index(key)]
         if isinstance(self.scalar, nn.Parameter):
-            self.scalar.copy_(torch.ones_like(self.scalar))
+            self.scalar.data.copy_(torch.ones_like(self.scalar))
         else:
             self.scalar = torch.ones_like(self.scalar)
 
     def get_weight(self, shape):
         if self.tucker:
-            weight = make_weight_tucker(
-                self.hada_t1,
-                self.hada_w1_a,
+            weight = loha_diff_weight(
                 self.hada_w1_b,
-                self.hada_t2,
-                self.hada_w2_a,
+                self.hada_w1_a,
                 self.hada_w2_b,
-                scale=torch.tensor(self.scale),
+                self.hada_w2_a,
+                self.hada_t1,
+                self.hada_t2,
+                gamma=torch.tensor(self.scale),
             )
         else:
-            weight = make_weight(
-                self.hada_w1_a,
+            weight = loha_diff_weight(
                 self.hada_w1_b,
-                self.hada_w2_a,
+                self.hada_w1_a,
                 self.hada_w2_b,
-                scale=torch.tensor(self.scale),
+                self.hada_w2_a,
+                None,
+                None,
+                gamma=torch.tensor(self.scale),
             )
         if shape is not None:
             weight = weight.reshape(shape)
@@ -339,20 +266,21 @@ class LohaModule(LycorisBaseModule):
 
 
 if __name__ == "__main__":
-    base = nn.Linear(128, 128).cuda()
-    lokr = LohaModule("test", base, 1, 4, 1, weight_decompose=True).cuda()
+    device = torch.device("mps")
+    base = nn.Linear(128, 128).to(device)
+    lokr = LohaModule("test", base, 1, 4, 1, weight_decompose=True).to(device)
     print(lokr)
-    test_input = torch.randn(1, 128).cuda()
+    test_input = torch.randn(1, 128).to(device)
     test_output = lokr(test_input)
     print(test_output.shape)
 
-    base_4bit = LinearNF4(128, 128)
-    base_4bit.load_state_dict(base.state_dict())
-    base_4bit.cuda()
-    qlocon = LohaModule("test", base_4bit, 1, 4, 1, weight_decompose=False).cuda()
-    print(qlocon)
-    test_input = torch.randn(1, 128).cuda()
-    test_output = qlocon(test_input)
+    # base_4bit = LinearNF4(128, 128)
+    # base_4bit.load_state_dict(base.state_dict())
+    # base_4bit.to(device)
+    # qlocon = LohaModule("test", base_4bit, 1, 4, 1, weight_decompose=False).to(device)
+    # print(qlocon)
+    # test_input = torch.randn(1, 128).to(device)
+    # test_output = qlocon(test_input)
     print(test_output.shape)
 
     base = nn.Conv2d(128, 128, 3, 1, 1)

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11}/lycoris/modules/lokr.py

@@ -280,7 +280,7 @@ class LokrModule(LycorisBaseModule):
             if "scalar" in key:
                 del missing_keys[missing_keys.index(key)]
         if isinstance(self.scalar, nn.Parameter):
-            self.scalar.copy_(torch.ones_like(self.scalar))
+            self.scalar.data.copy_(torch.ones_like(self.scalar))
         else:
             self.scalar = torch.ones_like(self.scalar)
 

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11}/lycoris/wrapper.py

@@ -331,6 +331,8 @@ class LycorisNetwork(torch.nn.Module):
             loras = {}
             lora_names = []
             for name, module in root_module.named_modules():
+                if module is root_module:
+                    continue
                 module_name = module.__class__.__name__
                 if module_name in self.MODULE_ALGO_MAP:
                     next_config = self.MODULE_ALGO_MAP[module_name]

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.3.0.dev9
+Version: 2.3.0.dev11
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11}/lycoris_lora.egg-info/SOURCES.txt

@@ -5,6 +5,10 @@ lycoris/__init__.py
 lycoris/config.py
 lycoris/logging.py
 lycoris/wrapper.py
+lycoris/functional/__init__.py
+lycoris/functional/general.py
+lycoris/functional/locon.py
+lycoris/functional/loha.py
 lycoris/hcp/__init__.py
 lycoris/hcp/base.py
 lycoris/hcp/diag_oft.py

{lycoris_lora-2.3.0.dev9 → lycoris_lora-2.3.0.dev11}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="2.3.0.dev9",
+    version="2.3.0.dev11",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",