lycoris-lora 3.0.0.dev1__tar.gz → 3.0.0.dev3__tar.gz

{lycoris_lora-3.0.0.dev1/lycoris_lora.egg-info → lycoris_lora-3.0.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.0.dev1
+Version: 3.0.0.dev3
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao
@@ -8,8 +8,5 @@ Author-email: apolloyeh0123@gmail.com
 Requires-Python: >=3.10
 License-File: LICENSE.md
 Requires-Dist: torch
-Requires-Dist: safetensors
-Requires-Dist: diffusers
-Requires-Dist: transformers
 Requires-Dist: einops
 Requires-Dist: toml

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/__init__.py

@@ -1,5 +1,8 @@
+try:
+    from . import kohya
+except:
+    pass
 from . import (
-    kohya,
     modules,
     utils,
 )

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/functional/general.py

@@ -59,6 +59,8 @@ def power2factorization(dimension: int, factor: int = -1) -> tuple[int, int]:
     n = 2**p
     m*n = dim
     """
+    if factor == -1:
+        factor = dimension
 
     # Find the first solution and check if it is even doable
     m = n = 0

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/modules/base.py

@@ -3,6 +3,7 @@ from collections import OrderedDict
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
+import torch.nn.utils.parametrize as parametrize
 
 from ..utils.bnb import QuantLinears, log_bypass
 
@@ -181,6 +182,38 @@ class LycorisBaseModule(ModuleCustomSD):
         self.org_forward = org_module.forward
         self.org_module = [org_module]
 
+    @classmethod
+    def parametrize(cls, org_module, attr, *args, **kwargs):
+        target_param = getattr(org_module, attr)
+        kwargs["bypass_mode"] = False
+        if target_param.dim() == 2:
+            proxy_module = nn.Linear(
+                target_param.shape[0], target_param.shape[1], bias=False
+            )
+            proxy_module.weight = target_param
+        elif target_param.dim() > 2:
+            module_type = [
+                None,
+                None,
+                None,
+                nn.Conv1d,
+                nn.Conv2d,
+                nn.Conv3d,
+                None,
+                None,
+            ][target_param.dim()]
+            proxy_module = module_type(
+                target_param.shape[0],
+                target_param.shape[1],
+                *target_param.shape[2:],
+            )
+            proxy_module.weight = target_param
+        module_obj = cls("", proxy_module, *args, **kwargs)
+        module_obj.forward = module_obj.parametrize_forward
+        module_obj.to(target_param.device)
+        parametrize.register_parametrization(org_module, attr, module_obj)
+        return module_obj
+
     @property
     def dtype(self):
         return self.dtype_tensor.dtype
@@ -231,5 +264,10 @@ class LycorisBaseModule(ModuleCustomSD):
     def bypass_forward(self, x, scale=1):
         raise NotImplementedError
 
+    def parametrize_forward(self, x: torch.Tensor, *args, **kwargs):
+        return self.get_merged_weight(
+            multiplier=self.multiplier, shape=x.shape, device=x.device
+        )[0].to(x.dtype)
+
     def forward(self, *args, **kwargs):
         raise NotImplementedError

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/modules/boft.py

@@ -136,7 +136,7 @@ class ButterflyOFTModule(LycorisBaseModule):
         diff = self.make_weight(scale=multiplier, device=device, diff=True)
         if shape is not None:
             diff = diff.view(shape)
-        return diff
+        return diff, None
 
     def get_merged_weight(self, multiplier=1, shape=None, device=None):
         diff = self.make_weight(scale=multiplier, device=device)
@@ -200,33 +200,43 @@ class ButterflyOFTModule(LycorisBaseModule):
 
 
 if __name__ == "__main__":
-    base = nn.Linear(128, 128).cuda()
-    boft = ButterflyOFTModule(
-        "test", base, 1, 4, 1, weight_decompose=True, factor=8
-    ).cuda()
-    print(boft)
-    test_input = torch.randn(1, 77, 128).cuda()
-    test_output = boft(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base_4bit = LinearNF4(128, 128)
-    base_4bit.load_state_dict(base.state_dict())
-    base_4bit.cuda()
-    qboft = ButterflyOFTModule(
-        "test", base_4bit, 1, 4, 1, weight_decompose=False
-    ).cuda()
-    print(qboft)
-    test_output = qboft(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base = nn.Conv2d(128, 128, 3, 1, 1)
-    boft = ButterflyOFTModule(
-        "test", base, 1, 4, 1, weight_decompose=True, use_tucker=True
-    )
-    print(boft)
-    test_input = torch.randn(1, 128, 16, 16)
-    test_output = boft(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
+    device = torch.device("cuda")
+    module = ButterflyOFTModule
+    with torch.autocast("cuda" if torch.cuda.is_available() else "cpu"):
+        base = nn.Linear(128, 128).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True).to(device)
+        print(net)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base_4bit = LinearNF4(128, 128, device="cuda")
+        base_4bit.load_state_dict(base.state_dict())
+        base_4bit.to(device)
+        qnet = module("test", base_4bit, 1, 4, 1, weight_decompose=False).to(device)
+        print(qnet)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = qnet(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True).to(
+            device
+        )
+        print(net)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module.parametrize(
+            base, "weight", 1, 4, 1, weight_decompose=True, use_tucker=True
+        ).to(device)
+        print(base)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = base(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/modules/diag_oft.py

@@ -119,7 +119,7 @@ class DiagOFTModule(LycorisBaseModule):
         diff = self.make_weight(scale=multiplier, device=device, diff=True)
         if shape is not None:
             diff = diff.view(shape)
-        return diff
+        return diff, None
 
     def get_merged_weight(self, multiplier=1, shape=None, device=None):
         diff = self.make_weight(scale=multiplier, device=device)
@@ -185,27 +185,43 @@ class DiagOFTModule(LycorisBaseModule):
 
 
 if __name__ == "__main__":
-    base = nn.Linear(128, 128).cuda()
-    doft = DiagOFTModule("test", base, 1, 4, 1, weight_decompose=True, factor=8).cuda()
-    print(doft)
-    test_input = torch.randn(1, 77, 128).cuda()
-    test_output = doft(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base_4bit = LinearNF4(128, 128)
-    base_4bit.load_state_dict(base.state_dict())
-    base_4bit.cuda()
-    qdoft = DiagOFTModule("test", base_4bit, 1, 4, 1, weight_decompose=False).cuda()
-    print(qdoft)
-    test_output = qdoft(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base = nn.Conv2d(128, 128, 3, 1, 1)
-    doft = DiagOFTModule("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True)
-    print(doft)
-    test_input = torch.randn(1, 128, 16, 16)
-    test_output = doft(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
+    device = torch.device("cuda")
+    module = DiagOFTModule
+    with torch.autocast("cuda" if torch.cuda.is_available() else "cpu"):
+        base = nn.Linear(128, 128).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True).to(device)
+        print(net)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base_4bit = LinearNF4(128, 128, device="cuda")
+        base_4bit.load_state_dict(base.state_dict())
+        base_4bit.to(device)
+        qnet = module("test", base_4bit, 1, 4, 1, weight_decompose=False).to(device)
+        print(qnet)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = qnet(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True).to(
+            device
+        )
+        print(net)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module.parametrize(
+            base, "weight", 1, 4, 1, weight_decompose=True, use_tucker=True
+        ).to(device)
+        print(base)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = base(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/modules/full.py

@@ -158,18 +158,33 @@ class FullModule(LycorisBaseModule):
 
 
 if __name__ == "__main__":
-    base = nn.Linear(128, 128).cuda()
-    full = FullModule("test", base, 1, 4, 1, weight_decompose=True, factor=8).cuda()
-    print(full)
-    test_input = torch.randn(1, 77, 128).cuda()
-    test_output = full(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base = nn.Conv2d(128, 128, 3, 1, 1)
-    full = FullModule("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True)
-    print(full)
-    test_input = torch.randn(1, 128, 16, 16)
-    test_output = full(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
+    device = torch.device("cuda")
+    module = FullModule
+    with torch.autocast("cuda" if torch.cuda.is_available() else "cpu"):
+        base = nn.Linear(128, 128).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True).to(device)
+        print(net)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True).to(
+            device
+        )
+        print(net)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module.parametrize(
+            base, "weight", 1, 4, 1, weight_decompose=True, use_tucker=True
+        ).to(device)
+        print(base)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = base(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/modules/glora.py

@@ -169,28 +169,43 @@ class GLoRAModule(LycorisBaseModule):
 
 
 if __name__ == "__main__":
-    base = nn.Linear(128, 128).cuda()
-    glora = GLoRAModule("test", base, 1, 4, 1, weight_decompose=True).cuda()
-    print(glora)
-    test_input = torch.randn(1, 128).cuda()
-    test_output = glora(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base_4bit = LinearNF4(128, 128)
-    base_4bit.load_state_dict(base.state_dict())
-    base_4bit.cuda()
-    qglora = GLoRAModule("test", base_4bit, 1, 4, 1, weight_decompose=False).cuda()
-    print(qglora)
-    test_input = torch.randn(1, 128).cuda()
-    test_output = qglora(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base = nn.Conv2d(128, 128, 3, 1, 1)
-    glora = GLoRAModule("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True)
-    print(glora)
-    test_input = torch.randn(1, 128, 16, 16)
-    test_output = glora(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
+    device = torch.device("cuda")
+    module = GLoRAModule
+    with torch.autocast("cuda" if torch.cuda.is_available() else "cpu"):
+        base = nn.Linear(128, 128).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True).to(device)
+        print(net)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base_4bit = LinearNF4(128, 128, device="cuda")
+        base_4bit.load_state_dict(base.state_dict())
+        base_4bit.to(device)
+        qnet = module("test", base_4bit, 1, 4, 1, weight_decompose=False).to(device)
+        print(qnet)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = qnet(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True).to(
+            device
+        )
+        print(net)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module.parametrize(
+            base, "weight", 1, 4, 1, weight_decompose=True, use_tucker=True
+        ).to(device)
+        print(base)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = base(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/modules/locon.py

@@ -183,11 +183,12 @@ class LoConModule(LycorisBaseModule):
             diff = diff.view(shape)
         if device is not None:
             diff = diff.to(device)
-        return diff
+        return diff, None
 
     def get_merged_weight(self, multiplier=1, shape=None, device=None):
-        merged = self.org_module[0].weight.data + self.get_diff_weight(
-            multiplier=multiplier, shape=shape, device=device
+        merged = (
+            self.org_module[0].weight.data
+            + self.get_diff_weight(multiplier=multiplier, shape=shape, device=device)[0]
         )
         if self.wd:
             merged = self.apply_weight_decompose(merged)
@@ -277,28 +278,43 @@ class LoConModule(LycorisBaseModule):
 
 
 if __name__ == "__main__":
-    base = nn.Linear(128, 128).cuda()
-    locon = LoConModule("test", base, 1, 4, 1, weight_decompose=True).cuda()
-    print(locon)
-    test_input = torch.randn(1, 128).cuda()
-    test_output = locon(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base_4bit = LinearNF4(128, 128)
-    base_4bit.load_state_dict(base.state_dict())
-    base_4bit.cuda()
-    qlocon = LoConModule("test", base_4bit, 1, 4, 1, weight_decompose=False).cuda()
-    print(qlocon)
-    test_input = torch.randn(1, 128).cuda()
-    test_output = qlocon(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base = nn.Conv2d(128, 128, 3, 1, 1)
-    locon = LoConModule("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True)
-    print(locon)
-    test_input = torch.randn(1, 128, 16, 16)
-    test_output = locon(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
+    device = torch.device("cuda")
+    module = LoConModule
+    with torch.autocast("cuda" if torch.cuda.is_available() else "cpu"):
+        base = nn.Linear(128, 128).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True).to(device)
+        print(net)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base_4bit = LinearNF4(128, 128, device="cuda")
+        base_4bit.load_state_dict(base.state_dict())
+        base_4bit.to(device)
+        qnet = module("test", base_4bit, 1, 4, 1, weight_decompose=False).to(device)
+        print(qnet)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = qnet(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True).to(
+            device
+        )
+        print(net)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module.parametrize(
+            base, "weight", 1, 4, 1, weight_decompose=True, use_tucker=True
+        )
+        print(base)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = base(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/modules/loha.py

@@ -181,15 +181,16 @@ class LohaModule(LycorisBaseModule):
         diff = self.get_weight(shape) * scale
         if device is not None:
             diff = diff.to(device)
-        return diff
+        return diff, None
 
     def get_merged_weight(self, multiplier=1, shape=None, device=None):
-        merged = self.org_module[0].weight.data + self.get_diff_weight(
-            multiplier=multiplier, shape=shape, device=device
+        merged = (
+            self.org_module[0].weight.data
+            + self.get_diff_weight(multiplier=multiplier, shape=shape, device=device)[0]
         )
         if self.wd:
             merged = self.apply_weight_decompose(merged)
-        return merged
+        return merged, None
 
     def apply_weight_decompose(self, weight):
         weight = weight.to(self.dora_scale.dtype)
@@ -269,28 +270,42 @@ class LohaModule(LycorisBaseModule):
 
 if __name__ == "__main__":
     device = torch.device("cuda")
-    base = nn.Linear(128, 128).to(device)
-    loha = LohaModule("test", base, 1, 4, 1, weight_decompose=True).to(device)
-    print(loha)
-    test_input = torch.randn(1, 128).to(device)
-    test_output = loha(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base_4bit = LinearNF4(128, 128)
-    base_4bit.load_state_dict(base.state_dict())
-    base_4bit.to(device)
-    qloha = LohaModule("test", base_4bit, 1, 4, 1, weight_decompose=False).to(device)
-    print(qloha)
-    test_input = torch.randn(1, 128).to(device)
-    test_output = qloha(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base = nn.Conv2d(128, 128, 3, 1, 1)
-    loha = LohaModule("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True)
-    print(loha)
-    test_input = torch.randn(1, 128, 16, 16)
-    test_output = loha(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
+    module = LohaModule
+    with torch.autocast("cuda" if torch.cuda.is_available() else "cpu"):
+        base = nn.Linear(128, 128).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True).to(device)
+        print(net)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base_4bit = LinearNF4(128, 128, device="cuda")
+        base_4bit.load_state_dict(base.state_dict())
+        base_4bit.to(device)
+        qnet = module("test", base_4bit, 1, 4, 1, weight_decompose=False).to(device)
+        print(qnet)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = qnet(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True).to(
+            device
+        )
+        print(net)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module.parametrize(
+            base, "weight", 1, 4, 1, weight_decompose=True, use_tucker=True
+        )
+        print(base)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = base(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/lycoris/modules/lokr.py

@@ -262,15 +262,21 @@ class LokrModule(LycorisBaseModule):
             weight *= drop
         return weight
 
-    def get_merged_weight(self, multiplier=1, shape=None, device=None):
+    def get_diff_weight(self, multiplier=1, shape=None, device=None):
         scale = self.scale * multiplier
         diff = self.get_weight(shape) * scale
         if device is not None:
             diff = diff.to(device)
-        merged = self.org_module[0].weight.data + diff
+        return diff, None
+
+    def get_merged_weight(self, multiplier=1, shape=None, device=None):
+        merged = (
+            self.org_module[0].weight.data
+            + self.get_diff_weight(multiplier=multiplier, shape=shape, device=device)[0]
+        )
         if self.wd:
             merged = self.apply_weight_decompose(merged)
-        return merged
+        return merged, None
 
     def apply_weight_decompose(self, weight):
         weight = weight.to(self.dora_scale.dtype)
@@ -394,27 +400,43 @@ class LokrModule(LycorisBaseModule):
 
 
 if __name__ == "__main__":
-    base = nn.Linear(128, 128).cuda()
-    lokr = LokrModule("test", base, 1, 4, 1, weight_decompose=True, factor=8).cuda()
-    print(lokr)
-    test_input = torch.randn(1, 77, 128).cuda()
-    test_output = lokr(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base_4bit = LinearNF4(128, 128)
-    base_4bit.load_state_dict(base.state_dict())
-    base_4bit.cuda()
-    qlokr = LokrModule("test", base_4bit, 1, 4, 1, weight_decompose=False).cuda()
-    print(qlokr)
-    test_output = qlokr(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
-
-    base = nn.Conv2d(128, 128, 3, 1, 1)
-    lokr = LokrModule("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True)
-    print(lokr)
-    test_input = torch.randn(1, 128, 16, 16)
-    test_output = lokr(test_input)
-    torch.sum(test_output).backward()
-    print(test_output.shape)
+    device = torch.device("cuda")
+    module = LokrModule
+    with torch.autocast("cuda" if torch.cuda.is_available() else "cpu"):
+        base = nn.Linear(128, 128).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True).to(device)
+        print(net)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base_4bit = LinearNF4(128, 128, device="cuda")
+        base_4bit.load_state_dict(base.state_dict())
+        base_4bit.to(device)
+        qnet = module("test", base_4bit, 1, 4, 1, weight_decompose=False).to(device)
+        print(qnet)
+        test_input = torch.randn(1, 128).to(device).half()
+        test_output = qnet(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module("test", base, 1, 4, 1, weight_decompose=True, use_tucker=True).to(
+            device
+        )
+        print(net)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = net(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)
+
+        base = nn.Conv2d(128, 128, 3, 1, 1).to(device).half()
+        net = module.parametrize(
+            base, "weight", 1, 4, 1, weight_decompose=True, use_tucker=True
+        )
+        print(base)
+        test_input = torch.randn(1, 128, 16, 16).to(device).half()
+        test_output = base(test_input)
+        torch.sum(test_output).backward()
+        print(test_output.shape)

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.0.dev1
+Version: 3.0.0.dev3
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao
@@ -8,8 +8,5 @@ Author-email: apolloyeh0123@gmail.com
 Requires-Python: >=3.10
 License-File: LICENSE.md
 Requires-Dist: torch
-Requires-Dist: safetensors
-Requires-Dist: diffusers
-Requires-Dist: transformers
 Requires-Dist: einops
 Requires-Dist: toml

lycoris_lora-3.0.0.dev3/lycoris_lora.egg-info/requires.txt

@@ -0,0 +1,3 @@
+torch
+einops
+toml

{lycoris_lora-3.0.0.dev1 → lycoris_lora-3.0.0.dev3}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.0.dev1",
+    version="3.0.0.dev3",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",
@@ -12,9 +12,6 @@ setup(
     zip_safe=False,
     install_requires=[
         "torch",
-        "safetensors",
-        "diffusers",
-        "transformers",
         "einops",
         "toml",
     ],

lycoris_lora-3.0.0.dev1/lycoris_lora.egg-info/requires.txt

@@ -1,6 +0,0 @@
-torch
-safetensors
-diffusers
-transformers
-einops
-toml