lycoris-lora 3.0.0.dev6__tar.gz → 3.0.0.dev8__tar.gz

{lycoris_lora-3.0.0.dev6/lycoris_lora.egg-info → lycoris_lora-3.0.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.0.dev6
+Version: 3.0.0.dev8
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/functional/loha.py

@@ -174,9 +174,7 @@ def loha_bypass_forward_diff(
 
 if __name__ == "__main__":
     w = torch.randn(128, 128, 3, 3, 3)
-    w1d, w2d, w1u, w2u, t1, t2 = loha_weight_gen(
-        w, 4, tucker=True
-    )
+    w1d, w2d, w1u, w2u, t1, t2 = loha_weight_gen(w, 4, tucker=True)
     w1u += 0.01
     extra_args = {"padding": 1}
 

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/kohya.py

@@ -611,11 +611,8 @@ class LycorisNetworkKohya(LycorisNetwork):
             # Precalculate model hashes to save time on indexing
             if metadata is None:
                 metadata = {}
-            model_hash, legacy_hash = precalculate_safetensors_hashes(
-                state_dict, metadata
-            )
+            model_hash = precalculate_safetensors_hashes(state_dict)
             metadata["sshs_model_hash"] = model_hash
-            metadata["sshs_legacy_hash"] = legacy_hash
 
             save_file(state_dict, file, metadata)
         else:
@@ -845,11 +842,8 @@ class IA3Network(torch.nn.Module):
             # Precalculate model hashes to save time on indexing
             if metadata is None:
                 metadata = {}
-            model_hash, legacy_hash = precalculate_safetensors_hashes(
-                state_dict, metadata
-            )
+            model_hash = precalculate_safetensors_hashes(state_dict)
             metadata["sshs_model_hash"] = model_hash
-            metadata["sshs_legacy_hash"] = legacy_hash
 
             save_file(state_dict, file, metadata)
         else:

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/modules/__init__.py

@@ -1,6 +1,7 @@
 import torch
 import torch.nn as nn
 
+from .base import LycorisBaseModule
 from .locon import LoConModule
 from .loha import LohaModule
 from .lokr import LokrModule
@@ -57,7 +58,7 @@ def make_module(lyco_type, params, lora_name, orig_module):
             module.dora_scale.copy_(dora_scale)
     elif lyco_type == "kron":
         w1, w1a, w1b, w2, w2a, w2b, _, t2, alpha, dora_scale = params
-        full_matrix=False
+        full_matrix = False
         if w1a is not None:
             lora_dim = w1a.size(1)
         elif w2a is not None:

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/modules/dylora.py

@@ -80,6 +80,9 @@ class DyLoraModule(LycorisBaseModule):
         for v in self.up_list:
             torch.nn.init.zeros_(v)
 
+    def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
+        return
+
     def custom_state_dict(self):
         destination = {}
         destination["alpha"] = self.alpha

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/modules/full.py

@@ -1,3 +1,4 @@
+from typing import Mapping
 import torch
 import torch.nn as nn
 
@@ -76,6 +77,9 @@ class FullModule(LycorisBaseModule):
         else:
             self.org_bias = None
 
+    def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
+        return
+
     def custom_state_dict(self):
         sd = {"diff": self.weight.data.cpu() - self._org_weight[0]}
         if self.bias is not None:

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/modules/glora.py

@@ -66,17 +66,17 @@ class GLoRAModule(LycorisBaseModule):
             stride = org_module.stride
             padding = org_module.padding
             out_dim = org_module.out_channels
-            self.tucker = use_tucker and k_size != (1, 1)
+            self.tucker = use_tucker and all(i == 1 for i in k_size)
             self.down_op = self.op
             self.up_op = self.op
 
             # A
-            self.a2 = self.module(in_dim, lora_dim, (1, 1), bias=False)
-            self.a1 = self.module(lora_dim, in_dim, (1, 1), bias=False)
+            self.a2 = self.module(in_dim, lora_dim, 1, bias=False)
+            self.a1 = self.module(lora_dim, in_dim, 1, bias=False)
 
             # B
-            if use_tucker and k_size != (1, 1):
-                self.b2 = self.module(in_dim, lora_dim, (1, 1), bias=False)
+            if use_tucker and any(i != 1 for i in k_size):
+                self.b2 = self.module(in_dim, lora_dim, 1, bias=False)
                 self.bm = self.module(
                     lora_dim, lora_dim, k_size, stride, padding, bias=False
                 )
@@ -85,7 +85,7 @@ class GLoRAModule(LycorisBaseModule):
                 self.b2 = self.module(
                     in_dim, lora_dim, k_size, stride, padding, bias=False
                 )
-            self.b1 = self.module(lora_dim, out_dim, (1, 1), bias=False)
+            self.b1 = self.module(lora_dim, out_dim, 1, bias=False)
         else:
             self.isconv = False
             self.down_op = F.linear
@@ -123,6 +123,7 @@ class GLoRAModule(LycorisBaseModule):
     def make_weight(self, device=None):
         wa1 = self.a1.weight.view(self.a1.weight.size(0), -1)
         wa2 = self.a2.weight.view(self.a2.weight.size(0), -1)
+        orig = self.org_weight
 
         if self.tucker:
             wb = tucker_weight_from_conv(self.b1.weight, self.b2.weight, self.bm.weight)
@@ -130,7 +131,7 @@ class GLoRAModule(LycorisBaseModule):
             wb1 = self.b1.weight.view(self.b1.weight.size(0), -1)
             wb2 = self.b2.weight.view(self.b2.weight.size(0), -1)
             wb = wb1 @ wb2
-        orig = self.org_weight
+            wb = wb.view(*orig.shape)
         if orig.dim() > 2:
             w_wa1 = torch.einsum("o i ..., i j -> o j ...", orig, wa1)
             w_wa2 = torch.einsum("o i ..., i j -> o j ...", w_wa1, wa2)

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/modules/locon.py

@@ -70,10 +70,10 @@ class LoConModule(LycorisBaseModule):
             stride = org_module.stride
             padding = org_module.padding
             out_dim = org_module.out_channels
-            self.tucker = use_tucker and k_size != (1, 1)
+            self.tucker = use_tucker and any(i != 1 for i in k_size)
             self.down_op = self.op
             self.up_op = self.op
-            if use_tucker and k_size != (1, 1):
+            if use_tucker and any(i != 1 for i in k_size):
                 self.lora_down = self.module(in_dim, lora_dim, (1, 1), bias=False)
                 self.lora_mid = self.module(
                     lora_dim, lora_dim, k_size, stride, padding, bias=False
@@ -97,7 +97,7 @@ class LoConModule(LycorisBaseModule):
 
         self.wd = weight_decompose
         if self.wd:
-            org_weight: nn.Parameter = org_module.weight
+            org_weight = org_module.weight.cpu().clone().float()
             self.dora_norm_dims = org_weight.dim() - 1
             self.dora_scale = nn.Parameter(
                 torch.norm(
@@ -148,6 +148,8 @@ class LoConModule(LycorisBaseModule):
                 del missing_keys[missing_keys.index(key)]
         if isinstance(self.scalar, nn.Parameter):
             self.scalar.data.copy_(torch.ones_like(self.scalar))
+        elif getattr(self, "scalar", None) is not None:
+            self.scalar.copy_(torch.ones_like(self.scalar))
         else:
             self.register_buffer(
                 "scalar", torch.ones_like(self.scalar), persistent=False

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/modules/loha.py

@@ -58,12 +58,11 @@ class LohaModule(LycorisBaseModule):
             k_size = org_module.kernel_size
             out_dim = org_module.out_channels
             self.shape = (out_dim, in_dim, *k_size)
-            self.tucker = use_tucker and k_size != (1, 1)
+            self.tucker = use_tucker and any(i != 1 for i in k_size)
             if self.tucker:
                 w_shape = (out_dim, in_dim, *k_size)
             else:
                 w_shape = (out_dim, in_dim * torch.tensor(k_size).prod().item())
-            self.op = F.conv2d
 
         if self.tucker:
             self.hada_t1 = nn.Parameter(torch.empty(lora_dim, lora_dim, *w_shape[2:]))
@@ -90,7 +89,7 @@ class LohaModule(LycorisBaseModule):
 
         self.wd = weight_decompose
         if self.wd:
-            org_weight: nn.Parameter = org_module.weight
+            org_weight = org_module.weight.cpu().clone().float()
             self.dora_norm_dims = org_weight.dim() - 1
             self.dora_scale = nn.Parameter(
                 torch.norm(
@@ -140,6 +139,8 @@ class LohaModule(LycorisBaseModule):
                 del missing_keys[missing_keys.index(key)]
         if isinstance(self.scalar, nn.Parameter):
             self.scalar.data.copy_(torch.ones_like(self.scalar))
+        elif getattr(self, "scalar", None) is not None:
+            self.scalar.copy_(torch.ones_like(self.scalar))
         else:
             self.register_buffer(
                 "scalar", torch.ones_like(self.scalar), persistent=False

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/modules/lokr.py

@@ -93,7 +93,7 @@ class LokrModule(LycorisBaseModule):
             if unbalanced_factorization:
                 out_l, out_k = out_k, out_l
             shape = ((out_l, out_k), (in_m, in_n), *k_size)  # ((a, b), (c, d), *k_size)
-            self.tucker = use_tucker and k_size != (1, 1)
+            self.tucker = use_tucker and any(i != 1 for i in k_size)
             if (
                 decompose_both
                 and lora_dim < max(shape[0][0], shape[1][0]) / 2
@@ -166,7 +166,7 @@ class LokrModule(LycorisBaseModule):
 
         self.wd = weight_decompose
         if self.wd:
-            org_weight: nn.Parameter = org_module.weight
+            org_weight = org_module.weight.cpu().clone().float()
             self.dora_norm_dims = org_weight.dim() - 1
             self.dora_scale = nn.Parameter(
                 torch.norm(
@@ -232,6 +232,8 @@ class LokrModule(LycorisBaseModule):
                 del missing_keys[missing_keys.index(key)]
         if isinstance(self.scalar, nn.Parameter):
             self.scalar.data.copy_(torch.ones_like(self.scalar))
+        elif getattr(self, "scalar", None) is not None:
+            self.scalar.copy_(torch.ones_like(self.scalar))
         else:
             self.register_buffer(
                 "scalar", torch.ones_like(self.scalar), persistent=False

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/lycoris/utils/__init__.py

@@ -469,12 +469,12 @@ def get_module(lyco_state_dict: Dict, lora_name):
 def tucker_weight_from_conv(up, down, mid):
     up = up.reshape(up.size(0), up.size(1))
     down = down.reshape(down.size(0), down.size(1))
-    return torch.einsum("m n w h, i m, n j -> i j w h", mid, up, down)
+    return torch.einsum("m n ..., i m, n j -> i j ...", mid, up, down)
 
 
 def tucker_weight(wa, wb, t):
-    temp = torch.einsum("i j k l, j r -> i r k l", t, wb)
-    return torch.einsum("i j k l, i r -> r j k l", temp, wa)
+    temp = torch.einsum("i j ..., j r -> i r ...", t, wb)
+    return torch.einsum("i j ..., i r -> r j ...", temp, wa)
 
 
 def apply_dora_scale(org_weight, rebuild, dora_scale, scale):

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.0.dev6
+Version: 3.0.0.dev8
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.0.dev6 → lycoris_lora-3.0.0.dev8}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.0.dev6",
+    version="3.0.0.dev8",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",