lycoris-lora 3.0.1.dev10__tar.gz → 3.0.1.dev12__tar.gz

{lycoris_lora-3.0.1.dev10/lycoris_lora.egg-info → lycoris_lora-3.0.1.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev10
+Version: 3.0.1.dev12
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev10 → lycoris_lora-3.0.1.dev12}/lycoris/functional/lokr.py

@@ -14,7 +14,10 @@ def make_kron(w1, w2, scale):
     w2 = w2.contiguous()
     rebuild = torch.kron(w1, w2)
 
-    return rebuild * scale
+    if scale != 1:
+        rebuild = rebuild * scale
+
+    return rebuild
 
 
 def weight_gen(

{lycoris_lora-3.0.1.dev10 → lycoris_lora-3.0.1.dev12}/lycoris/modules/loha.py

@@ -182,6 +182,9 @@ class LohaModule(LycorisBaseModule):
             )
 
     def get_weight(self, shape):
+        scale = torch.tensor(
+            self.scale, dtype=self.hada_w1_b.dtype, device=self.hada_w1_b.device
+        )
         if self.tucker:
             weight = loha_diff_weight(
                 self.hada_w1_b,
@@ -190,7 +193,7 @@ class LohaModule(LycorisBaseModule):
                 self.hada_w2_a,
                 self.hada_t1,
                 self.hada_t2,
-                gamma=torch.tensor(self.scale),
+                gamma=scale,
             )
         else:
             weight = loha_diff_weight(
@@ -200,7 +203,7 @@ class LohaModule(LycorisBaseModule):
                 self.hada_w2_a,
                 None,
                 None,
-                gamma=torch.tensor(self.scale),
+                gamma=scale,
             )
         if shape is not None:
             weight = weight.reshape(shape)

{lycoris_lora-3.0.1.dev10 → lycoris_lora-3.0.1.dev12}/lycoris/modules/lokr.py

@@ -281,7 +281,29 @@ class LokrModule(LycorisBaseModule):
         ):
             factor = -1
         else:
-            factor = max(w1.shape) if w1 is not None else max(w1a.size(0), w1b.size(1))
+            w1_shape = w1.shape if w1 is not None else (w1a.size(0), w1b.size(1))
+            w2_shape = w2.shape if w2 is not None else (w2a.size(0), w2b.size(1))
+            shape_group_1 = (w1_shape[0], w2_shape[0])
+            shape_group_2 = (w1_shape[1], w2_shape[1])
+            w_shape = (w1_shape[0] * w2_shape[0], w1_shape[1] * w2_shape[1])
+            factor1 = max(w1.shape) if w1 is not None else max(w1a.size(0), w1b.size(1))
+            factor2 = max(w2.shape) if w2 is not None else max(w2a.size(0), w2b.size(1))
+            if (
+                w_shape[0] % factor1 == 0
+                and w_shape[1] % factor1 == 0
+                and factor1 in shape_group_1
+                and factor1 in shape_group_2
+            ):
+                factor = factor1
+            elif (
+                w_shape[0] % factor2 == 0
+                and w_shape[1] % factor2 == 0
+                and factor2 in shape_group_1
+                and factor2 in shape_group_2
+            ):
+                factor = factor2
+            else:
+                factor = max(factor1, factor2)
 
         module = cls(
             lora_name,
@@ -337,7 +359,7 @@ class LokrModule(LycorisBaseModule):
                     else self.lokr_w2_a @ self.lokr_w2_b
                 )
             ),
-            torch.tensor(self.scale),
+            self.scale,
         )
         dtype = weight.dtype
         if shape is not None:
@@ -516,6 +538,8 @@ class LokrModule(LycorisBaseModule):
                 weight = self.apply_weight_decompose(
                     weight + diff_weight, self.multiplier
                 )
+            elif self.multiplier == 1:
+                weight = weight + diff_weight
             else:
                 weight = weight + diff_weight * self.multiplier
             bias = (

{lycoris_lora-3.0.1.dev10 → lycoris_lora-3.0.1.dev12/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev10
+Version: 3.0.1.dev12
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev10 → lycoris_lora-3.0.1.dev12}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.1.dev10",
+    version="3.0.1.dev12",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",

{lycoris_lora-3.0.1.dev10 → lycoris_lora-3.0.1.dev12}/test/kohya.py

@@ -104,9 +104,11 @@ class LycorisKohyaWrapperTests(unittest.TestCase):
         )
         network.apply_to([sd_te1, sd_te2], sdxl_unet, True, True)
         network.restore()
-        create_network_from_weights(
+        network = create_network_from_weights(
             1, "", vae, [sd_te1, sd_te2], sdxl_unet, weights_sd=network.state_dict()
         )
+        network.merge_to()
+        del network
         torch.cuda.empty_cache()
 
     @parameterized.expand(extract_param_list)

{lycoris_lora-3.0.1.dev10 → lycoris_lora-3.0.1.dev12}/test/wrapper.py

@@ -206,6 +206,7 @@ class LycorisWrapperTests(unittest.TestCase):
             test_input = torch.randn(1, 16, 8, 8, 8).to(device, dtype)
             test_output = test_net(test_input)
             test_lycoris.restore()
+            test_lycoris.merge_to()
 
             state_dict = test_lycoris.state_dict()
             test_lycoris_from_weights: LycorisNetwork
@@ -219,9 +220,13 @@ class LycorisWrapperTests(unittest.TestCase):
             test_lycoris_from_weights.load_state_dict(test_lycoris.state_dict())
 
             self.assertTrue(
-                len(test_lycoris.loras) == len(test_lycoris_from_weights.loras)
+                len(test_lycoris.loras) == len(test_lycoris_from_weights.loras),
+                f"{len(test_lycoris.loras)} != {len(test_lycoris_from_weights.loras)}",
+            )
+            self.assertTrue(
+                torch.allclose(test_output, test_output_from_weights),
+                f"diff: {torch.nn.functional.mse_loss(test_output, test_output_from_weights).item()}",
             )
-            self.assertTrue(torch.allclose(test_output, test_output_from_weights))
         finally:
             reset_globals()