lycoris-lora 3.0.1.dev11__tar.gz → 3.0.1.dev12__tar.gz

{lycoris_lora-3.0.1.dev11/lycoris_lora.egg-info → lycoris_lora-3.0.1.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev11
+Version: 3.0.1.dev12
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev12}/lycoris/functional/lokr.py

@@ -14,7 +14,10 @@ def make_kron(w1, w2, scale):
     w2 = w2.contiguous()
     rebuild = torch.kron(w1, w2)
 
-    return rebuild * scale
+    if scale != 1:
+        rebuild = rebuild * scale
+
+    return rebuild
 
 
 def weight_gen(

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev12}/lycoris/modules/loha.py

@@ -182,6 +182,9 @@ class LohaModule(LycorisBaseModule):
             )
 
     def get_weight(self, shape):
+        scale = torch.tensor(
+            self.scale, dtype=self.hada_w1_b.dtype, device=self.hada_w1_b.device
+        )
         if self.tucker:
             weight = loha_diff_weight(
                 self.hada_w1_b,
@@ -190,7 +193,7 @@ class LohaModule(LycorisBaseModule):
                 self.hada_w2_a,
                 self.hada_t1,
                 self.hada_t2,
-                gamma=torch.tensor(self.scale),
+                gamma=scale,
             )
         else:
             weight = loha_diff_weight(
@@ -200,7 +203,7 @@ class LohaModule(LycorisBaseModule):
                 self.hada_w2_a,
                 None,
                 None,
-                gamma=torch.tensor(self.scale),
+                gamma=scale,
             )
         if shape is not None:
             weight = weight.reshape(shape)

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev12}/lycoris/modules/lokr.py

@@ -288,9 +288,19 @@ class LokrModule(LycorisBaseModule):
             w_shape = (w1_shape[0] * w2_shape[0], w1_shape[1] * w2_shape[1])
             factor1 = max(w1.shape) if w1 is not None else max(w1a.size(0), w1b.size(1))
             factor2 = max(w2.shape) if w2 is not None else max(w2a.size(0), w2b.size(1))
-            if w_shape[0] % factor1 == 0 and w_shape[1] % factor1 == 0 and factor1 in shape_group_1 and factor1 in shape_group_2:
+            if (
+                w_shape[0] % factor1 == 0
+                and w_shape[1] % factor1 == 0
+                and factor1 in shape_group_1
+                and factor1 in shape_group_2
+            ):
                 factor = factor1
-            elif w_shape[0] % factor2 == 0 and w_shape[1] % factor2 == 0 and factor2 in shape_group_1 and factor2 in shape_group_2:
+            elif (
+                w_shape[0] % factor2 == 0
+                and w_shape[1] % factor2 == 0
+                and factor2 in shape_group_1
+                and factor2 in shape_group_2
+            ):
                 factor = factor2
             else:
                 factor = max(factor1, factor2)
@@ -349,7 +359,7 @@ class LokrModule(LycorisBaseModule):
                     else self.lokr_w2_a @ self.lokr_w2_b
                 )
             ),
-            torch.tensor(self.scale),
+            self.scale,
         )
         dtype = weight.dtype
         if shape is not None:
@@ -528,6 +538,8 @@ class LokrModule(LycorisBaseModule):
                 weight = self.apply_weight_decompose(
                     weight + diff_weight, self.multiplier
                 )
+            elif self.multiplier == 1:
+                weight = weight + diff_weight
             else:
                 weight = weight + diff_weight * self.multiplier
             bias = (

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev12/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev11
+Version: 3.0.1.dev12
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev12}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.1.dev11",
+    version="3.0.1.dev12",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev12}/test/kohya.py

@@ -104,9 +104,11 @@ class LycorisKohyaWrapperTests(unittest.TestCase):
         )
         network.apply_to([sd_te1, sd_te2], sdxl_unet, True, True)
         network.restore()
-        create_network_from_weights(
+        network = create_network_from_weights(
             1, "", vae, [sd_te1, sd_te2], sdxl_unet, weights_sd=network.state_dict()
         )
+        network.merge_to()
+        del network
         torch.cuda.empty_cache()
 
     @parameterized.expand(extract_param_list)

{lycoris_lora-3.0.1.dev11 → lycoris_lora-3.0.1.dev12}/test/wrapper.py

@@ -206,6 +206,7 @@ class LycorisWrapperTests(unittest.TestCase):
             test_input = torch.randn(1, 16, 8, 8, 8).to(device, dtype)
             test_output = test_net(test_input)
             test_lycoris.restore()
+            test_lycoris.merge_to()
 
             state_dict = test_lycoris.state_dict()
             test_lycoris_from_weights: LycorisNetwork
@@ -219,9 +220,13 @@ class LycorisWrapperTests(unittest.TestCase):
             test_lycoris_from_weights.load_state_dict(test_lycoris.state_dict())
 
             self.assertTrue(
-                len(test_lycoris.loras) == len(test_lycoris_from_weights.loras)
+                len(test_lycoris.loras) == len(test_lycoris_from_weights.loras),
+                f"{len(test_lycoris.loras)} != {len(test_lycoris_from_weights.loras)}",
+            )
+            self.assertTrue(
+                torch.allclose(test_output, test_output_from_weights),
+                f"diff: {torch.nn.functional.mse_loss(test_output, test_output_from_weights).item()}",
             )
-            self.assertTrue(torch.allclose(test_output, test_output_from_weights))
         finally:
             reset_globals()