lycoris-lora 2.2.0.dev5__tar.gz → 2.2.0.dev7__tar.gz

{lycoris_lora-2.2.0.dev5/lycoris_lora.egg-info → lycoris_lora-2.2.0.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.2.0.dev5
+Version: 2.2.0.dev7
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.2.0.dev5 → lycoris_lora-2.2.0.dev7}/lycoris/kohya/__init__.py

@@ -74,6 +74,9 @@ def create_network(
     weight_decompose = str_bool(kwargs.get("dora_wd", False))
     full_matrix = str_bool(kwargs.get("full_matrix", False))
 
+    if weight_decompose:
+        logger.info("Weight decomposition is enabled")
+
     if full_matrix:
         logger.info("Full matrix mode for LoKr is enabled")
 

{lycoris_lora-2.2.0.dev5 → lycoris_lora-2.2.0.dev7}/lycoris/modules/locon.py

@@ -122,7 +122,8 @@ class LoConModule(ModuleCustomSD):
         if use_scalar:
             self.scalar = nn.Parameter(torch.tensor(0.0))
         else:
-            self.scalar = torch.tensor(1.0)
+            self.scalar: torch.FloatTensor
+            self.register_buffer("scalar", torch.tensor(1.0), persistent=False)
         # same as microsoft's
         torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5))
         if use_scalar:
@@ -176,7 +177,7 @@ class LoConModule(ModuleCustomSD):
                 drop /= drop.mean()
             weight *= drop
 
-        return weight * self.scalar
+        return weight * self.scalar.to(device)
 
     def apply_weight_decompose(self, weight):
         return (
@@ -185,6 +186,8 @@ class LoConModule(ModuleCustomSD):
 
     def custom_state_dict(self):
         destination = {}
+        if self.wd:
+            destination["dora_scale"] = self.dora_scale
         destination["alpha"] = self.alpha
         destination["lora_up.weight"] = self.lora_up.weight * self.scalar
         destination["lora_down.weight"] = self.lora_down.weight
@@ -324,10 +327,11 @@ class LoConModule(ModuleCustomSD):
                 return self.org_forward(x)
         scale = self.scale * self.multiplier
 
+        dtype_device = next(self.parameters())
         if self.wd:
             weight = (
-                self.org_module[0].weight.data.to(self.scalar)
-                + self.make_weight(x.device).to(self.scalar) * scale
+                self.org_module[0].weight.data.to(dtype_device)
+                + self.make_weight(x.device).to(dtype_device) * scale
             )
             if self.wd:
                 weight = self.apply_weight_decompose(weight)

{lycoris_lora-2.2.0.dev5 → lycoris_lora-2.2.0.dev7}/lycoris/modules/lokr.py

@@ -111,7 +111,7 @@ class LokrModule(ModuleCustomSD):
         self.tucker = False
         self.use_w1 = False
         self.use_w2 = False
-        self.full_matrix = False
+        self.full_matrix = full_matrix
 
         self.shape = org_module.weight.shape
         if org_module.__class__.__name__ == "Conv2d":

{lycoris_lora-2.2.0.dev5 → lycoris_lora-2.2.0.dev7/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.2.0.dev5
+Version: 2.2.0.dev7
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.2.0.dev5 → lycoris_lora-2.2.0.dev7}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="2.2.0.dev5",
+    version="2.2.0.dev7",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",