lycoris-lora 2.0.0.dev9__tar.gz → 2.0.0.dev11__tar.gz

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.0.0.dev9
+Version: 2.0.0.dev11
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/lycoris/kohya/__init__.py

@@ -63,7 +63,7 @@ def create_network(multiplier, network_dim, network_alpha, vae, text_encoder, un
     use_scalar = str_bool(kwargs.get('use_scalar', False))
     block_size = int(kwargs.get('block_size', 4) or 4)
     train_norm = str_bool(kwargs.get('train_norm', False))
-    constrain = int(kwargs.get('constrain', 0) or 0)
+    constrain = float(kwargs.get('constrain', 0) or 0)
     rescaled = str_bool(kwargs.get('rescaled', False))
     
     if algo == 'glora' and conv_dim>0:

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/lycoris/kohya/model_utils.py

@@ -1028,7 +1028,7 @@ def load_models_from_stable_diffusion_checkpoint(v2, ckpt_path, device="cpu", dt
             transformers_version="4.25.0.dev0",
         )
         text_model = CLIPTextModel._from_config(cfg)
-        info = text_model.load_state_dict(converted_text_encoder_checkpoint)
+        info = text_model.load_state_dict(converted_text_encoder_checkpoint, strict=False)
     else:
         converted_text_encoder_checkpoint = convert_ldm_clip_checkpoint_v1(state_dict)
 
@@ -1057,7 +1057,7 @@ def load_models_from_stable_diffusion_checkpoint(v2, ckpt_path, device="cpu", dt
             torch_dtype="float32",
         )
         text_model = CLIPTextModel._from_config(cfg)
-        info = text_model.load_state_dict(converted_text_encoder_checkpoint)
+        info = text_model.load_state_dict(converted_text_encoder_checkpoint, strict=False)
     print("loading text encoder:", info)
 
     return text_model, vae, unet

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/lycoris/kohya/sdxl_model_util.py

@@ -228,11 +228,11 @@ def load_models_from_sdxl_checkpoint(model_version, ckpt_path, map_location):
         elif k.startswith("conditioner.embedders.1.model."):
             te2_sd[k] = state_dict.pop(k)
 
-    info1 = text_model1.load_state_dict(te1_sd)
+    info1 = text_model1.load_state_dict(te1_sd, strict=False)
     print("text encoder 1:", info1)
 
     converted_sd, logit_scale = convert_sdxl_text_encoder_2_checkpoint(te2_sd, max_length=77)
-    info2 = text_model2.load_state_dict(converted_sd)
+    info2 = text_model2.load_state_dict(converted_sd, strict=False)
     print("text encoder 2:", info2)
 
     # prepare vae

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/lycoris/modules/diag_oft.py

@@ -47,15 +47,10 @@ class DiagOFTModule(ModuleCustomSD):
         self.block_size, self.block_num = factorization(out_dim, lora_dim)
         # block_num > block_size
         self.rescaled = rescaled
-        self.constrain = constrain
-        if self.constrain>0:
-            #follow kohya's naming to reduce more works in inference
-            self.oft_blocks = nn.Parameter(torch.zeros(self.block_num, self.block_size, self.block_size))
-        else:
-            # For non-constrained OFT, different formulation so use different naming
-            self.oft_diag = nn.Parameter(torch.zeros(self.block_num, self.block_size, self.block_size))
+        self.constrain = constrain * out_dim
+        self.oft_blocks = nn.Parameter(torch.zeros(self.block_num, self.block_size, self.block_size))
         if rescaled:
-            self.rescale = nn.Parameter(torch.ones(self.block_num, self.block_size))
+            self.rescale = nn.Parameter(torch.ones(self.block_num, self.block_size, 1))
         
         self.rank_dropout = rank_dropout
         self.rank_dropout_scale = rank_dropout_scale
@@ -78,24 +73,20 @@ class DiagOFTModule(ModuleCustomSD):
         }
     
     def get_r(self):
+        I = self.I
+        # for Q = -Q^T
+        q = self.oft_blocks - self.oft_blocks.transpose(1, 2)
+        normed_q = q
         if self.constrain > 0:
-            I = self.I
-            # for Q = -Q^T
-            q = self.oft_blocks - self.oft_blocks.transpose(1, 2)
             q_norm = torch.norm(q) + 1e-8
             if q_norm > self.constrain:
                 normed_q = q * self.constrain / q_norm
-            else:
-                normed_q = q
-            # assume we have autocast enabled
-            # use float() to prevent unsupported type
-            r = (I + normed_q) @ (I - normed_q).float().inverse()
-        else:
-            r = self.oft_diag
+        # use float() to prevent unsupported type
+        r = (I + normed_q) @ (I - normed_q).float().inverse()
         
         if self.rescaled:
             # Noted: not implemented in Kohya
-            r = self.rescale.unsqueeze(0) * r
+            r = self.rescale * r
         return r
 
     def make_weight(self, scale = 1, device=None):
@@ -112,7 +103,7 @@ class DiagOFTModule(ModuleCustomSD):
         # Init R=0, so add I on it to ensure the output of step0 is original model output
         weight = torch.einsum(
             "k n m, k n ... -> k m ...", 
-            r * scale + self.I, org_weight
+            r * scale + (1-scale) * self.I, org_weight
         )
         weight = rearrange(weight, 'k m ... -> (k m) ...')
         return weight
@@ -137,5 +128,5 @@ class DiagOFTModule(ModuleCustomSD):
         scale = self.multiplier
         
         w = self.make_weight(scale, x.device)
-        kw_dict = self.kw_dict | {"weight": w}
+        kw_dict = self.kw_dict | {"weight": w, "bias": self.org_module[0].bias}
         return self.op(x, **kw_dict)

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/lycoris/modules/full.py

@@ -15,6 +15,7 @@ class FullModule(ModuleCustomSD):
         use_tucker=False, use_scalar=False, rank_dropout_scale=False,
         **kwargs,
     ):
+        super().__init__()
         self.lora_name = lora_name
 
         if isinstance(org_module, nn.Linear):

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/lycoris/utils/__init__.py

@@ -462,7 +462,8 @@ def rebuild_weight(module_type, params, orig_weight, orig_bias, scale=1):
         merged = orig_weight + rebuild * scale
         merged_bias = orig_bias + rebuild_b * scale
     else:
-        raise NotImplementedError
+        merged = orig_weight
+        merged_bias = orig_bias
     
     return merged, merged_bias
 

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/lycoris_lora.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris-lora
-Version: 2.0.0.dev9
+Version: 2.0.0.dev11
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.0.0.dev9 → lycoris_lora-2.0.0.dev11}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
     name='lycoris_lora',
     packages=find_packages(),
-    version='2.0.0.dev9',
+    version='2.0.0.dev11',
     url='https://github.com/KohakuBlueleaf/LyCORIS',
     description='Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion',
     author='Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao',