lycoris-lora 3.0.0.dev10__tar.gz → 3.0.0.dev12__tar.gz

{lycoris_lora-3.0.0.dev10/lycoris_lora.egg-info → lycoris_lora-3.0.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.0.dev10
+Version: 3.0.0.dev12
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.0.dev10 → lycoris_lora-3.0.0.dev12}/lycoris/config.py

@@ -63,7 +63,11 @@ PRESET = {
             "SelfAttention",
         ],
         "unet_target_name": [],
-        "text_encoder_target_module": ["CLIPAttention", "BertAttention", "MT5LayerSelfAttention"],
+        "text_encoder_target_module": [
+            "CLIPAttention",
+            "BertAttention",
+            "MT5LayerSelfAttention",
+        ],
         "text_encoder_target_name": [],
     },
     "unet-only": {

{lycoris_lora-3.0.0.dev10 → lycoris_lora-3.0.0.dev12}/lycoris/logging.py

@@ -1,6 +1,7 @@
 import sys
 import copy
 import logging
+from functools import cache
 
 
 class ColoredFormatter(logging.Formatter):
@@ -34,3 +35,18 @@ if not logger.handlers:
         )
     )
     logger.addHandler(handler)
+
+
+@cache
+def info_once(msg):
+    logger.info(msg)
+
+
+@cache
+def warning_once(msg):
+    logger.warning(msg)
+
+
+@cache
+def error_once(msg):
+    logger.error(msg)

{lycoris_lora-3.0.0.dev10 → lycoris_lora-3.0.0.dev12}/lycoris/modules/__init__.py

@@ -102,6 +102,7 @@ def make_module(lyco_type, params, lora_name, orig_module):
             weight_decompose=dora_scale is not None,
             full_matrix=full_matrix,
         )
+        print(module.use_w1, module.use_w2)
         if w1 is not None:
             module.lokr_w1.copy_(w1)
         else:

{lycoris_lora-3.0.0.dev10 → lycoris_lora-3.0.0.dev12}/lycoris/modules/base.py

@@ -80,6 +80,7 @@ class LycorisBaseModule(ModuleCustomSD):
         """if alpha == 0 or None, alpha is rank (no scaling)."""
         super().__init__()
         self.lora_name = lora_name
+        self.not_supported = False
 
         self.module = type(org_module)
         if isinstance(org_module, nn.Linear):
@@ -150,6 +151,7 @@ class LycorisBaseModule(ModuleCustomSD):
             self.dim = org_module.num_channels
             self.kw_dict = {"num_groups": org_module.num_groups, "eps": org_module.eps}
         else:
+            self.not_supported = True
             self.module_type = "unknown"
 
         self.register_buffer("dtype_tensor", torch.tensor(0.0), persistent=False)
@@ -236,13 +238,19 @@ class LycorisBaseModule(ModuleCustomSD):
         self.org_module[0].weight.data.copy_(value)
 
     def apply_to(self, **kwargs):
+        if self.not_supported:
+            return
         self.org_forward = self.org_module[0].forward
         self.org_module[0].forward = self.forward
 
     def restore(self):
+        if self.not_supported:
+            return
         self.org_module[0].forward = self.org_forward
 
     def merge_to(self, multiplier=1.0):
+        if self.not_supported:
+            return
         weight, bias = self.get_merged_weight(
             multiplier, self.org_weight.shape, self.org_weight.device
         )

{lycoris_lora-3.0.0.dev10 → lycoris_lora-3.0.0.dev12}/lycoris/modules/lokr.py

@@ -137,7 +137,11 @@ class LokrModule(LycorisBaseModule):
                 (in_m, in_n),
             )  # ((a, b), (c, d)), out_dim = a*c, in_dim = b*d
             # smaller part. weight scale
-            if decompose_both and lora_dim < max(shape[0][0], shape[1][0]) / 2:
+            if (
+                decompose_both
+                and lora_dim < max(shape[0][0], shape[1][0]) / 2
+                and not self.full_matrix
+            ):
                 self.lokr_w1_a = nn.Parameter(torch.empty(shape[0][0], lora_dim))
                 self.lokr_w1_b = nn.Parameter(torch.empty(lora_dim, shape[1][0]))
             else:
@@ -145,7 +149,7 @@ class LokrModule(LycorisBaseModule):
                 self.lokr_w1 = nn.Parameter(
                     torch.empty(shape[0][0], shape[1][0])
                 )  # a*c, 1-mode
-            if lora_dim < max(shape[0][1], shape[1][1]) / 2:
+            if lora_dim < max(shape[0][1], shape[1][1]) / 2 and not self.full_matrix:
                 # bigger part. weight and LoRA. [b, dim] x [dim, d]
                 self.lokr_w2_a = nn.Parameter(torch.empty(shape[0][1], lora_dim))
                 self.lokr_w2_b = nn.Parameter(torch.empty(lora_dim, shape[1][1]))

{lycoris_lora-3.0.0.dev10 → lycoris_lora-3.0.0.dev12}/lycoris/modules/norms.py

@@ -2,6 +2,7 @@ import torch
 import torch.nn as nn
 
 from .base import LycorisBaseModule
+from ..logging import warning_once
 
 
 class NormModule(LycorisBaseModule):
@@ -30,15 +31,31 @@ class NormModule(LycorisBaseModule):
             rank_dropout_scale=rank_dropout_scale,
             **kwargs,
         )
-        if self.module_type not in self.support_module:
-            raise ValueError(f"{self.module_type} is not supported in Norm algo.")
+        if self.module_type == "unknown":
+            if not hasattr(org_module, "weight") or not hasattr(org_module, "_norm"):
+                warning_once(f"{type(org_module)} is not supported in Norm algo.")
+                return
+            else:
+                self.dim = org_module.weight.numel()
+                self.not_supported = False
+        elif self.module_type not in self.support_module:
+            warning_once(f"{self.module_type} is not supported in Norm algo.")
+            return
 
         self.w_norm = nn.Parameter(torch.zeros(self.dim))
-        self.b_norm = nn.Parameter(torch.zeros(self.dim))
+        if hasattr(org_module, "bias"):
+            self.b_norm = nn.Parameter(torch.zeros(self.dim))
+        if hasattr(org_module, "_norm"):
+            self.org_norm = org_module._norm
+        else:
+            self.org_norm = None
 
     def make_weight(self, scale=1, device=None):
         org_weight = self.org_module[0].weight.to(device, dtype=self.w_norm.dtype)
-        org_bias = self.org_module[0].bias.to(device, dtype=self.b_norm.dtype)
+        if hasattr(self.org_module[0], "bias"):
+            org_bias = self.org_module[0].bias.to(device, dtype=self.b_norm.dtype)
+        else:
+            org_bias = None
         if self.rank_dropout and self.training:
             drop = (torch.rand(self.dim, device=device) < self.rank_dropout).to(
                 self.w_norm.device
@@ -53,35 +70,58 @@ class NormModule(LycorisBaseModule):
             else 1
         )
         weight = self.w_norm.to(device) * drop * scale
-        bias = self.b_norm.to(device) * drop * scale
-        return org_weight + weight, org_bias + bias
+        if org_bias is not None:
+            bias = self.b_norm.to(device) * drop * scale
+        return org_weight + weight, org_bias + bias if org_bias is not None else None
 
     def get_diff_weight(self, multiplier=1, shape=None, device=None):
+        if self.not_supported:
+            return 0, 0
         w = self.w_norm * multiplier
-        b = self.b_norm * multiplier
         if device is not None:
             w = w.to(device)
-            b = b.to(device)
         if shape is not None:
             w = w.view(shape)
-            b = b.view(shape)
+        if self.b_norm is not None:
+            b = self.b_norm * multiplier
+            if device is not None:
+                b = b.to(device)
+            if shape is not None:
+                b = b.view(shape)
+        else:
+            b = None
         return w, b
 
     def get_merged_weight(self, multiplier=1, shape=None, device=None):
-        org_w = self.org_module[0].weight.to(device, dtype=self.w_norm.dtype)
-        org_b = self.org_module[0].bias.to(device, dtype=self.b_norm.dtype)
+        if self.not_supported:
+            return None, None
         diff_w, diff_b = self.get_diff_weight(multiplier, shape, device)
+        org_w = self.org_module[0].weight.to(device, dtype=self.w_norm.dtype)
         weight = org_w + diff_w
-        bias = org_b + diff_b
+        if diff_b is not None:
+            org_b = self.org_module[0].bias.to(device, dtype=self.b_norm.dtype)
+            bias = org_b + diff_b
+        else:
+            bias = None
         return weight, bias
 
     def forward(self, x):
-        if self.module_dropout and self.training:
-            if torch.rand(1) < self.module_dropout:
-                return self.org_forward(x)
+        if self.not_supported or (
+            self.module_dropout
+            and self.training
+            and torch.rand(1) < self.module_dropout
+        ):
+            return self.org_forward(x)
         scale = self.multiplier
 
         w, b = self.make_weight(scale, x.device)
+        if self.org_norm is not None:
+            normed = self.org_norm(x)
+            scaled = normed * w
+            if b is not None:
+                scaled += b
+            return scaled
+
         kw_dict = self.kw_dict | {"weight": w, "bias": b}
         return self.op(x, **kw_dict)
 

{lycoris_lora-3.0.0.dev10 → lycoris_lora-3.0.0.dev12/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.0.dev10
+Version: 3.0.0.dev12
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.0.dev10 → lycoris_lora-3.0.0.dev12}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.0.dev10",
+    version="3.0.0.dev12",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",