lycoris-lora 2.1.0.dev7__tar.gz → 2.1.0.dev9__tar.gz

{lycoris_lora-2.1.0.dev7 → lycoris_lora-2.1.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.1.0.dev7
+Version: 2.1.0.dev9
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.1.0.dev7 → lycoris_lora-2.1.0.dev9}/lycoris/__init__.py

@@ -19,3 +19,5 @@ from .wrapper import (
     create_lycoris,
     create_lycoris_from_weights,
 )
+
+from .logging import logger

{lycoris_lora-2.1.0.dev7 → lycoris_lora-2.1.0.dev9}/lycoris/kohya/__init__.py

@@ -4,13 +4,13 @@
 # https://github.com/cloneofsimo/lora/blob/master/lora_diffusion/lora.py
 # https://github.com/kohya-ss/sd-scripts/blob/main/networks/lora.py
 
-import math
-from warnings import warn
 import os
+import re
 import sys
+from typing import List
 
 sys.setrecursionlimit(10000)
-from typing import List
+
 import torch
 import torch.utils.checkpoint as checkpoint
 
@@ -25,11 +25,13 @@ from ..modules.glora import GLoRAModule
 from ..modules.norms import NormModule
 from ..modules.full import FullModule
 from ..modules.diag_oft import DiagOFTModule
+from ..modules.boft import ButterflyOFTModule
 from ..modules import make_module
 
 from ..config import PRESET
 from ..utils.preset import read_preset
 from ..utils import get_module, str_bool
+from ..logging import logger
 
 
 network_module_dict = {
@@ -42,6 +44,7 @@ network_module_dict = {
     "glora": GLoRAModule,
     "full": FullModule,
     "diag-oft": DiagOFTModule,
+    "boft": ButterflyOFTModule,
 }
 
 
@@ -63,7 +66,7 @@ def create_network(
         or kwargs.get("use_tucker", False)
     )
     if "disable_conv_cp" in kwargs or "use_cp" in kwargs or "use_conv_cp" in kwargs:
-        warn(
+        logger.warning(
             "disable_conv_cp and use_cp are deprecated. Please use use_tucker instead.",
             stacklevel=2,
         )
@@ -75,7 +78,7 @@ def create_network(
 
     if algo == "glora" and conv_dim > 0:
         conv_dim = 0
-        print("Disable conv layer for GLoRA")
+        logger.info("Disable conv layer for GLoRA")
 
     preset = kwargs.get("preset", "full")
     if preset not in PRESET:
@@ -85,22 +88,20 @@ def create_network(
     assert preset is not None
     LycorisNetworkKohya.apply_preset(preset)
 
-    print(f"Using rank adaptation algo: {algo}")
+    logger.info(f"Using rank adaptation algo: {algo}")
 
     if (
         (algo == "loha")
         and not kwargs.get("no_dim_warn", False)
         and (network_dim > 64 or conv_dim > 64)
     ):
-        print("=" * 20 + "WARNING" + "=" * 20)
         warning_type = {"loha": "Hadamard Product representation"}
         warning_msg = f"""You are not supposed to use dim>64 (64*64 = 4096, it already has enough rank)\n
             in {warning_type[algo]}!\n
             Please consider use lower dim or disable this warning with --network_args no_dim_warn=True\n
             If you just want to use high dim {algo}, please consider use lower lr.
         """
-        warn(warning_msg, stacklevel=2)
-        print("=" * 20 + "WARNING" + "=" * 20)
+        logger.warning(warning_msg, stacklevel=2)
 
     if algo == "ia3":
         network = IA3Network(
@@ -240,7 +241,7 @@ def create_hypernetwork(
         or kwargs.get("use_tucker", False)
     )
     if "disable_conv_cp" in kwargs or "use_cp" in kwargs or "use_conv_cp" in kwargs:
-        warn(
+        logger.warning(
             "disable_conv_cp and use_cp are deprecated. Please use use_tucker instead.",
             stacklevel=2,
         )
@@ -254,7 +255,7 @@ def create_hypernetwork(
         "locon": LoConModule,
     }[algo]
 
-    print(f"Using rank adaptation algo: {algo}")
+    logger.info(f"Using rank adaptation algo: {algo}")
 
     return HyperDreamNetwork(
         text_encoder,
@@ -352,19 +353,19 @@ class LycorisNetworkKohya(LycorisNetwork):
 
         self.conv_lora_dim = int(conv_lora_dim)
         if self.conv_lora_dim and self.conv_lora_dim != self.lora_dim:
-            print("Apply different lora dim for conv layer")
-            print(f"Conv Dim: {conv_lora_dim}, Linear Dim: {lora_dim}")
+            logger.info("Apply different lora dim for conv layer")
+            logger.info(f"Conv Dim: {conv_lora_dim}, Linear Dim: {lora_dim}")
         elif self.conv_lora_dim == 0:
-            print("Disable conv layer")
+            logger.info("Disable conv layer")
 
         self.alpha = alpha
         self.conv_alpha = float(conv_alpha)
         if self.conv_lora_dim and self.alpha != self.conv_alpha:
-            print("Apply different alpha value for conv layer")
-            print(f"Conv alpha: {conv_alpha}, Linear alpha: {alpha}")
+            logger.info("Apply different alpha value for conv layer")
+            logger.info(f"Conv alpha: {conv_alpha}, Linear alpha: {alpha}")
 
         if 1 >= dropout >= 0:
-            print(f"Use Dropout value: {dropout}")
+            logger.info(f"Use Dropout value: {dropout}")
         self.dropout = dropout
         self.rank_dropout = rank_dropout
         self.module_dropout = module_dropout
@@ -465,7 +466,7 @@ class LycorisNetworkKohya(LycorisNetwork):
             target_replace_modules,
             target_replace_names=[],
         ) -> List:
-            print("Create LyCORIS Module")
+            logger.info("Create LyCORIS Module")
             loras = []
             next_config = {}
             for name, module in root_module.named_modules():
@@ -482,7 +483,9 @@ class LycorisNetworkKohya(LycorisNetwork):
                         ]
                     )
                     next_config = {}
-                elif name in target_replace_names:
+                elif name in target_replace_names or any(
+                    re.match(t, name) for t in target_replace_names
+                ):
                     if name in self.NAME_ALGO_MAP:
                         next_config = self.NAME_ALGO_MAP[name]
                         algo = next_config.get("algo", network_module)
@@ -500,7 +503,7 @@ class LycorisNetworkKohya(LycorisNetwork):
             return loras
 
         if network_module == GLoRAModule:
-            print("GLoRA enabled, only train transformer")
+            logger.info("GLoRA enabled, only train transformer")
             # only train transformer (for GLoRA)
             LycorisNetworkKohya.UNET_TARGET_REPLACE_MODULE = [
                 "Transformer2DModel",
@@ -526,7 +529,7 @@ class LycorisNetworkKohya(LycorisNetwork):
                     LycorisNetworkKohya.TEXT_ENCODER_TARGET_REPLACE_NAME,
                 )
             )
-        print(
+        logger.info(
             f"create LyCORIS for Text Encoder: {len(self.text_encoder_loras)} modules."
         )
 
@@ -536,14 +539,14 @@ class LycorisNetworkKohya(LycorisNetwork):
             LycorisNetworkKohya.UNET_TARGET_REPLACE_MODULE,
             LycorisNetworkKohya.UNET_TARGET_REPLACE_NAME,
         )
-        print(f"create LyCORIS for U-Net: {len(self.unet_loras)} modules.")
+        logger.info(f"create LyCORIS for U-Net: {len(self.unet_loras)} modules.")
 
         algo_table = {}
         for lora in self.text_encoder_loras + self.unet_loras:
             algo_table[lora.__class__.__name__] = (
                 algo_table.get(lora.__class__.__name__, 0) + 1
             )
-        print(f"module type table: {algo_table}")
+        logger.info(f"module type table: {algo_table}")
 
         self.weights_sd = None
 
@@ -577,12 +580,12 @@ class LycorisNetworkKohya(LycorisNetwork):
         ), f"internal error: flag not set"
 
         if apply_text_encoder:
-            print("enable LyCORIS for text encoder")
+            logger.info("enable LyCORIS for text encoder")
         else:
             self.text_encoder_loras = []
 
         if apply_unet:
-            print("enable LyCORIS for U-Net")
+            logger.info("enable LyCORIS for U-Net")
         else:
             self.unet_loras = []
 
@@ -595,7 +598,7 @@ class LycorisNetworkKohya(LycorisNetwork):
         if self.weights_sd:
             # if some weights are not in state dict, it is ok because initial LoRA does nothing (lora_up is initialized by zeros)
             info = self.load_state_dict(self.weights_sd, False)
-            print(f"weights are loaded: {info}")
+            logger.info(f"weights are loaded: {info}")
 
     # TODO refactor to common function with apply_to
     def merge_to(self, text_encoder, unet, weights_sd, dtype, device):
@@ -607,12 +610,12 @@ class LycorisNetworkKohya(LycorisNetwork):
                 apply_unet = True
 
         if apply_text_encoder:
-            print("enable LoRA for text encoder")
+            logger.info("enable LoRA for text encoder")
         else:
             self.text_encoder_loras = []
 
         if apply_unet:
-            print("enable LoRA for U-Net")
+            logger.info("enable LoRA for U-Net")
         else:
             self.unet_loras = []
 
@@ -714,12 +717,12 @@ class HyperDreamNetwork(torch.nn.Module):
         self.alpha = alpha
 
         if 1 >= dropout >= 0:
-            print(f"Use Dropout value: {dropout}")
+            logger.info(f"Use Dropout value: {dropout}")
         if network_module != LoConModule:
-            print("HyperDreamBooth only support LoRA at this time")
+            logger.info("HyperDreamBooth only support LoRA at this time")
             raise NotImplementedError
         if lora_dim * (down_dim + up_dim) > 4096:
-            print(
+            logger.info(
                 "weight elements > 4096 (dim * (down_dim + up_dim)) is not recommended!"
             )
 
@@ -734,7 +737,7 @@ class HyperDreamNetwork(torch.nn.Module):
             target_replace_modules,
             target_replace_names=[],
         ) -> List[network_module]:
-            print("Create LyCORIS Module")
+            logger.info("Create LyCORIS Module")
             loras = []
             for name, module in root_module.named_modules():
                 if module.__class__.__name__ in target_replace_modules:
@@ -829,7 +832,7 @@ class HyperDreamNetwork(torch.nn.Module):
                     LycorisNetworkKohya.TEXT_ENCODER_TARGET_REPLACE_MODULE,
                 )
             )
-        print(
+        logger.info(
             f"create LyCORIS for Text Encoder: "
             f"{len(self.text_encoder_loras)} modules."
         )
@@ -839,7 +842,7 @@ class HyperDreamNetwork(torch.nn.Module):
             unet,
             LycorisNetworkKohya.UNET_TARGET_REPLACE_MODULE,
         )
-        print(f"create LyCORIS for U-Net: {len(self.unet_loras)} modules.")
+        logger.info(f"create LyCORIS for U-Net: {len(self.unet_loras)} modules.")
 
         self.loras: list[LoConModule] = self.text_encoder_loras + self.unet_loras
         self.img_weight_generater = ImgWeightGenerator(
@@ -965,12 +968,12 @@ class HyperDreamNetwork(torch.nn.Module):
             ), f"internal error: flag not set"
 
         if apply_text_encoder:
-            print("enable LyCORIS for text encoder")
+            logger.info("enable LyCORIS for text encoder")
         else:
             self.text_encoder_loras = []
 
         if apply_unet:
-            print("enable LyCORIS for U-Net")
+            logger.info("enable LyCORIS for U-Net")
         else:
             self.unet_loras = []
 
@@ -1122,7 +1125,7 @@ class IA3Network(torch.nn.Module):
             target_replace_names=[],
             target_train_input=[],
         ) -> List[IA3Module]:
-            print("Create LyCORIS Module")
+            logger.info("Create LyCORIS Module")
             loras = []
             for name, module in root_module.named_modules():
                 if module.__class__.__name__ in target_replace_modules:
@@ -1171,7 +1174,7 @@ class IA3Network(torch.nn.Module):
                     IA3Network.TRAIN_INPUT,
                 )
             )
-        print(
+        logger.info(
             f"create LyCORIS for Text Encoder: {len(self.text_encoder_loras)} modules."
         )
 
@@ -1182,7 +1185,7 @@ class IA3Network(torch.nn.Module):
             IA3Network.UNET_TARGET_REPLACE_NAME,
             IA3Network.TRAIN_INPUT,
         )
-        print(f"create LyCORIS for U-Net: {len(self.unet_loras)} modules.")
+        logger.info(f"create LyCORIS for U-Net: {len(self.unet_loras)} modules.")
 
         self.weights_sd = None
 
@@ -1235,12 +1238,12 @@ class IA3Network(torch.nn.Module):
             ), f"internal error: flag not set"
 
         if apply_text_encoder:
-            print("enable LyCORIS for text encoder")
+            logger.info("enable LyCORIS for text encoder")
         else:
             self.text_encoder_loras = []
 
         if apply_unet:
-            print("enable LyCORIS for U-Net")
+            logger.info("enable LyCORIS for U-Net")
         else:
             self.unet_loras = []
 
@@ -1251,7 +1254,7 @@ class IA3Network(torch.nn.Module):
         if self.weights_sd:
             # if some weights are not in state dict, it is ok because initial LoRA does nothing (lora_up is initialized by zeros)
             info = self.load_state_dict(self.weights_sd, False)
-            print(f"weights are loaded: {info}")
+            logger.info(f"weights are loaded: {info}")
 
     def enable_gradient_checkpointing(self):
         # not supported

lycoris_lora-2.1.0.dev9/lycoris/logging.py

@@ -0,0 +1,36 @@
+import sys
+import copy
+import logging
+
+
+class ColoredFormatter(logging.Formatter):
+    COLORS = {
+        "DEBUG": "\033[0;36m",  # CYAN
+        "INFO": "\033[0;32m",  # GREEN
+        "WARNING": "\033[0;33m",  # YELLOW
+        "ERROR": "\033[0;31m",  # RED
+        "CRITICAL": "\033[0;37;41m",  # WHITE ON RED
+        "RESET": "\033[0m",  # RESET COLOR
+    }
+
+    def format(self, record):
+        colored_record = copy.copy(record)
+        levelname = colored_record.levelname
+        seq = self.COLORS.get(levelname, self.COLORS["RESET"])
+        colored_record.levelname = f"{seq}{levelname}{self.COLORS['RESET']}"
+        return super().format(colored_record)
+
+
+logger = logging.getLogger("LyCORIS")
+logger.propagate = False
+logger.setLevel(logging.INFO)
+
+
+if not logger.handlers:
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(
+        ColoredFormatter(
+            "%(asctime)s|[%(name)s]-%(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S"
+        )
+    )
+    logger.addHandler(handler)

lycoris_lora-2.1.0.dev9/lycoris/modules/boft.py

@@ -0,0 +1,211 @@
+from functools import cache
+from math import log2, ceil
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from einops import rearrange
+
+from .base import ModuleCustomSD
+from lycoris.logging import logger
+
+
+@cache
+def log_butterfly_factorize(dim, factor, result):
+    logger.info(
+        f"Use BOFT({int(log2(result[1]))}, {result[0]//2})"
+        f" (equivalent to factor={result[0]}) "
+        f"for {dim=} and {factor=}"
+    )
+
+
+def butterfly_factor(dimension: int, factor: int = -1) -> tuple[int, int]:
+    """
+    m = 2k
+    n = 2**p
+    m*n = dim
+    """
+
+    # Find the first solution and check if it is even doable
+    m = n = 0
+    while m <= factor:
+        m += 2
+        while dimension % m != 0 and m < dimension:
+            m += 2
+        if m > factor:
+            break
+        if sum(int(i) for i in f"{dimension//m:b}") == 1:
+            n = dimension // m
+
+    if n == 0:
+        raise ValueError(
+            f"It is impossible to decompose {dimension} with factor {factor} under BOFT constrains."
+        )
+
+    log_butterfly_factorize(dimension, factor, (dimension // n, n))
+    return dimension // n, n
+
+
+class ButterflyOFTModule(ModuleCustomSD):
+    """
+    modifed from kohya-ss/sd-scripts/networks/lora:LoRAModule
+    """
+
+    def __init__(
+        self,
+        lora_name,
+        org_module: nn.Module,
+        multiplier=1.0,
+        lora_dim=4,
+        alpha=1,
+        dropout=0.0,
+        rank_dropout=0.0,
+        module_dropout=0.0,
+        use_tucker=False,
+        use_scalar=False,
+        rank_dropout_scale=False,
+        constrain=0,
+        rescaled=False,
+        **kwargs,
+    ):
+        """if alpha == 0 or None, alpha is rank (no scaling)."""
+        super().__init__()
+        self.lora_name = lora_name
+
+        if isinstance(org_module, nn.Linear):
+            self.op = F.linear
+            self.dim = org_module.out_features
+            self.kw_dict = {}
+        elif isinstance(org_module, nn.Conv2d):
+            self.op = F.conv2d
+            self.dim = org_module.out_channels
+            self.kw_dict = {
+                "stride": org_module.stride,
+                "padding": org_module.padding,
+                "dilation": org_module.dilation,
+                "groups": org_module.groups,
+            }
+        else:
+            raise NotImplementedError
+
+        out_dim = org_module.weight.shape[0]
+        b, m_exp = butterfly_factor(out_dim, lora_dim)
+        self.block_size = b
+        self.block_num = m_exp
+        #BOFT(m, b)
+        self.boft_b = b
+        self.boft_m = sum(int(i) for i in f"{m_exp-1:b}") + 1
+        # block_num > block_size
+        self.rescaled = rescaled
+        self.constrain = constrain * out_dim
+        self.oft_blocks = nn.Parameter(
+            torch.zeros(self.boft_m, self.block_num, self.block_size, self.block_size)
+        )
+        if rescaled:
+            self.rescale = nn.Parameter(torch.ones(self.block_num, self.block_size, 1))
+
+        self.rank_dropout = rank_dropout
+        self.rank_dropout_scale = rank_dropout_scale
+        self.module_dropout = module_dropout
+
+        self.multiplier = multiplier
+        self.org_module = [org_module]
+        self.org_forward = self.org_module[0].forward
+
+    @property
+    def I(self):
+        return torch.eye(self.block_size, device=next(self.parameters()).device)
+
+    def apply_to(self, **kwargs):
+        self.org_forward = self.org_module[0].forward
+        self.org_module[0].forward = self.forward
+
+    def restore(self):
+        self.org_module[0].forward = self.org_forward
+
+    def merge_to(self, multiplier=1.0):
+        weight = self.make_weight(scale=multiplier)
+        self.org_module[0].weight.data.copy_(weight)
+
+    def custom_state_dict(self):
+        return {"oft_diag": self.get_r()}
+
+    def get_r(self):
+        I = self.I
+        # for Q = -Q^T
+        q = self.oft_blocks - self.oft_blocks.transpose(-1, -2)
+        normed_q = q
+        # Diag OFT style constrain
+        if self.constrain > 0:
+            q_norm = torch.norm(q) + 1e-8
+            if q_norm > self.constrain:
+                normed_q = q * self.constrain / q_norm
+        # use float() to prevent unsupported type
+        r = (I + normed_q) @ (I - normed_q).float().inverse()
+        return r
+
+    def make_weight(self, scale=1, device=None):
+        if self.rank_dropout and self.training:
+            drop = torch.rand(self.dim, device=device) < self.rank_dropout
+            drop = drop.to(self.oft_blocks.dtype)
+            if self.rank_dropout_scale:
+                drop /= drop.mean()
+        else:
+            drop = 1
+
+        m = self.boft_m
+        b = self.boft_b
+        r_b = b // 2
+        r = self.get_r()
+        inp = self.org_module[0].weight.to(device, dtype=r.dtype)
+        
+        for i in range(m):
+            bi = r[i] # b_num, b_size, b_size
+            if i == 0:
+                # Apply multiplier/scale and rescale into first weight
+                bi = bi * scale + (1 - scale) * self.I
+                if self.rescaled:
+                    bi = bi * self.rescale
+            inp = rearrange(inp, "(c g k) ... -> (c k g) ...", g=2, k=2**i * r_b)
+            inp = rearrange(inp, "(d b) ... -> d b ...", b=b)
+            inp = torch.einsum("b i j, b j ... -> b i ...", bi, inp)
+            inp = rearrange(inp, "d b ... -> (d b) ...")
+            inp = rearrange(inp, "(c k g) ... -> (c g k) ...", g=2, k=2**i * r_b)
+        
+        weight = inp
+        return weight*drop
+
+    @torch.no_grad()
+    def apply_max_norm(self, max_norm, device=None):
+        orig_norm = self.oft_blocks.to(device).norm()
+        norm = torch.clamp(orig_norm, max_norm / 2)
+        desired = torch.clamp(norm, max=max_norm)
+        ratio = desired / norm
+
+        scaled = ratio.item() != 1.0
+        if scaled:
+            self.oft_blocks *= ratio
+
+        return scaled, orig_norm * ratio
+
+    def forward(self, x):
+        if self.module_dropout and self.training:
+            if torch.rand(1) < self.module_dropout:
+                return self.org_forward(x)
+        scale = self.multiplier
+
+        w = self.make_weight(scale, x.device)
+        kw_dict = self.kw_dict | {"weight": w, "bias": self.org_module[0].bias}
+        return self.op(x, **kw_dict)
+
+
+if __name__ == '__main__':
+    dim = 320
+    test_layer = nn.Linear(dim, dim)
+    test_layer.weight = nn.Parameter(torch.eye(dim))
+    test_boft = ButterflyOFTModule('test', test_layer, lora_dim=16)
+    test_boft.oft_blocks = nn.Parameter(torch.randn_like(test_boft.oft_blocks))
+    
+    print(test_boft.make_weight(1, 'cpu').shape)
+    print(torch.count_nonzero(test_boft.make_weight(1, 'cpu')), dim*dim)

{lycoris_lora-2.1.0.dev7 → lycoris_lora-2.1.0.dev9}/lycoris/utils/__init__.py

@@ -230,12 +230,14 @@ def extract_diff(
                         extract_c.detach().cpu().contiguous().half()
                     )
                     diff = (
-                        root_weight
-                        - torch.einsum(
-                            "i j k l, j r, p i -> p r k l",
-                            extract_c,
-                            extract_a.flatten(1, -1),
-                            extract_b.flatten(1, -1),
+                        (
+                            root_weight
+                            - torch.einsum(
+                                "i j k l, j r, p i -> p r k l",
+                                extract_c,
+                                extract_a.flatten(1, -1),
+                                extract_b.flatten(1, -1),
+                            )
                         )
                         .detach()
                         .cpu()

{lycoris_lora-2.1.0.dev7 → lycoris_lora-2.1.0.dev9}/lycoris/wrapper.py

@@ -7,7 +7,6 @@ import sys
 
 sys.setrecursionlimit(10000)
 from typing import List
-from warnings import warn
 
 import torch
 
@@ -20,11 +19,13 @@ from .modules.glora import GLoRAModule
 from .modules.norms import NormModule
 from .modules.full import FullModule
 from .modules.diag_oft import DiagOFTModule
+from .modules.boft import ButterflyOFTModule
 from .modules import make_module
 
 from .config import PRESET
 from .utils.preset import read_preset
 from .utils import get_module, str_bool
+from .logging import logger
 
 
 network_module_dict = {
@@ -36,6 +37,7 @@ network_module_dict = {
     "glora": GLoRAModule,
     "full": FullModule,
     "diag-oft": DiagOFTModule,
+    "boft": ButterflyOFTModule,
 }
 
 
@@ -55,7 +57,7 @@ def create_lycoris(module, multiplier, linear_dim, linear_alpha, **kwargs):
         or kwargs.get("use_tucker", False)
     )
     if "disable_conv_cp" in kwargs or "use_cp" in kwargs or "use_conv_cp" in kwargs:
-        warn(
+        logger.warning(
             "disable_conv_cp and use_cp are deprecated. Please use use_tucker instead.",
             stacklevel=2,
         )
@@ -67,7 +69,7 @@ def create_lycoris(module, multiplier, linear_dim, linear_alpha, **kwargs):
 
     if algo == "glora" and conv_dim > 0:
         conv_dim = 0
-        print("Disable conv layer for GLoRA")
+        logger.info("Disable conv layer for GLoRA")
 
     preset = kwargs.get("preset", "full")
     if preset not in PRESET:
@@ -77,14 +79,13 @@ def create_lycoris(module, multiplier, linear_dim, linear_alpha, **kwargs):
     assert preset is not None
     LycorisNetwork.apply_preset(preset)
 
-    print(f"Using rank adaptation algo: {algo}")
+    logger.info(f"Using rank adaptation algo: {algo}")
 
     if (
         (algo == "loha")
         and not kwargs.get("no_dim_warn", False)
         and (linear_dim > 64 or conv_dim > 64)
     ):
-        print("=" * 20 + "WARNING" + "=" * 20)
         warning_type = {"loha": "Hadamard Product representation"}
         warning_msg = (
             "You are not supposed to use dim>64 (64*64 = 4096, it already has enough rank)\n"
@@ -92,8 +93,7 @@ def create_lycoris(module, multiplier, linear_dim, linear_alpha, **kwargs):
             "Please consider use lower dim or disable this warning with --network_args no_dim_warn=True\n"
             f"If you just want to use high dim {algo}, please consider use lower lr."
         )
-        warn(warning_msg, stacklevel=2)
-        print("=" * 20 + "WARNING" + "=" * 20)
+        logger.warning(warning_msg, stacklevel=2)
 
     network = LycorisNetwork(
         module,
@@ -230,19 +230,19 @@ class LycorisNetwork(torch.nn.Module):
 
         self.conv_lora_dim = int(conv_lora_dim)
         if self.conv_lora_dim and self.conv_lora_dim != self.lora_dim:
-            print("Apply different lora dim for conv layer")
-            print(f"Conv Dim: {conv_lora_dim}, Linear Dim: {lora_dim}")
+            logger.info("Apply different lora dim for conv layer")
+            logger.info(f"Conv Dim: {conv_lora_dim}, Linear Dim: {lora_dim}")
         elif self.conv_lora_dim == 0:
-            print("Disable conv layer")
+            logger.info("Disable conv layer")
 
         self.alpha = alpha
         self.conv_alpha = float(conv_alpha)
         if self.conv_lora_dim and self.alpha != self.conv_alpha:
-            print("Apply different alpha value for conv layer")
-            print(f"Conv alpha: {conv_alpha}, Linear alpha: {alpha}")
+            logger.info("Apply different alpha value for conv layer")
+            logger.info(f"Conv alpha: {conv_alpha}, Linear alpha: {alpha}")
 
         if 1 >= dropout >= 0:
-            print(f"Use Dropout value: {dropout}")
+            logger.info(f"Use Dropout value: {dropout}")
         self.dropout = dropout
         self.rank_dropout = rank_dropout
         self.module_dropout = module_dropout
@@ -341,7 +341,7 @@ class LycorisNetwork(torch.nn.Module):
             target_replace_modules,
             target_replace_names=[],
         ) -> List:
-            print("Create LyCORIS Module")
+            logger.info("Create LyCORIS Module")
             loras = []
             next_config = {}
             for name, module in root_module.named_modules():
@@ -383,14 +383,14 @@ class LycorisNetwork(torch.nn.Module):
             LycorisNetwork.TARGET_REPLACE_MODULE,
             LycorisNetwork.TARGET_REPLACE_NAME,
         )
-        print(f"create LyCORIS: {len(self.loras)} modules.")
+        logger.info(f"create LyCORIS: {len(self.loras)} modules.")
 
         algo_table = {}
         for lora in self.loras:
             algo_table[lora.__class__.__name__] = (
                 algo_table.get(lora.__class__.__name__, 0) + 1
             )
-        print(f"module type table: {algo_table}")
+        logger.info(f"module type table: {algo_table}")
 
         self.weights_sd = None
 
@@ -430,7 +430,7 @@ class LycorisNetwork(torch.nn.Module):
         if self.weights_sd:
             # if some weights are not in state dict, it is ok because initial LoRA does nothing (lora_up is initialized by zeros)
             info = self.load_state_dict(self.weights_sd, False)
-            print(f"weights are loaded: {info}")
+            logger.info(f"weights are loaded: {info}")
 
     def is_mergeable(self):
         return True

{lycoris_lora-2.1.0.dev7 → lycoris_lora-2.1.0.dev9}/lycoris_lora.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris-lora
-Version: 2.1.0.dev7
+Version: 2.1.0.dev9
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.1.0.dev7 → lycoris_lora-2.1.0.dev9}/lycoris_lora.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ README.md
 setup.py
 lycoris/__init__.py
 lycoris/config.py
+lycoris/logging.py
 lycoris/wrapper.py
 lycoris/hcp/__init__.py
 lycoris/hcp/base.py
@@ -18,6 +19,7 @@ lycoris/kohya/utils.py
 lycoris/modules/__init__.py
 lycoris/modules/attention.py
 lycoris/modules/base.py
+lycoris/modules/boft.py
 lycoris/modules/diag_oft.py
 lycoris/modules/dylora.py
 lycoris/modules/full.py

{lycoris_lora-2.1.0.dev7 → lycoris_lora-2.1.0.dev9}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="2.1.0.dev7",
+    version="2.1.0.dev9",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",