lycoris-lora 3.0.1.dev4__tar.gz → 3.0.1.dev6__tar.gz

{lycoris_lora-3.0.1.dev4/lycoris_lora.egg-info → lycoris_lora-3.0.1.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev4
+Version: 3.0.1.dev6
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev4 → lycoris_lora-3.0.1.dev6}/lycoris/kohya.py

@@ -1,7 +1,9 @@
 import os
+import fnmatch
 import re
 import logging
-from typing import List
+
+from typing import Any, List
 
 import torch
 
@@ -234,6 +236,7 @@ class LycorisNetworkKohya(LycorisNetwork):
     LORA_PREFIX_TEXT_ENCODER = "lora_te"
     MODULE_ALGO_MAP = {}
     NAME_ALGO_MAP = {}
+    USE_FNMATCH = False
 
     @classmethod
     def apply_preset(cls, preset):
@@ -253,6 +256,8 @@ class LycorisNetworkKohya(LycorisNetwork):
             cls.MODULE_ALGO_MAP = preset["module_algo_map"]
         if "name_algo_map" in preset:
             cls.NAME_ALGO_MAP = preset["name_algo_map"]
+        if "use_fnmatch" in preset:
+            cls.USE_FNMATCH = preset["use_fnmatch"]
         return cls
 
     def __init__(
@@ -404,7 +409,9 @@ class LycorisNetworkKohya(LycorisNetwork):
             next_config = {}
             for name, module in root_module.named_modules():
                 module_name = module.__class__.__name__
-                if module_name in target_replace_modules:
+                if module_name in target_replace_modules and not any(
+                    self.match_fn(t, name) for t in target_replace_names
+                ):
                     if module_name in self.MODULE_ALGO_MAP:
                         next_config = self.MODULE_ALGO_MAP[module_name]
                         algo = next_config.get("algo", network_module)
@@ -417,10 +424,11 @@ class LycorisNetworkKohya(LycorisNetwork):
                     )
                     next_config = {}
                 elif name in target_replace_names or any(
-                    re.match(t, name) for t in target_replace_names
+                    self.match_fn(t, name) for t in target_replace_names
                 ):
-                    if name in self.NAME_ALGO_MAP:
-                        next_config = self.NAME_ALGO_MAP[name]
+                    conf_from_name = self.find_conf_for_name(name)
+                    if conf_from_name is not None:
+                        next_config = conf_from_name
                         algo = next_config.get("algo", network_module)
                     elif module_name in self.MODULE_ALGO_MAP:
                         next_config = self.MODULE_ALGO_MAP[module_name]
@@ -492,6 +500,24 @@ class LycorisNetworkKohya(LycorisNetwork):
             ), f"duplicated lora name: {lora.lora_name}"
             names.add(lora.lora_name)
 
+    def match_fn(self, pattern: str, name: str) -> bool:
+        if self.USE_FNMATCH:
+            return fnmatch.fnmatch(name, pattern)
+        return re.match(pattern, name)
+
+    def find_conf_for_name(
+        self,
+        name: str,
+    ) -> dict[str, Any]:
+        if name in self.NAME_ALGO_MAP.keys():
+            return self.NAME_ALGO_MAP[name]
+
+        for key, value in self.NAME_ALGO_MAP.items():
+            if self.match_fn(key, name):
+                return value
+
+        return None
+
     def load_weights(self, file):
         if os.path.splitext(file)[1] == ".safetensors":
             from safetensors.torch import load_file, safe_open

{lycoris_lora-3.0.1.dev4 → lycoris_lora-3.0.1.dev6}/lycoris/modules/norms.py

@@ -37,14 +37,14 @@ class NormModule(LycorisBaseModule):
         if self.module_type == "unknown":
             if not hasattr(org_module, "weight") or not hasattr(org_module, "_norm"):
                 warning_once(f"{type(org_module)} is not supported in Norm algo.")
-                self.not_supported=True
+                self.not_supported = True
                 return
             else:
                 self.dim = org_module.weight.numel()
                 self.not_supported = False
         elif self.module_type not in self.support_module:
             warning_once(f"{self.module_type} is not supported in Norm algo.")
-            self.not_supported=True
+            self.not_supported = True
             return
 
         self.w_norm = nn.Parameter(torch.zeros(self.dim))

{lycoris_lora-3.0.1.dev4 → lycoris_lora-3.0.1.dev6}/lycoris/utils/bnb.py

@@ -74,4 +74,6 @@ QuantLinears = (
 
 @cache
 def log_bypass():
-    return logger.warning("Using bnb/quanto/optimum-quanto with LyCORIS will enable force-bypass mode.")
+    return logger.warning(
+        "Using bnb/quanto/optimum-quanto with LyCORIS will enable force-bypass mode."
+    )

{lycoris_lora-3.0.1.dev4 → lycoris_lora-3.0.1.dev6}/lycoris/wrapper.py

@@ -1,9 +1,10 @@
 # General LyCORIS wrapper based on kohya-ss/sd-scripts' style
 import os
-import regex as re
+import fnmatch
+import re
 import logging
 
-from typing import List
+from typing import Any, List
 
 import torch
 import torch.nn as nn
@@ -25,6 +26,22 @@ from .utils import str_bool
 from .logging import logger
 
 
+VALID_PRESET_KEYS = [
+    "enable_conv",
+    "target_module",
+    "target_name",
+    "module_algo_map",
+    "name_algo_map",
+    "lora_prefix",
+    "use_fnmatch",
+    "unet_target_module",
+    "unet_target_name",
+    "text_encoder_target_module",
+    "text_encoder_target_name",
+    "exclude_name",
+]
+
+
 network_module_dict = {
     "lora": LoConModule,
     "locon": LoConModule,
@@ -189,9 +206,17 @@ class LycorisNetwork(torch.nn.Module):
     LORA_PREFIX = "lycoris"
     MODULE_ALGO_MAP = {}
     NAME_ALGO_MAP = {}
+    USE_FNMATCH = False
+    TARGET_EXCLUDE_NAME = []
 
     @classmethod
     def apply_preset(cls, preset):
+        for preset_key in preset.keys():
+            if preset_key not in VALID_PRESET_KEYS:
+                raise KeyError(
+                    f'Unknown preset key "{preset_key}". Valid keys: {VALID_PRESET_KEYS}'
+                )
+
         if "enable_conv" in preset:
             cls.ENABLE_CONV = preset["enable_conv"]
         if "target_module" in preset:
@@ -202,6 +227,12 @@ class LycorisNetwork(torch.nn.Module):
             cls.MODULE_ALGO_MAP = preset["module_algo_map"]
         if "name_algo_map" in preset:
             cls.NAME_ALGO_MAP = preset["name_algo_map"]
+        if "lora_prefix" in preset:
+            cls.LORA_PREFIX = preset["lora_prefix"]
+        if "use_fnmatch" in preset:
+            cls.USE_FNMATCH = preset["use_fnmatch"]
+        if "exclude_name" in preset:
+            cls.TARGET_EXCLUDE_NAME = preset["exclude_name"]
         return cls
 
     def __init__(
@@ -324,27 +355,42 @@ class LycorisNetwork(torch.nn.Module):
             prefix: str,
             root_module: torch.nn.Module,
             algo,
+            current_lora_map: dict[str, Any],
             configs={},
         ):
-            loras = {}
+            assert current_lora_map is not None, "No mapping supplied"
+            loras = current_lora_map
             lora_names = []
             for name, module in root_module.named_modules():
                 module_name = module.__class__.__name__
                 if module_name in self.MODULE_ALGO_MAP and module is not root_module:
                     next_config = self.MODULE_ALGO_MAP[module_name]
                     next_algo = next_config.get("algo", algo)
-                    new_loras, new_lora_names = create_modules_(
-                        f"{prefix}_{name}" if name else prefix, module, next_algo, next_config
+                    new_loras, new_lora_names, new_lora_map = create_modules_(
+                        f"{prefix}_{name}" if name else prefix,
+                        module,
+                        next_algo,
+                        loras,
+                        configs=next_config,
                     )
+                    loras = {**loras, **new_lora_map}
                     for lora_name, lora in zip(new_lora_names, new_loras):
-                        if lora_name not in loras:
+                        if lora_name not in loras and lora_name not in current_lora_map:
                             loras[lora_name] = lora
                             lora_names.append(lora_name)
                     continue
+
                 if name:
                     lora_name = prefix + "." + name
                 else:
                     lora_name = prefix
+
+                if f"{self.LORA_PREFIX}_." in lora_name:
+                    lora_name = lora_name.replace(
+                        f"{self.LORA_PREFIX}_.",
+                        f"{self.LORA_PREFIX}.",
+                    )
+
                 lora_name = lora_name.replace(".", "_")
                 if lora_name in loras:
                     continue
@@ -353,7 +399,7 @@ class LycorisNetwork(torch.nn.Module):
                 if lora is not None:
                     loras[lora_name] = lora
                     lora_names.append(lora_name)
-            return [loras[lora_name] for lora_name in lora_names], lora_names
+            return [loras[lora_name] for lora_name in lora_names], lora_names, loras
 
         # create module instances
         def create_modules(
@@ -361,29 +407,44 @@ class LycorisNetwork(torch.nn.Module):
             root_module: torch.nn.Module,
             target_replace_modules,
             target_replace_names=[],
+            target_exclude_names=[],
         ) -> List:
             logger.info("Create LyCORIS Module")
             loras = []
+            lora_map = {}
             next_config = {}
             for name, module in root_module.named_modules():
+                if name in target_exclude_names or any(
+                    self.match_fn(t, name) for t in target_exclude_names
+                ):
+                    continue
+
                 module_name = module.__class__.__name__
-                if module_name in target_replace_modules:
+                if module_name in target_replace_modules and not any(
+                    self.match_fn(t, name) for t in target_replace_names
+                ):
                     if module_name in self.MODULE_ALGO_MAP:
                         next_config = self.MODULE_ALGO_MAP[module_name]
                         algo = next_config.get("algo", network_module)
                     else:
                         algo = network_module
-                    loras.extend(
-                        create_modules_(f"{prefix}_{name}", module, algo, next_config)[
-                            0
-                        ]
+
+                    lora_lst, _, _lora_map = create_modules_(
+                        f"{prefix}_{name}",
+                        module,
+                        algo,
+                        lora_map,
+                        configs=next_config,
                     )
+                    lora_map = {**lora_map, **_lora_map}
+                    loras.extend(lora_lst)
                     next_config = {}
                 elif name in target_replace_names or any(
-                    re.match(t, name) for t in target_replace_names
+                    self.match_fn(t, name) for t in target_replace_names
                 ):
-                    if name in self.NAME_ALGO_MAP:
-                        next_config = self.NAME_ALGO_MAP[name]
+                    conf_from_name = self.find_conf_for_name(name)
+                    if conf_from_name is not None:
+                        next_config = conf_from_name
                         algo = next_config.get("algo", network_module)
                     elif module_name in self.MODULE_ALGO_MAP:
                         next_config = self.MODULE_ALGO_MAP[module_name]
@@ -392,17 +453,37 @@ class LycorisNetwork(torch.nn.Module):
                         algo = network_module
                     lora_name = prefix + "." + name
                     lora_name = lora_name.replace(".", "_")
+
+                    if lora_name in lora_map:
+                        continue
+
                     lora = create_single_module(lora_name, module, algo, **next_config)
                     next_config = {}
                     if lora is not None:
+                        lora_map[lora.lora_name] = lora
                         loras.append(lora)
             return loras
 
         self.loras = create_modules(
             LycorisNetwork.LORA_PREFIX,
             module,
-            LycorisNetwork.TARGET_REPLACE_MODULE,
-            LycorisNetwork.TARGET_REPLACE_NAME,
+            list(
+                set(
+                    [
+                        *LycorisNetwork.TARGET_REPLACE_MODULE,
+                        *LycorisNetwork.MODULE_ALGO_MAP.keys(),
+                    ]
+                )
+            ),
+            list(
+                set(
+                    [
+                        *LycorisNetwork.TARGET_REPLACE_NAME,
+                        *LycorisNetwork.NAME_ALGO_MAP.keys(),
+                    ]
+                )
+            ),
+            target_exclude_names=LycorisNetwork.TARGET_EXCLUDE_NAME,
         )
         logger.info(f"create LyCORIS: {len(self.loras)} modules.")
 
@@ -413,7 +494,8 @@ class LycorisNetwork(torch.nn.Module):
             )
         logger.info(f"module type table: {algo_table}")
 
-        # assertion
+        # Assertion to ensure we have not accidentally wrapped some layers
+        # multiple times.
         names = set()
         for lora in self.loras:
             assert (
@@ -421,6 +503,24 @@ class LycorisNetwork(torch.nn.Module):
             ), f"duplicated lora name: {lora.lora_name}"
             names.add(lora.lora_name)
 
+    def match_fn(self, pattern: str, name: str) -> bool:
+        if self.USE_FNMATCH:
+            return fnmatch.fnmatch(name, pattern)
+        return bool(re.match(pattern, name))
+
+    def find_conf_for_name(
+        self,
+        name: str,
+    ) -> dict[str, Any]:
+        if name in self.NAME_ALGO_MAP.keys():
+            return self.NAME_ALGO_MAP[name]
+
+        for key, value in self.NAME_ALGO_MAP.items():
+            if self.match_fn(key, name):
+                return value
+
+        return None
+
     def set_multiplier(self, multiplier):
         self.multiplier = multiplier
         for lora in self.loras:
@@ -442,6 +542,9 @@ class LycorisNetwork(torch.nn.Module):
         return state
 
     def apply_to(self):
+        """
+        Register to modules to the subclass so that torch sees them.
+        """
         for lora in self.loras:
             lora.apply_to()
             self.add_module(lora.lora_name, lora)

{lycoris_lora-3.0.1.dev4 → lycoris_lora-3.0.1.dev6/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 3.0.1.dev4
+Version: 3.0.1.dev6
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-3.0.1.dev4 → lycoris_lora-3.0.1.dev6}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="3.0.1.dev4",
+    version="3.0.1.dev6",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",