lycoris-lora 2.2.0.dev12__tar.gz → 2.2.0.post1__tar.gz

{lycoris_lora-2.2.0.dev12/lycoris_lora.egg-info → lycoris_lora-2.2.0.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.2.0.dev12
+Version: 2.2.0.post1
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.2.0.dev12 → lycoris_lora-2.2.0.post1}/README.md

@@ -153,6 +153,10 @@ forward_with_lyco = your_model(x)
 
 You can check my [HakuPhi](https://github.com/KohakuBlueleaf/HakuPhi) project to see how I utilize LyCORIS to finetune the Phi-1.5 models.
 
+#### Bitsandbytes support
+
+See [bnb_example.py](bnb_example.py) for example. Basically as same as standalone wrapper.
+
 #### Graphical interfaces and Colabs (via kohya trainer)
 
 You can also train LyCORIS with the following graphical interfaces
@@ -236,30 +240,30 @@ See [docs/Conversion-scripts.md](docs/Conversion-scripts.md) for more informatio
 
 For full log, please see [Change.md](Change.md)
 
-## 2024/02/18 update to 2.1.0
+## 2024/03/15 update to 2.2.0 - QLyCORIS and DoRA
 
 #### New Algo
 
-* [BOFT (Butterfly OFT)](https://arxiv.org/abs/2311.06243)
+* DoRA
+  * Ref: [DoRA: Weight-Decomposed Low-Rank Adaptation](https://github.com/KohakuBlueleaf/LyCORIS)
+* Weight decompose for LoHa and LoKr. (A.K.A DoHa/DoKr)
+  * DoRA/DoHa/DoKr will require smaller Learning rate!
 
-#### Improvements
+#### New Features
 
-* Faster, better extract script
-* support kohya-ss/sd-scripts image gen
-* support regex name in kohya-ss/sd-scripts
-* support resume on:
-  * full
-  * loha
-  * oft
-  * boft
-* Add logger into LyCORIS
+* Support "bypass" (a.k.a. adapter) mode for LoHa/LoKr/OFT/BOFT
+  * LoHa will require 2xFLOPs since we rebuild full diff weight and then do one more forward.
+  * LoKr, OFT, BOFT should be more efficient than LoHa in bypass mode.
+* Support [bnb 8bit/4bit Linear layer](https://github.com/TimDettmers/bitsandbytes) (a.k.a. QLyCORIS) with LoHa/LoKr/OFT/BOFT.
+  * This will force module to enable bypass mode.
 
 #### Fixes, slight changes
 
-* Update HCP convert for the case where only UNet or TE is trained.
-* Change arg names for conversion scripts.
-* Fix wrong TE prefix in merge scripts.
-* Fix warnings and confusing logging.
+* Refine some details about code quality. Based on the report from GitRoll. (Thx you gitroll!)
+* Remove redundant calculation in BOFT
+* rank_dropout has been removed from OFT/BOFT temporarily untill we ensure how to apply it.
+* Fix bugs in lokr when `lokr_w1_a` not exist.
+* Fix bugs in conversion scritps.
 
 ## Todo list
 

{lycoris_lora-2.2.0.dev12 → lycoris_lora-2.2.0.post1}/lycoris/kohya/__init__.py

@@ -350,10 +350,10 @@ class LycorisNetworkKohya(LycorisNetwork):
                     **kwargs,
                 )
             lora = None
-            if module.__class__.__name__ == "Linear" and lora_dim > 0:
+            if isinstance(module, torch.nn.Linear) and lora_dim > 0:
                 dim = dim or lora_dim
                 alpha = alpha or self.alpha
-            elif module.__class__.__name__ == "Conv2d":
+            elif isinstance(module, torch.nn.Conv2d):
                 k_size, *_ = module.kernel_size
                 if k_size == 1 and lora_dim > 0:
                     dim = dim or lora_dim

{lycoris_lora-2.2.0.dev12 → lycoris_lora-2.2.0.post1}/lycoris/modules/locon.py

@@ -133,7 +133,6 @@ class LoConModule(ModuleCustomSD):
         if use_scalar:
             self.scalar = nn.Parameter(torch.tensor(0.0))
         else:
-            self.scalar: torch.FloatTensor
             self.register_buffer("scalar", torch.tensor(1.0), persistent=False)
         # same as microsoft's
         torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5))
@@ -225,11 +224,11 @@ class LoConModule(ModuleCustomSD):
                 return self.org_forward(x)
         scale = self.scale * self.multiplier
 
-        dtype_device = next(self.parameters())
+        dtype = next(self.parameters()).dtype
         if not self.bypass_mode:
             weight = (
-                self.org_module[0].weight.data.to(dtype_device)
-                + self.make_weight(x.device).to(dtype_device) * scale
+                self.org_module[0].weight.data.to(device=x.device, dtype=dtype)
+                + self.make_weight(x.device).to(device=x.device, dtype=dtype) * scale
             )
             if self.wd:
                 weight = self.apply_weight_decompose(weight)

{lycoris_lora-2.2.0.dev12 → lycoris_lora-2.2.0.post1}/lycoris/modules/loha.py

@@ -281,7 +281,7 @@ class LohaModule(ModuleCustomSD):
 
     @torch.no_grad()
     def apply_max_norm(self, max_norm, device=None):
-        orig_norm = (self.get_weight() * self.scalar).norm()
+        orig_norm = (self.get_weight(self.shape) * self.scalar).norm()
         norm = torch.clamp(orig_norm, max_norm / 2)
         desired = torch.clamp(norm, max=max_norm)
         ratio = desired.cpu() / norm.cpu()

{lycoris_lora-2.2.0.dev12 → lycoris_lora-2.2.0.post1}/lycoris/modules/lokr.py

@@ -352,7 +352,7 @@ class LokrModule(ModuleCustomSD):
 
     @torch.no_grad()
     def apply_max_norm(self, max_norm, device=None):
-        orig_norm = self.get_weight().norm()
+        orig_norm = self.get_weight(self.shape).norm()
         norm = torch.clamp(orig_norm, max_norm / 2)
         desired = torch.clamp(norm, max=max_norm)
         ratio = desired.cpu() / norm.cpu()

{lycoris_lora-2.2.0.dev12 → lycoris_lora-2.2.0.post1}/lycoris/wrapper.py

@@ -292,10 +292,10 @@ class LycorisNetwork(torch.nn.Module):
                     **kwargs,
                 )
             lora = None
-            if module.__class__.__name__ == "Linear" and lora_dim > 0:
+            if isinstance(module, torch.nn.Linear) and lora_dim > 0:
                 dim = dim or lora_dim
                 alpha = alpha or self.alpha
-            elif module.__class__.__name__ == "Conv2d":
+            elif isinstance(module, torch.nn.Conv2d):
                 k_size, *_ = module.kernel_size
                 if k_size == 1 and lora_dim > 0:
                     dim = dim or lora_dim

{lycoris_lora-2.2.0.dev12 → lycoris_lora-2.2.0.post1/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.2.0.dev12
+Version: 2.2.0.post1
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.2.0.dev12 → lycoris_lora-2.2.0.post1}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="2.2.0.dev12",
+    version="2.2.0.post1",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",