lycoris-lora 2.3.0.dev2__tar.gz → 2.3.0.dev4__tar.gz

{lycoris_lora-2.3.0.dev2/lycoris_lora.egg-info → lycoris_lora-2.3.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.3.0.dev2
+Version: 2.3.0.dev4
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.3.0.dev2 → lycoris_lora-2.3.0.dev4}/lycoris/modules/__init__.py

@@ -12,7 +12,7 @@ from .ia3 import IA3Module
 def make_module(lyco_type, params, lora_name, orig_module):
     module = None
     if lyco_type == "locon":
-        up, down, mid, alpha = params
+        up, down, mid, alpha, dora_scale = params
         module = LoConModule(
             lora_name,
             orig_module,
@@ -20,13 +20,16 @@ def make_module(lyco_type, params, lora_name, orig_module):
             down.size(0),
             float(alpha),
             use_tucker=mid is not None,
+            weight_decompose=dora_scale is not None,
         )
         module.lora_up.weight.data.copy_(up)
         module.lora_down.weight.data.copy_(down)
         if mid is not None:
             module.lora_mid.weight.data.copy_(mid)
+        if dora_scale is not None:
+            module.dora_scale.copy_(dora_scale)
     elif lyco_type == "hada":
-        w1a, w1b, w2a, w2b, t1, t2, alpha = params
+        w1a, w1b, w2a, w2b, t1, t2, alpha, dora_scale = params
         module = LohaModule(
             lora_name,
             orig_module,
@@ -34,6 +37,7 @@ def make_module(lyco_type, params, lora_name, orig_module):
             w1b.size(0),
             float(alpha),
             use_tucker=t1 is not None,
+            weight_decompose=dora_scale is not None,
         )
         module.hada_w1_a.copy_(w1a)
         module.hada_w1_b.copy_(w1b)
@@ -42,8 +46,10 @@ def make_module(lyco_type, params, lora_name, orig_module):
         if t1 is not None:
             module.hada_t1.copy_(t1)
             module.hada_t2.copy_(t2)
+        if dora_scale is not None:
+            module.dora_scale.copy_(dora_scale)
     elif lyco_type == "kron":
-        w1, w1a, w1b, w2, w2a, w2b, _, t2, alpha = params
+        w1, w1a, w1b, w2, w2a, w2b, _, t2, alpha, dora_scale = params
         if w1a is not None:
             lora_dim = w1a.size(1)
         elif w2a is not None:
@@ -83,6 +89,7 @@ def make_module(lyco_type, params, lora_name, orig_module):
             use_tucker=t2 is not None,
             decompose_both=w1 is None and w2 is None,
             factor=factor,
+            weight_decompose=dora_scale is not None,
         )
         if w1 is not None:
             module.lokr_w1.copy_(w1)
@@ -96,6 +103,8 @@ def make_module(lyco_type, params, lora_name, orig_module):
             module.lokr_w2_b.copy_(w2b)
         if t2 is not None:
             module.lokr_t2.copy_(t2)
+        if dora_scale is not None:
+            module.dora_scale.copy_(dora_scale)
     elif lyco_type == "norm":
         w_norm, b_norm = params
         module = NormModule(

{lycoris_lora-2.3.0.dev2 → lycoris_lora-2.3.0.dev4}/lycoris/modules/locon.py

@@ -196,6 +196,7 @@ class LoConModule(ModuleCustomSD):
         return weight * self.scalar.to(device)
 
     def apply_weight_decompose(self, weight):
+        weight = weight.to(self.dora_scale.dtype)
         weight_norm = (
             weight.transpose(0, 1)
             .reshape(weight.shape[1], -1)
@@ -204,7 +205,7 @@ class LoConModule(ModuleCustomSD):
             .transpose(0, 1)
         )
 
-        return weight * (self.dora_scale / weight_norm)
+        return weight * (self.dora_scale.to(weight.device) / weight_norm)
 
     def custom_state_dict(self):
         destination = {}

{lycoris_lora-2.3.0.dev2 → lycoris_lora-2.3.0.dev4}/lycoris/modules/loha.py

@@ -267,6 +267,7 @@ class LohaModule(ModuleCustomSD):
         return weight
 
     def apply_weight_decompose(self, weight):
+        weight = weight.to(self.dora_scale.dtype)
         weight_norm = (
             weight.transpose(0, 1)
             .reshape(weight.shape[1], -1)
@@ -275,7 +276,7 @@ class LohaModule(ModuleCustomSD):
             .transpose(0, 1)
         )
 
-        return weight * (self.dora_scale / weight_norm)
+        return weight * (self.dora_scale.to(weight.device) / weight_norm)
 
     def custom_state_dict(self):
         destination = {}

{lycoris_lora-2.3.0.dev2 → lycoris_lora-2.3.0.dev4}/lycoris/modules/lokr.py

@@ -337,6 +337,7 @@ class LokrModule(ModuleCustomSD):
         return weight
 
     def apply_weight_decompose(self, weight):
+        weight = weight.to(self.dora_scale.dtype)
         weight_norm = (
             weight.transpose(0, 1)
             .reshape(weight.shape[1], -1)
@@ -345,7 +346,7 @@ class LokrModule(ModuleCustomSD):
             .transpose(0, 1)
         )
 
-        return weight * (self.dora_scale / weight_norm)
+        return weight * (self.dora_scale.to(weight.device) / weight_norm)
 
     def custom_state_dict(self):
         destination = {}

{lycoris_lora-2.3.0.dev2 → lycoris_lora-2.3.0.dev4}/lycoris/utils/__init__.py

@@ -449,9 +449,17 @@ def tucker_weight(wa, wb, t):
 
 
 def apply_dora_scale(org_weight, rebuild, dora_scale, scale):
-    dora_mean_dim = tuple(i for i in range(org_weight.dim()) if i != 1)
+    dora_norm_dims = org_weight.dim() - 1
     weight = org_weight + rebuild
-    merged_scale1 = weight / weight.mean(dim=dora_mean_dim, keepdim=True) * dora_scale
+    weight = weight.to(dora_scale.dtype)
+    weight_norm = (
+        weight.transpose(0, 1)
+        .reshape(weight.shape[1], -1)
+        .norm(dim=1, keepdim=True)
+        .reshape(weight.shape[1], *[1] * dora_norm_dims)
+        .transpose(0, 1)
+    )
+    merged_scale1 = weight / weight_norm * dora_scale
     diff_weight = merged_scale1 - org_weight
     return org_weight + diff_weight * scale
 

lycoris_lora-2.3.0.dev4/lycoris/utils/tensor_norm.py

@@ -0,0 +1,7 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+def multi_dim_norm(x: torch.Tensor, dim: int = 1) -> torch.Tensor:
+    return torch.norm(x, dim=dim, p=2, keepdim=True)

{lycoris_lora-2.3.0.dev2 → lycoris_lora-2.3.0.dev4/lycoris_lora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 2.3.0.dev2
+Version: 2.3.0.dev4
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-2.3.0.dev2 → lycoris_lora-2.3.0.dev4}/lycoris_lora.egg-info/SOURCES.txt

@@ -39,6 +39,7 @@ lycoris/utils/__init__.py
 lycoris/utils/bnb.py
 lycoris/utils/logger.py
 lycoris/utils/preset.py
+lycoris/utils/tensor_norm.py
 lycoris/utils/xformers_utils.py
 lycoris_lora.egg-info/PKG-INFO
 lycoris_lora.egg-info/SOURCES.txt

{lycoris_lora-2.3.0.dev2 → lycoris_lora-2.3.0.dev4}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 setup(
     name="lycoris_lora",
     packages=find_packages(),
-    version="2.3.0.dev2",
+    version="2.3.0.dev4",
     url="https://github.com/KohakuBlueleaf/LyCORIS",
     description="Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion",
     author="Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao",