lycoris-lora 1.9.0.dev6__tar.gz → 1.9.0.dev8__tar.gz

{lycoris_lora-1.9.0.dev6 → lycoris_lora-1.9.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 1.9.0.dev6
+Version: 1.9.0.dev8
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-1.9.0.dev6 → lycoris_lora-1.9.0.dev8}/lycoris/kohya/__init__.py

@@ -19,6 +19,7 @@ from ..modules.lokr import LokrModule
 from ..modules.dylora import DyLoraModule
 from ..modules.glora import GLoRAModule
 from ..modules.norms import NormModule
+from ..modules.full import FullModule
 
 from ..config import PRESET
 from ..utils.preset import read_preset
@@ -46,6 +47,7 @@ def create_network(multiplier, network_dim, network_alpha, vae, text_encoder, un
         'lokr': LokrModule,
         'dylora': DyLoraModule,
         'glora': GLoRAModule,
+        'full': FullModule,
     }[algo]
     
     if algo == 'glora' and conv_dim>0:
@@ -434,7 +436,7 @@ class LycorisNetwork(torch.nn.Module):
         self.apply(make_ckpt)
         pass
 
-    def prepare_optimizer_params(self, text_encoder_lr, unet_lr):
+    def prepare_optimizer_params(self, text_encoder_lr, unet_lr, learning_rate):
         def enumerate_params(loras):
             params = []
             for lora in loras:
@@ -981,7 +983,7 @@ class IA3Network(torch.nn.Module):
         self.apply(make_ckpt)
         pass
 
-    def prepare_optimizer_params(self, text_encoder_lr, unet_lr):
+    def prepare_optimizer_params(self, text_encoder_lr, unet_lr, learning_rate):
         def enumerate_params(loras):
             params = []
             for lora in loras:

lycoris_lora-1.9.0.dev8/lycoris/modules/full.py

@@ -0,0 +1,85 @@
+import math
+from weakref import ref
+from collections import OrderedDict, abc as container_abcs
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class FullModule(nn.Module):
+    """
+    modifed from kohya-ss/sd-scripts/networks/lora:LoRAModule
+    """
+
+    def __init__(
+        self, 
+        lora_name, org_module: nn.Module, 
+        multiplier=1.0, 
+        lora_dim=4, alpha=1, 
+        dropout=0., rank_dropout=0., module_dropout=0.,
+        use_cp=False, use_scalar=False,
+        **kwargs,
+    ):
+        """ if alpha == 0 or None, alpha is rank (no scaling). """
+        super().__init__()
+        self.lora_name = lora_name
+
+        if isinstance(org_module, nn.Linear):
+            self.op = F.linear
+            self.dim = org_module.out_features
+            self.kw_dict = {}
+        elif isinstance(org_module, nn.Conv2d):
+            self.op = F.conv2d
+            self.dim = org_module.out_channels
+            self.kw_dict = {
+                "stride": org_module.stride,
+                "padding": org_module.padding,
+                "dilation": org_module.dilation,
+                "groups": org_module.groups,
+            }
+        else:
+            raise NotImplementedError
+        self.diff = nn.Parameter(torch.zeros_like(org_module.weight))
+        if org_module.bias is not None:
+            self.diff_b = nn.Parameter(torch.zeros_like(org_module.bias))
+        else:
+            self.diff_b = None
+        
+        self.rank_dropout = rank_dropout
+        self.module_dropout = module_dropout
+        
+        self.multiplier = multiplier
+        self.org_module = [org_module]
+
+    def apply_to(self, **kwargs):
+        self.org_forward = self.org_module[0].forward
+        self.org_module[0].forward = self.forward
+
+    def make_weight(self, scale = 1, device=None):
+        drop = (
+            torch.rand(self.dim, device=device) < self.rank_dropout 
+            if self.rank_dropout and self.training 
+            else 1
+        )
+        org_weight = self.org_module[0].weight.to(device)
+        weight = self.diff.to(device) * drop * scale
+        weight = weight + org_weight
+        
+        if self.diff_b is not None:
+            org_bias = self.org_module[0].bias.to(device)
+            bias = self.diff_b.to(device) * drop * scale
+            bias = bias + org_bias
+        else:
+            bias = None
+        return weight, bias
+
+    def forward(self, x):
+        if self.module_dropout and self.training:
+            if torch.rand(1) < self.module_dropout:
+                return self.org_forward(x)
+        scale = self.multiplier
+        
+        w, b = self.make_weight(scale, x.device)
+        kw_dict = self.kw_dict | {"weight": w, "bias": b}
+        return self.op(x, **kw_dict)

{lycoris_lora-1.9.0.dev6 → lycoris_lora-1.9.0.dev8}/lycoris/modules/locon.py

@@ -72,7 +72,7 @@ class LoConModule(nn.Module):
         if use_scalar:
             self.scalar = nn.Parameter(torch.tensor(0.0))
         else:
-            self.scalar = 1.0
+            self.scalar = torch.tensor(1.0)
         # same as microsoft's
         torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5))
         if use_scalar:

{lycoris_lora-1.9.0.dev6 → lycoris_lora-1.9.0.dev8}/lycoris/modules/loha.py

@@ -157,7 +157,7 @@ class LohaModule(nn.Module):
         if use_scalar:
             self.scalar = nn.Parameter(torch.tensor(0.0))
         else:
-            self.scalar = 1.0
+            self.scalar = torch.tensor(1.0)
         # Need more experiments on init method
         if self.cp:
             torch.nn.init.normal_(self.hada_t1, std=0.1)
@@ -262,4 +262,4 @@ class LohaModule(nn.Module):
             weight.view(self.shape),
             bias,
             **self.extra_args
-        )
+        )

{lycoris_lora-1.9.0.dev6 → lycoris_lora-1.9.0.dev8}/lycoris/modules/lokr.py

@@ -179,7 +179,7 @@ class LokrModule(nn.Module):
         if use_scalar:
             self.scalar = nn.Parameter(torch.tensor(0.0))
         else:
-            self.scalar = 1.0
+            self.scalar = torch.tensor(1.0)
         
         if self.use_w2:
             if use_scalar:

{lycoris_lora-1.9.0.dev6 → lycoris_lora-1.9.0.dev8}/lycoris_lora.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris-lora
-Version: 1.9.0.dev6
+Version: 1.9.0.dev8
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-1.9.0.dev6 → lycoris_lora-1.9.0.dev8}/lycoris_lora.egg-info/SOURCES.txt

@@ -30,6 +30,7 @@ lycoris/kohya/utils.py
 lycoris/modules/__init__.py
 lycoris/modules/attention.py
 lycoris/modules/dylora.py
+lycoris/modules/full.py
 lycoris/modules/glokr.py
 lycoris/modules/glora.py
 lycoris/modules/hyperlycoris.py

{lycoris_lora-1.9.0.dev6 → lycoris_lora-1.9.0.dev8}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
     name='lycoris_lora',
     packages=find_packages(),
-    version='1.9.0.dev6',
+    version='1.9.0.dev8',
     url='https://github.com/KohakuBlueleaf/LyCORIS',
     description='Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion',
     author='Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao',
@@ -15,4 +15,4 @@ setup(
         'diffusers',
         'transformers'
     ],
-)
+)