lycoris-lora 1.8.0.dev7__tar.gz → 1.8.0.dev9__tar.gz

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 1.8.0.dev7
+Version: 1.8.0.dev9
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev9}/lycoris/modules/locon.py

@@ -1,5 +1,6 @@
 import math
 from weakref import ref
+from collections import OrderedDict, abc as container_abcs
 
 import torch
 import torch.nn as nn
@@ -26,6 +27,7 @@ class LoConModule(nn.Module):
         self.lora_dim = lora_dim
         self.cp = False
 
+        self.scalar = nn.Parameter(torch.tensor(0.0))
         if isinstance(org_module, nn.Conv2d):
             # For general LoCon
             in_dim = org_module.in_channels
@@ -64,7 +66,7 @@ class LoConModule(nn.Module):
 
         # same as microsoft's
         torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5))
-        torch.nn.init.zeros_(self.lora_up.weight)
+        torch.nn.init.kaiming_uniform_(self.lora_up.weight)
         if self.cp:
             torch.nn.init.kaiming_uniform_(self.lora_mid.weight, a=math.sqrt(5))
 
@@ -78,7 +80,33 @@ class LoConModule(nn.Module):
     def make_weight(self, device=None):
         wa = self.lora_up.weight.to(device)
         wb = self.lora_down.weight.to(device)
-        return (wa.view(wa.size(0), -1) @ wb.view(wb.size(0), -1)).view(self.shape)
+        return (wa.view(wa.size(0), -1) @ wb.view(wb.size(0), -1)).view(self.shape) * self.scalar
+    
+    def state_dict(self, *args, destination=None, prefix='', keep_vars=False):
+        # TODO: Remove `args` and the parsing logic when BC allows.
+        if len(args) > 0:
+            if destination is None:
+                destination = args[0]
+            if len(args) > 1 and prefix == '':
+                prefix = args[1]
+            if len(args) > 2 and keep_vars is False:
+                keep_vars = args[2]
+            # DeprecationWarning is ignored by default
+
+        if destination is None:
+            destination = OrderedDict()
+            destination._metadata = OrderedDict()
+
+        local_metadata = dict(version=self._version)
+        if hasattr(destination, "_metadata"):
+            destination._metadata[prefix[:-1]] = local_metadata
+
+        destination[f'{prefix}alpha'] = self.alpha
+        destination[f'{prefix}lora_up.weight'] = self.lora_up.weight * self.scalar
+        destination[f'{prefix}lora_down.weight'] = self.lora_down.weight
+        if self.cp:
+            destination[f'{prefix}lora_mid.weight'] = self.lora_mid.weight
+        return destination
 
     @torch.no_grad()
     def apply_max_norm(self, max_norm, device=None):
@@ -89,11 +117,7 @@ class LoConModule(nn.Module):
         
         scaled = ratio.item() != 1.0
         if scaled:
-            modules = self.cp + 2
-            self.lora_up.weight *= ratio**(1/modules)
-            self.lora_down.weight *= ratio**(1/modules)
-            if self.cp:
-                self.lora_mid.weight *= ratio**(1/modules)
+            self.scalar *= ratio
         
         return scaled, orig_norm*ratio
 
@@ -115,4 +139,4 @@ class LoConModule(nn.Module):
                 drop = drop.view(*[1]*(dims-1), -1)
             mid = mid * drop
         
-        return self.org_forward(x) + self.dropout(self.lora_up(mid) * scale)
+        return self.org_forward(x) + self.dropout(self.lora_up(mid) * self.scalar * scale)

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev9}/lycoris/modules/loha.py

@@ -1,4 +1,5 @@
 import math
+from collections import OrderedDict, abc as container_abcs
 
 import torch
 import torch.nn as nn
@@ -102,6 +103,7 @@ class LohaModule(nn.Module):
         self.cp=False
         
         self.shape = org_module.weight.shape
+        self.scalar = nn.Parameter(torch.tensor(0.0))
         if org_module.__class__.__name__ == 'Conv2d':
             in_dim = org_module.in_channels
             k_size = org_module.kernel_size
@@ -159,7 +161,7 @@ class LohaModule(nn.Module):
         torch.nn.init.normal_(self.hada_w1_b, std=1)
         torch.nn.init.normal_(self.hada_w1_a, std=0.1)
         torch.nn.init.normal_(self.hada_w2_b, std=1)
-        torch.nn.init.constant_(self.hada_w2_a, 0)
+        torch.nn.init.normal_(self.hada_w2_a, std=0.1)
 
         self.multiplier = multiplier
         self.org_module = [org_module] # remove in applying
@@ -187,25 +189,46 @@ class LohaModule(nn.Module):
             drop = torch.rand(weight.size(0)) < self.rank_dropout
             weight *= drop.view(-1, [1]*len(weight.shape[1:])).to(weight.device)
         return weight
+    
+    def state_dict(self, *args, destination=None, prefix='', keep_vars=False):
+        # TODO: Remove `args` and the parsing logic when BC allows.
+        if len(args) > 0:
+            if destination is None:
+                destination = args[0]
+            if len(args) > 1 and prefix == '':
+                prefix = args[1]
+            if len(args) > 2 and keep_vars is False:
+                keep_vars = args[2]
+            # DeprecationWarning is ignored by default
+
+        if destination is None:
+            destination = OrderedDict()
+            destination._metadata = OrderedDict()
+
+        local_metadata = dict(version=self._version)
+        if hasattr(destination, "_metadata"):
+            destination._metadata[prefix[:-1]] = local_metadata
+
+        destination[f'{prefix}alpha'] = self.alpha
+        destination[f'{prefix}hada_w1_a'] = self.hada_w1_a * self.scalar
+        destination[f'{prefix}hada_w1_b'] = self.hada_w1_b
+        destination[f'{prefix}hada_w2_a'] = self.hada_w2_a
+        destination[f'{prefix}hada_w2_b'] = self.hada_w2_b
+        if self.cp:
+            destination[f'{prefix}hada_t1'] = self.hada_t1
+            destination[f'{prefix}hada_t2'] = self.hada_t2
+        return destination
 
     @torch.no_grad()
     def apply_max_norm(self, max_norm, device=None):
-        orig_norm = self.get_weight().norm()
+        orig_norm = (self.get_weight()*self.scalar).norm()
         norm = torch.clamp(orig_norm, max_norm/2)
         desired = torch.clamp(norm, max=max_norm)
         ratio = desired.cpu()/norm.cpu()
         
         scaled = ratio.item() != 1.0
         if scaled:
-            modules = (self.cp + 2)*2
-            self.hada_w1_a *= ratio**(1/modules)
-            self.hada_w1_b *= ratio**(1/modules)
-            self.hada_w2_a *= ratio**(1/modules)
-            self.hada_w2_b *= ratio**(1/modules)
-            
-            if self.cp:
-                self.hada_t1 *= ratio**(1/modules)
-                self.hada_t2 *= ratio**(1/modules)
+            self.scalar *= ratio
         
         return scaled, orig_norm*ratio
 
@@ -220,7 +243,7 @@ class LohaModule(nn.Module):
                 )
         weight = (
             self.org_module[0].weight.data 
-            + self.get_weight(self.org_module[0].weight.data) * self.multiplier
+            + self.get_weight(self.org_module[0].weight.data) * self.scalar * self.multiplier
         )
         bias = None if self.org_module[0].bias is None else self.org_module[0].bias.data
         return self.op(

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev9}/lycoris_lora.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris-lora
-Version: 1.8.0.dev7
+Version: 1.8.0.dev9
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev9}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
     name='lycoris_lora',
     packages=find_packages(),
-    version='1.8.0.dev7',
+    version='1.8.0.dev9',
     url='https://github.com/KohakuBlueleaf/LyCORIS',
     description='Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion',
     author='Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao',