lycoris-lora 1.8.0.dev7__tar.gz → 1.8.0.dev8__tar.gz

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris_lora
-Version: 1.8.0.dev7
+Version: 1.8.0.dev8
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev8}/lycoris/modules/locon.py

@@ -1,5 +1,6 @@
 import math
 from weakref import ref
+from collections import OrderedDict, abc as container_abcs
 
 import torch
 import torch.nn as nn
@@ -26,6 +27,7 @@ class LoConModule(nn.Module):
         self.lora_dim = lora_dim
         self.cp = False
 
+        self.scalar = nn.Parameter(torch.tensor(0.0))
         if isinstance(org_module, nn.Conv2d):
             # For general LoCon
             in_dim = org_module.in_channels
@@ -64,7 +66,7 @@ class LoConModule(nn.Module):
 
         # same as microsoft's
         torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5))
-        torch.nn.init.zeros_(self.lora_up.weight)
+        torch.nn.init.kaiming_uniform_(self.lora_up.weight)
         if self.cp:
             torch.nn.init.kaiming_uniform_(self.lora_mid.weight, a=math.sqrt(5))
 
@@ -78,7 +80,33 @@ class LoConModule(nn.Module):
     def make_weight(self, device=None):
         wa = self.lora_up.weight.to(device)
         wb = self.lora_down.weight.to(device)
-        return (wa.view(wa.size(0), -1) @ wb.view(wb.size(0), -1)).view(self.shape)
+        return (wa.view(wa.size(0), -1) @ wb.view(wb.size(0), -1)).view(self.shape) * self.scalar
+    
+    def state_dict(self, *args, destination=None, prefix='', keep_vars=False):
+        # TODO: Remove `args` and the parsing logic when BC allows.
+        if len(args) > 0:
+            if destination is None:
+                destination = args[0]
+            if len(args) > 1 and prefix == '':
+                prefix = args[1]
+            if len(args) > 2 and keep_vars is False:
+                keep_vars = args[2]
+            # DeprecationWarning is ignored by default
+
+        if destination is None:
+            destination = OrderedDict()
+            destination._metadata = OrderedDict()
+
+        local_metadata = dict(version=self._version)
+        if hasattr(destination, "_metadata"):
+            destination._metadata[prefix[:-1]] = local_metadata
+
+        destination[f'{prefix}alpha'] = self.alpha
+        destination[f'{prefix}lora_up.weight'] = self.lora_up.weight * self.scalar
+        destination[f'{prefix}lora_down.weight'] = self.lora_down.weight
+        if self.cp:
+            destination[f'{prefix}lora_mid.weight'] = self.lora_mid.weight
+        return destination
 
     @torch.no_grad()
     def apply_max_norm(self, max_norm, device=None):
@@ -89,11 +117,7 @@ class LoConModule(nn.Module):
         
         scaled = ratio.item() != 1.0
         if scaled:
-            modules = self.cp + 2
-            self.lora_up.weight *= ratio**(1/modules)
-            self.lora_down.weight *= ratio**(1/modules)
-            if self.cp:
-                self.lora_mid.weight *= ratio**(1/modules)
+            self.scalar *= ratio
         
         return scaled, orig_norm*ratio
 
@@ -115,4 +139,4 @@ class LoConModule(nn.Module):
                 drop = drop.view(*[1]*(dims-1), -1)
             mid = mid * drop
         
-        return self.org_forward(x) + self.dropout(self.lora_up(mid) * scale)
+        return self.org_forward(x) + self.dropout(self.lora_up(mid) * self.scalar * scale)

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev8}/lycoris_lora.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lycoris-lora
-Version: 1.8.0.dev7
+Version: 1.8.0.dev8
 Summary: Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion
 Home-page: https://github.com/KohakuBlueleaf/LyCORIS
 Author: Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao

{lycoris_lora-1.8.0.dev7 → lycoris_lora-1.8.0.dev8}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
     name='lycoris_lora',
     packages=find_packages(),
-    version='1.8.0.dev7',
+    version='1.8.0.dev8',
     url='https://github.com/KohakuBlueleaf/LyCORIS',
     description='Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion',
     author='Shih-Ying Yeh(KohakuBlueLeaf), Yu-Guan Hsieh, Zhidong Gao',