deeplotx 0.9.11__py3-none-any.whl → 0.9.13__py3-none-any.whl

deeplotx/__init__.py

@@ -15,6 +15,7 @@ from .nn import (
     RecursiveSequential,
     LongContextRecursiveSequential,
     RoPE,
+    LoRA,
     Attention,
     MultiHeadAttention,
     RoFormerEncoder,

deeplotx/encoder/long_text_encoder.py

@@ -7,7 +7,7 @@ from vortezwohl.concurrent import ThreadPool
 from vortezwohl.cache import LRUCache
 
 from deeplotx.encoder.encoder import Encoder, DEFAULT_BERT
-from deeplotx.util.hash import sha512
+from deeplotx.util import sha512
 
 logger = logging.getLogger('deeplotx.embedding')
 

deeplotx/nn/__init__.py

@@ -7,6 +7,7 @@ from .softmax_regression import SoftmaxRegression
 from .recursive_sequential import RecursiveSequential
 from .long_context_recursive_sequential import LongContextRecursiveSequential
 from .rope import RoPE
+from .lora import LoRA
 from .attention import Attention
 from .multi_head_attention import MultiHeadAttention
 from .roformer_encoder import RoFormerEncoder

deeplotx/nn/base_neural_network.py

@@ -10,13 +10,15 @@ DEFAULT_SUFFIX = 'dlx'
 
 class BaseNeuralNetwork(nn.Module):
     def __init__(self, in_features: int, out_features: int, model_name: str | None = None,
-                 device: str | None = None, dtype: torch.dtype | None = None):
+                 device: str | torch.device | None = None, dtype: torch.dtype | None = None):
         super().__init__()
         self._model_name = model_name \
             if model_name is not None \
             else self.__class__.__name__
-        self.device = torch.device(device) if device is not None \
-            else torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.device = device if isinstance(device, torch.device) else None
+        if self.device is None:
+            self.device = torch.device(device) if device is not None \
+                else torch.device('cuda' if torch.cuda.is_available() else 'cpu')
         self.dtype = dtype if dtype is not None else torch.float32
         self._in_features = in_features
         self._out_features = out_features

deeplotx/nn/lora.py

@@ -0,0 +1,56 @@
+from typing_extensions import override
+
+import torch
+from torch import nn
+
+from deeplotx.nn.base_neural_network import BaseNeuralNetwork
+
+
+class LoRA(BaseNeuralNetwork):
+    def __init__(self, input_dim: int, output_dim: int, rank: int = 8, alpha: int = 16,
+                 dropout_rate: float = .0, model_name: str | None = None, device: str | torch.device | None = None,
+                 dtype: torch.dtype | None = None):
+        super().__init__(in_features=input_dim, out_features=output_dim, model_name=model_name,
+                         device=device, dtype=dtype)
+        self._rank = rank
+        self._alpha = alpha
+        self._scaling = self._alpha / self._rank
+        self._dropout = nn.Dropout(p=dropout_rate) if dropout_rate > .0 else nn.Identity()
+        self.lora_A = nn.Linear(in_features=input_dim, out_features=rank, bias=False,
+                                device=self.device, dtype=self.dtype)
+        self.lora_B = nn.Linear(in_features=rank, out_features=output_dim, bias=False,
+                                device=self.device, dtype=self.dtype)
+        nn.init.normal_(self.lora_A.weight, mean=.0, std=.01)
+        nn.init.zeros_(self.lora_B.weight)
+        self.w0 = None
+
+    @override
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if not isinstance(self.w0, nn.Module):
+            raise ValueError('LoRA adapter was not mounted successfully.')
+        original_out = self.w0(x)
+        lora_out = self.lora_B(self._dropout(self.lora_A(x))) * self._scaling
+        return original_out + lora_out
+
+    @staticmethod
+    def apply_to(model: nn.Module, target_modules: list[str] | str, rank: int = 8, alpha: int = 16,
+                 dropout_rate: float = .0) -> nn.Module:
+        if isinstance(target_modules, str):
+            target_modules = [target_modules]
+        for layer_name, module in model.named_modules():
+            if any(_name in layer_name.split('.')[-1] for _name in target_modules):
+                lora = LoRA(input_dim=module.in_features, output_dim=module.out_features,
+                            rank=rank, alpha=alpha, dropout_rate=dropout_rate,
+                            device=next(module.parameters()).device,
+                            dtype=next(module.parameters()).dtype)
+                lora.w0 = module
+                parent_name = layer_name.rsplit('.', 1)[0] if '.' in layer_name else ''
+                child_name = layer_name.split('.')[-1]
+                parent_module = dict(model.named_modules())[parent_name] if parent_name else model
+                setattr(parent_module, child_name, lora)
+        for param in model.parameters():
+            param.requires_grad = False
+        for name, param in model.named_parameters():
+            if 'lora_A.weight' in name or 'lora_B.weight' in name:
+                param.requires_grad = True
+        return model

{deeplotx-0.9.11.dist-info → deeplotx-0.9.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.11
+Version: 0.9.13
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -15,7 +15,7 @@ Requires-Dist: tiktoken
 Requires-Dist: torch
 Requires-Dist: transformers
 Requires-Dist: typing-extensions
-Requires-Dist: vortezwohl>=0.0.10
+Requires-Dist: vortezwohl>=0.0.17
 Requires-Dist: name4py>=0.1.4
 Dynamic: license-file
 

{deeplotx-0.9.11.dist-info → deeplotx-0.9.13.dist-info}/RECORD

@@ -1,22 +1,23 @@
-deeplotx/__init__.py,sha256=0OWLsgXlStzwm0m9ScaoZvBnsx3a0xTmlzYBUgarl-g,1306
+deeplotx/__init__.py,sha256=9vGW9dz8aY9GMR0ctU0mBXvzhZR4a7P0CUG8fB1YQRc,1317
 deeplotx/encoder/__init__.py,sha256=BrsF5_4O-4pfihYF2wjExDOoAY-03kGJTH-Mhez4tsE,129
 deeplotx/encoder/encoder.py,sha256=uJswUSrYVDWP84HeCD40R9KgXGPUEa080konv7jEp8I,3539
-deeplotx/encoder/long_text_encoder.py,sha256=4oRa9FqfGNZ8-gq14UKuhDkZC0A1Xi-wKmbQsn-uZ58,3966
+deeplotx/encoder/long_text_encoder.py,sha256=eabhgTMhJrAvRC7YyXAAR_LRv9-LULzo8S0uybWTcwM,3961
 deeplotx/encoder/longformer_encoder.py,sha256=mfAI_NE3QQZhvGHbZkP7S6g0Jj59wmLWQ9QW7HOjqm0,2876
 deeplotx/ner/__init__.py,sha256=Rss1pup9HzHZCG8U9ub8niWa9zRjWCy3Z7zg378KZQg,114
 deeplotx/ner/base_ner.py,sha256=pZTl50OrHH_FJm4rKp9iuixeOE6FX_AzgDXD32aXsN0,204
 deeplotx/ner/bert_ner.py,sha256=tfbM3CQBEpZsD0KYVA7GVNJax-7kzNOQgwlpo2S8h-c,8986
 deeplotx/ner/named_entity.py,sha256=c6XufIwH6yloJ-ccUjagf4mBl1XbbYDT8xyEJJ_-ZNs,269
 deeplotx/ner/n2g/__init__.py,sha256=L1IJ8W1nApzqHx2u7JMtPCLfABm5qKJvh_bHMWdvdLY,3538
-deeplotx/nn/__init__.py,sha256=YILwbxb-NHdiJjfOwBKH8F7PuZSDZSrGpTznPDucTro,710
+deeplotx/nn/__init__.py,sha256=9v-cftkJe7MSRkqfvBwu1kZx4nEosT0d36xi3KQtzXw,734
 deeplotx/nn/attention.py,sha256=R-i-Rd7gnsh6hwXDeYfqLQOJvfSZIGfQbFzRlC91XLo,2879
 deeplotx/nn/auto_regression.py,sha256=j_R7WGPq9REngjpLuX5c0AaNqOpgGm2Vfrolw-XjWXw,877
-deeplotx/nn/base_neural_network.py,sha256=QCyB1dxOs4I8vpu6PCshrZs0infoHXS9IErw6tN-dhs,6060
+deeplotx/nn/base_neural_network.py,sha256=w98m1qOSw4C9zFu0uI4k3z8zeXrn5NiAZruyCe2ZQjs,6192
 deeplotx/nn/feed_forward.py,sha256=kGWEUo8J7jrhSSWlitNnj-AcitNiLz6eOCvUcEuWlVs,2949
 deeplotx/nn/linear_regression.py,sha256=LWrrdAIw32KIT1bdr7q6HczdpEiCgb-R8BCNXGywMxE,1763
 deeplotx/nn/logistic_regression.py,sha256=nipWD3ZPRub2Cx0rU2zxYQyG0COn3NJvew8b2gbJy24,998
 deeplotx/nn/long_context_auto_regression.py,sha256=uy0k_g8wEfMH5nd5HCfrHA8dgEsuWBA2x8U-g3h4vQc,1054
 deeplotx/nn/long_context_recursive_sequential.py,sha256=pcZfnrIHBqbp2BssfUTS1klpuykZwowikfAIaOnvRUI,2674
+deeplotx/nn/lora.py,sha256=oy3HltuDily4NLSbUTNvS0i4Kq2kiwl92rBsIt8UOIA,2919
 deeplotx/nn/multi_head_attention.py,sha256=3z73uGbvy3jszRy1B9nxGOJjlttHpcpRF8Qd09OEams,2267
 deeplotx/nn/multi_head_feed_forward.py,sha256=hD9ScrVJZ9kNksoFASf0xaPgEnNgCeRivW-XjYOPjj8,1908
 deeplotx/nn/recursive_sequential.py,sha256=sNvAs9iVCuWIgx0_6TizDq41hJpFbfKT3kyDHE86wRM,2928
@@ -28,8 +29,8 @@ deeplotx/similarity/distribution.py,sha256=wQGouuuW531pZeBRKBujXsdsoz4fDnPw7_GW8
 deeplotx/similarity/set.py,sha256=zhGFxtSIXlWqvipBYzoiPahp4g0boAIoUiMfG0wl07A,686
 deeplotx/similarity/vector.py,sha256=WVbDHqykt-fvuILVrhUCtIFAOEjY_zvttrXGM9eylG0,1125
 deeplotx/util/__init__.py,sha256=d1qelOGVTLSsHp1R_gsP_FSMAtAxUxWMwiPrTS58RSg,66
-deeplotx-0.9.11.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.9.11.dist-info/METADATA,sha256=yw4xLxeBE-LP-6QO9WvG2B5npZN0qvy0QmtGHrwxdJw,14444
-deeplotx-0.9.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deeplotx-0.9.11.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.9.11.dist-info/RECORD,,
+deeplotx-0.9.13.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.9.13.dist-info/METADATA,sha256=_pdwcpo6iRSfeaD8VFxg4tIjfAxOXyeOFtRZRWQVLNw,14444
+deeplotx-0.9.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deeplotx-0.9.13.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.9.13.dist-info/RECORD,,