deeplotx 0.9.12__py3-none-any.whl → 0.9.15__py3-none-any.whl

deeplotx/__init__.py

@@ -15,6 +15,7 @@ from .nn import (
     RecursiveSequential,
     LongContextRecursiveSequential,
     RoPE,
+    LoRA,
     Attention,
     MultiHeadAttention,
     RoFormerEncoder,

deeplotx/nn/__init__.py

@@ -7,6 +7,7 @@ from .softmax_regression import SoftmaxRegression
 from .recursive_sequential import RecursiveSequential
 from .long_context_recursive_sequential import LongContextRecursiveSequential
 from .rope import RoPE
+from .lora import LoRA
 from .attention import Attention
 from .multi_head_attention import MultiHeadAttention
 from .roformer_encoder import RoFormerEncoder

deeplotx/nn/base_neural_network.py

@@ -10,13 +10,15 @@ DEFAULT_SUFFIX = 'dlx'
 
 class BaseNeuralNetwork(nn.Module):
     def __init__(self, in_features: int, out_features: int, model_name: str | None = None,
-                 device: str | None = None, dtype: torch.dtype | None = None):
+                 device: str | torch.device | None = None, dtype: torch.dtype | None = None):
         super().__init__()
         self._model_name = model_name \
             if model_name is not None \
             else self.__class__.__name__
-        self.device = torch.device(device) if device is not None \
-            else torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.device = device if isinstance(device, torch.device) else None
+        if self.device is None:
+            self.device = torch.device(device) if device is not None \
+                else torch.device('cuda' if torch.cuda.is_available() else 'cpu')
         self.dtype = dtype if dtype is not None else torch.float32
         self._in_features = in_features
         self._out_features = out_features

deeplotx/nn/lora.py

@@ -0,0 +1,56 @@
+from typing_extensions import override
+
+import torch
+from torch import nn
+
+from deeplotx.nn.base_neural_network import BaseNeuralNetwork
+
+
+class LoRA(BaseNeuralNetwork):
+    def __init__(self, input_dim: int, output_dim: int, rank: int = 8, alpha: int = 16,
+                 dropout_rate: float = .0, model_name: str | None = None, device: str | torch.device | None = None,
+                 dtype: torch.dtype | None = None):
+        super().__init__(in_features=input_dim, out_features=output_dim, model_name=model_name,
+                         device=device, dtype=dtype)
+        self._rank = rank
+        self._alpha = alpha
+        self._scaling = self._alpha / self._rank
+        self._dropout = nn.Dropout(p=dropout_rate) if dropout_rate > .0 else nn.Identity()
+        self.lora_A = nn.Linear(in_features=input_dim, out_features=rank, bias=False,
+                                device=self.device, dtype=self.dtype)
+        self.lora_B = nn.Linear(in_features=rank, out_features=output_dim, bias=False,
+                                device=self.device, dtype=self.dtype)
+        nn.init.normal_(self.lora_A.weight, mean=.0, std=.01)
+        nn.init.zeros_(self.lora_B.weight)
+        self.w0 = None
+
+    @override
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if not isinstance(self.w0, nn.Module):
+            raise ValueError('LoRA adapter was not mounted successfully.')
+        original_out = self.w0(x)
+        lora_out = self.lora_B(self._dropout(self.lora_A(x))) * self._scaling
+        return original_out + lora_out
+
+    @staticmethod
+    def apply_to(model: nn.Module, target_modules: list[str] | str, rank: int = 8, alpha: int = 16,
+                 dropout_rate: float = .0) -> nn.Module:
+        if isinstance(target_modules, str):
+            target_modules = [target_modules]
+        for layer_name, module in model.named_modules():
+            if any(_name in layer_name.split('.')[-1] for _name in target_modules):
+                lora = LoRA(input_dim=module.in_features, output_dim=module.out_features,
+                            rank=rank, alpha=alpha, dropout_rate=dropout_rate,
+                            device=next(module.parameters()).device,
+                            dtype=next(module.parameters()).dtype)
+                lora.w0 = module
+                parent_name = layer_name.rsplit('.', 1)[0] if '.' in layer_name else ''
+                child_name = layer_name.split('.')[-1]
+                parent_module = dict(model.named_modules())[parent_name] if parent_name else model
+                setattr(parent_module, child_name, lora)
+        for param in model.parameters():
+            param.requires_grad = False
+        for name, param in model.named_parameters():
+            if 'lora_A.weight' in name or 'lora_B.weight' in name:
+                param.requires_grad = True
+        return model

{deeplotx-0.9.12.dist-info → deeplotx-0.9.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.12
+Version: 0.9.15
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -15,7 +15,7 @@ Requires-Dist: tiktoken
 Requires-Dist: torch
 Requires-Dist: transformers
 Requires-Dist: typing-extensions
-Requires-Dist: vortezwohl>=0.0.10
+Requires-Dist: vortezwohl>=0.0.17
 Requires-Dist: name4py>=0.1.4
 Dynamic: license-file
 
@@ -157,6 +157,21 @@ year = {2025}
     (<Gender.Male: 'male'>, 1.0)
     ```
 
+- ### Apply LoRA to a model
+
+    Import dependencies
+
+    ```python
+    from deeplotx import LoRA
+    ```
+
+    Assumed that the `model` has been loaded
+
+    ```python
+    model = ...  # Maybe an LLM or some other deep neural network models
+    lora_model = LoRA.apply_to(model, target_modules=['q_proj'], rank=16, alpha=32, dropout_rate=.05)
+    ```
+
 - ### Long text embedding
 
     - **BERT based long text embedding**
@@ -273,6 +288,7 @@ year = {2025}
         RecursiveSequential, 
         LongContextRecursiveSequential, 
         RoPE, 
+        LoRA, 
         Attention, 
         MultiHeadAttention, 
         RoFormerEncoder, 
@@ -391,35 +407,3 @@ year = {2025}
             v = self.v_proj(y)
             return torch.matmul(self._attention(x, y, mask), v)
     ```
-
-- ### Text binary classification task with predefined trainer
-
-    ```python
-    from deeplotx import TextBinaryClassifierTrainer, LongTextEncoder
-    from deeplotx.util import get_files, read_file
-
-    long_text_encoder = LongTextEncoder(
-        max_length=2048, 
-        chunk_size=448, 
-        overlapping=32, 
-        cache_capacity=512 
-    )
-    trainer = TextBinaryClassifierTrainer(
-        long_text_encoder=long_text_encoder,
-        batch_size=2,
-        train_ratio=0.9 
-    )
-    pos_data_path = 'path/to/pos_dir'
-    neg_data_path = 'path/to/neg_dir'
-    pos_data = [read_file(x) for x in get_files(pos_data_path)]
-    neg_data = [read_file(x) for x in get_files(neg_data_path)]
-    model = trainer.train(pos_data, neg_data, 
-                        num_epochs=36, learning_rate=2e-5, 
-                        balancing_dataset=True, alpha=1e-4, 
-                        rho=.2, encoder_layers=2, 
-                        attn_heads=8, 
-                        recursive_layers=2) 
-    model.save(model_name='test_model', model_dir='model')
-    model = model.load(model_name='test_model', model_dir='model')
-    model.predict(long_text_encoder.encode('这是一个测试文本.', flatten=False))
-    ```

{deeplotx-0.9.12.dist-info → deeplotx-0.9.15.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deeplotx/__init__.py,sha256=0OWLsgXlStzwm0m9ScaoZvBnsx3a0xTmlzYBUgarl-g,1306
+deeplotx/__init__.py,sha256=9vGW9dz8aY9GMR0ctU0mBXvzhZR4a7P0CUG8fB1YQRc,1317
 deeplotx/encoder/__init__.py,sha256=BrsF5_4O-4pfihYF2wjExDOoAY-03kGJTH-Mhez4tsE,129
 deeplotx/encoder/encoder.py,sha256=uJswUSrYVDWP84HeCD40R9KgXGPUEa080konv7jEp8I,3539
 deeplotx/encoder/long_text_encoder.py,sha256=eabhgTMhJrAvRC7YyXAAR_LRv9-LULzo8S0uybWTcwM,3961
@@ -8,15 +8,16 @@ deeplotx/ner/base_ner.py,sha256=pZTl50OrHH_FJm4rKp9iuixeOE6FX_AzgDXD32aXsN0,204
 deeplotx/ner/bert_ner.py,sha256=tfbM3CQBEpZsD0KYVA7GVNJax-7kzNOQgwlpo2S8h-c,8986
 deeplotx/ner/named_entity.py,sha256=c6XufIwH6yloJ-ccUjagf4mBl1XbbYDT8xyEJJ_-ZNs,269
 deeplotx/ner/n2g/__init__.py,sha256=L1IJ8W1nApzqHx2u7JMtPCLfABm5qKJvh_bHMWdvdLY,3538
-deeplotx/nn/__init__.py,sha256=YILwbxb-NHdiJjfOwBKH8F7PuZSDZSrGpTznPDucTro,710
+deeplotx/nn/__init__.py,sha256=9v-cftkJe7MSRkqfvBwu1kZx4nEosT0d36xi3KQtzXw,734
 deeplotx/nn/attention.py,sha256=R-i-Rd7gnsh6hwXDeYfqLQOJvfSZIGfQbFzRlC91XLo,2879
 deeplotx/nn/auto_regression.py,sha256=j_R7WGPq9REngjpLuX5c0AaNqOpgGm2Vfrolw-XjWXw,877
-deeplotx/nn/base_neural_network.py,sha256=QCyB1dxOs4I8vpu6PCshrZs0infoHXS9IErw6tN-dhs,6060
+deeplotx/nn/base_neural_network.py,sha256=w98m1qOSw4C9zFu0uI4k3z8zeXrn5NiAZruyCe2ZQjs,6192
 deeplotx/nn/feed_forward.py,sha256=kGWEUo8J7jrhSSWlitNnj-AcitNiLz6eOCvUcEuWlVs,2949
 deeplotx/nn/linear_regression.py,sha256=LWrrdAIw32KIT1bdr7q6HczdpEiCgb-R8BCNXGywMxE,1763
 deeplotx/nn/logistic_regression.py,sha256=nipWD3ZPRub2Cx0rU2zxYQyG0COn3NJvew8b2gbJy24,998
 deeplotx/nn/long_context_auto_regression.py,sha256=uy0k_g8wEfMH5nd5HCfrHA8dgEsuWBA2x8U-g3h4vQc,1054
 deeplotx/nn/long_context_recursive_sequential.py,sha256=pcZfnrIHBqbp2BssfUTS1klpuykZwowikfAIaOnvRUI,2674
+deeplotx/nn/lora.py,sha256=oy3HltuDily4NLSbUTNvS0i4Kq2kiwl92rBsIt8UOIA,2919
 deeplotx/nn/multi_head_attention.py,sha256=3z73uGbvy3jszRy1B9nxGOJjlttHpcpRF8Qd09OEams,2267
 deeplotx/nn/multi_head_feed_forward.py,sha256=hD9ScrVJZ9kNksoFASf0xaPgEnNgCeRivW-XjYOPjj8,1908
 deeplotx/nn/recursive_sequential.py,sha256=sNvAs9iVCuWIgx0_6TizDq41hJpFbfKT3kyDHE86wRM,2928
@@ -28,8 +29,8 @@ deeplotx/similarity/distribution.py,sha256=wQGouuuW531pZeBRKBujXsdsoz4fDnPw7_GW8
 deeplotx/similarity/set.py,sha256=zhGFxtSIXlWqvipBYzoiPahp4g0boAIoUiMfG0wl07A,686
 deeplotx/similarity/vector.py,sha256=WVbDHqykt-fvuILVrhUCtIFAOEjY_zvttrXGM9eylG0,1125
 deeplotx/util/__init__.py,sha256=d1qelOGVTLSsHp1R_gsP_FSMAtAxUxWMwiPrTS58RSg,66
-deeplotx-0.9.12.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.9.12.dist-info/METADATA,sha256=dDoMn1gtn1dTfc7KhfGmyuZ1i_YycYVmh0QxnRh5g5I,14444
-deeplotx-0.9.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deeplotx-0.9.12.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.9.12.dist-info/RECORD,,
+deeplotx-0.9.15.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.9.15.dist-info/METADATA,sha256=8y2RddiUJGYpRITudVOnD4KnynqJNKfIEKYR6qietWI,13573
+deeplotx-0.9.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deeplotx-0.9.15.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.9.15.dist-info/RECORD,,