deeplotx 0.4.15__py3-none-any.whl → 0.5.1__py3-none-any.whl

deeplotx/__init__.py

@@ -9,7 +9,10 @@ from .nn import (
     LogisticRegression,
     SoftmaxRegression,
     RecursiveSequential,
-    AutoRegression
+    LongContextRecursiveSequential,
+    SelfAttention,
+    AutoRegression,
+    LongContextAutoRegression
 )
 from .trainer import TextBinaryClassifierTrainer
 

deeplotx/nn/__init__.py

@@ -2,4 +2,7 @@ from .linear_regression import LinearRegression
 from .logistic_regression import LogisticRegression
 from .softmax_regression import SoftmaxRegression
 from .recursive_sequential import RecursiveSequential
+from .long_context_recursive_sequential import LongContextRecursiveSequential
+from .self_attention import SelfAttention
 from .auto_regression import AutoRegression
+from .long_context_auto_regression import LongContextAutoRegression

deeplotx/nn/base_neural_network.py

@@ -1,8 +1,11 @@
+import os
 from abc import abstractmethod
 
 import torch
 from torch import nn
 
+DEFAULT_SUFFIX = 'dlx'
+
 
 class BaseNeuralNetwork(nn.Module):
     def __init__(self, model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
@@ -44,7 +47,7 @@ class BaseNeuralNetwork(nn.Module):
     @abstractmethod
     def forward(self, *args, **kwargs) -> torch.Tensor: ...
 
-    def predict(self, x) -> torch.Tensor:
+    def predict(self, x: torch.Tensor) -> torch.Tensor:
         x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
         __train = self.training
         self.training = False
@@ -53,10 +56,13 @@ class BaseNeuralNetwork(nn.Module):
         self.training = __train
         return res
 
-    def save(self):
-        torch.save(self.state_dict(), f'{self._model_name}.deeplotx')
+    def save(self, model_name: str | None = None, model_dir: str = '.', _suffix: str = DEFAULT_SUFFIX):
+        os.makedirs(model_dir, exist_ok=True)
+        model_file_name = f'{model_name}.{_suffix}' if model_name is not None else f'{self._model_name}.{_suffix}'
+        torch.save(self.state_dict(), os.path.join(model_dir, model_file_name))
         return self
 
-    def load(self):
-        self.load_state_dict(torch.load(f'{self._model_name}.deeplotx', map_location=self.device, weights_only=True))
+    def load(self, model_name: str | None = None, model_dir: str = '.', _suffix: str = DEFAULT_SUFFIX):
+        model_file_name = f'{model_name}.{_suffix}' if model_name is not None else f'{self._model_name}.{_suffix}'
+        self.load_state_dict(torch.load(os.path.join(model_dir, model_file_name), map_location=self.device, weights_only=True))
         return self

deeplotx/nn/linear_regression.py

@@ -22,7 +22,7 @@ class LinearRegression(BaseNeuralNetwork):
         self.parametric_relu_4 = nn.PReLU(num_parameters=1, init=5e-3, device=self.device, dtype=self.dtype)
 
     @override
-    def forward(self, x) -> torch.Tensor:
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
         x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
         fc1_out = self.parametric_relu_1(self.fc1(x))
         x = nn.LayerNorm(normalized_shape=1024, eps=1e-9, device=self.device, dtype=self.dtype)(fc1_out)

deeplotx/nn/logistic_regression.py

@@ -11,6 +11,6 @@ class LogisticRegression(LinearRegression):
         super().__init__(input_dim=input_dim, output_dim=output_dim, model_name=model_name, device=device, dtype=dtype)
 
     @override
-    def forward(self, x) -> torch.Tensor:
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
         x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
         return torch.sigmoid(super().forward(x))

deeplotx/nn/long_context_auto_regression.py

@@ -0,0 +1,12 @@
+import torch
+
+from deeplotx.nn import LongContextRecursiveSequential
+
+
+class LongContextAutoRegression(LongContextRecursiveSequential):
+    def __init__(self, feature_dim: int, hidden_dim: int | None = None,
+                 recursive_layers: int = 2, model_name: str | None = None,
+                 device: str | None = None, dtype: torch.dtype | None = None):
+        super().__init__(input_dim=feature_dim, output_dim=feature_dim,
+                         hidden_dim=hidden_dim, recursive_layers=recursive_layers,
+                         model_name=model_name, device=device, dtype=dtype)

deeplotx/nn/long_context_recursive_sequential.py

@@ -0,0 +1,28 @@
+from typing_extensions import override
+
+import torch
+from torch import nn
+
+from deeplotx.nn.recursive_sequential import RecursiveSequential
+from deeplotx.nn.self_attention import SelfAttention
+
+
+class LongContextRecursiveSequential(RecursiveSequential):
+    def __init__(self, input_dim: int, output_dim: int,
+                 hidden_dim: int | None = None, recursive_layers: int = 2,
+                 model_name: str | None = None, device: str | None = None,
+                 dtype: torch.dtype | None = None):
+        super().__init__(input_dim=input_dim, output_dim=output_dim,
+                         hidden_dim=hidden_dim, recursive_layers=recursive_layers,
+                         model_name=model_name, device=device, dtype=dtype)
+        self._feature_dim = input_dim
+        self.self_attention = SelfAttention(feature_dim=input_dim)
+        self.proj = nn.Linear(in_features=input_dim * 2, out_features=input_dim,
+                              bias=True, device=self.device, dtype=self.dtype)
+
+    @override
+    def forward(self, x: torch.Tensor, state: tuple[torch.Tensor, torch.Tensor]) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+        x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
+        x = torch.cat([self.self_attention(x), x], dim=-1)
+        x = nn.LayerNorm(normalized_shape=x.shape[-1], eps=1e-9, device=self.device, dtype=self.dtype)(x)
+        return super().forward(self.proj(x), state)

deeplotx/nn/recursive_sequential.py

@@ -27,7 +27,7 @@ class RecursiveSequential(BaseNeuralNetwork):
         return zeros, zeros
 
     @override
-    def forward(self, x, state: tuple[torch.Tensor, torch.Tensor]) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+    def forward(self, x: torch.Tensor, state: tuple[torch.Tensor, torch.Tensor]) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
         x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
         state = (self.ensure_device_and_dtype(state[0], device=self.device, dtype=self.dtype),
                  self.ensure_device_and_dtype(state[1], device=self.device, dtype=self.dtype))
@@ -36,7 +36,7 @@ class RecursiveSequential(BaseNeuralNetwork):
         return x, (hidden_state, cell_state)
 
     @override
-    def predict(self, x) -> torch.Tensor:
+    def predict(self, x: torch.Tensor) -> torch.Tensor:
         __train = self.training
         self.training = False
         with torch.no_grad():

deeplotx/nn/self_attention.py

@@ -0,0 +1,34 @@
+from typing_extensions import override
+
+import torch
+from torch import nn, softmax
+
+from deeplotx.nn.base_neural_network import BaseNeuralNetwork
+
+
+class SelfAttention(BaseNeuralNetwork):
+    def __init__(self, feature_dim: int, model_name: str | None = None,
+                 device: str | None = None, dtype: torch.dtype | None = None):
+        super().__init__(model_name=model_name, device=device, dtype=dtype)
+        self._feature_dim = feature_dim
+        self.q_proj = nn.Linear(in_features=self._feature_dim, out_features=self._feature_dim,
+                                bias=True, device=self.device, dtype=self.dtype)
+        self.k_proj = nn.Linear(in_features=self._feature_dim, out_features=self._feature_dim,
+                                bias=True, device=self.device, dtype=self.dtype)
+        self.v_proj = nn.Linear(in_features=self._feature_dim, out_features=self._feature_dim,
+                                bias=True, device=self.device, dtype=self.dtype)
+
+    def _attention(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
+        q, k = self.q_proj(x), self.k_proj(x)
+        attn = torch.matmul(q, k.transpose(-2, -1))
+        attn = attn / (self._feature_dim ** 0.5)
+        attn = attn.masked_fill(mask == 0, -1e9) if mask is not None else attn
+        return softmax(attn, dim=-1)
+
+    @override
+    def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
+        x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
+        if mask is not None:
+            mask = self.ensure_device_and_dtype(mask, device=self.device, dtype=self.dtype)
+        v = self.v_proj(x)
+        return torch.matmul(self._attention(x, mask), v)

deeplotx/nn/softmax_regression.py

@@ -11,6 +11,6 @@ class SoftmaxRegression(LinearRegression):
         super().__init__(input_dim=input_dim, output_dim=output_dim, model_name=model_name, device=device, dtype=dtype)
 
     @override
-    def forward(self, x) -> torch.Tensor:
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
         x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
         return torch.softmax(super().forward(x), dim=-1, dtype=self.dtype)

deeplotx/trainer/text_binary_classification_trainer.py

@@ -6,7 +6,7 @@ from torch import nn, optim
 from torch.utils.data import DataLoader, TensorDataset
 
 from deeplotx.encoder.long_text_encoder import LongTextEncoder
-from deeplotx.nn.recursive_sequential import RecursiveSequential
+from deeplotx.nn.long_context_recursive_sequential import LongContextRecursiveSequential
 from deeplotx.trainer.base_trainer import BaseTrainer
 
 logger = logging.getLogger('deeplotx.trainer')
@@ -23,7 +23,7 @@ class TextBinaryClassifierTrainer(BaseTrainer):
               num_epochs: int, learning_rate: float = 2e-6, balancing_dataset: bool = True,
               train_loss_threshold: float = 0.0, valid_loss_threshold: float = 0.0,
               alpha: float = 1e-4, rho: float = 0.2,
-              hidden_dim: int = 256, recursive_layers: int = 2) -> RecursiveSequential:
+              hidden_dim: int = 256, recursive_layers: int = 2) -> LongContextRecursiveSequential:
         if balancing_dataset:
             min_length = min(len(positive_texts), len(negative_texts))
             positive_texts = positive_texts[:min_length]
@@ -46,10 +46,10 @@ class TextBinaryClassifierTrainer(BaseTrainer):
             logger.warning("The dimension of features doesn't match. A new model instance will be created.")
             self.model = None
         if self.model is None:
-            self.model = RecursiveSequential(input_dim=feature_dim, output_dim=1,
-                                             hidden_dim=hidden_dim,
-                                             recursive_layers=recursive_layers,
-                                             device=self.device, dtype=dtype)
+            self.model = LongContextRecursiveSequential(input_dim=feature_dim, output_dim=1,
+                                                        hidden_dim=hidden_dim,
+                                                        recursive_layers=recursive_layers,
+                                                        device=self.device, dtype=dtype)
         loss_function = nn.BCELoss()
         optimizer = optim.Adamax(self.model.parameters(), lr=learning_rate)
         for epoch in range(num_epochs):
@@ -76,7 +76,8 @@ class TextBinaryClassifierTrainer(BaseTrainer):
                              f"Valid Loss: {total_valid_loss:.4f}")
                 if total_valid_loss < valid_loss_threshold:
                     break
-            logger.debug(f"Epoch {epoch + 1}/{num_epochs} | Train Loss: {total_loss:.4f}")
+            else:
+                logger.debug(f"Epoch {epoch + 1}/{num_epochs} | Train Loss: {total_loss:.4f}")
             if total_loss < train_loss_threshold:
                 break
         return self.model

{deeplotx-0.4.15.dist-info → deeplotx-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.4.15
+Version: 0.5.1
 Summary: Easy-2-use long text NLP toolkit.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

deeplotx-0.5.1.dist-info/RECORD

@@ -0,0 +1,30 @@
+deeplotx/__init__.py,sha256=3rVjGSRdcxpxZzHIQohT8dheB5mVdeXIrBkfH2yorcQ,1091
+deeplotx/encoder/__init__.py,sha256=EM-xrTsHoGaiiFpj-iFAxilMHXC_sQKWYrcq1qCnI3U,138
+deeplotx/encoder/bert_encoder.py,sha256=uLqGcXH6AGY6CcjjbYbh09VWYqSpsg-y-jHYB6Fmp3w,2377
+deeplotx/encoder/long_text_encoder.py,sha256=hl_O8kR9o1kcII9YfSx2rf_Pk0l_Rv7LNbsS9UsTU0c,3373
+deeplotx/encoder/longformer_encoder.py,sha256=A8FXqd4mdHxSn_o_R689XtpT73ISDT788EgMQRGLC2g,1822
+deeplotx/nn/__init__.py,sha256=CS0UwyYKa8wI6vu6FBIYxvm-HAmw39MTMFlZDtqi6UA,444
+deeplotx/nn/auto_regression.py,sha256=7P63opWCWMqE2DigwbsL6kfXtFtJPz00Yo1RqflBz4A,572
+deeplotx/nn/base_neural_network.py,sha256=o9s0NqxkDcFZdipX8UrlbBmwYHOg7wPmzbjBEeGw63s,2902
+deeplotx/nn/linear_regression.py,sha256=7TbbplBgY70b1l5lKvTJMzDWQ8khQfnRCyMjObhVdEc,2180
+deeplotx/nn/logistic_regression.py,sha256=YiSLAon8gLDtMXAkPQ210sauod24eyJYYH50fPhj6T8,667
+deeplotx/nn/long_context_auto_regression.py,sha256=Z67Enq1kc1bERIrQW4jHeDQQmisOXhhjrtaPklnHkyw,605
+deeplotx/nn/long_context_recursive_sequential.py,sha256=_fKpPA7wt6B0kPyyig4xuhmLxygK19FSLgxW1Xa453M,1487
+deeplotx/nn/recursive_sequential.py,sha256=8YHZ-IdLyMJN5QVWPMuizDxLodAE9Bgdg1_YtIxFw7o,2247
+deeplotx/nn/self_attention.py,sha256=fb34wXnfgAGYJEhqa1l9AxMa-AHcCTOLbUlAfaGIK7Q,1766
+deeplotx/nn/softmax_regression.py,sha256=BeVk0G2H3zKG6bsQgPRNWuTxnnNmVI2zFZtCHgARAAc,688
+deeplotx/similarity/__init__.py,sha256=s3u-KSgxjnMcWpIItKgXNltFMPQ7YY3CqsqHI-5F1c8,724
+deeplotx/similarity/distribution.py,sha256=wQGouuuW531pZeBRKBujXsdsoz4fDnPw7_GW81jwepc,1066
+deeplotx/similarity/set.py,sha256=zhGFxtSIXlWqvipBYzoiPahp4g0boAIoUiMfG0wl07A,686
+deeplotx/similarity/vector.py,sha256=WVbDHqykt-fvuILVrhUCtIFAOEjY_zvttrXGM9eylG0,1125
+deeplotx/trainer/__init__.py,sha256=Fl5DR9UecQc5VtBcczU9sx_HtPNoFohpuELOh-Jrsks,77
+deeplotx/trainer/base_trainer.py,sha256=z0MeAT-rRYmjeBXt0ckt7J1itYArR0Cx02wHesXUoZE,385
+deeplotx/trainer/text_binary_classification_trainer.py,sha256=Ktdk4rCNHgTFdXVFmbTnvIlGIJi1gphGRkuRgL2bVOo,4793
+deeplotx/util/__init__.py,sha256=JxqAK_WOOHcYVSTHBT1-WuBwWrPEVDTV3titeVWvNUM,74
+deeplotx/util/hash.py,sha256=wwsC6kOQvbpuvwKsNQOARd78_wePmW9i3oaUuXRUnpc,352
+deeplotx/util/read_file.py,sha256=ptzouvEQeeW8KU5BrWNJlXw-vFXVrpS9SkAUxsu6A8A,612
+deeplotx-0.5.1.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.5.1.dist-info/METADATA,sha256=LatUJZ1YzKrlPMDNI2UiOqSf5h9mP57kf4f5ngnfa8Q,6954
+deeplotx-0.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deeplotx-0.5.1.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.5.1.dist-info/RECORD,,

deeplotx-0.4.15.dist-info/RECORD

@@ -1,27 +0,0 @@
-deeplotx/__init__.py,sha256=wMN_AI14V-0BPbQghYpvd2y7eUGfhr7jKTTuur-5Upg,1002
-deeplotx/encoder/__init__.py,sha256=EM-xrTsHoGaiiFpj-iFAxilMHXC_sQKWYrcq1qCnI3U,138
-deeplotx/encoder/bert_encoder.py,sha256=uLqGcXH6AGY6CcjjbYbh09VWYqSpsg-y-jHYB6Fmp3w,2377
-deeplotx/encoder/long_text_encoder.py,sha256=hl_O8kR9o1kcII9YfSx2rf_Pk0l_Rv7LNbsS9UsTU0c,3373
-deeplotx/encoder/longformer_encoder.py,sha256=A8FXqd4mdHxSn_o_R689XtpT73ISDT788EgMQRGLC2g,1822
-deeplotx/nn/__init__.py,sha256=oQ-vYXyuaGelfCOs2im_gZXAiiBlCCVXh1uw9yjvRMs,253
-deeplotx/nn/auto_regression.py,sha256=7P63opWCWMqE2DigwbsL6kfXtFtJPz00Yo1RqflBz4A,572
-deeplotx/nn/base_neural_network.py,sha256=oGlqY6ZZ3DGOXWQQ9nZ7ktJpfyIaUrSraGnFRbGD1jM,2384
-deeplotx/nn/linear_regression.py,sha256=_LQFrOKBbQxvuNzb_B8Mr6PAQJUg-pFeu3h7_jQz04o,2166
-deeplotx/nn/logistic_regression.py,sha256=j8QGe0e7In97RMOXApJRID85qf1rOUCOk3V368CBfqs,653
-deeplotx/nn/recursive_sequential.py,sha256=pHZChjzw9cuMQ0lmv42lxxVgxCU6D7owOgph0Irj-w4,2219
-deeplotx/nn/softmax_regression.py,sha256=SlhvHho-Oufp7adAjm1t1ygidu-FrnHQ9aleMXyS_s8,674
-deeplotx/similarity/__init__.py,sha256=s3u-KSgxjnMcWpIItKgXNltFMPQ7YY3CqsqHI-5F1c8,724
-deeplotx/similarity/distribution.py,sha256=wQGouuuW531pZeBRKBujXsdsoz4fDnPw7_GW81jwepc,1066
-deeplotx/similarity/set.py,sha256=zhGFxtSIXlWqvipBYzoiPahp4g0boAIoUiMfG0wl07A,686
-deeplotx/similarity/vector.py,sha256=WVbDHqykt-fvuILVrhUCtIFAOEjY_zvttrXGM9eylG0,1125
-deeplotx/trainer/__init__.py,sha256=Fl5DR9UecQc5VtBcczU9sx_HtPNoFohpuELOh-Jrsks,77
-deeplotx/trainer/base_trainer.py,sha256=z0MeAT-rRYmjeBXt0ckt7J1itYArR0Cx02wHesXUoZE,385
-deeplotx/trainer/text_binary_classification_trainer.py,sha256=Wq_pGO78zgdXxFeBjam4yp__-dTvsuwl4H81HSl_kjE,4691
-deeplotx/util/__init__.py,sha256=JxqAK_WOOHcYVSTHBT1-WuBwWrPEVDTV3titeVWvNUM,74
-deeplotx/util/hash.py,sha256=wwsC6kOQvbpuvwKsNQOARd78_wePmW9i3oaUuXRUnpc,352
-deeplotx/util/read_file.py,sha256=ptzouvEQeeW8KU5BrWNJlXw-vFXVrpS9SkAUxsu6A8A,612
-deeplotx-0.4.15.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.4.15.dist-info/METADATA,sha256=HB6VHdLgyuMclJYLykBMnbnLa7s-rwfHyhrgjNdoRFQ,6955
-deeplotx-0.4.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deeplotx-0.4.15.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.4.15.dist-info/RECORD,,