deeplotx 0.8.3__tar.gz → 0.8.5__tar.gz

{deeplotx-0.8.3 → deeplotx-0.8.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.8.3
+Version: 0.8.5
 Summary: Easy-2-use long text NLP toolkit.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -168,6 +168,7 @@ Dynamic: license-file
     ```python
     from deeplotx import (
         FeedForward,  # 前馈神经网络
+        MultiHeadFeedForward,  # 多头前馈神经网络
         LinearRegression,  # 线性回归
         LogisticRegression,  # 逻辑回归 / 二分类 / 多标签分类
         SoftmaxRegression,  # Softmax 回归 / 多分类

{deeplotx-0.8.3 → deeplotx-0.8.5}/README.md

@@ -150,6 +150,7 @@
     ```python
     from deeplotx import (
         FeedForward,  # 前馈神经网络
+        MultiHeadFeedForward,  # 多头前馈神经网络
         LinearRegression,  # 线性回归
         LogisticRegression,  # 逻辑回归 / 二分类 / 多标签分类
         SoftmaxRegression,  # Softmax 回归 / 多分类

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/__init__.py

@@ -6,6 +6,7 @@ __ROOT__ = os.path.dirname(os.path.abspath(__file__))
 from .encoder import Encoder, LongTextEncoder, LongformerEncoder
 from .nn import (
     FeedForward,
+    MultiHeadFeedForward,
     LinearRegression,
     LogisticRegression,
     SoftmaxRegression,

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/nn/__init__.py

@@ -1,5 +1,6 @@
 from .base_neural_network import BaseNeuralNetwork
 from .feed_forward import FeedForward
+from .multi_head_feed_forward import MultiHeadFeedForward
 from .linear_regression import LinearRegression
 from .logistic_regression import LogisticRegression
 from .softmax_regression import SoftmaxRegression

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/nn/auto_regression.py

@@ -7,8 +7,8 @@ class AutoRegression(RecursiveSequential):
     def __init__(self, feature_dim: int, bias: bool = True,
                  recursive_layers: int = 1, recursive_hidden_dim: int | None = None,
                  ffn_layers: int = 1, ffn_expansion_factor: int | float = 2, dropout_rate: float = 0.05,
-                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
+                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None, **kwargs):
         super().__init__(input_dim=feature_dim, output_dim=feature_dim, bias=bias,
                          recursive_layers=recursive_layers, recursive_hidden_dim=recursive_hidden_dim,
                          ffn_layers=ffn_layers, ffn_expansion_factor=ffn_expansion_factor,
-                         dropout_rate=dropout_rate, model_name=model_name, device=device, dtype=dtype)
+                         dropout_rate=dropout_rate, model_name=model_name, device=device, dtype=dtype, **kwargs)

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/nn/linear_regression.py

@@ -4,16 +4,17 @@ import torch
 from torch import nn
 
 from deeplotx.nn.base_neural_network import BaseNeuralNetwork
-from deeplotx.nn.feed_forward import FeedForward
+from deeplotx.nn.multi_head_feed_forward import MultiHeadFeedForward
 
 
 class LinearRegression(BaseNeuralNetwork):
-    def __init__(self, input_dim: int, output_dim: int, num_layers: int = 1,
+    def __init__(self, input_dim: int, output_dim: int, num_heads: int = 1, num_layers: int = 1,
                  expansion_factor: int | float = 1.5, bias: bool = True, dropout_rate: float = 0.1,
                  model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
         super().__init__(in_features=input_dim, out_features=output_dim, model_name=model_name, device=device, dtype=dtype)
-        self.ffn = FeedForward(feature_dim=input_dim, num_layers=num_layers, expansion_factor=expansion_factor,
-                               bias=bias, dropout_rate=dropout_rate, device=self.device, dtype=self.dtype)
+        self.ffn = MultiHeadFeedForward(feature_dim=input_dim, num_heads=num_heads,
+                                        num_layers=num_layers, expansion_factor=expansion_factor,
+                                        bias=bias, dropout_rate=dropout_rate, device=self.device, dtype=self.dtype)
         self.proj = nn.Linear(in_features=input_dim, out_features=output_dim,
                               bias=bias, device=self.device, dtype=self.dtype)
 

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/nn/logistic_regression.py

@@ -6,10 +6,10 @@ from deeplotx.nn.linear_regression import LinearRegression
 
 
 class LogisticRegression(LinearRegression):
-    def __init__(self, input_dim: int, output_dim: int = 1, num_layers: int = 1, expansion_factor: int | float = 1.5,
-                 bias: bool = True, dropout_rate: float = 0.1, model_name: str | None = None,
-                 device: str | None = None, dtype: torch.dtype | None = None):
-        super().__init__(input_dim=input_dim, output_dim=output_dim, num_layers=num_layers,
+    def __init__(self, input_dim: int, output_dim: int = 1, num_heads: int = 1, num_layers: int = 1,
+                 expansion_factor: int | float = 1.5, bias: bool = True, dropout_rate: float = 0.1,
+                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
+        super().__init__(input_dim=input_dim, output_dim=output_dim, num_heads=num_heads, num_layers=num_layers,
                          expansion_factor=expansion_factor, bias=bias, dropout_rate=dropout_rate,
                          model_name=model_name, device=device, dtype=dtype)
 

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/nn/long_context_recursive_sequential.py

@@ -12,12 +12,11 @@ class LongContextRecursiveSequential(RecursiveSequential):
     def __init__(self, input_dim: int, output_dim: int, bias: bool = True,
                  encoder_layers: int = 1, attn_heads: int = 1, recursive_layers: int = 2, recursive_hidden_dim: int | None = None,
                  ffn_layers: int = 1, ffn_expansion_factor: int | float = 2, dropout_rate: float = 0.05,
-                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None,
-                 **kwargs):
+                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None, **kwargs):
         super().__init__(input_dim=input_dim, output_dim=output_dim, bias=bias,
                          recursive_layers=recursive_layers, recursive_hidden_dim=recursive_hidden_dim,
                          ffn_layers=ffn_layers, ffn_expansion_factor=ffn_expansion_factor, dropout_rate=dropout_rate,
-                         model_name=model_name, device=device, dtype=dtype)
+                         model_name=model_name, device=device, dtype=dtype, **kwargs)
         self.roformer_encoders = nn.ModuleList([RoFormerEncoder(feature_dim=input_dim, attn_heads=attn_heads, bias=bias,
                                                                 ffn_layers=kwargs.get('encoder_ffn_layers', ffn_layers),
                                                                 ffn_expansion_factor=kwargs.get('encoder_expansion_factor', ffn_expansion_factor),

deeplotx-0.8.5/deeplotx/nn/multi_head_feed_forward.py

@@ -0,0 +1,32 @@
+from typing_extensions import override
+
+import torch
+from torch import nn
+
+from deeplotx.nn.base_neural_network import BaseNeuralNetwork
+from deeplotx.nn.feed_forward import FeedForward
+
+
+class MultiHeadFeedForward(BaseNeuralNetwork):
+    def __init__(self, feature_dim: int, num_heads: int = 1, num_layers: int = 1, expansion_factor: int | float = 2,
+                 bias: bool = True, dropout_rate: float = 0.05, model_name: str | None = None,
+                 device: str | None = None, dtype: torch.dtype | None = None):
+        super().__init__(in_features=feature_dim, out_features=feature_dim, model_name=model_name,
+                         device=device, dtype=dtype)
+        self._num_heads = num_heads
+        self.expand_proj = nn.Linear(in_features=feature_dim, out_features=feature_dim * self._num_heads, bias=bias,
+                                     device=self.device, dtype=self.dtype)
+        self.ffn_heads = nn.ModuleList([FeedForward(feature_dim=feature_dim, num_layers=num_layers,
+                                                    expansion_factor=expansion_factor, bias=bias,
+                                                    dropout_rate=dropout_rate, device=self.device,
+                                                    dtype=self.dtype) for _ in range(self._num_heads)])
+        self.out_proj = nn.Linear(in_features=feature_dim * self._num_heads, out_features=feature_dim, bias=bias,
+                                  device=self.device, dtype=self.dtype)
+
+    @override
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
+        x = self.expand_proj(x)
+        x_heads = x.split(self.in_features, dim=-1)
+        head_outs = [self.ffn_heads[_](x_heads[_]) for _ in range(self._num_heads)]
+        return self.out_proj(torch.concat(head_outs, dim=-1))

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/nn/recursive_sequential.py

@@ -4,14 +4,14 @@ import torch
 from torch import nn
 
 from deeplotx.nn.base_neural_network import BaseNeuralNetwork
-from deeplotx.nn.feed_forward import FeedForward
+from deeplotx.nn.multi_head_feed_forward import MultiHeadFeedForward
 
 
 class RecursiveSequential(BaseNeuralNetwork):
     def __init__(self, input_dim: int, output_dim: int, bias: bool = True,
                  recursive_layers: int = 1, recursive_hidden_dim: int | None = None,
                  ffn_layers: int = 1, ffn_expansion_factor: int | float = 2, dropout_rate: float = 0.05,
-                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
+                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None, **kwargs):
         super().__init__(in_features=input_dim, out_features=output_dim, model_name=model_name,
                          device=device, dtype=dtype)
         if recursive_hidden_dim is None:
@@ -20,9 +20,9 @@ class RecursiveSequential(BaseNeuralNetwork):
                             num_layers=recursive_layers, batch_first=True,
                             bias=True, bidirectional=True, device=self.device,
                             dtype=self.dtype)
-        self.ffn = FeedForward(feature_dim=recursive_hidden_dim * 2, num_layers=ffn_layers,
-                               expansion_factor=ffn_expansion_factor, bias=bias, dropout_rate=dropout_rate,
-                               device=self.device, dtype=self.dtype)
+        self.ffn = MultiHeadFeedForward(feature_dim=recursive_hidden_dim * 2, num_heads=kwargs.get('ffn_heads', 1),
+                                        num_layers=ffn_layers, expansion_factor=ffn_expansion_factor,
+                                        bias=bias, dropout_rate=dropout_rate, device=self.device, dtype=self.dtype)
         self.__proj = nn.Linear(in_features=recursive_hidden_dim * 2, out_features=output_dim, bias=bias,
                                 device=self.device, dtype=self.dtype)
 

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/nn/softmax_regression.py

@@ -6,10 +6,10 @@ from deeplotx.nn.linear_regression import LinearRegression
 
 
 class SoftmaxRegression(LinearRegression):
-    def __init__(self, input_dim: int, output_dim: int, num_layers: int = 1, expansion_factor: int | float = 1.5,
-                 bias: bool = True, dropout_rate: float = 0.1, model_name: str | None = None,
-                 device: str | None = None, dtype: torch.dtype | None = None):
-        super().__init__(input_dim=input_dim, output_dim=output_dim, num_layers=num_layers,
+    def __init__(self, input_dim: int, output_dim: int, num_heads: int = 1, num_layers: int = 1,
+                 expansion_factor: int | float = 1.5, bias: bool = True, dropout_rate: float = 0.1,
+                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
+        super().__init__(input_dim=input_dim, output_dim=output_dim, num_heads=num_heads, num_layers=num_layers,
                          expansion_factor=expansion_factor, bias=bias, dropout_rate=dropout_rate,
                          model_name=model_name, device=device, dtype=dtype)
 

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx/trainer/text_binary_classification_trainer.py

@@ -49,6 +49,7 @@ class TextBinaryClassifierTrainer(BaseTrainer):
             logger.warning("The dimension of features doesn't match. A new model instance will be created.")
             self.model = None
         if self.model is None:
+            ffn_heads = kwargs.get('ffn_heads', 2)
             ffn_layers = kwargs.get('ffn_layers', 5)
             ffn_expansion_factor = kwargs.get('ffn_expansion_factor', 2)
             bias = kwargs.get('bias', True)
@@ -63,11 +64,11 @@ class TextBinaryClassifierTrainer(BaseTrainer):
             self.model = LongContextRecursiveSequential(input_dim=feature_dim, output_dim=1, bias=bias,
                                                         encoder_layers=encoder_layers, attn_heads=attn_heads,
                                                         recursive_layers=recursive_layers, recursive_hidden_dim=recursive_hidden_dim,
-                                                        ffn_layers=ffn_layers, ffn_expansion_factor=ffn_expansion_factor, dropout_rate=dropout_rate,
-                                                        encoder_ffn_layers=encoder_ffn_layers, encoder_expansion_factor=encoder_expansion_factor,
-                                                        encoder_dropout_rate=encoder_dropout_rate, attn_ffn_layers=attn_ffn_layers,
-                                                        attn_expansion_factor=attn_expansion_factor, attn_dropout_rate=attn_dropout_rate,
-                                                        theta=theta).initialize_weights()
+                                                        ffn_layers=ffn_layers, ffn_heads=ffn_heads, ffn_expansion_factor=ffn_expansion_factor,
+                                                        dropout_rate=dropout_rate, encoder_ffn_layers=encoder_ffn_layers,
+                                                        encoder_expansion_factor=encoder_expansion_factor, encoder_dropout_rate=encoder_dropout_rate,
+                                                        attn_ffn_layers=attn_ffn_layers, attn_expansion_factor=attn_expansion_factor,
+                                                        attn_dropout_rate=attn_dropout_rate, theta=theta).initialize_weights()
         logger.debug(f'Training Model: \n{self.model}')
         loss_function = nn.BCELoss()
         optimizer = optim.Adamax(self.model.parameters(), lr=learning_rate)

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.8.3
+Version: 0.8.5
 Summary: Easy-2-use long text NLP toolkit.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -168,6 +168,7 @@ Dynamic: license-file
     ```python
     from deeplotx import (
         FeedForward,  # 前馈神经网络
+        MultiHeadFeedForward,  # 多头前馈神经网络
         LinearRegression,  # 线性回归
         LogisticRegression,  # 逻辑回归 / 二分类 / 多标签分类
         SoftmaxRegression,  # Softmax 回归 / 多分类

{deeplotx-0.8.3 → deeplotx-0.8.5}/deeplotx.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ deeplotx/nn/logistic_regression.py
 deeplotx/nn/long_context_auto_regression.py
 deeplotx/nn/long_context_recursive_sequential.py
 deeplotx/nn/multi_head_attention.py
+deeplotx/nn/multi_head_feed_forward.py
 deeplotx/nn/recursive_sequential.py
 deeplotx/nn/roformer_encoder.py
 deeplotx/nn/rope.py

{deeplotx-0.8.3 → deeplotx-0.8.5}/pyproject.toml

@@ -1,17 +1,17 @@
-[project]
-name = "deeplotx"
-version = "0.8.3"
-description = "Easy-2-use long text NLP toolkit."
-readme = "README.md"
-requires-python = ">=3.10"
-dependencies = [
-    "hf-xet",
-    "jupyter",
-    "numpy",
-    "protobuf",
-    "python-dotenv",
-    "torch",
-    "transformers",
-    "typing-extensions",
-    "vortezwohl>=0.0.8",
-]
+[project]
+name = "deeplotx"
+version = "0.8.5"
+description = "Easy-2-use long text NLP toolkit."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "hf-xet",
+    "jupyter",
+    "numpy",
+    "protobuf",
+    "python-dotenv",
+    "torch",
+    "transformers",
+    "typing-extensions",
+    "vortezwohl>=0.0.8",
+]