deeplotx 0.8.5__py3-none-any.whl → 0.8.7__py3-none-any.whl

deeplotx/encoder/encoder.py

@@ -48,7 +48,7 @@ class Encoder(nn.Module):
             return self.encoder.forward(_input_tup[0], attention_mask=_input_tup[1]).last_hidden_state[:, 0, :]
 
         num_chunks = math.ceil(input_ids.shape[-1] / self.embed_dim)
-        chunks = chunk_results = []
+        chunks, chunk_results = [], []
         for i in range(num_chunks):
             start_idx = i * self.embed_dim
             end_idx = min(start_idx + self.embed_dim, input_ids.shape[-1])

deeplotx/nn/base_neural_network.py

@@ -99,11 +99,14 @@ class BaseNeuralNetwork(nn.Module):
 
     def predict(self, x: torch.Tensor) -> torch.Tensor:
         x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
-        __train = self.training
-        self.training = False
+        training_state_dict = dict()
+        for m in self.modules():
+            training_state_dict[m] = m.training
+            m.training = False
         with torch.no_grad():
             res = self.forward(x)
-        self.training = __train
+        for m, training_state in training_state_dict.items():
+            m.training = training_state
         return res
 
     def save(self, model_name: str | None = None, model_dir: str = '.', _suffix: str = DEFAULT_SUFFIX):

deeplotx/nn/feed_forward.py

@@ -28,7 +28,7 @@ class FeedForwardUnit(BaseNeuralNetwork):
         x = self.layer_norm(x)
         x = self.up_proj(x)
         x = self.parametric_relu(x)
-        if self._dropout_rate > .0:
+        if self._dropout_rate > .0 and self.training:
             x = torch.dropout(x, p=self._dropout_rate, train=self.training)
         return self.down_proj(x) + residual
 

deeplotx/nn/linear_regression.py

@@ -10,17 +10,20 @@ from deeplotx.nn.multi_head_feed_forward import MultiHeadFeedForward
 class LinearRegression(BaseNeuralNetwork):
     def __init__(self, input_dim: int, output_dim: int, num_heads: int = 1, num_layers: int = 1,
                  expansion_factor: int | float = 1.5, bias: bool = True, dropout_rate: float = 0.1,
-                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
+                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None, **kwargs):
         super().__init__(in_features=input_dim, out_features=output_dim, model_name=model_name, device=device, dtype=dtype)
-        self.ffn = MultiHeadFeedForward(feature_dim=input_dim, num_heads=num_heads,
-                                        num_layers=num_layers, expansion_factor=expansion_factor,
-                                        bias=bias, dropout_rate=dropout_rate, device=self.device, dtype=self.dtype)
-        self.proj = nn.Linear(in_features=input_dim, out_features=output_dim,
-                              bias=bias, device=self.device, dtype=self.dtype)
+        self.multi_head_ffn_layers = nn.ModuleList([MultiHeadFeedForward(feature_dim=input_dim, num_heads=num_heads,
+                                                                         num_layers=kwargs.get('head_layers', 1),
+                                                                         expansion_factor=expansion_factor,
+                                                                         bias=bias, dropout_rate=dropout_rate,
+                                                                         device=self.device, dtype=self.dtype) for _ in range(num_layers)])
+        self.out_proj = nn.Linear(in_features=input_dim, out_features=output_dim,
+                                  bias=bias, device=self.device, dtype=self.dtype)
 
     @override
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         x = self.ensure_device_and_dtype(x, device=self.device, dtype=self.dtype)
         residual = x
-        x = self.ffn(x) + residual
-        return self.proj(x)
+        for ffn in self.multi_head_ffn_layers:
+            x = ffn(x)
+        return self.out_proj(x + residual)

deeplotx/nn/logistic_regression.py

@@ -8,10 +8,10 @@ from deeplotx.nn.linear_regression import LinearRegression
 class LogisticRegression(LinearRegression):
     def __init__(self, input_dim: int, output_dim: int = 1, num_heads: int = 1, num_layers: int = 1,
                  expansion_factor: int | float = 1.5, bias: bool = True, dropout_rate: float = 0.1,
-                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
+                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None, **kwargs):
         super().__init__(input_dim=input_dim, output_dim=output_dim, num_heads=num_heads, num_layers=num_layers,
                          expansion_factor=expansion_factor, bias=bias, dropout_rate=dropout_rate,
-                         model_name=model_name, device=device, dtype=dtype)
+                         model_name=model_name, device=device, dtype=dtype, **kwargs)
 
     @override
     def forward(self, x: torch.Tensor) -> torch.Tensor:

deeplotx/nn/recursive_sequential.py

@@ -4,7 +4,7 @@ import torch
 from torch import nn
 
 from deeplotx.nn.base_neural_network import BaseNeuralNetwork
-from deeplotx.nn.multi_head_feed_forward import MultiHeadFeedForward
+from deeplotx.nn.linear_regression import LinearRegression
 
 
 class RecursiveSequential(BaseNeuralNetwork):
@@ -20,11 +20,10 @@ class RecursiveSequential(BaseNeuralNetwork):
                             num_layers=recursive_layers, batch_first=True,
                             bias=True, bidirectional=True, device=self.device,
                             dtype=self.dtype)
-        self.ffn = MultiHeadFeedForward(feature_dim=recursive_hidden_dim * 2, num_heads=kwargs.get('ffn_heads', 1),
-                                        num_layers=ffn_layers, expansion_factor=ffn_expansion_factor,
-                                        bias=bias, dropout_rate=dropout_rate, device=self.device, dtype=self.dtype)
-        self.__proj = nn.Linear(in_features=recursive_hidden_dim * 2, out_features=output_dim, bias=bias,
-                                device=self.device, dtype=self.dtype)
+        self.out_proj = LinearRegression(input_dim=recursive_hidden_dim * 2, output_dim=output_dim,
+                                         num_heads=kwargs.get('ffn_heads', 1), head_layers=kwargs.get('ffn_head_layers', 1),
+                                         num_layers=ffn_layers, expansion_factor=ffn_expansion_factor,
+                                         bias=bias, dropout_rate=dropout_rate, device=self.device, dtype=self.dtype)
 
     def initial_state(self, batch_size: int = 1) -> tuple[torch.Tensor, torch.Tensor]:
         zeros = torch.zeros(self.lstm.num_layers * 2, batch_size, self.lstm.hidden_size, device=self.device, dtype=self.dtype)
@@ -37,16 +36,17 @@ class RecursiveSequential(BaseNeuralNetwork):
                  self.ensure_device_and_dtype(state[1], device=self.device, dtype=self.dtype))
         x, (hidden_state, cell_state) = self.lstm(x, state)
         x = x[:, -1, :]
-        residual = x
-        x = self.ffn(x) + residual
-        x = self.__proj(x)
+        x = self.out_proj(x)
         return x, (hidden_state, cell_state)
 
     @override
     def predict(self, x: torch.Tensor) -> torch.Tensor:
-        __train = self.training
-        self.training = False
+        training_state_dict = dict()
+        for m in self.modules():
+            training_state_dict[m] = m.training
+            m.training = False
         with torch.no_grad():
             res = self.forward(x.unsqueeze(0), self.initial_state(batch_size=1))[0]
-        self.training = __train
+        for m, training_state in training_state_dict.items():
+            m.training = training_state
         return res

deeplotx/nn/roformer_encoder.py

@@ -27,8 +27,8 @@ class RoFormerEncoder(BaseNeuralNetwork):
                                device=self.device, dtype=self.dtype)
         self.layer_norm = nn.LayerNorm(normalized_shape=feature_dim, eps=1e-9,
                                        device=self.device, dtype=self.dtype)
-        self.__proj = nn.Linear(in_features=feature_dim * 2, out_features=feature_dim,
-                                bias=bias, device=self.device, dtype=self.dtype)
+        self.out_proj = nn.Linear(in_features=feature_dim * 2, out_features=feature_dim,
+                                  bias=bias, device=self.device, dtype=self.dtype)
 
     @override
     def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
@@ -37,4 +37,4 @@ class RoFormerEncoder(BaseNeuralNetwork):
             mask = self.ensure_device_and_dtype(mask, device=self.device, dtype=self.dtype)
         attn = self.attn(x=self.layer_norm(x), y=None, mask=mask)
         x = torch.concat([attn, x], dim=-1)
-        return self.__proj(self.ffn(x))
+        return self.out_proj(self.ffn(x))

deeplotx/nn/softmax_regression.py

@@ -8,10 +8,10 @@ from deeplotx.nn.linear_regression import LinearRegression
 class SoftmaxRegression(LinearRegression):
     def __init__(self, input_dim: int, output_dim: int, num_heads: int = 1, num_layers: int = 1,
                  expansion_factor: int | float = 1.5, bias: bool = True, dropout_rate: float = 0.1,
-                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None):
+                 model_name: str | None = None, device: str | None = None, dtype: torch.dtype | None = None, **kwargs):
         super().__init__(input_dim=input_dim, output_dim=output_dim, num_heads=num_heads, num_layers=num_layers,
                          expansion_factor=expansion_factor, bias=bias, dropout_rate=dropout_rate,
-                         model_name=model_name, device=device, dtype=dtype)
+                         model_name=model_name, device=device, dtype=dtype, **kwargs)
 
     @override
     def forward(self, x: torch.Tensor) -> torch.Tensor:

{deeplotx-0.8.5.dist-info → deeplotx-0.8.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.8.5
+Version: 0.8.7
 Summary: Easy-2-use long text NLP toolkit.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{deeplotx-0.8.5.dist-info → deeplotx-0.8.7.dist-info}/RECORD

@@ -1,23 +1,23 @@
 deeplotx/__init__.py,sha256=xEq8WQ2LpEZoLX_Z464d0dy4aemFGrEV6ZMJr6ioFnQ,1186
 deeplotx/encoder/__init__.py,sha256=BrsF5_4O-4pfihYF2wjExDOoAY-03kGJTH-Mhez4tsE,129
-deeplotx/encoder/encoder.py,sha256=oSBdA-MiwMKNfTFJWR-RdvNS0G0qfX-Qchwy4LuwB00,3985
+deeplotx/encoder/encoder.py,sha256=tksTtmz9JRDSimCdhMkxpbGUHNWhARGaeKh2pBvLgEI,3988
 deeplotx/encoder/long_text_encoder.py,sha256=3ScdKDi65J5tdO8PFCXBjCzNUCLlJRwVhpDR0BrphG4,3951
 deeplotx/encoder/longformer_encoder.py,sha256=NNYLr5I9tdeh0C8Ir7QcbEMU9gDk6U7CiF3Tbg6NEsE,3372
 deeplotx/nn/__init__.py,sha256=YILwbxb-NHdiJjfOwBKH8F7PuZSDZSrGpTznPDucTro,710
 deeplotx/nn/attention.py,sha256=R-i-Rd7gnsh6hwXDeYfqLQOJvfSZIGfQbFzRlC91XLo,2879
 deeplotx/nn/auto_regression.py,sha256=j_R7WGPq9REngjpLuX5c0AaNqOpgGm2Vfrolw-XjWXw,877
-deeplotx/nn/base_neural_network.py,sha256=FjQEDFH810fJS7JV3aLgJZnaMqC6DH--wlBvuj-ghTc,5900
-deeplotx/nn/feed_forward.py,sha256=4ozj7EDalO9pb6JUhZtsJqE0r8bIHFApHRt2zTrl4ho,2931
-deeplotx/nn/linear_regression.py,sha256=EotBCCam7FH5iaAv0ma4TfYId2YfhBnrQlMirF0xoq4,1400
-deeplotx/nn/logistic_regression.py,sha256=6vlXuP5el6EdXEhUbpVTKstcf-pikD50Xezw66l-aUc,978
+deeplotx/nn/base_neural_network.py,sha256=QCyB1dxOs4I8vpu6PCshrZs0infoHXS9IErw6tN-dhs,6060
+deeplotx/nn/feed_forward.py,sha256=kGWEUo8J7jrhSSWlitNnj-AcitNiLz6eOCvUcEuWlVs,2949
+deeplotx/nn/linear_regression.py,sha256=LWrrdAIw32KIT1bdr7q6HczdpEiCgb-R8BCNXGywMxE,1763
+deeplotx/nn/logistic_regression.py,sha256=nipWD3ZPRub2Cx0rU2zxYQyG0COn3NJvew8b2gbJy24,998
 deeplotx/nn/long_context_auto_regression.py,sha256=uy0k_g8wEfMH5nd5HCfrHA8dgEsuWBA2x8U-g3h4vQc,1054
 deeplotx/nn/long_context_recursive_sequential.py,sha256=pcZfnrIHBqbp2BssfUTS1klpuykZwowikfAIaOnvRUI,2674
 deeplotx/nn/multi_head_attention.py,sha256=3z73uGbvy3jszRy1B9nxGOJjlttHpcpRF8Qd09OEams,2267
 deeplotx/nn/multi_head_feed_forward.py,sha256=hD9ScrVJZ9kNksoFASf0xaPgEnNgCeRivW-XjYOPjj8,1908
-deeplotx/nn/recursive_sequential.py,sha256=crD3rEUPPjwu-uSJSiX9kqaM8OPI8SYspbDPlZb2J2Y,2900
-deeplotx/nn/roformer_encoder.py,sha256=UJjKniNdMd0rfoYQcsX6bPo6Ceq_Z6EhwHe2kgqWC_k,2426
+deeplotx/nn/recursive_sequential.py,sha256=sNvAs9iVCuWIgx0_6TizDq41hJpFbfKT3kyDHE86wRM,2928
+deeplotx/nn/roformer_encoder.py,sha256=BAPAMS5-qiM3i2FUyIW-ZTc7og4gZzwlu5LniqzaymY,2432
 deeplotx/nn/rope.py,sha256=RTOjnllubktdy2rzFWxBfkuLuGjhEMyDd06uojdqPhM,1848
-deeplotx/nn/softmax_regression.py,sha256=1brNbnj8qI0VfycZmZQlfn52myKZZe8BF_ziq1JQfPY,999
+deeplotx/nn/softmax_regression.py,sha256=xe2etxSfN0e9XZ4E6Uyz5ThWWzAdQVjYIvN24j8kfNY,1019
 deeplotx/similarity/__init__.py,sha256=s3u-KSgxjnMcWpIItKgXNltFMPQ7YY3CqsqHI-5F1c8,724
 deeplotx/similarity/distribution.py,sha256=wQGouuuW531pZeBRKBujXsdsoz4fDnPw7_GW81jwepc,1066
 deeplotx/similarity/set.py,sha256=zhGFxtSIXlWqvipBYzoiPahp4g0boAIoUiMfG0wl07A,686
@@ -28,8 +28,8 @@ deeplotx/trainer/text_binary_classification_trainer.py,sha256=TFxOX8rWU_zKliI9zm
 deeplotx/util/__init__.py,sha256=5CH4MTeSgsmCe3LPMfvKoSBpwh6jDSBuHVElJvzQzgs,90
 deeplotx/util/hash.py,sha256=qbNU3RLBWGQYFVte9WZBAkZ1BkdjCXiKLDaKPN54KFk,662
 deeplotx/util/read_file.py,sha256=ptzouvEQeeW8KU5BrWNJlXw-vFXVrpS9SkAUxsu6A8A,612
-deeplotx-0.8.5.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.8.5.dist-info/METADATA,sha256=aM49grLNXqwEDdA4PwOEgiBKH1uCPjFuu7OCf5-_5aU,13138
-deeplotx-0.8.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deeplotx-0.8.5.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.8.5.dist-info/RECORD,,
+deeplotx-0.8.7.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.8.7.dist-info/METADATA,sha256=fGyVnmSy3YKst_ZpwtMQhCq_-yxp5pvf-4zcQlhxNBA,13138
+deeplotx-0.8.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deeplotx-0.8.7.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.8.7.dist-info/RECORD,,