deeplotx 0.4.5__py3-none-any.whl → 0.4.7__py3-none-any.whl

deeplotx/encoder/bert_encoder.py

@@ -1,11 +1,13 @@
+import os
 import math
+
 import torch
 from torch import nn
 from transformers import BertTokenizer, BertModel
 
 from deeplotx import __ROOT__
 
-CACHE_PATH = f'{__ROOT__}\\.cache'
+CACHE_PATH = os.path.join(__ROOT__, '.cache')
 DEFAULT_BERT = 'bert-base-uncased'
 
 

deeplotx/encoder/long_text_encoder.py

@@ -24,13 +24,21 @@ class LongTextEncoder(BertEncoder):
         return input_tup[0], super().forward(input_tup[1], attention_mask=input_tup[2])
 
     @override
-    def encode(self, text: str, use_cache: bool = True) -> torch.Tensor:
+    def encode(self, text: str, flatten: bool = True, use_cache: bool = True) -> torch.Tensor:
+        def postprocess(tensors: list[torch.Tensor], _flatten: bool) -> torch.Tensor:
+            if not _flatten:
+                return torch.stack(tensors, dim=0).squeeze()
+            _fin_emb_tensor = torch.tensor([], dtype=torch.float32)
+            for _emb in fin_embedding:
+                _fin_emb_tensor = torch.cat((_fin_emb_tensor.detach().clone(), _emb.detach().clone()), dim=-1)
+            return _fin_emb_tensor.squeeze()
+
         _text_to_show = text.replace("\n", str())
         logger.debug(f'Embedding \"{_text_to_show if len(_text_to_show) < 128 else _text_to_show[:128] + "..."}\".')
         # read cache
         _text_hash = md5(text)
         if _text_hash in self._cache.keys():
-            return self._cache[_text_hash]
+            return postprocess(self._cache[_text_hash], flatten)
         _text_to_input_ids = self.tokenizer.encode(text.strip())[:self._max_length]
         _text_to_input_ids_att_mask = []
         # padding
@@ -53,11 +61,7 @@ class LongTextEncoder(BertEncoder):
             embeddings = list(executor.map(self.__chunk_embedding, chunks))
         embeddings.sort(key=lambda x: x[0])
         fin_embedding = [x[1] for x in embeddings]
-        fin_emb_tensor = torch.tensor([], dtype=torch.float32)
-        for emb in fin_embedding:
-            fin_emb_tensor = torch.cat((fin_emb_tensor.detach().clone(), emb.detach().clone()), dim=-1)
-        fin_emb_tensor = fin_emb_tensor.squeeze()
         # write cache
         if use_cache:
-            self._cache[_text_hash] = fin_emb_tensor
-        return fin_emb_tensor
+            self._cache[_text_hash] = fin_embedding
+        return postprocess(fin_embedding, flatten)

deeplotx/encoder/longformer_encoder.py

@@ -1,10 +1,12 @@
+import os
+
 import torch
 from torch import nn
 from transformers import LongformerTokenizer, LongformerModel
 
 from deeplotx import __ROOT__
 
-CACHE_PATH = f'{__ROOT__}\\.cache'
+CACHE_PATH = os.path.join(__ROOT__, '.cache')
 DEFAULT_LONGFORMER = 'allenai/longformer-base-4096'
 
 

{deeplotx-0.4.5.dist-info → deeplotx-0.4.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.4.5
+Version: 0.4.7
 Summary: Easy-2-use long text NLP toolkit.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{deeplotx-0.4.5.dist-info → deeplotx-0.4.7.dist-info}/RECORD

@@ -1,8 +1,8 @@
 deeplotx/__init__.py,sha256=wMN_AI14V-0BPbQghYpvd2y7eUGfhr7jKTTuur-5Upg,1002
 deeplotx/encoder/__init__.py,sha256=EM-xrTsHoGaiiFpj-iFAxilMHXC_sQKWYrcq1qCnI3U,138
-deeplotx/encoder/bert_encoder.py,sha256=A-B7Gj94xv6UhvsFTBH7tnkAdGHRhfUZA2QjSnTKB6c,1970
-deeplotx/encoder/long_text_encoder.py,sha256=V6VxaHW6bMMaZHgU1UZ8n19UfSIV2f2sarWXquiFffQ,3018
-deeplotx/encoder/longformer_encoder.py,sha256=mZpC5TrGHQo98-ydGtVQQ9KRHgCGl1sRoxcQs7r4SSo,1409
+deeplotx/encoder/bert_encoder.py,sha256=6QY2pOvayWNz4w749JAGndvQ-jeKJgy3BalQl2JCkgk,1994
+deeplotx/encoder/long_text_encoder.py,sha256=KEld6LRHuUc2nP_t_KTVfb3F3TYkIieS4jSzkR0tEVs,3275
+deeplotx/encoder/longformer_encoder.py,sha256=4avKYsLN6TTpPoky8BQ0nIhQm8lVxMvvzqkrdKCWj3Q,1433
 deeplotx/nn/__init__.py,sha256=oQ-vYXyuaGelfCOs2im_gZXAiiBlCCVXh1uw9yjvRMs,253
 deeplotx/nn/auto_regression.py,sha256=o82C9TREZbhGdj2knSVGTXhjJne0LGEqc7BllByJJWE,449
 deeplotx/nn/base_neural_network.py,sha256=xWKG4FX6Jzdlrfc1HOW1aO9uh0Af3D-dB5Jl7eCxsAk,1635
@@ -20,8 +20,8 @@ deeplotx/trainer/text_binary_classification_trainer.py,sha256=5O-5dwVMCj5EDX9gjJ
 deeplotx/util/__init__.py,sha256=JxqAK_WOOHcYVSTHBT1-WuBwWrPEVDTV3titeVWvNUM,74
 deeplotx/util/hash.py,sha256=wwsC6kOQvbpuvwKsNQOARd78_wePmW9i3oaUuXRUnpc,352
 deeplotx/util/read_file.py,sha256=ptzouvEQeeW8KU5BrWNJlXw-vFXVrpS9SkAUxsu6A8A,612
-deeplotx-0.4.5.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.4.5.dist-info/METADATA,sha256=DpSM3W-OH0ayM1AOt8OYfbpRHqZLkUpPf5qc0PEROzs,1656
-deeplotx-0.4.5.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-deeplotx-0.4.5.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.4.5.dist-info/RECORD,,
+deeplotx-0.4.7.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.4.7.dist-info/METADATA,sha256=oXBTSPd2wYHvbcM7A_-fTiln6CjxVkLQLF9nApv5B7g,1656
+deeplotx-0.4.7.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+deeplotx-0.4.7.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.4.7.dist-info/RECORD,,