deeplotx 0.4.6__py3-none-any.whl → 0.4.8__py3-none-any.whl

deeplotx/encoder/long_text_encoder.py

@@ -24,13 +24,21 @@ class LongTextEncoder(BertEncoder):
         return input_tup[0], super().forward(input_tup[1], attention_mask=input_tup[2])
 
     @override
-    def encode(self, text: str, use_cache: bool = True) -> torch.Tensor:
+    def encode(self, text: str, flatten: bool = True, use_cache: bool = True) -> torch.Tensor:
+        def postprocess(tensors: list[torch.Tensor], _flatten: bool) -> torch.Tensor:
+            if not _flatten:
+                return torch.stack(tensors, dim=0).squeeze()
+            _fin_emb_tensor = torch.tensor([], dtype=tensors[0].dtype)
+            for _emb in tensors:
+                _fin_emb_tensor = torch.cat((_fin_emb_tensor.detach().clone(), _emb.detach().clone()), dim=-1)
+            return _fin_emb_tensor.squeeze()
+
         _text_to_show = text.replace("\n", str())
         logger.debug(f'Embedding \"{_text_to_show if len(_text_to_show) < 128 else _text_to_show[:128] + "..."}\".')
         # read cache
         _text_hash = md5(text)
         if _text_hash in self._cache.keys():
-            return self._cache[_text_hash]
+            return postprocess(self._cache[_text_hash], flatten)
         _text_to_input_ids = self.tokenizer.encode(text.strip())[:self._max_length]
         _text_to_input_ids_att_mask = []
         # padding
@@ -47,17 +55,13 @@ class LongTextEncoder(BertEncoder):
         for i in range(num_chunks):
             _tmp_left = max(i * self._chunk_size - self._overlapping, 0)
             _tmp_right = (i + 1) * self._chunk_size + self._overlapping
-            chunks.append((i, torch.tensor([_text_to_input_ids[_tmp_left: _tmp_right]], dtype=torch.long),
+            chunks.append((i, torch.tensor([_text_to_input_ids[_tmp_left: _tmp_right]], dtype=torch.int),
                            torch.tensor([_text_to_input_ids_att_mask[_tmp_left: _tmp_right]], dtype=torch.int)))
         with ThreadPoolExecutor(max_workers=min(num_chunks + 1, 3)) as executor:
             embeddings = list(executor.map(self.__chunk_embedding, chunks))
         embeddings.sort(key=lambda x: x[0])
         fin_embedding = [x[1] for x in embeddings]
-        fin_emb_tensor = torch.tensor([], dtype=torch.float32)
-        for emb in fin_embedding:
-            fin_emb_tensor = torch.cat((fin_emb_tensor.detach().clone(), emb.detach().clone()), dim=-1)
-        fin_emb_tensor = fin_emb_tensor.squeeze()
         # write cache
         if use_cache:
-            self._cache[_text_hash] = fin_emb_tensor
-        return fin_emb_tensor
+            self._cache[_text_hash] = fin_embedding
+        return postprocess(fin_embedding, flatten)

{deeplotx-0.4.6.dist-info → deeplotx-0.4.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.4.6
+Version: 0.4.8
 Summary: Easy-2-use long text NLP toolkit.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{deeplotx-0.4.6.dist-info → deeplotx-0.4.8.dist-info}/RECORD

@@ -1,7 +1,7 @@
 deeplotx/__init__.py,sha256=wMN_AI14V-0BPbQghYpvd2y7eUGfhr7jKTTuur-5Upg,1002
 deeplotx/encoder/__init__.py,sha256=EM-xrTsHoGaiiFpj-iFAxilMHXC_sQKWYrcq1qCnI3U,138
 deeplotx/encoder/bert_encoder.py,sha256=6QY2pOvayWNz4w749JAGndvQ-jeKJgy3BalQl2JCkgk,1994
-deeplotx/encoder/long_text_encoder.py,sha256=V6VxaHW6bMMaZHgU1UZ8n19UfSIV2f2sarWXquiFffQ,3018
+deeplotx/encoder/long_text_encoder.py,sha256=FP0ACiOaOCjK2buRSWqBs-peg3IWQKuIdP2S00LNvSs,3271
 deeplotx/encoder/longformer_encoder.py,sha256=4avKYsLN6TTpPoky8BQ0nIhQm8lVxMvvzqkrdKCWj3Q,1433
 deeplotx/nn/__init__.py,sha256=oQ-vYXyuaGelfCOs2im_gZXAiiBlCCVXh1uw9yjvRMs,253
 deeplotx/nn/auto_regression.py,sha256=o82C9TREZbhGdj2knSVGTXhjJne0LGEqc7BllByJJWE,449
@@ -20,8 +20,8 @@ deeplotx/trainer/text_binary_classification_trainer.py,sha256=5O-5dwVMCj5EDX9gjJ
 deeplotx/util/__init__.py,sha256=JxqAK_WOOHcYVSTHBT1-WuBwWrPEVDTV3titeVWvNUM,74
 deeplotx/util/hash.py,sha256=wwsC6kOQvbpuvwKsNQOARd78_wePmW9i3oaUuXRUnpc,352
 deeplotx/util/read_file.py,sha256=ptzouvEQeeW8KU5BrWNJlXw-vFXVrpS9SkAUxsu6A8A,612
-deeplotx-0.4.6.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.4.6.dist-info/METADATA,sha256=3SBHwAo506Q9kkzfv2ZEd2AUKRD7q--3DLc7hRR8wms,1656
-deeplotx-0.4.6.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-deeplotx-0.4.6.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.4.6.dist-info/RECORD,,
+deeplotx-0.4.8.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.4.8.dist-info/METADATA,sha256=KMyt-My-d5261MGBfC1_HsyqbJ_KVEvZ--kZNFq8B2A,1656
+deeplotx-0.4.8.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+deeplotx-0.4.8.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.4.8.dist-info/RECORD,,