deeplotx 0.9.6__py3-none-any.whl → 0.9.7b0__py3-none-any.whl

deeplotx/encoder/encoder.py

@@ -1,10 +1,10 @@
 import logging
 import os
 import math
-from requests.exceptions import ConnectTimeout, SSLError
 
 import torch
 from torch import nn
+from requests.exceptions import RequestException
 from transformers import AutoTokenizer, AutoModel
 
 from deeplotx import __ROOT__
@@ -26,14 +26,7 @@ class Encoder(nn.Module):
             self.encoder = AutoModel.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                      cache_dir=CACHE_PATH, _from_auto=True,
                                                      trust_remote_code=True).to(self.device)
-        except ConnectTimeout:
-            self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                           cache_dir=CACHE_PATH, _from_auto=True,
-                                                           trust_remote_code=True, local_files_only=True)
-            self.encoder = AutoModel.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                     cache_dir=CACHE_PATH, _from_auto=True,
-                                                     trust_remote_code=True, local_files_only=True).to(self.device)
-        except SSLError:
+        except RequestException:
             self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                            cache_dir=CACHE_PATH, _from_auto=True,
                                                            trust_remote_code=True, local_files_only=True)

deeplotx/encoder/longformer_encoder.py

@@ -3,8 +3,8 @@ import os
 
 import torch
 from torch import nn
+from requests.exceptions import RequestException
 from transformers import AutoModel, AutoTokenizer
-from requests.exceptions import ConnectTimeout, SSLError
 
 from deeplotx import __ROOT__
 
@@ -25,14 +25,7 @@ class LongformerEncoder(nn.Module):
             self.encoder = AutoModel.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                      cache_dir=CACHE_PATH, _from_auto=True,
                                                      trust_remote_code=True).to(self.device)
-        except ConnectTimeout:
-            self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                           cache_dir=CACHE_PATH, _from_auto=True,
-                                                           trust_remote_code=True, local_files_only=True)
-            self.encoder = AutoModel.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                     cache_dir=CACHE_PATH, _from_auto=True,
-                                                     trust_remote_code=True, local_files_only=True).to(self.device)
-        except SSLError:
+        except RequestException:
             self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                            cache_dir=CACHE_PATH, _from_auto=True,
                                                            trust_remote_code=True, local_files_only=True)

deeplotx/ner/bert_ner.py

@@ -1,8 +1,8 @@
 import logging
 import os
-from requests.exceptions import ConnectTimeout, SSLError
 
 import torch
+from requests.exceptions import RequestException
 from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
 
 from deeplotx import __ROOT__
@@ -30,14 +30,7 @@ class BertNER(BaseNER):
             self.encoder = AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                                            cache_dir=CACHE_PATH, _from_auto=True,
                                                                            trust_remote_code=True).to(self.device)
-        except ConnectTimeout:
-            self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                           cache_dir=CACHE_PATH, _from_auto=True,
-                                                           trust_remote_code=True, local_files_only=True)
-            self.encoder = AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                                           cache_dir=CACHE_PATH, _from_auto=True,
-                                                                           trust_remote_code=True, local_files_only=True).to(self.device)
-        except SSLError:
+        except RequestException:
             self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                            cache_dir=CACHE_PATH, _from_auto=True,
                                                            trust_remote_code=True, local_files_only=True)
@@ -74,6 +67,8 @@ class BertNER(BaseNER):
                 break
         for ent in entities:
             ent[0] = ent[0].strip()
+            if len(ent[0]) < 1:
+                ent[0] = ' '
             # stripping
             while not ent[0][0].isalpha():
                 if len(ent[0]) < 2:

deeplotx/util/read_file.py

@@ -13,6 +13,17 @@ def read_file(path: str, encoding: str = 'utf-8') -> str:
             pass
 
 
+def write_file(content: str | bytes, path: str, encoding: str = 'utf-8') -> str:
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    if isinstance(content, bytes):
+        with open(path, mode='wb') as f:
+            f.write(content)
+        return path
+    with open(path, mode='w', encoding=encoding) as f:
+        f.write(content)
+    return path
+
+
 def get_files(path: str) -> list:
     if os.path.exists(path):
         entries = os.listdir(path)

{deeplotx-0.9.6.dist-info → deeplotx-0.9.7b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.6
+Version: 0.9.7b0
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{deeplotx-0.9.6.dist-info → deeplotx-0.9.7b0.dist-info}/RECORD

@@ -1,11 +1,11 @@
 deeplotx/__init__.py,sha256=x4CbJuW20al6S5KkKyrReeuwNGv04JGoqtGUyx-ACtg,1356
 deeplotx/encoder/__init__.py,sha256=BrsF5_4O-4pfihYF2wjExDOoAY-03kGJTH-Mhez4tsE,129
-deeplotx/encoder/encoder.py,sha256=wVRl3p_7eg7qT_tJEit5qnmZx7dXkMVLxAtao5vImkk,4201
+deeplotx/encoder/encoder.py,sha256=uJswUSrYVDWP84HeCD40R9KgXGPUEa080konv7jEp8I,3539
 deeplotx/encoder/long_text_encoder.py,sha256=4oRa9FqfGNZ8-gq14UKuhDkZC0A1Xi-wKmbQsn-uZ58,3966
-deeplotx/encoder/longformer_encoder.py,sha256=7Lm65AUD3qwbrzrhJ3dPZkyHeNRSapga3f-5QJCxV5A,3538
+deeplotx/encoder/longformer_encoder.py,sha256=mfAI_NE3QQZhvGHbZkP7S6g0Jj59wmLWQ9QW7HOjqm0,2876
 deeplotx/ner/__init__.py,sha256=Rss1pup9HzHZCG8U9ub8niWa9zRjWCy3Z7zg378KZQg,114
 deeplotx/ner/base_ner.py,sha256=pZTl50OrHH_FJm4rKp9iuixeOE6FX_AzgDXD32aXsN0,204
-deeplotx/ner/bert_ner.py,sha256=_IpR2AqXzEP_QBUj8KKVS7jO9mHZyFGDxr7q1LUgJf4,9246
+deeplotx/ner/bert_ner.py,sha256=6al5iMc8gb3XuJf7dbJd-noXShLYmRoLXTd8L1-wYMM,8581
 deeplotx/ner/named_entity.py,sha256=c6XufIwH6yloJ-ccUjagf4mBl1XbbYDT8xyEJJ_-ZNs,269
 deeplotx/ner/n2g/__init__.py,sha256=b6fOWJVLaOCtoz8Qlp8NWQbL5lUSbn6H3-8fnVNIPi0,3940
 deeplotx/nn/__init__.py,sha256=YILwbxb-NHdiJjfOwBKH8F7PuZSDZSrGpTznPDucTro,710
@@ -32,9 +32,9 @@ deeplotx/trainer/base_trainer.py,sha256=z0MeAT-rRYmjeBXt0ckt7J1itYArR0Cx02wHesXU
 deeplotx/trainer/text_binary_classification_trainer.py,sha256=TFxOX8rWU_zKliI9zm7F5ZH7snR2d-sk95s3pfTmm78,6601
 deeplotx/util/__init__.py,sha256=5CH4MTeSgsmCe3LPMfvKoSBpwh6jDSBuHVElJvzQzgs,90
 deeplotx/util/hash.py,sha256=qbNU3RLBWGQYFVte9WZBAkZ1BkdjCXiKLDaKPN54KFk,662
-deeplotx/util/read_file.py,sha256=ptzouvEQeeW8KU5BrWNJlXw-vFXVrpS9SkAUxsu6A8A,612
-deeplotx-0.9.6.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.9.6.dist-info/METADATA,sha256=yjG3QmoZfUJwenoR76b7MXAjzCt7Qh9UMr1CishZFG4,14442
-deeplotx-0.9.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deeplotx-0.9.6.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.9.6.dist-info/RECORD,,
+deeplotx/util/read_file.py,sha256=O9nieNgAGQ7Ct1EFxCdcgL6hVs5s2Vw_ItcUK6VeTwY,981
+deeplotx-0.9.7b0.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.9.7b0.dist-info/METADATA,sha256=TFuaNIUCzu6JefhA33sDyBV-Wh3PHgCWclCDKKICfT4,14444
+deeplotx-0.9.7b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deeplotx-0.9.7b0.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.9.7b0.dist-info/RECORD,,