deeplotx 0.9.6__tar.gz → 0.9.7b0__tar.gz

{deeplotx-0.9.6 → deeplotx-0.9.7b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.6
+Version: 0.9.7b0
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{deeplotx-0.9.6 → deeplotx-0.9.7b0}/deeplotx/encoder/encoder.py

@@ -1,10 +1,10 @@
 import logging
 import os
 import math
-from requests.exceptions import ConnectTimeout, SSLError
 
 import torch
 from torch import nn
+from requests.exceptions import RequestException
 from transformers import AutoTokenizer, AutoModel
 
 from deeplotx import __ROOT__
@@ -26,14 +26,7 @@ class Encoder(nn.Module):
             self.encoder = AutoModel.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                      cache_dir=CACHE_PATH, _from_auto=True,
                                                      trust_remote_code=True).to(self.device)
-        except ConnectTimeout:
-            self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                           cache_dir=CACHE_PATH, _from_auto=True,
-                                                           trust_remote_code=True, local_files_only=True)
-            self.encoder = AutoModel.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                     cache_dir=CACHE_PATH, _from_auto=True,
-                                                     trust_remote_code=True, local_files_only=True).to(self.device)
-        except SSLError:
+        except RequestException:
             self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                            cache_dir=CACHE_PATH, _from_auto=True,
                                                            trust_remote_code=True, local_files_only=True)

{deeplotx-0.9.6 → deeplotx-0.9.7b0}/deeplotx/encoder/longformer_encoder.py

@@ -3,8 +3,8 @@ import os
 
 import torch
 from torch import nn
+from requests.exceptions import RequestException
 from transformers import AutoModel, AutoTokenizer
-from requests.exceptions import ConnectTimeout, SSLError
 
 from deeplotx import __ROOT__
 
@@ -25,14 +25,7 @@ class LongformerEncoder(nn.Module):
             self.encoder = AutoModel.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                      cache_dir=CACHE_PATH, _from_auto=True,
                                                      trust_remote_code=True).to(self.device)
-        except ConnectTimeout:
-            self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                           cache_dir=CACHE_PATH, _from_auto=True,
-                                                           trust_remote_code=True, local_files_only=True)
-            self.encoder = AutoModel.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                     cache_dir=CACHE_PATH, _from_auto=True,
-                                                     trust_remote_code=True, local_files_only=True).to(self.device)
-        except SSLError:
+        except RequestException:
             self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                            cache_dir=CACHE_PATH, _from_auto=True,
                                                            trust_remote_code=True, local_files_only=True)

{deeplotx-0.9.6 → deeplotx-0.9.7b0}/deeplotx/ner/bert_ner.py

@@ -1,8 +1,8 @@
 import logging
 import os
-from requests.exceptions import ConnectTimeout, SSLError
 
 import torch
+from requests.exceptions import RequestException
 from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
 
 from deeplotx import __ROOT__
@@ -30,14 +30,7 @@ class BertNER(BaseNER):
             self.encoder = AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                                            cache_dir=CACHE_PATH, _from_auto=True,
                                                                            trust_remote_code=True).to(self.device)
-        except ConnectTimeout:
-            self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                           cache_dir=CACHE_PATH, _from_auto=True,
-                                                           trust_remote_code=True, local_files_only=True)
-            self.encoder = AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
-                                                                           cache_dir=CACHE_PATH, _from_auto=True,
-                                                                           trust_remote_code=True, local_files_only=True).to(self.device)
-        except SSLError:
+        except RequestException:
             self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path,
                                                            cache_dir=CACHE_PATH, _from_auto=True,
                                                            trust_remote_code=True, local_files_only=True)
@@ -74,6 +67,8 @@ class BertNER(BaseNER):
                 break
         for ent in entities:
             ent[0] = ent[0].strip()
+            if len(ent[0]) < 1:
+                ent[0] = ' '
             # stripping
             while not ent[0][0].isalpha():
                 if len(ent[0]) < 2:

{deeplotx-0.9.6 → deeplotx-0.9.7b0}/deeplotx/util/read_file.py

@@ -13,6 +13,17 @@ def read_file(path: str, encoding: str = 'utf-8') -> str:
             pass
 
 
+def write_file(content: str | bytes, path: str, encoding: str = 'utf-8') -> str:
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    if isinstance(content, bytes):
+        with open(path, mode='wb') as f:
+            f.write(content)
+        return path
+    with open(path, mode='w', encoding=encoding) as f:
+        f.write(content)
+    return path
+
+
 def get_files(path: str) -> list:
     if os.path.exists(path):
         entries = os.listdir(path)

{deeplotx-0.9.6 → deeplotx-0.9.7b0}/deeplotx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.6
+Version: 0.9.7b0
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{deeplotx-0.9.6 → deeplotx-0.9.7b0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deeplotx"
-version = '0.9.6'
+version = '0.9.7.beta'
 description = "An out-of-the-box long-text NLP framework."
 readme = "README.md"
 requires-python = ">=3.10"