deeplotx 0.9.5__tar.gz → 0.9.7__tar.gz

{deeplotx-0.9.5 → deeplotx-0.9.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.5
+Version: 0.9.7
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -27,6 +27,28 @@ Dynamic: license-file
 
 > Author: [vortezwohl](https://github.com/vortezwohl)
 
+## Citation
+
+If you are incorporating the `DeepLoTX` framework into your research, please remember to properly cite it to acknowledge its contribution to your work.
+
+Если вы интегрируете фреймворк `DeepLoTX` в своё исследование, пожалуйста, не забудьте правильно сослаться на него, указывая его вклад в вашу работу.
+
+もしあなたが研究に `DeepLoTX` フレームワークを組み入れているなら、その貢献を認めるために適切に引用することを忘れないでください.
+
+如果您正在將 `DeepLoTX` 框架整合到您的研究中，請務必正確引用它，以聲明它對您工作的貢獻.
+
+```bibtex
+@software{Wu_DeepLoTX_2025,
+author = {Wu, Zihao},
+license = {GPL-3.0},
+month = aug,
+title = {{DeepLoTX}},
+url = {https://github.com/vortezwohl/DeepLoTX},
+version = {0.9.5},
+year = {2025}
+}
+```
+
 ## Installation
 
 - **With pip**

{deeplotx-0.9.5 → deeplotx-0.9.7}/README.md

@@ -6,6 +6,28 @@
 
 > Author: [vortezwohl](https://github.com/vortezwohl)
 
+## Citation
+
+If you are incorporating the `DeepLoTX` framework into your research, please remember to properly cite it to acknowledge its contribution to your work.
+
+Если вы интегрируете фреймворк `DeepLoTX` в своё исследование, пожалуйста, не забудьте правильно сослаться на него, указывая его вклад в вашу работу.
+
+もしあなたが研究に `DeepLoTX` フレームワークを組み入れているなら、その貢献を認めるために適切に引用することを忘れないでください.
+
+如果您正在將 `DeepLoTX` 框架整合到您的研究中，請務必正確引用它，以聲明它對您工作的貢獻.
+
+```bibtex
+@software{Wu_DeepLoTX_2025,
+author = {Wu, Zihao},
+license = {GPL-3.0},
+month = aug,
+title = {{DeepLoTX}},
+url = {https://github.com/vortezwohl/DeepLoTX},
+version = {0.9.5},
+year = {2025}
+}
+```
+
 ## Installation
 
 - **With pip**

{deeplotx-0.9.5 → deeplotx-0.9.7}/deeplotx/ner/bert_ner.py

@@ -12,7 +12,7 @@ from deeplotx.ner.named_entity import NamedEntity, NamedPerson
 
 CACHE_PATH = os.path.join(__ROOT__, '.cache')
 NEW_LINE, BLANK = '\n', ' '
-DEFAULT_LENGTH_THRESHOLD = 384
+DEFAULT_LENGTH_THRESHOLD = 448
 DEFAULT_BERT_NER = 'Davlan/xlm-roberta-base-ner-hrl'
 N2G_MODEL: list[Name2Gender] = []
 logger = logging.getLogger('deeplotx.ner')
@@ -50,7 +50,7 @@ class BertNER(BaseNER):
 
     def _fast_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0) -> list[NamedEntity]:
         assert prob_threshold <= 1., f'prob_threshold ({prob_threshold}) cannot be larger than 1.'
-        s = f' {s.replace(NEW_LINE, BLANK)} '
+        s = f' {s.replace(NEW_LINE, BLANK * 2)} '
         raw_entities = self._ner_pipeline(s)
         entities = []
         for ent in raw_entities:
@@ -60,7 +60,10 @@ class BertNER(BaseNER):
                 if len(ent[0].strip()) < 1:
                     del entities[i]
                 if ent[1].upper().startswith('I') and entities[i - 1][1].upper().startswith('B'):
-                    entities[i - 1][0] += ent[0]
+                    if entities[i - 1][0] + ent[0] in s:
+                        entities[i - 1][0] += ent[0]
+                    else:
+                        entities[i - 1][0] += f' {ent[0]}'
                     entities[i - 1][2] *= ent[2]
                     del entities[i]
             _continue = False
@@ -71,6 +74,8 @@ class BertNER(BaseNER):
                 break
         for ent in entities:
             ent[0] = ent[0].strip()
+            if len(ent[0]) < 1:
+                ent[0] = ' '
             # stripping
             while not ent[0][0].isalpha():
                 if len(ent[0]) < 2:
@@ -102,18 +107,20 @@ class BertNER(BaseNER):
                                           gender_probability=gender_prob)
         return entities
 
-    def _slow_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0, deduplicate: bool = True) -> list[NamedEntity]:
-        _length_threshold = DEFAULT_LENGTH_THRESHOLD
+    def _slow_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0,
+                      window_size: int = DEFAULT_LENGTH_THRESHOLD, deduplicate: bool = True) -> list[NamedEntity]:
         _s_seq = self.tokenizer.encode(s, add_special_tokens=False)
         _entities = self._fast_extract(self.tokenizer.decode(_s_seq, skip_special_tokens=True),
                                        with_gender=with_gender,
-                                       prob_threshold=prob_threshold) if len(_s_seq) < _length_threshold else []
+                                       prob_threshold=prob_threshold) if len(_s_seq) < window_size else []
         # sliding window extracting
-        if len(_s_seq) >= _length_threshold:
-            _window_size = _length_threshold
-            _stride = _length_threshold // 4
-            for i in range(0, len(_s_seq) + _stride, _stride):
-                _window_text = self.tokenizer.decode(_s_seq[i: i + _window_size], skip_special_tokens=True)
+        if len(_s_seq) >= window_size:
+            _stride = window_size // 4
+            for i in range(0, len(_s_seq) + window_size, _stride):
+                _tmp_s_seq = _s_seq[i: i + window_size]
+                if len(_tmp_s_seq) < 1:
+                    continue
+                _window_text = self.tokenizer.decode(_tmp_s_seq, skip_special_tokens=True)
                 _entities.extend(self._fast_extract(_window_text, with_gender=with_gender, prob_threshold=prob_threshold))
         # entity combination
         _tmp_entities = sorted(_entities, key=lambda x: len(x.text), reverse=True)
@@ -155,4 +162,5 @@ class BertNER(BaseNER):
             return self._fast_extract(s=s, with_gender=with_gender, prob_threshold=prob_threshold)
         else:
             return self._slow_extract(s=s, with_gender=with_gender, prob_threshold=prob_threshold,
+                                      window_size=kwargs.get('window_size', DEFAULT_LENGTH_THRESHOLD),
                                       deduplicate=kwargs.get('deduplicate', True))

{deeplotx-0.9.5 → deeplotx-0.9.7}/deeplotx/util/read_file.py

@@ -13,6 +13,17 @@ def read_file(path: str, encoding: str = 'utf-8') -> str:
             pass
 
 
+def write_file(content: str | bytes, path: str, encoding: str = 'utf-8') -> str:
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    if isinstance(content, bytes):
+        with open(path, mode='wb') as f:
+            f.write(content)
+        return path
+    with open(path, mode='w', encoding=encoding) as f:
+        f.write(content)
+    return path
+
+
 def get_files(path: str) -> list:
     if os.path.exists(path):
         entries = os.listdir(path)

{deeplotx-0.9.5 → deeplotx-0.9.7}/deeplotx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.5
+Version: 0.9.7
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -27,6 +27,28 @@ Dynamic: license-file
 
 > Author: [vortezwohl](https://github.com/vortezwohl)
 
+## Citation
+
+If you are incorporating the `DeepLoTX` framework into your research, please remember to properly cite it to acknowledge its contribution to your work.
+
+Если вы интегрируете фреймворк `DeepLoTX` в своё исследование, пожалуйста, не забудьте правильно сослаться на него, указывая его вклад в вашу работу.
+
+もしあなたが研究に `DeepLoTX` フレームワークを組み入れているなら、その貢献を認めるために適切に引用することを忘れないでください.
+
+如果您正在將 `DeepLoTX` 框架整合到您的研究中，請務必正確引用它，以聲明它對您工作的貢獻.
+
+```bibtex
+@software{Wu_DeepLoTX_2025,
+author = {Wu, Zihao},
+license = {GPL-3.0},
+month = aug,
+title = {{DeepLoTX}},
+url = {https://github.com/vortezwohl/DeepLoTX},
+version = {0.9.5},
+year = {2025}
+}
+```
+
 ## Installation
 
 - **With pip**

{deeplotx-0.9.5 → deeplotx-0.9.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deeplotx"
-version = "0.9.5"
+version = '0.9.7'
 description = "An out-of-the-box long-text NLP framework."
 readme = "README.md"
 requires-python = ">=3.10"