deeplotx 0.9.4__py3-none-any.whl → 0.9.6__py3-none-any.whl

deeplotx/ner/bert_ner.py

@@ -11,6 +11,8 @@ from deeplotx.ner.base_ner import BaseNER
 from deeplotx.ner.named_entity import NamedEntity, NamedPerson
 
 CACHE_PATH = os.path.join(__ROOT__, '.cache')
+NEW_LINE, BLANK = '\n', ' '
+DEFAULT_LENGTH_THRESHOLD = 448
 DEFAULT_BERT_NER = 'Davlan/xlm-roberta-base-ner-hrl'
 N2G_MODEL: list[Name2Gender] = []
 logger = logging.getLogger('deeplotx.ner')
@@ -44,11 +46,11 @@ class BertNER(BaseNER):
                                                                            trust_remote_code=True, local_files_only=True).to(self.device)
         self.embed_dim = self.encoder.config.max_position_embeddings
         self._ner_pipeline = pipeline(task='ner', model=self.encoder, tokenizer=self.tokenizer, trust_remote_code=True)
-        logger.debug(f'{BaseNER.__name__} initialized on device: {self.device}.')
+        logger.debug(f'{BertNER.__name__} initialized on device: {self.device}.')
 
     def _fast_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0) -> list[NamedEntity]:
         assert prob_threshold <= 1., f'prob_threshold ({prob_threshold}) cannot be larger than 1.'
-        s = f' {s} '
+        s = f' {s.replace(NEW_LINE, BLANK * 2)} '
         raw_entities = self._ner_pipeline(s)
         entities = []
         for ent in raw_entities:
@@ -58,7 +60,10 @@ class BertNER(BaseNER):
                 if len(ent[0].strip()) < 1:
                     del entities[i]
                 if ent[1].upper().startswith('I') and entities[i - 1][1].upper().startswith('B'):
-                    entities[i - 1][0] += ent[0]
+                    if entities[i - 1][0] + ent[0] in s:
+                        entities[i - 1][0] += ent[0]
+                    else:
+                        entities[i - 1][0] += f' {ent[0]}'
                     entities[i - 1][2] *= ent[2]
                     del entities[i]
             _continue = False
@@ -69,8 +74,21 @@ class BertNER(BaseNER):
                 break
         for ent in entities:
             ent[0] = ent[0].strip()
+            # stripping
+            while not ent[0][0].isalpha():
+                if len(ent[0]) < 2:
+                    break
+                if not ent[0][0].isnumeric():
+                    ent[0] = ent[0][1:]
+            while not ent[0][-1].isalpha():
+                if len(ent[0]) < 2:
+                    break
+                if not ent[0][-1].isnumeric():
+                    ent[0] = ent[0][:-1]
             if ent[1].upper().startswith('B'):
                 ent[1] = ent[1].upper()[1:].strip('-')
+        if len(entities) > 0:
+            logger.debug(f'Entities: {[_[0] for _ in entities]}, extracted from: "{s.strip()}".')
         entities = [NamedEntity(*_) for _ in entities if _[2] >= prob_threshold]
         if not with_gender:
             return entities
@@ -87,15 +105,22 @@ class BertNER(BaseNER):
                                           gender_probability=gender_prob)
         return entities
 
-    def _slow_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0, deduplicate: bool = True) -> list[NamedEntity]:
-        _entities = self._fast_extract(s, with_gender=with_gender, prob_threshold=prob_threshold) if len(s) < 512 else []
-        if len(s) >= 512:
-            window_size: int = 512
-            offset = window_size // 6
-            for _offset in [- offset, offset]:
-                _window_size = window_size + _offset
-                for i in range(0, len(s) + _window_size, _window_size):
-                    _entities.extend(self._fast_extract(s[i: i + _window_size], with_gender=with_gender, prob_threshold=prob_threshold))
+    def _slow_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0,
+                      window_size: int = DEFAULT_LENGTH_THRESHOLD, deduplicate: bool = True) -> list[NamedEntity]:
+        _s_seq = self.tokenizer.encode(s, add_special_tokens=False)
+        _entities = self._fast_extract(self.tokenizer.decode(_s_seq, skip_special_tokens=True),
+                                       with_gender=with_gender,
+                                       prob_threshold=prob_threshold) if len(_s_seq) < window_size else []
+        # sliding window extracting
+        if len(_s_seq) >= window_size:
+            _stride = window_size // 4
+            for i in range(0, len(_s_seq) + window_size, _stride):
+                _tmp_s_seq = _s_seq[i: i + window_size]
+                if len(_tmp_s_seq) < 1:
+                    continue
+                _window_text = self.tokenizer.decode(_tmp_s_seq, skip_special_tokens=True)
+                _entities.extend(self._fast_extract(_window_text, with_gender=with_gender, prob_threshold=prob_threshold))
+        # entity combination
         _tmp_entities = sorted(_entities, key=lambda x: len(x.text), reverse=True)
         for _ent_i in _tmp_entities:
             for _ent_j in _entities:
@@ -103,6 +128,7 @@ class BertNER(BaseNER):
                         and len(_ent_j.text) != len(_ent_i.text)
                         and _ent_j in _tmp_entities):
                     _tmp_entities.remove(_ent_j)
+        # entity cleaning
         while True:
             for _ent in _tmp_entities:
                 if _ent.text not in s or len(_ent.text) < 2:
@@ -116,6 +142,7 @@ class BertNER(BaseNER):
                 break
         if not deduplicate:
             return sorted(_tmp_entities, key=lambda _: _.text[0], reverse=False)
+        # entity deduplication
         _fin_entities = dict()
         texts = set([text.text for text in _tmp_entities])
         for text in texts:
@@ -133,4 +160,5 @@ class BertNER(BaseNER):
             return self._fast_extract(s=s, with_gender=with_gender, prob_threshold=prob_threshold)
         else:
             return self._slow_extract(s=s, with_gender=with_gender, prob_threshold=prob_threshold,
+                                      window_size=kwargs.get('window_size', DEFAULT_LENGTH_THRESHOLD),
                                       deduplicate=kwargs.get('deduplicate', True))

{deeplotx-0.9.4.dist-info → deeplotx-0.9.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.4
+Version: 0.9.6
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -27,6 +27,28 @@ Dynamic: license-file
 
 > Author: [vortezwohl](https://github.com/vortezwohl)
 
+## Citation
+
+If you are incorporating the `DeepLoTX` framework into your research, please remember to properly cite it to acknowledge its contribution to your work.
+
+Если вы интегрируете фреймворк `DeepLoTX` в своё исследование, пожалуйста, не забудьте правильно сослаться на него, указывая его вклад в вашу работу.
+
+もしあなたが研究に `DeepLoTX` フレームワークを組み入れているなら、その貢献を認めるために適切に引用することを忘れないでください.
+
+如果您正在將 `DeepLoTX` 框架整合到您的研究中，請務必正確引用它，以聲明它對您工作的貢獻.
+
+```bibtex
+@software{Wu_DeepLoTX_2025,
+author = {Wu, Zihao},
+license = {GPL-3.0},
+month = aug,
+title = {{DeepLoTX}},
+url = {https://github.com/vortezwohl/DeepLoTX},
+version = {0.9.5},
+year = {2025}
+}
+```
+
 ## Installation
 
 - **With pip**

{deeplotx-0.9.4.dist-info → deeplotx-0.9.6.dist-info}/RECORD

@@ -5,7 +5,7 @@ deeplotx/encoder/long_text_encoder.py,sha256=4oRa9FqfGNZ8-gq14UKuhDkZC0A1Xi-wKmb
 deeplotx/encoder/longformer_encoder.py,sha256=7Lm65AUD3qwbrzrhJ3dPZkyHeNRSapga3f-5QJCxV5A,3538
 deeplotx/ner/__init__.py,sha256=Rss1pup9HzHZCG8U9ub8niWa9zRjWCy3Z7zg378KZQg,114
 deeplotx/ner/base_ner.py,sha256=pZTl50OrHH_FJm4rKp9iuixeOE6FX_AzgDXD32aXsN0,204
-deeplotx/ner/bert_ner.py,sha256=bp2oFhNdyiI_VORNBppo8HDc3SI0YBsWH0IrZIMXkk4,7866
+deeplotx/ner/bert_ner.py,sha256=_IpR2AqXzEP_QBUj8KKVS7jO9mHZyFGDxr7q1LUgJf4,9246
 deeplotx/ner/named_entity.py,sha256=c6XufIwH6yloJ-ccUjagf4mBl1XbbYDT8xyEJJ_-ZNs,269
 deeplotx/ner/n2g/__init__.py,sha256=b6fOWJVLaOCtoz8Qlp8NWQbL5lUSbn6H3-8fnVNIPi0,3940
 deeplotx/nn/__init__.py,sha256=YILwbxb-NHdiJjfOwBKH8F7PuZSDZSrGpTznPDucTro,710
@@ -33,8 +33,8 @@ deeplotx/trainer/text_binary_classification_trainer.py,sha256=TFxOX8rWU_zKliI9zm
 deeplotx/util/__init__.py,sha256=5CH4MTeSgsmCe3LPMfvKoSBpwh6jDSBuHVElJvzQzgs,90
 deeplotx/util/hash.py,sha256=qbNU3RLBWGQYFVte9WZBAkZ1BkdjCXiKLDaKPN54KFk,662
 deeplotx/util/read_file.py,sha256=ptzouvEQeeW8KU5BrWNJlXw-vFXVrpS9SkAUxsu6A8A,612
-deeplotx-0.9.4.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.9.4.dist-info/METADATA,sha256=FkHL9xnVxqKEE22X2Dpfcf6RJlHhVkMJoFXVwa7ufGA,13472
-deeplotx-0.9.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deeplotx-0.9.4.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.9.4.dist-info/RECORD,,
+deeplotx-0.9.6.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.9.6.dist-info/METADATA,sha256=yjG3QmoZfUJwenoR76b7MXAjzCt7Qh9UMr1CishZFG4,14442
+deeplotx-0.9.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deeplotx-0.9.6.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.9.6.dist-info/RECORD,,