deeplotx 0.9.2__py3-none-any.whl → 0.9.3__py3-none-any.whl

deeplotx/ner/base_ner.py

@@ -4,7 +4,4 @@ from deeplotx.ner.named_entity import NamedEntity, NamedPerson
 class BaseNER:
     def __init__(self): ...
 
-    def __call__(self, s: str, *args, **kwargs) -> list[NamedEntity | NamedPerson]:
-        return self.extract_entities(s=s, *args, **kwargs)
-
-    def extract_entities(self, s: str, *args, **kwargs) -> list[NamedEntity | NamedPerson]: ...
+    def __call__(self, s: str, *args, **kwargs) -> list[NamedEntity | NamedPerson]: ...

deeplotx/ner/bert_ner.py

@@ -46,9 +46,9 @@ class BertNER(BaseNER):
         self._ner_pipeline = pipeline(task='ner', model=self.encoder, tokenizer=self.tokenizer, trust_remote_code=True)
         logger.debug(f'{BaseNER.__name__} initialized on device: {self.device}.')
 
-    def extract_entities(self, s: str, with_gender: bool = True, prob_threshold: float = .0, *args, **kwargs) -> list[NamedEntity]:
+    def _fast_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0) -> list[NamedEntity]:
         assert prob_threshold <= 1., f'prob_threshold ({prob_threshold}) cannot be larger than 1.'
-        s = ' ' + s
+        s = f' {s} '
         raw_entities = self._ner_pipeline(s)
         entities = []
         for ent in raw_entities:
@@ -87,5 +87,49 @@ class BertNER(BaseNER):
                                           gender_probability=gender_prob)
         return entities
 
-    def __call__(self, s: str, with_gender: bool = True, prob_threshold: float = .0, *args, **kwargs):
-        return self.extract_entities(s=s, with_gender=with_gender, prob_threshold=prob_threshold, *args, **kwargs)
+    def _slow_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0, deduplicate: bool = True) -> list[NamedEntity]:
+        _entities = self._fast_extract(s, with_gender=with_gender, prob_threshold=prob_threshold) if len(s) < 512 else []
+        if len(s) >= 512:
+            window_size: int = 512
+            offset = window_size // 6
+            for _offset in [- offset, offset]:
+                _window_size = window_size + _offset
+                for i in range(0, len(s) + _window_size, _window_size):
+                    _entities.extend(self._fast_extract(s[i: i + _window_size], with_gender=with_gender, prob_threshold=prob_threshold))
+        _tmp_entities = sorted(_entities, key=lambda x: len(x.text), reverse=True)
+        for _ent_i in _tmp_entities:
+            for _ent_j in _entities:
+                if (_ent_j.text in _ent_i.text
+                        and len(_ent_j.text) != len(_ent_i.text)
+                        and _ent_j in _tmp_entities):
+                    _tmp_entities.remove(_ent_j)
+        while True:
+            for _ent in _tmp_entities:
+                if _ent.text not in s or len(_ent.text) < 2:
+                    _tmp_entities.remove(_ent)
+            _continue = False
+            for _ent in _tmp_entities:
+                if _ent.text not in s or len(_ent.text) < 2:
+                    _continue = True
+                    break
+            if not _continue:
+                break
+        if not deduplicate:
+            return _tmp_entities
+        _fin_entities = dict()
+        texts = set([text.text for text in _tmp_entities])
+        for text in texts:
+            for _ent in _tmp_entities:
+                if _ent.text == text:
+                    if _ent.text not in _fin_entities.keys():
+                        _fin_entities[_ent.text] = _ent
+                    else:
+                        if _ent.base_probability > _fin_entities[_ent.text].base_probability:
+                            _fin_entities[_ent.text] = _ent
+        return [v for k, v in _fin_entities.items()]
+
+    def __call__(self, s: str, with_gender: bool = True, prob_threshold: float = .0, fast_mode: bool = False, *args, **kwargs):
+        if fast_mode:
+            return self._fast_extract(s=s, with_gender=with_gender, prob_threshold=prob_threshold)
+        else:
+            return self._slow_extract(s=s, with_gender=with_gender, prob_threshold=prob_threshold, deduplicate=True)

{deeplotx-0.9.2.dist-info → deeplotx-0.9.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.2
+Version: 0.9.3
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{deeplotx-0.9.2.dist-info → deeplotx-0.9.3.dist-info}/RECORD

@@ -4,8 +4,8 @@ deeplotx/encoder/encoder.py,sha256=wVRl3p_7eg7qT_tJEit5qnmZx7dXkMVLxAtao5vImkk,4
 deeplotx/encoder/long_text_encoder.py,sha256=4oRa9FqfGNZ8-gq14UKuhDkZC0A1Xi-wKmbQsn-uZ58,3966
 deeplotx/encoder/longformer_encoder.py,sha256=7Lm65AUD3qwbrzrhJ3dPZkyHeNRSapga3f-5QJCxV5A,3538
 deeplotx/ner/__init__.py,sha256=Rss1pup9HzHZCG8U9ub8niWa9zRjWCy3Z7zg378KZQg,114
-deeplotx/ner/base_ner.py,sha256=bAp7R6mawsfO7owBONXtbPN0rzMSltMJVEGGNKhi41A,359
-deeplotx/ner/bert_ner.py,sha256=RkqHVBY4SBJtHHR0YuR006v5gFmAaKqJCCKkOOs9ulY,5458
+deeplotx/ner/base_ner.py,sha256=pZTl50OrHH_FJm4rKp9iuixeOE6FX_AzgDXD32aXsN0,204
+deeplotx/ner/bert_ner.py,sha256=I8yFsarsLEQv0vcnNU2JIc0-LuPJcxaO-mLhDFCh1PI,7704
 deeplotx/ner/named_entity.py,sha256=c6XufIwH6yloJ-ccUjagf4mBl1XbbYDT8xyEJJ_-ZNs,269
 deeplotx/ner/n2g/__init__.py,sha256=b6fOWJVLaOCtoz8Qlp8NWQbL5lUSbn6H3-8fnVNIPi0,3940
 deeplotx/nn/__init__.py,sha256=YILwbxb-NHdiJjfOwBKH8F7PuZSDZSrGpTznPDucTro,710
@@ -33,8 +33,8 @@ deeplotx/trainer/text_binary_classification_trainer.py,sha256=TFxOX8rWU_zKliI9zm
 deeplotx/util/__init__.py,sha256=5CH4MTeSgsmCe3LPMfvKoSBpwh6jDSBuHVElJvzQzgs,90
 deeplotx/util/hash.py,sha256=qbNU3RLBWGQYFVte9WZBAkZ1BkdjCXiKLDaKPN54KFk,662
 deeplotx/util/read_file.py,sha256=ptzouvEQeeW8KU5BrWNJlXw-vFXVrpS9SkAUxsu6A8A,612
-deeplotx-0.9.2.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
-deeplotx-0.9.2.dist-info/METADATA,sha256=lA_h92G6v6cT3ff94pmxVAi0LLj-qO2qrEjAVLFTYHw,13472
-deeplotx-0.9.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deeplotx-0.9.2.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
-deeplotx-0.9.2.dist-info/RECORD,,
+deeplotx-0.9.3.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+deeplotx-0.9.3.dist-info/METADATA,sha256=Fg0KzWIxFcMtuTfmuQ9BBJDFXjNTWtl9l3Cuuc1sX3I,13472
+deeplotx-0.9.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deeplotx-0.9.3.dist-info/top_level.txt,sha256=hKg4pVDXZ-WWxkRfJFczRIll1Sv7VyfKCmzHLXbuh1U,9
+deeplotx-0.9.3.dist-info/RECORD,,