deeplotx 0.9.2__tar.gz → 0.9.4__tar.gz

{deeplotx-0.9.2 → deeplotx-0.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.2
+Version: 0.9.4
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

deeplotx-0.9.4/deeplotx/ner/base_ner.py

@@ -0,0 +1,7 @@
+from deeplotx.ner.named_entity import NamedEntity, NamedPerson
+
+
+class BaseNER:
+    def __init__(self): ...
+
+    def __call__(self, s: str, *args, **kwargs) -> list[NamedEntity | NamedPerson]: ...

{deeplotx-0.9.2 → deeplotx-0.9.4}/deeplotx/ner/bert_ner.py

@@ -46,9 +46,9 @@ class BertNER(BaseNER):
         self._ner_pipeline = pipeline(task='ner', model=self.encoder, tokenizer=self.tokenizer, trust_remote_code=True)
         logger.debug(f'{BaseNER.__name__} initialized on device: {self.device}.')
 
-    def extract_entities(self, s: str, with_gender: bool = True, prob_threshold: float = .0, *args, **kwargs) -> list[NamedEntity]:
+    def _fast_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0) -> list[NamedEntity]:
         assert prob_threshold <= 1., f'prob_threshold ({prob_threshold}) cannot be larger than 1.'
-        s = ' ' + s
+        s = f' {s} '
         raw_entities = self._ner_pipeline(s)
         entities = []
         for ent in raw_entities:
@@ -87,5 +87,50 @@ class BertNER(BaseNER):
                                           gender_probability=gender_prob)
         return entities
 
-    def __call__(self, s: str, with_gender: bool = True, prob_threshold: float = .0, *args, **kwargs):
-        return self.extract_entities(s=s, with_gender=with_gender, prob_threshold=prob_threshold, *args, **kwargs)
+    def _slow_extract(self, s: str, with_gender: bool = True, prob_threshold: float = .0, deduplicate: bool = True) -> list[NamedEntity]:
+        _entities = self._fast_extract(s, with_gender=with_gender, prob_threshold=prob_threshold) if len(s) < 512 else []
+        if len(s) >= 512:
+            window_size: int = 512
+            offset = window_size // 6
+            for _offset in [- offset, offset]:
+                _window_size = window_size + _offset
+                for i in range(0, len(s) + _window_size, _window_size):
+                    _entities.extend(self._fast_extract(s[i: i + _window_size], with_gender=with_gender, prob_threshold=prob_threshold))
+        _tmp_entities = sorted(_entities, key=lambda x: len(x.text), reverse=True)
+        for _ent_i in _tmp_entities:
+            for _ent_j in _entities:
+                if (_ent_j.text in _ent_i.text
+                        and len(_ent_j.text) != len(_ent_i.text)
+                        and _ent_j in _tmp_entities):
+                    _tmp_entities.remove(_ent_j)
+        while True:
+            for _ent in _tmp_entities:
+                if _ent.text not in s or len(_ent.text) < 2:
+                    _tmp_entities.remove(_ent)
+            _continue = False
+            for _ent in _tmp_entities:
+                if _ent.text not in s or len(_ent.text) < 2:
+                    _continue = True
+                    break
+            if not _continue:
+                break
+        if not deduplicate:
+            return sorted(_tmp_entities, key=lambda _: _.text[0], reverse=False)
+        _fin_entities = dict()
+        texts = set([text.text for text in _tmp_entities])
+        for text in texts:
+            for _ent in _tmp_entities:
+                if _ent.text == text:
+                    if _ent.text not in _fin_entities.keys():
+                        _fin_entities[_ent.text] = _ent
+                    else:
+                        if _ent.base_probability > _fin_entities[_ent.text].base_probability:
+                            _fin_entities[_ent.text] = _ent
+        return sorted([v for k, v in _fin_entities.items()], key=lambda _: _.text[0], reverse=False)
+
+    def __call__(self, s: str, with_gender: bool = True, prob_threshold: float = .0, fast_mode: bool = False, *args, **kwargs):
+        if fast_mode:
+            return self._fast_extract(s=s, with_gender=with_gender, prob_threshold=prob_threshold)
+        else:
+            return self._slow_extract(s=s, with_gender=with_gender, prob_threshold=prob_threshold,
+                                      deduplicate=kwargs.get('deduplicate', True))

{deeplotx-0.9.2 → deeplotx-0.9.4}/deeplotx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeplotx
-Version: 0.9.2
+Version: 0.9.4
 Summary: An out-of-the-box long-text NLP framework.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{deeplotx-0.9.2 → deeplotx-0.9.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deeplotx"
-version = "0.9.2"
+version = "0.9.4"
 description = "An out-of-the-box long-text NLP framework."
 readme = "README.md"
 requires-python = ">=3.10"

deeplotx-0.9.2/deeplotx/ner/base_ner.py

@@ -1,10 +0,0 @@
-from deeplotx.ner.named_entity import NamedEntity, NamedPerson
-
-
-class BaseNER:
-    def __init__(self): ...
-
-    def __call__(self, s: str, *args, **kwargs) -> list[NamedEntity | NamedPerson]:
-        return self.extract_entities(s=s, *args, **kwargs)
-
-    def extract_entities(self, s: str, *args, **kwargs) -> list[NamedEntity | NamedPerson]: ...