arekit 0.25.0__tar.gz → 0.25.2__tar.gz

arekit-0.25.2/PKG-INFO

@@ -0,0 +1,80 @@
+Metadata-Version: 2.1
+Name: arekit
+Version: 0.25.2
+Summary: Document level Attitude and Relation Extraction toolkit (AREkit) for sampling and prompting mass-media news into datasets for ML-model training
+Home-page: https://github.com/nicolay-r/AREkit
+Author: Nicolay Rusnachenko
+Author-email: rusnicolay@gmail.com
+License: MIT License
+Description: # AREkit 0.25.2
+        
+        ![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+        [![PyPI downloads](https://img.shields.io/pypi/dm/arekit.svg)](https://pypistats.org/packages/arekit)
+        
+        
+        <p align="center">
+            <img src="logo.png"/>
+        </p>
+        
+        **AREkit** (Attitude and Relation Extraction Toolkit) --
+        is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+        
+        ## Description
+        
+        
+        This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
+        
+        <p align="center">
+            <img src="docs/arekit-pipeline-concept.png"/>
+        </p>
+        
+        > Figure: AREkit pipelines design. More on 
+        > **[ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction](https://link.springer.com/chapter/10.1007/978-3-031-56069-9_23)** paper
+        
+        In particular, this framework serves the following features: 
+        * ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+        * 🔗 EL (entity-linking) API support for objects, 
+        * ➰ avoidance of cyclic connections,
+        * :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+        * 📑 relations annotations and filtering rules,
+        * *️⃣ entities formatting or masking, and more.
+        
+        The core functionality includes: 
+        * API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+        for sentence level relations preparation (dubbed as contexts);
+        * API for contexts extraction;
+        * Relations transferring from sentence-level onto document-level, and more.
+        
+        ## Installation 
+        
+        ```bash
+        pip install git+https://github.com/nicolay-r/AREkit.git@0.25.2-rc
+        ```
+        
+        ## Usage
+        
+        Please follow the **[tutorial section on project Wiki](https://github.com/nicolay-r/AREkit/wiki/Tutorials)** for mode details.
+        
+        ## How to cite
+        A great research is also accompanied by the faithful reference. 
+        if you use or extend our work, please cite as follows:
+        
+        ```bibtex
+        @inproceedings{rusnachenko2024arelight,
+          title={ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction},
+          author={Rusnachenko, Nicolay and Liang, Huizhi and Kolomeets, Maxim and Shi, Lei},
+          booktitle={European Conference on Information Retrieval},
+          year={2024},
+          organization={Springer}
+        }
+        ```
+        
+Keywords: natural language processing,relation extraction,sentiment analysis
+Platform: UNKNOWN
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown

{arekit-0.25.0 → arekit-0.25.2}/README.md

@@ -1,6 +1,8 @@
-# AREkit 0.25.0
+# AREkit 0.25.2
 
 ![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+[![PyPI downloads](https://img.shields.io/pypi/dm/arekit.svg)](https://pypistats.org/packages/arekit)
+
 
 <p align="center">
     <img src="logo.png"/>
@@ -12,7 +14,7 @@ is a python toolkit, devoted to document level Attitude and Relation Extraction
 ## Description
 
 
-This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
 
 <p align="center">
     <img src="docs/arekit-pipeline-concept.png"/>
@@ -38,7 +40,7 @@ for sentence level relations preparation (dubbed as contexts);
 ## Installation 
 
 ```bash
-pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.2-rc
 ```
 
 ## Usage
@@ -57,4 +59,4 @@ if you use or extend our work, please cite as follows:
   year={2024},
   organization={Springer}
 }
-```
+```

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/context/terms_mapper.py

@@ -1,12 +1,15 @@
 from collections.abc import Iterable
 
 from arekit.common.context.token import Token
-from arekit.common.entities.base import Entity
 from arekit.common.frames.text_variant import TextFrameVariant
 
 
 class TextTermsMapper(object):
 
+    def __init__(self, is_entity_func):
+        assert(callable(is_entity_func))
+        self.__is_entity_func = is_entity_func
+
     def iter_mapped(self, terms):
         """ Performs mapping operation of each terms in a sequence
         """
@@ -22,7 +25,7 @@ class TextTermsMapper(object):
                 m_term = self.map_token(i, term)
             elif isinstance(term, TextFrameVariant):
                 m_term = self.map_text_frame_variant(i, term)
-            elif isinstance(term, Entity):
+            elif self.__is_entity_func(term):
                 m_term = self.map_entity(i, term)
             else:
                 raise Exception("Unsupported type {}".format(term))

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/data/input/providers/rows/samples.py

@@ -9,13 +9,11 @@ from arekit.common.data.input.providers.label.multiple import MultipleLabelProvi
 from arekit.common.data.input.providers.rows.base import BaseRowProvider
 from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
 from arekit.common.data.rows_fmt import create_base_column_fmt
-from arekit.common.entities.base import Entity
-from arekit.common.labels.base import Label
-
-from arekit.common.linkage.text_opinions import TextOpinionsLinkage
 from arekit.common.docs.parsed.base import ParsedDocument
 from arekit.common.docs.parsed.providers.entity_service import EntityEndType, EntityServiceProvider
 from arekit.common.docs.parsed.term_position import TermPositionTypes
+from arekit.common.labels.base import Label
+from arekit.common.linkage.text_opinions import TextOpinionsLinkage
 from arekit.common.text_opinions.base import TextOpinion
 
 
@@ -26,13 +24,15 @@ class BaseSampleRowProvider(BaseRowProvider):
     """ Rows provider for samples storage.
     """
 
-    def __init__(self, label_provider, text_provider):
+    def __init__(self, is_entity_func, label_provider, text_provider):
+        assert(callable(is_entity_func))
         assert(isinstance(label_provider, LabelProvider))
         assert(isinstance(text_provider, BaseSingleTextProvider))
         super(BaseSampleRowProvider, self).__init__()
 
         self._label_provider = label_provider
         self.__text_provider = text_provider
+        self.__is_entity_func = is_entity_func
         self.__instances_provider = self.__create_instances_provider(label_provider)
         self.__store_labels = None
         self._val_fmt = create_base_column_fmt(fmt_type="writer")
@@ -65,7 +65,7 @@ class BaseSampleRowProvider(BaseRowProvider):
             parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
 
         # Entity indices from the related context.
-        entities = list(filter(lambda term: isinstance(term, Entity), sentence_terms))
+        entities = list(filter(self.__is_entity_func, sentence_terms))
 
         # Values mapping.
         vm = {
@@ -76,7 +76,7 @@ class BaseSampleRowProvider(BaseRowProvider):
             const.SENT_IND: sentence_ind,
             const.ENTITY_VALUES: entities,
             const.ENTITY_TYPES: entities,
-            const.ENTITIES: [str(i) for i, t in enumerate(sentence_terms) if isinstance(t, Entity)],
+            const.ENTITIES: [str(i) for i, t in enumerate(sentence_terms) if self.__is_entity_func(t)],
             const.S_IND: actual_s_ind,
             const.T_IND: actual_t_ind,
             const.LABEL_UINT: None,
@@ -143,9 +143,6 @@ class BaseSampleRowProvider(BaseRowProvider):
 
     def __provide_rows(self, row_dict, parsed_doc, entity_service,
                        text_opinion_linkage, index_in_linked, idle_mode):
-        """
-        Providing Rows depending on row_id_formatter type
-        """
         assert(isinstance(parsed_doc, ParsedDocument))
         assert(isinstance(row_dict, OrderedDict))
         assert(isinstance(text_opinion_linkage, TextOpinionsLinkage))
@@ -153,7 +150,6 @@ class BaseSampleRowProvider(BaseRowProvider):
         etalon_label = self.__instances_provider.provide_label(text_opinion_linkage)
         for instance in self.__instances_provider.iter_instances(text_opinion_linkage):
             yield self.__create_row(row=row_dict,
-                                    row_id=0,
                                     parsed_doc=parsed_doc,
                                     entity_service=entity_service,
                                     text_opinions_linkage=instance,
@@ -162,7 +158,7 @@ class BaseSampleRowProvider(BaseRowProvider):
                                     etalon_label=etalon_label,
                                     idle_mode=idle_mode)
 
-    def __create_row(self, row, row_id, parsed_doc, entity_service, text_opinions_linkage,
+    def __create_row(self, row, parsed_doc, entity_service, text_opinions_linkage,
                      index_in_linked, etalon_label, idle_mode):
         """
         Composing row in following format:

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/data/input/providers/sample/cropped.py

@@ -8,10 +8,11 @@ class CroppedSampleRowProvider(BaseSampleRowProvider):
         attitude inside.
     """
 
-    def __init__(self, crop_window_size, label_scaler, text_provider):
+    def __init__(self, crop_window_size, label_scaler, **kwargs):
         assert(isinstance(crop_window_size, int) and crop_window_size > 0)
-        super(CroppedSampleRowProvider, self).__init__(label_provider=MultipleLabelProvider(label_scaler),
-                                                       text_provider=text_provider)
+        super(CroppedSampleRowProvider, self).__init__(
+            label_provider=MultipleLabelProvider(label_scaler),
+            **kwargs)
         self.__crop_window_size = crop_window_size
 
     @staticmethod

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/data/input/terms_mapper.py

@@ -1,6 +1,5 @@
 from arekit.common.context.terms_mapper import TextTermsMapper
 from arekit.common.context.token import Token
-from arekit.common.entities.base import Entity
 from arekit.common.entities.str_fmt import StringEntitiesFormatter
 from arekit.common.entities.types import OpinionEntityType
 from arekit.common.frames.text_variant import TextFrameVariant
@@ -12,9 +11,12 @@ class OpinionContainingTextTermsMapper(TextTermsMapper):
     The latter might be utilized with synonyms collection
     """
 
-    def __init__(self, entity_formatter):
+    def __init__(self, entity_formatter, entity_group_ind_func, **kwargs):
         assert(isinstance(entity_formatter, StringEntitiesFormatter))
+        assert(callable(entity_group_ind_func))
+        super(OpinionContainingTextTermsMapper, self).__init__(**kwargs)
         self.__entities_formatter = entity_formatter
+        self.__syn_group = entity_group_ind_func
         self.__s_ind = None
         self.__t_ind = None
         self.__s_group = None
@@ -24,12 +26,6 @@ class OpinionContainingTextTermsMapper(TextTermsMapper):
     def StringEntitiesFormatter(self):
         return self.__entities_formatter
 
-    def __syn_group(self, entity):
-        """ Note: here we guarantee that entity has GroupIndex.
-        """
-        assert(isinstance(entity, Entity))
-        return entity.GroupIndex if entity is not None else None
-
     def set_s_ind(self, s_ind):
         assert(isinstance(s_ind, int))
         self.__s_ind = s_ind

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/data/storages/base.py

@@ -10,6 +10,9 @@ logger = logging.getLogger(__name__)
 
 class BaseRowsStorage(object):
 
+    def __init__(self, log_out=None):
+        self.__log_out = log_out
+
     # region protected methods
 
     def _begin_filling_row(self, row_ind):
@@ -31,27 +34,9 @@ class BaseRowsStorage(object):
     def _get_rows_count(self):
         raise NotImplemented()
 
-    def find_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def find_first_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def iter_column_values(self, column_name, dtype=None):
-        raise NotImplemented()
-
-    def get_row(self, row_index):
-        raise NotImplemented()
-
-    def get_cell(self, row_index, column_name):
-        raise NotImplemented()
-
     def init_empty(self, columns_provider):
         raise NotImplemented()
 
-    def iter_shuffled(self):
-        raise NotImplemented()
-
     def iter_column_names(self):
         raise NotImplemented()
 
@@ -81,6 +66,7 @@ class BaseRowsStorage(object):
             condition_func=lambda item: not isinstance(item[1], MetaEmptyLinkedDataWrapper),
             postfix_func=postfix_func,
             desc="{fmt}".format(fmt=desc),
+            file=self.__log_out,
             total=rows_count)
 
         for row_index, item in enumerate(pbar_it):

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/docs/entities_grouping.py

@@ -1,17 +1,19 @@
-from arekit.common.entities.base import Entity
 from arekit.common.pipeline.items.base import BasePipelineItem
 
 
 class EntitiesGroupingPipelineItem(BasePipelineItem):
 
-    def __init__(self, value_to_group_id_func, **kwargs):
+    def __init__(self, value_to_group_id_func, is_entity_func, **kwargs):
         assert(callable(value_to_group_id_func))
+        assert(callable(is_entity_func))
         super(EntitiesGroupingPipelineItem, self).__init__(**kwargs)
+
         self.__value_to_group_id_func = value_to_group_id_func
+        self.__is_entity_func = is_entity_func
 
     def apply_core(self, input_data, pipeline_ctx):
         assert(isinstance(input_data, list))
 
-        for entity in filter(lambda term: isinstance(term, Entity), input_data):
+        for entity in filter(lambda term: self.__is_entity_func(term), input_data):
             group_index = self.__value_to_group_id_func(entity.Value)
             entity.set_group_index(group_index)

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/docs/parsed/base.py

@@ -1,6 +1,5 @@
 from collections.abc import Iterable
 
-from arekit.common.entities.base import Entity
 from arekit.common.text.enums import TermFormat
 from arekit.common.text.parsed import BaseParsedText
 
@@ -73,8 +72,9 @@ class ParsedDocument(object):
         assert(isinstance(s_ind, int))
         return self.__parsed_sentences[s_ind]
 
-    def iter_entities(self):
-        for entity in self.__iter_all_raw_terms(term_only=True, filter_func=lambda t: isinstance(t, Entity)):
+    def iter_entities(self, is_entity_func):
+        assert(callable(is_entity_func))
+        for entity in self.__iter_all_raw_terms(term_only=True, filter_func=is_entity_func):
             yield entity
 
     def iter_terms(self, filter_func=None, term_only=True):

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/docs/parsed/providers/base.py

@@ -1,4 +1,3 @@
-from arekit.common.entities.base import Entity
 from arekit.common.docs.entity import DocumentEntity
 from arekit.common.docs.parsed.base import ParsedDocument
 
@@ -6,7 +5,7 @@ from arekit.common.docs.parsed.base import ParsedDocument
 class BaseParsedDocumentServiceProvider(object):
 
     def __init__(self, entity_index_func=None):
-        """ Outside enity indexing function
+        """ Outside entity indexing function
             entity_index_func: provides id for a given entity, i.e.
                 func(entity) -> int (id)
         """
@@ -19,7 +18,7 @@ class BaseParsedDocumentServiceProvider(object):
     def Name(self):
         raise NotImplementedError()
 
-    def init_parsed_doc(self, parsed_doc):
+    def init_parsed_doc(self, parsed_doc, is_entity_func):
         assert(isinstance(parsed_doc, ParsedDocument))
 
         def __iter_childs_and_root_node(entity):
@@ -37,7 +36,7 @@ class BaseParsedDocumentServiceProvider(object):
         self.__entity_map.clear()
 
         current_id = 0
-        for _, entity in enumerate(parsed_doc.iter_entities()):
+        for _, entity in enumerate(parsed_doc.iter_entities(is_entity_func=is_entity_func)):
 
             child_doc_entities = []
             for tree_entity, is_child in __iter_childs_and_root_node(entity):
@@ -61,7 +60,6 @@ class BaseParsedDocumentServiceProvider(object):
     def get_document_entity(self, entity):
         """ Maps entity to the related one with DocumentEntity type
         """
-        assert(isinstance(entity, Entity))
         return self.__entity_map[self.__entity_index_func(entity)]
 
     def contains_entity(self, entity):

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/docs/parsed/providers/entity_service.py

@@ -1,8 +1,6 @@
 from enum import Enum
 
-from arekit.common.entities.base import Entity
 from arekit.common.docs.entity import DocumentEntity
-from arekit.common.docs.parsed.base import ParsedDocument
 from arekit.common.docs.parsed.providers.base import BaseParsedDocumentServiceProvider
 from arekit.common.docs.parsed.term_position import TermPositionTypes, TermPosition
 from arekit.common.text_opinions.base import TextOpinion
@@ -41,9 +39,8 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
 
     NAME = "entity-service-provider"
 
-    def __init__(self, entity_index_func):
-        assert(callable(entity_index_func))
-        super(EntityServiceProvider, self).__init__(entity_index_func=entity_index_func)
+    def __init__(self, **kwargs):
+        super(EntityServiceProvider, self).__init__(**kwargs)
         # Initialize API.
         self.__iter_raw_terms_func = None
         # Initialize entity positions.
@@ -53,24 +50,16 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
     def Name(self):
         return self.NAME
 
-    def init_parsed_doc(self, parsed_doc):
-        super(EntityServiceProvider, self).init_parsed_doc(parsed_doc)
-        assert(isinstance(parsed_doc, ParsedDocument))
+    def init_parsed_doc(self, parsed_doc, is_entity_func):
+        super(EntityServiceProvider, self).init_parsed_doc(parsed_doc=parsed_doc, is_entity_func=is_entity_func)
         self.__iter_raw_terms_func = lambda: parsed_doc.iter_terms(filter_func=None, term_only=False)
-        self.__entity_positions = self.__calculate_entity_positions()
-
-    # region public 'extract' methods
-
-    def extract_entity_value(self, text_opinion, end_type):
-        return self.__extract_entity_value(text_opinion=text_opinion, end_type=end_type)
+        self.__entity_positions = self.__calculate_entity_positions(is_entity_func=is_entity_func)
 
     def extract_entity_position(self, text_opinion, end_type, position_type=None):
         return self.__get_entity_position(text_opinion=text_opinion,
                                           end_type=end_type,
                                           position_type=position_type)
 
-    # endregion
-
     # region public 'calculate' methods
 
     @staticmethod
@@ -112,20 +101,10 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
 
         return e_pos.get_index(position_type)
 
-    def get_entity_value(self, id_in_document):
-        entity = self._doc_entities[id_in_document]
-        assert(isinstance(entity, Entity))
-        return entity.Value
-
     # endregion
 
     # region private methods
 
-    def __extract_entity_value(self, text_opinion, end_type):
-        assert(isinstance(text_opinion, TextOpinion))
-        end_id = self.__get_end_id(text_opinion=text_opinion, end_type=end_type)
-        return self.get_entity_value(end_id)
-
     def __get_entity_position(self, text_opinion, end_type, position_type=None):
         assert(isinstance(text_opinion, TextOpinion))
         end_id = self.__get_end_id(text_opinion=text_opinion, end_type=end_type)
@@ -147,7 +126,7 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
         assert(end_type == EntityEndType.Source or end_type == EntityEndType.Target)
         return text_opinion.SourceId if end_type == EntityEndType.Source else text_opinion.TargetId
 
-    def __calculate_entity_positions(self):
+    def __calculate_entity_positions(self, is_entity_func):
         """ Note: here we consider the same order as in self._entities.
         """
         t_ind_in_doc = -1
@@ -157,7 +136,7 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
 
             t_ind_in_doc += 1
 
-            if not isinstance(term, Entity):
+            if not is_entity_func(term):
                 continue
 
             # We consider that entities within a single tree has the same positions.

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/docs/parsed/providers/opinion_pairs.py

@@ -1,4 +1,3 @@
-from arekit.common.entities.base import Entity
 from arekit.common.docs.parsed.providers.base_pairs import BasePairProvider
 from arekit.common.opinions.base import Opinion
 
@@ -7,14 +6,15 @@ class OpinionPairsProvider(BasePairProvider):
 
     NAME = "opinion-pairs-provider"
 
+    def __init__(self, entity_value_func, **kwargs):
+        super(OpinionPairsProvider, self).__init__(**kwargs)
+        self.__entity_value_func = entity_value_func
+
     @property
     def Name(self):
         return self.NAME
 
     def _create_pair(self, source_entity, target_entity, label):
-        assert(isinstance(source_entity, Entity))
-        assert(isinstance(target_entity, Entity))
-
-        return Opinion(source_value=source_entity.Value,
-                       target_value=target_entity.Value,
+        return Opinion(source_value=self.__entity_value_func(source_entity),
+                       target_value=self.__entity_value_func(target_entity),
                        label=label)

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/docs/parsed/providers/text_opinion_pairs.py

@@ -16,8 +16,8 @@ class TextOpinionPairsProvider(BasePairProvider):
 
     NAME = "text-opinion-pairs-provider"
 
-    def __init__(self, value_to_group_id_func):
-        super(TextOpinionPairsProvider, self).__init__()
+    def __init__(self, value_to_group_id_func, **kwargs):
+        super(TextOpinionPairsProvider, self).__init__(**kwargs)
         self.__value_to_group_id_func = value_to_group_id_func
         self.__doc_id = None
         self.__entities_collection = None
@@ -36,8 +36,8 @@ class TextOpinionPairsProvider(BasePairProvider):
                            label=label,
                            text_opinion_id=None)
 
-    def init_parsed_doc(self, parsed_doc):
-        super(TextOpinionPairsProvider, self).init_parsed_doc(parsed_doc)
+    def init_parsed_doc(self, parsed_doc, is_entity_func):
+        super(TextOpinionPairsProvider, self).init_parsed_doc(parsed_doc=parsed_doc, is_entity_func=is_entity_func)
         self.__doc_id = parsed_doc.RelatedDocID
         self.__entities_collection = EntityCollection(
             entities=list(self._doc_entities),

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/docs/parsed/service.py

@@ -6,7 +6,7 @@ class ParsedDocumentService(object):
     """ Represents a collection of providers, combined with the parsed doc.
     """
 
-    def __init__(self, parsed_doc, providers):
+    def __init__(self, parsed_doc, providers, is_entity_func):
         assert(isinstance(parsed_doc, ParsedDocument))
         assert(isinstance(providers, list))
         self.__parsed_doc = parsed_doc
@@ -20,7 +20,7 @@ class ParsedDocumentService(object):
             self.__providers[provider.Name] = provider
 
             # Post initialize with the related parsed doc.
-            provider.init_parsed_doc(self.__parsed_doc)
+            provider.init_parsed_doc(self.__parsed_doc, is_entity_func=is_entity_func)
 
 
     @property

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/docs/parser.py

@@ -1,42 +1,14 @@
-from tqdm import tqdm
 from arekit.common.docs.base import Document
 from arekit.common.docs.parsed.base import ParsedDocument
-from arekit.common.pipeline.base import BasePipelineLauncher
 from arekit.common.pipeline.batching import BatchingPipelineLauncher
 from arekit.common.pipeline.context import PipelineContext
 from arekit.common.pipeline.utils import BatchIterator
 from arekit.common.text.parsed import BaseParsedText
+from arekit.common.utils import progress_bar_defined
 
 
 class DocumentParsers(object):
 
-    @staticmethod
-    def parse(doc, pipeline_items, parent_ppl_ctx=None, src_key="input", show_progress=False):
-        """ This document parser is based on single text parts (sentences)
-            that passes sequentially through the pipeline of transformations.
-        """
-        assert(isinstance(doc, Document))
-        assert(isinstance(pipeline_items, list))
-        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
-
-        parsed_sentences = []
-
-        data_it = range(doc.SentencesCount)
-        progress_it = tqdm(data_it, disable=not show_progress)
-
-        for sent_ind in progress_it:
-
-            # Composing the context from a single sentence.
-            ctx = PipelineContext({src_key: doc.get_sentence(sent_ind)}, parent_ctx=parent_ppl_ctx)
-
-            # Apply all the operations.
-            BasePipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key=src_key)
-
-            # Collecting the result.
-            parsed_sentences.append(BaseParsedText(terms=ctx.provide("result")))
-
-        return ParsedDocument(doc_id=doc.ID, parsed_sentences=parsed_sentences)
-
     @staticmethod
     def parse_batch(doc, pipeline_items, batch_size, parent_ppl_ctx=None, src_key="input", show_progress=False):
         """ This document parser is based on batch of sentences.
@@ -49,7 +21,8 @@ class DocumentParsers(object):
         parsed_sentences = []
 
         data_it = BatchIterator(data_iter=iter(range(doc.SentencesCount)), batch_size=batch_size)
-        progress_it = tqdm(data_it, total=round(doc.SentencesCount / batch_size), disable=not show_progress)
+        progress_it = progress_bar_defined(data_it, total=round(doc.SentencesCount / batch_size),
+                                           disable=not show_progress)
 
         for batch in progress_it:
 

{arekit-0.25.0 → arekit-0.25.2}/arekit/common/model/labeling/single.py

@@ -1,11 +1,15 @@
-import numpy as np
-
 from arekit.common.model.labeling.base import LabelsHelper
 from arekit.common.model.labeling.modes import LabelCalculationMode
 
 
 class SingleLabelsHelper(LabelsHelper):
 
+    @staticmethod
+    def __sign(x):
+        if x == 0:
+            return 0
+        return -1 if x < 0 else 1
+
     def aggregate_labels(self, labels_list, label_calc_mode):
         assert(isinstance(labels_list, list))
         assert(isinstance(label_calc_mode, LabelCalculationMode))
@@ -18,7 +22,7 @@ class SingleLabelsHelper(LabelsHelper):
         if label_calc_mode == LabelCalculationMode.AVERAGE:
             int_labels = [self._label_scaler.label_to_int(label)
                           for label in labels_list]
-            label = self._label_scaler.int_to_label(int(np.sign(sum(int_labels))))
+            label = self._label_scaler.int_to_label(SingleLabelsHelper.__sign(sum(int_labels)))
 
         return label