arekit 0.25.0__py3-none-any.whl → 0.25.2__py3-none-any.whl

arekit/common/context/terms_mapper.py

@@ -1,12 +1,15 @@
 from collections.abc import Iterable
 
 from arekit.common.context.token import Token
-from arekit.common.entities.base import Entity
 from arekit.common.frames.text_variant import TextFrameVariant
 
 
 class TextTermsMapper(object):
 
+    def __init__(self, is_entity_func):
+        assert(callable(is_entity_func))
+        self.__is_entity_func = is_entity_func
+
     def iter_mapped(self, terms):
         """ Performs mapping operation of each terms in a sequence
         """
@@ -22,7 +25,7 @@ class TextTermsMapper(object):
                 m_term = self.map_token(i, term)
             elif isinstance(term, TextFrameVariant):
                 m_term = self.map_text_frame_variant(i, term)
-            elif isinstance(term, Entity):
+            elif self.__is_entity_func(term):
                 m_term = self.map_entity(i, term)
             else:
                 raise Exception("Unsupported type {}".format(term))

arekit/common/data/input/providers/rows/samples.py

@@ -9,13 +9,11 @@ from arekit.common.data.input.providers.label.multiple import MultipleLabelProvi
 from arekit.common.data.input.providers.rows.base import BaseRowProvider
 from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
 from arekit.common.data.rows_fmt import create_base_column_fmt
-from arekit.common.entities.base import Entity
-from arekit.common.labels.base import Label
-
-from arekit.common.linkage.text_opinions import TextOpinionsLinkage
 from arekit.common.docs.parsed.base import ParsedDocument
 from arekit.common.docs.parsed.providers.entity_service import EntityEndType, EntityServiceProvider
 from arekit.common.docs.parsed.term_position import TermPositionTypes
+from arekit.common.labels.base import Label
+from arekit.common.linkage.text_opinions import TextOpinionsLinkage
 from arekit.common.text_opinions.base import TextOpinion
 
 
@@ -26,13 +24,15 @@ class BaseSampleRowProvider(BaseRowProvider):
     """ Rows provider for samples storage.
     """
 
-    def __init__(self, label_provider, text_provider):
+    def __init__(self, is_entity_func, label_provider, text_provider):
+        assert(callable(is_entity_func))
         assert(isinstance(label_provider, LabelProvider))
         assert(isinstance(text_provider, BaseSingleTextProvider))
         super(BaseSampleRowProvider, self).__init__()
 
         self._label_provider = label_provider
         self.__text_provider = text_provider
+        self.__is_entity_func = is_entity_func
         self.__instances_provider = self.__create_instances_provider(label_provider)
         self.__store_labels = None
         self._val_fmt = create_base_column_fmt(fmt_type="writer")
@@ -65,7 +65,7 @@ class BaseSampleRowProvider(BaseRowProvider):
             parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
 
         # Entity indices from the related context.
-        entities = list(filter(lambda term: isinstance(term, Entity), sentence_terms))
+        entities = list(filter(self.__is_entity_func, sentence_terms))
 
         # Values mapping.
         vm = {
@@ -76,7 +76,7 @@ class BaseSampleRowProvider(BaseRowProvider):
             const.SENT_IND: sentence_ind,
             const.ENTITY_VALUES: entities,
             const.ENTITY_TYPES: entities,
-            const.ENTITIES: [str(i) for i, t in enumerate(sentence_terms) if isinstance(t, Entity)],
+            const.ENTITIES: [str(i) for i, t in enumerate(sentence_terms) if self.__is_entity_func(t)],
             const.S_IND: actual_s_ind,
             const.T_IND: actual_t_ind,
             const.LABEL_UINT: None,
@@ -143,9 +143,6 @@ class BaseSampleRowProvider(BaseRowProvider):
 
     def __provide_rows(self, row_dict, parsed_doc, entity_service,
                        text_opinion_linkage, index_in_linked, idle_mode):
-        """
-        Providing Rows depending on row_id_formatter type
-        """
         assert(isinstance(parsed_doc, ParsedDocument))
         assert(isinstance(row_dict, OrderedDict))
         assert(isinstance(text_opinion_linkage, TextOpinionsLinkage))
@@ -153,7 +150,6 @@ class BaseSampleRowProvider(BaseRowProvider):
         etalon_label = self.__instances_provider.provide_label(text_opinion_linkage)
         for instance in self.__instances_provider.iter_instances(text_opinion_linkage):
             yield self.__create_row(row=row_dict,
-                                    row_id=0,
                                     parsed_doc=parsed_doc,
                                     entity_service=entity_service,
                                     text_opinions_linkage=instance,
@@ -162,7 +158,7 @@ class BaseSampleRowProvider(BaseRowProvider):
                                     etalon_label=etalon_label,
                                     idle_mode=idle_mode)
 
-    def __create_row(self, row, row_id, parsed_doc, entity_service, text_opinions_linkage,
+    def __create_row(self, row, parsed_doc, entity_service, text_opinions_linkage,
                      index_in_linked, etalon_label, idle_mode):
         """
         Composing row in following format:

arekit/common/data/input/providers/sample/cropped.py

@@ -8,10 +8,11 @@ class CroppedSampleRowProvider(BaseSampleRowProvider):
         attitude inside.
     """
 
-    def __init__(self, crop_window_size, label_scaler, text_provider):
+    def __init__(self, crop_window_size, label_scaler, **kwargs):
         assert(isinstance(crop_window_size, int) and crop_window_size > 0)
-        super(CroppedSampleRowProvider, self).__init__(label_provider=MultipleLabelProvider(label_scaler),
-                                                       text_provider=text_provider)
+        super(CroppedSampleRowProvider, self).__init__(
+            label_provider=MultipleLabelProvider(label_scaler),
+            **kwargs)
         self.__crop_window_size = crop_window_size
 
     @staticmethod

arekit/common/data/input/terms_mapper.py

@@ -1,6 +1,5 @@
 from arekit.common.context.terms_mapper import TextTermsMapper
 from arekit.common.context.token import Token
-from arekit.common.entities.base import Entity
 from arekit.common.entities.str_fmt import StringEntitiesFormatter
 from arekit.common.entities.types import OpinionEntityType
 from arekit.common.frames.text_variant import TextFrameVariant
@@ -12,9 +11,12 @@ class OpinionContainingTextTermsMapper(TextTermsMapper):
     The latter might be utilized with synonyms collection
     """
 
-    def __init__(self, entity_formatter):
+    def __init__(self, entity_formatter, entity_group_ind_func, **kwargs):
         assert(isinstance(entity_formatter, StringEntitiesFormatter))
+        assert(callable(entity_group_ind_func))
+        super(OpinionContainingTextTermsMapper, self).__init__(**kwargs)
         self.__entities_formatter = entity_formatter
+        self.__syn_group = entity_group_ind_func
         self.__s_ind = None
         self.__t_ind = None
         self.__s_group = None
@@ -24,12 +26,6 @@ class OpinionContainingTextTermsMapper(TextTermsMapper):
     def StringEntitiesFormatter(self):
         return self.__entities_formatter
 
-    def __syn_group(self, entity):
-        """ Note: here we guarantee that entity has GroupIndex.
-        """
-        assert(isinstance(entity, Entity))
-        return entity.GroupIndex if entity is not None else None
-
     def set_s_ind(self, s_ind):
         assert(isinstance(s_ind, int))
         self.__s_ind = s_ind

arekit/common/data/storages/base.py

@@ -10,6 +10,9 @@ logger = logging.getLogger(__name__)
 
 class BaseRowsStorage(object):
 
+    def __init__(self, log_out=None):
+        self.__log_out = log_out
+
     # region protected methods
 
     def _begin_filling_row(self, row_ind):
@@ -31,27 +34,9 @@ class BaseRowsStorage(object):
     def _get_rows_count(self):
         raise NotImplemented()
 
-    def find_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def find_first_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def iter_column_values(self, column_name, dtype=None):
-        raise NotImplemented()
-
-    def get_row(self, row_index):
-        raise NotImplemented()
-
-    def get_cell(self, row_index, column_name):
-        raise NotImplemented()
-
     def init_empty(self, columns_provider):
         raise NotImplemented()
 
-    def iter_shuffled(self):
-        raise NotImplemented()
-
     def iter_column_names(self):
         raise NotImplemented()
 
@@ -81,6 +66,7 @@ class BaseRowsStorage(object):
             condition_func=lambda item: not isinstance(item[1], MetaEmptyLinkedDataWrapper),
             postfix_func=postfix_func,
             desc="{fmt}".format(fmt=desc),
+            file=self.__log_out,
             total=rows_count)
 
         for row_index, item in enumerate(pbar_it):

arekit/common/docs/entities_grouping.py

@@ -1,17 +1,19 @@
-from arekit.common.entities.base import Entity
 from arekit.common.pipeline.items.base import BasePipelineItem
 
 
 class EntitiesGroupingPipelineItem(BasePipelineItem):
 
-    def __init__(self, value_to_group_id_func, **kwargs):
+    def __init__(self, value_to_group_id_func, is_entity_func, **kwargs):
         assert(callable(value_to_group_id_func))
+        assert(callable(is_entity_func))
         super(EntitiesGroupingPipelineItem, self).__init__(**kwargs)
+
         self.__value_to_group_id_func = value_to_group_id_func
+        self.__is_entity_func = is_entity_func
 
     def apply_core(self, input_data, pipeline_ctx):
         assert(isinstance(input_data, list))
 
-        for entity in filter(lambda term: isinstance(term, Entity), input_data):
+        for entity in filter(lambda term: self.__is_entity_func(term), input_data):
             group_index = self.__value_to_group_id_func(entity.Value)
             entity.set_group_index(group_index)

arekit/common/docs/parsed/base.py

@@ -1,6 +1,5 @@
 from collections.abc import Iterable
 
-from arekit.common.entities.base import Entity
 from arekit.common.text.enums import TermFormat
 from arekit.common.text.parsed import BaseParsedText
 
@@ -73,8 +72,9 @@ class ParsedDocument(object):
         assert(isinstance(s_ind, int))
         return self.__parsed_sentences[s_ind]
 
-    def iter_entities(self):
-        for entity in self.__iter_all_raw_terms(term_only=True, filter_func=lambda t: isinstance(t, Entity)):
+    def iter_entities(self, is_entity_func):
+        assert(callable(is_entity_func))
+        for entity in self.__iter_all_raw_terms(term_only=True, filter_func=is_entity_func):
             yield entity
 
     def iter_terms(self, filter_func=None, term_only=True):

arekit/common/docs/parsed/providers/base.py

@@ -1,4 +1,3 @@
-from arekit.common.entities.base import Entity
 from arekit.common.docs.entity import DocumentEntity
 from arekit.common.docs.parsed.base import ParsedDocument
 
@@ -6,7 +5,7 @@ from arekit.common.docs.parsed.base import ParsedDocument
 class BaseParsedDocumentServiceProvider(object):
 
     def __init__(self, entity_index_func=None):
-        """ Outside enity indexing function
+        """ Outside entity indexing function
             entity_index_func: provides id for a given entity, i.e.
                 func(entity) -> int (id)
         """
@@ -19,7 +18,7 @@ class BaseParsedDocumentServiceProvider(object):
     def Name(self):
         raise NotImplementedError()
 
-    def init_parsed_doc(self, parsed_doc):
+    def init_parsed_doc(self, parsed_doc, is_entity_func):
         assert(isinstance(parsed_doc, ParsedDocument))
 
         def __iter_childs_and_root_node(entity):
@@ -37,7 +36,7 @@ class BaseParsedDocumentServiceProvider(object):
         self.__entity_map.clear()
 
         current_id = 0
-        for _, entity in enumerate(parsed_doc.iter_entities()):
+        for _, entity in enumerate(parsed_doc.iter_entities(is_entity_func=is_entity_func)):
 
             child_doc_entities = []
             for tree_entity, is_child in __iter_childs_and_root_node(entity):
@@ -61,7 +60,6 @@ class BaseParsedDocumentServiceProvider(object):
     def get_document_entity(self, entity):
         """ Maps entity to the related one with DocumentEntity type
         """
-        assert(isinstance(entity, Entity))
         return self.__entity_map[self.__entity_index_func(entity)]
 
     def contains_entity(self, entity):

arekit/common/docs/parsed/providers/entity_service.py

@@ -1,8 +1,6 @@
 from enum import Enum
 
-from arekit.common.entities.base import Entity
 from arekit.common.docs.entity import DocumentEntity
-from arekit.common.docs.parsed.base import ParsedDocument
 from arekit.common.docs.parsed.providers.base import BaseParsedDocumentServiceProvider
 from arekit.common.docs.parsed.term_position import TermPositionTypes, TermPosition
 from arekit.common.text_opinions.base import TextOpinion
@@ -41,9 +39,8 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
 
     NAME = "entity-service-provider"
 
-    def __init__(self, entity_index_func):
-        assert(callable(entity_index_func))
-        super(EntityServiceProvider, self).__init__(entity_index_func=entity_index_func)
+    def __init__(self, **kwargs):
+        super(EntityServiceProvider, self).__init__(**kwargs)
         # Initialize API.
         self.__iter_raw_terms_func = None
         # Initialize entity positions.
@@ -53,24 +50,16 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
     def Name(self):
         return self.NAME
 
-    def init_parsed_doc(self, parsed_doc):
-        super(EntityServiceProvider, self).init_parsed_doc(parsed_doc)
-        assert(isinstance(parsed_doc, ParsedDocument))
+    def init_parsed_doc(self, parsed_doc, is_entity_func):
+        super(EntityServiceProvider, self).init_parsed_doc(parsed_doc=parsed_doc, is_entity_func=is_entity_func)
         self.__iter_raw_terms_func = lambda: parsed_doc.iter_terms(filter_func=None, term_only=False)
-        self.__entity_positions = self.__calculate_entity_positions()
-
-    # region public 'extract' methods
-
-    def extract_entity_value(self, text_opinion, end_type):
-        return self.__extract_entity_value(text_opinion=text_opinion, end_type=end_type)
+        self.__entity_positions = self.__calculate_entity_positions(is_entity_func=is_entity_func)
 
     def extract_entity_position(self, text_opinion, end_type, position_type=None):
         return self.__get_entity_position(text_opinion=text_opinion,
                                           end_type=end_type,
                                           position_type=position_type)
 
-    # endregion
-
     # region public 'calculate' methods
 
     @staticmethod
@@ -112,20 +101,10 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
 
         return e_pos.get_index(position_type)
 
-    def get_entity_value(self, id_in_document):
-        entity = self._doc_entities[id_in_document]
-        assert(isinstance(entity, Entity))
-        return entity.Value
-
     # endregion
 
     # region private methods
 
-    def __extract_entity_value(self, text_opinion, end_type):
-        assert(isinstance(text_opinion, TextOpinion))
-        end_id = self.__get_end_id(text_opinion=text_opinion, end_type=end_type)
-        return self.get_entity_value(end_id)
-
     def __get_entity_position(self, text_opinion, end_type, position_type=None):
         assert(isinstance(text_opinion, TextOpinion))
         end_id = self.__get_end_id(text_opinion=text_opinion, end_type=end_type)
@@ -147,7 +126,7 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
         assert(end_type == EntityEndType.Source or end_type == EntityEndType.Target)
         return text_opinion.SourceId if end_type == EntityEndType.Source else text_opinion.TargetId
 
-    def __calculate_entity_positions(self):
+    def __calculate_entity_positions(self, is_entity_func):
         """ Note: here we consider the same order as in self._entities.
         """
         t_ind_in_doc = -1
@@ -157,7 +136,7 @@ class EntityServiceProvider(BaseParsedDocumentServiceProvider):
 
             t_ind_in_doc += 1
 
-            if not isinstance(term, Entity):
+            if not is_entity_func(term):
                 continue
 
             # We consider that entities within a single tree has the same positions.

arekit/common/docs/parsed/providers/opinion_pairs.py

@@ -1,4 +1,3 @@
-from arekit.common.entities.base import Entity
 from arekit.common.docs.parsed.providers.base_pairs import BasePairProvider
 from arekit.common.opinions.base import Opinion
 
@@ -7,14 +6,15 @@ class OpinionPairsProvider(BasePairProvider):
 
     NAME = "opinion-pairs-provider"
 
+    def __init__(self, entity_value_func, **kwargs):
+        super(OpinionPairsProvider, self).__init__(**kwargs)
+        self.__entity_value_func = entity_value_func
+
     @property
     def Name(self):
         return self.NAME
 
     def _create_pair(self, source_entity, target_entity, label):
-        assert(isinstance(source_entity, Entity))
-        assert(isinstance(target_entity, Entity))
-
-        return Opinion(source_value=source_entity.Value,
-                       target_value=target_entity.Value,
+        return Opinion(source_value=self.__entity_value_func(source_entity),
+                       target_value=self.__entity_value_func(target_entity),
                        label=label)

arekit/common/docs/parsed/providers/text_opinion_pairs.py

@@ -16,8 +16,8 @@ class TextOpinionPairsProvider(BasePairProvider):
 
     NAME = "text-opinion-pairs-provider"
 
-    def __init__(self, value_to_group_id_func):
-        super(TextOpinionPairsProvider, self).__init__()
+    def __init__(self, value_to_group_id_func, **kwargs):
+        super(TextOpinionPairsProvider, self).__init__(**kwargs)
         self.__value_to_group_id_func = value_to_group_id_func
         self.__doc_id = None
         self.__entities_collection = None
@@ -36,8 +36,8 @@ class TextOpinionPairsProvider(BasePairProvider):
                            label=label,
                            text_opinion_id=None)
 
-    def init_parsed_doc(self, parsed_doc):
-        super(TextOpinionPairsProvider, self).init_parsed_doc(parsed_doc)
+    def init_parsed_doc(self, parsed_doc, is_entity_func):
+        super(TextOpinionPairsProvider, self).init_parsed_doc(parsed_doc=parsed_doc, is_entity_func=is_entity_func)
         self.__doc_id = parsed_doc.RelatedDocID
         self.__entities_collection = EntityCollection(
             entities=list(self._doc_entities),

arekit/common/docs/parsed/service.py

@@ -6,7 +6,7 @@ class ParsedDocumentService(object):
     """ Represents a collection of providers, combined with the parsed doc.
     """
 
-    def __init__(self, parsed_doc, providers):
+    def __init__(self, parsed_doc, providers, is_entity_func):
         assert(isinstance(parsed_doc, ParsedDocument))
         assert(isinstance(providers, list))
         self.__parsed_doc = parsed_doc
@@ -20,7 +20,7 @@ class ParsedDocumentService(object):
             self.__providers[provider.Name] = provider
 
             # Post initialize with the related parsed doc.
-            provider.init_parsed_doc(self.__parsed_doc)
+            provider.init_parsed_doc(self.__parsed_doc, is_entity_func=is_entity_func)
 
 
     @property

arekit/common/docs/parser.py

@@ -1,42 +1,14 @@
-from tqdm import tqdm
 from arekit.common.docs.base import Document
 from arekit.common.docs.parsed.base import ParsedDocument
-from arekit.common.pipeline.base import BasePipelineLauncher
 from arekit.common.pipeline.batching import BatchingPipelineLauncher
 from arekit.common.pipeline.context import PipelineContext
 from arekit.common.pipeline.utils import BatchIterator
 from arekit.common.text.parsed import BaseParsedText
+from arekit.common.utils import progress_bar_defined
 
 
 class DocumentParsers(object):
 
-    @staticmethod
-    def parse(doc, pipeline_items, parent_ppl_ctx=None, src_key="input", show_progress=False):
-        """ This document parser is based on single text parts (sentences)
-            that passes sequentially through the pipeline of transformations.
-        """
-        assert(isinstance(doc, Document))
-        assert(isinstance(pipeline_items, list))
-        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
-
-        parsed_sentences = []
-
-        data_it = range(doc.SentencesCount)
-        progress_it = tqdm(data_it, disable=not show_progress)
-
-        for sent_ind in progress_it:
-
-            # Composing the context from a single sentence.
-            ctx = PipelineContext({src_key: doc.get_sentence(sent_ind)}, parent_ctx=parent_ppl_ctx)
-
-            # Apply all the operations.
-            BasePipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key=src_key)
-
-            # Collecting the result.
-            parsed_sentences.append(BaseParsedText(terms=ctx.provide("result")))
-
-        return ParsedDocument(doc_id=doc.ID, parsed_sentences=parsed_sentences)
-
     @staticmethod
     def parse_batch(doc, pipeline_items, batch_size, parent_ppl_ctx=None, src_key="input", show_progress=False):
         """ This document parser is based on batch of sentences.
@@ -49,7 +21,8 @@ class DocumentParsers(object):
         parsed_sentences = []
 
         data_it = BatchIterator(data_iter=iter(range(doc.SentencesCount)), batch_size=batch_size)
-        progress_it = tqdm(data_it, total=round(doc.SentencesCount / batch_size), disable=not show_progress)
+        progress_it = progress_bar_defined(data_it, total=round(doc.SentencesCount / batch_size),
+                                           disable=not show_progress)
 
         for batch in progress_it:
 

arekit/common/model/labeling/single.py

@@ -1,11 +1,15 @@
-import numpy as np
-
 from arekit.common.model.labeling.base import LabelsHelper
 from arekit.common.model.labeling.modes import LabelCalculationMode
 
 
 class SingleLabelsHelper(LabelsHelper):
 
+    @staticmethod
+    def __sign(x):
+        if x == 0:
+            return 0
+        return -1 if x < 0 else 1
+
     def aggregate_labels(self, labels_list, label_calc_mode):
         assert(isinstance(labels_list, list))
         assert(isinstance(label_calc_mode, LabelCalculationMode))
@@ -18,7 +22,7 @@ class SingleLabelsHelper(LabelsHelper):
         if label_calc_mode == LabelCalculationMode.AVERAGE:
             int_labels = [self._label_scaler.label_to_int(label)
                           for label in labels_list]
-            label = self._label_scaler.int_to_label(int(np.sign(sum(int_labels))))
+            label = self._label_scaler.int_to_label(SingleLabelsHelper.__sign(sum(int_labels)))
 
         return label
 

arekit/common/opinions/annot/algo/pair_based.py

@@ -16,8 +16,9 @@ class PairBasedOpinionAnnotationAlgorithm(BaseOpinionAnnotationAlgorithm):
             [1] Extracting Sentiment Attitudes from Analytical Texts https://arxiv.org/pdf/1808.08932.pdf
     """
 
-    def __init__(self, dist_in_terms_bound, label_provider, entity_index_func, dist_in_sents=0,
-                 is_entity_ignored_func=None):
+    def __init__(self, dist_in_terms_bound, label_provider,
+                 is_entity_func, entity_index_func, entity_value_func,
+                 dist_in_sents=0, is_entity_ignored_func=None):
         """
         dist_in_terms_bound: int
             max allowed distance in term (less than passed value)
@@ -34,7 +35,9 @@ class PairBasedOpinionAnnotationAlgorithm(BaseOpinionAnnotationAlgorithm):
         self.__dist_in_terms_bound = dist_in_terms_bound
         self.__dist_in_sents = dist_in_sents
         self.__is_entity_ignored_func = is_entity_ignored_func
+        self.__is_entity_func = is_entity_func
         self.__entity_index_func = entity_index_func
+        self.__entity_value_func = entity_value_func
 
     # region private methods
 
@@ -90,10 +93,11 @@ class PairBasedOpinionAnnotationAlgorithm(BaseOpinionAnnotationAlgorithm):
             return key is not None
 
         # Initialize providers.
-        opinions_provider = OpinionPairsProvider(entity_index_func=self.__entity_index_func)
+        opinions_provider = OpinionPairsProvider(entity_index_func=self.__entity_index_func,
+                                                 entity_value_func=self.__entity_value_func)
         entity_service_provider = EntityServiceProvider(entity_index_func=self.__entity_index_func)
-        opinions_provider.init_parsed_doc(parsed_doc)
-        entity_service_provider.init_parsed_doc(parsed_doc)
+        opinions_provider.init_parsed_doc(parsed_doc=parsed_doc, is_entity_func=self.__is_entity_func)
+        entity_service_provider.init_parsed_doc(parsed_doc=parsed_doc, is_entity_func=self.__is_entity_func)
 
         return opinions_provider.iter_from_all(label_provider=self.__label_provider,
                                                filter_func=__filter_pair_func)

arekit/common/pipeline/base.py

@@ -1,5 +1,4 @@
 from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
 
 
 class BasePipelineLauncher:
@@ -11,7 +10,6 @@ class BasePipelineLauncher:
         assert(isinstance(src_key, str) or src_key is None)
 
         for ind, item in enumerate(filter(lambda itm: itm is not None, pipeline)):
-            assert(isinstance(item, BasePipelineItem))
             do_force_key = src_key is not None and ind == 0
             input_data = item.get_source(pipeline_ctx, force_key=src_key if do_force_key else None) \
                 if has_input or ind > 0 else None

arekit/common/pipeline/batching.py

@@ -1,5 +1,4 @@
 from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
 
 
 class BatchingPipelineLauncher:
@@ -11,8 +10,6 @@ class BatchingPipelineLauncher:
         assert(isinstance(src_key, str) or src_key is None)
 
         for ind, item in enumerate(filter(lambda itm: itm is not None, pipeline)):
-            assert (isinstance(item, BasePipelineItem))
-
             # Handle the content of the batch or batch itself.
             content = item.get_source(pipeline_ctx, call_func=False, force_key=src_key if ind == 0 else None)
             handled_batch = [item._src_func(i) if item._src_func is not None else i for i in content]

arekit/common/pipeline/items/base.py

@@ -2,7 +2,7 @@ from arekit.common.pipeline.context import PipelineContext
 
 
 class BasePipelineItem(object):
-    """ Single pipeline item that might be instatiated and embedded into pipeline.
+    """ Single pipeline item that might be instantiated and embedded into pipeline.
     """
 
     def __init__(self, src_key="result", result_key="result", src_func=None):

arekit/common/utils.py

@@ -1,4 +1,3 @@
-import sys
 import os
 from tqdm import tqdm
 
@@ -27,14 +26,14 @@ def split_by_whitespaces(text):
     return text.split()
 
 
-def progress_bar(iterable, total, desc="", unit="it"):
+def progress_bar(iterable, total, desc="", unit="it", file=None, disable=False):
     if total is not None:
-        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit)
+        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit, file=file, disable=disable)
     else:
-        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit)
+        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit, file=file, disable=disable)
 
 
-def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it"):
+def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it", file=None):
     """ This progress-bar updates only on the
         specific conditions during the iteration process.
     """
@@ -47,7 +46,7 @@ def progress_bar_conditional(iterable, condition_func, total, postfix_func=None,
             yield 0
 
     pbar_it = progress_bar(iterable=__iter_infinite_placeholder(),
-                           desc=desc, unit=unit, total=total)
+                           desc=desc, unit=unit, total=total, file=file)
     element = iter(pbar_it)
 
     # Initialize with 0.
@@ -65,7 +64,7 @@ def progress_bar_conditional(iterable, condition_func, total, postfix_func=None,
             pbar_it.set_postfix(postfix_func(item))
 
 
-def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
+def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it", file=None, disable=False):
     return tqdm(iterable=iterable,
                 total=total,
                 desc=desc,
@@ -73,13 +72,17 @@ def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
                 position=0,
                 leave=True,
                 unit=unit,
+                file=file,
+                disable=disable,
                 miniters=total / miniters if total is not None else total)
 
 
-def progress_bar_iter(iterable, desc="", unit='it'):
+def progress_bar_iter(iterable, desc="", unit='it', file=None, disable=False):
     return tqdm(iterable=iterable,
                 desc=desc,
                 position=0,
                 leave=True,
                 ncols=120,
+                file=file,
+                disable=disable,
                 unit=unit)