arekit 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/utils/nn/rows.py

@@ -1,83 +0,0 @@
-import collections
-
-from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
-from arekit.common.data.input.terms_mapper import OpinionContainingTextTermsMapper
-from arekit.common.entities.str_fmt import StringEntitiesFormatter
-from arekit.contrib.networks.input.ctx_serialization import NetworkSerializationContext
-from arekit.contrib.networks.input.formatters.pos_mapper import PosTermsMapper
-from arekit.contrib.networks.input.providers.sample import NetworkSampleRowProvider
-from arekit.contrib.networks.input.providers.text import NetworkSingleTextProvider
-from arekit.contrib.networks.input.term_types import TermTypes
-from arekit.contrib.networks.input.terms_mapping import VectorizedNetworkTermMapping
-from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
-from arekit.contrib.utils.resources import load_embedding_news_mystem_skipgram_1000_20_2015
-from arekit.contrib.utils.vectorizers.bpe import BPEVectorizer
-from arekit.contrib.utils.vectorizers.random_norm import RandomNormalVectorizer
-
-
-def __add_term_embedding(dict_data, term, emb_vector):
-    if term in dict_data:
-        return
-    dict_data[term] = emb_vector
-
-
-def create_rows_provider(str_entity_fmt, ctx, vectorizers="default"):
-    """ This method is corresponds to the default initialization of
-        the rows provider for data sampling pipeline.
-
-        vectorizers:
-            NONE: no need to vectorize, just provide text (using SingleTextProvider).
-            DEFAULT: we consider an application of stemmer for Russian Language.
-            DICT: in which for every type there is an assigned Vectorizer
-                vectorization of term types.
-                {
-                    TermType.Word: Vectorizer,
-                    TermType.Entity: Vectorizer,
-                    ...
-                }
-    """
-    assert(isinstance(str_entity_fmt, StringEntitiesFormatter))
-    assert(isinstance(ctx, NetworkSerializationContext))
-    assert(isinstance(vectorizers, dict) or vectorizers == "default" or vectorizers is None)
-
-    term_embedding_pairs = None
-
-    if vectorizers is not None:
-
-        if vectorizers == "default":
-            # initialize default vectorizer for Russian language.
-            embedding = load_embedding_news_mystem_skipgram_1000_20_2015(stemmer=MystemWrapper(), auto_download=True)
-            bpe_vectorizer = BPEVectorizer(embedding=embedding, max_part_size=3)
-            norm_vectorizer = RandomNormalVectorizer(vector_size=embedding.VectorSize,
-                                                     token_offset=12345)
-            vectorizers = {
-                TermTypes.WORD: bpe_vectorizer,
-                TermTypes.ENTITY: bpe_vectorizer,
-                TermTypes.FRAME: bpe_vectorizer,
-                TermTypes.TOKEN: norm_vectorizer
-            }
-
-        # Setup term-embedding pairs collection instance.
-        term_embedding_pairs = collections.OrderedDict()
-
-        # Use text provider with vectorizers.
-        text_provider = NetworkSingleTextProvider(
-            text_terms_mapper=VectorizedNetworkTermMapping(
-                vectorizers=vectorizers,
-                string_entities_formatter=str_entity_fmt),
-            pair_handling_func=lambda pair: __add_term_embedding(
-                dict_data=term_embedding_pairs,
-                term=pair[0],
-                emb_vector=pair[1]))
-    else:
-        # Create text provider which without vectorizers.
-        text_provider = BaseSingleTextProvider(
-            text_terms_mapper=OpinionContainingTextTermsMapper(str_entity_fmt))
-
-    return NetworkSampleRowProvider(
-        label_provider=ctx.LabelProvider,
-        text_provider=text_provider,
-        frames_connotation_provider=ctx.FramesConnotationProvider,
-        frame_role_label_scaler=ctx.FrameRolesLabelScaler,
-        pos_terms_mapper=PosTermsMapper(ctx.PosTagger) if ctx.PosTagger is not None else None,
-        term_embedding_pairs=term_embedding_pairs)

arekit/contrib/utils/pipelines/items/text/terms_splitter.py

@@ -1,10 +0,0 @@
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.utils import split_by_whitespaces
-
-
-class TermsSplitterParser(BasePipelineItem):
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        return split_by_whitespaces(input_data)

arekit/contrib/utils/pipelines/sources/nerel/doc_provider.py

@@ -1,27 +0,0 @@
-from arekit.common.data.doc_provider import DocumentProvider
-from arekit.contrib.source.nerel.reader import NerelDocReader
-from arekit.contrib.source.nerel.versions import NerelVersions
-
-
-class NERELDocProvider(DocumentProvider):
-    """ A Russian dataset with nested named entities, relations, events and linked entities.
-        https://github.com/nerel-ds/NEREL
-    """
-
-    def __init__(self, filename_by_id, version):
-        """ filename_ids: dict
-                Dictionary of {id: filename}, where
-                    - id: int
-                    - filename: str
-            version: NerelVersions
-                Specify the appropriate version of teh NEREL collection.
-        """
-        assert(isinstance(filename_by_id, dict))
-        assert(isinstance(version, NerelVersions))
-        super(NERELDocProvider, self).__init__()
-        self.__filename_by_id = filename_by_id
-        self.__version = version
-        self.__doc_reader = NerelDocReader(version)
-
-    def by_id(self, doc_id):
-        return self.__doc_reader.read_document(doc_id=doc_id, filename=self.__filename_by_id[doc_id])

arekit/contrib/utils/pipelines/sources/nerel/extract_text_relations.py

@@ -1,65 +0,0 @@
-from arekit.common.data.doc_provider import DocumentProvider
-from arekit.common.experiment.data_type import DataType
-from arekit.contrib.source.nerel.io_utils import NerelIOUtils
-from arekit.contrib.source.nerel.versions import NerelVersions
-from arekit.contrib.utils.pipelines.sources.nerel.doc_provider import NERELDocProvider
-from arekit.contrib.utils.pipelines.sources.nerel.labels_fmt import NerelAnyLabelFormatter
-from arekit.contrib.utils.pipelines.text_opinion.annot.predefined import PredefinedTextOpinionAnnotator
-from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
-from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
-
-
-def create_text_relation_extraction_pipeline(nerel_version,
-                                             text_parser,
-                                             label_formatter=NerelAnyLabelFormatter(),
-                                             terms_per_context=50,
-                                             doc_ops=None,
-                                             docs_limit=None,
-                                             custom_text_opinion_filters=None):
-    assert(isinstance(nerel_version, NerelVersions))
-    assert(isinstance(doc_ops, DocumentProvider) or doc_ops is None)
-    assert(isinstance(custom_text_opinion_filters, list) or custom_text_opinion_filters is None)
-
-    data_folding = None
-
-    if doc_ops is None:
-        # Default Initialization.
-        filenames_by_ids, data_folding = NerelIOUtils.read_dataset_split(version=nerel_version,
-                                                                         docs_limit=docs_limit)
-        doc_ops = NERELDocProvider(filename_by_id=filenames_by_ids, version=nerel_version)
-
-    # Default text opinion filters.
-    text_opinion_filters = [
-        DistanceLimitedTextOpinionFilter(terms_per_context)
-    ]
-
-    # Append with the custom filters afterwards.
-    if custom_text_opinion_filters is not None:
-        text_opinion_filters += custom_text_opinion_filters
-
-    predefined_annot = PredefinedTextOpinionAnnotator(doc_ops, label_formatter)
-
-    pipelines = {
-        DataType.Train: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                         get_doc_by_id_func=doc_ops.by_id,
-                                                         annotators=[predefined_annot],
-                                                         entity_index_func=lambda brat_entity: brat_entity.ID,
-                                                         text_opinion_filters=text_opinion_filters),
-        DataType.Test: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                        get_doc_by_id_func=doc_ops.by_id,
-                                                        annotators=[predefined_annot],
-                                                        entity_index_func=lambda brat_entity: brat_entity.ID,
-                                                        text_opinion_filters=text_opinion_filters),
-        DataType.Dev: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                       get_doc_by_id_func=doc_ops.by_id,
-                                                       annotators=[predefined_annot],
-                                                       entity_index_func=lambda brat_entity: brat_entity.ID,
-                                                       text_opinion_filters=text_opinion_filters),
-    }
-
-    # In the case when we setup a default data-folding.
-    # There is a need to provide it, due to the needs in further.
-    if data_folding is not None:
-        return pipelines, data_folding
-
-    return pipelines

arekit/contrib/utils/pipelines/sources/nerel/labels_fmt.py

@@ -1,60 +0,0 @@
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.contrib.source.nerel import labels
-
-
-class NerelAnyLabelFormatter(StringLabelsFormatter):
-
-    def __init__(self):
-
-        stol = {
-            "OPINION_BELONGS_TO": labels.OpinionBelongsTo,
-            "OPINION_RELATES_TO": labels.OpinionRelatesTo,
-            "NEG_EFFECT_FROM": labels.NegEffectFrom,
-            "POS_EFFECT_FROM": labels.PosEffectFrom,
-            "NEG_STATE_FROM": labels.NegStateFrom,
-            "POS_STATE_FROM": labels.PosStateFrom,
-            "NEGATIVE_TO": labels.NegativeTo,
-            "POSITIVE_TO": labels.PositiveTo,
-            "STATE_BELONGS_TO": labels.STATE_BELONGS_TO,
-            "POS_AUTHOR_FROM": labels.PosAuthorFrom,
-            "NEG_AUTHOR_FROM": labels.NegAuthorFrom,
-            "ALTERNATIVE_NAME": labels.ALTERNATIVE_NAME,
-            "ORIGINS_FROM": labels.ORIGINS_FROM,
-            "START_TIME": labels.START_TIME,
-            "OWNER_OF": labels.OWNER_OF,
-            "SUBEVENT_OF": labels.SUBEVENT_OF,
-            "PARENT_OF": labels.PARENT_OF,
-            "SUBORDINATE_OF": labels.SUBORDINATE_OF,
-            "PART_OF": labels.PART_OF,
-            "TAKES_PLACE_IN": labels.TAKES_PLACE_IN,
-            "PARTICIPANT_IN": labels.PARTICIPANT_IN,
-            "WORKPLACE": labels.WORKPLACE,
-            "PENALIZED_AS": labels.PENALIZED_AS,
-            "WORKS_AS": labels.WORKS_AS,
-            "PLACE_OF_DEATH": labels.PLACE_OF_DEATH,
-            "PLACE_OF_BIRTH": labels.PLACE_OF_BIRTH,
-            "HAS_CAUSE": labels.HAS_CAUSE,
-            "AWARDED_WITH": labels.AWARDED_WITH,
-            "CAUSE_OF_DEATH": labels.CAUSE_OF_DEATH,
-            "CONVICTED_OF": labels.CONVICTED_OF,
-            "DATE_DEFUNCT_IN": labels.DATE_DEFUNCT_IN,
-            "DATE_FOUNDED_IN": labels.DATE_FOUNDED_IN,
-            "DATE_OF_BIRTH": labels.DATE_OF_BIRTH,
-            "DATE_OF_CREATION": labels.DATE_OF_CREATION,
-            "DATE_OF_DEATH": labels.DATE_OF_DEATH,
-            "END_TIME": labels.END_TIME,
-            "EXPENDITURE": labels.EXPENDITURE,
-            "FOUNDED_BY": labels.FOUNDED_BY,
-            "KNOWS": labels.KNOWS,
-            "RELATIVE": labels.RELATIVE,
-            "LOCATED_IN": labels.LOCATED_IN,
-            "RELIGION_OF": labels.RELIGION_OF,
-            "MEDICAL_CONDITION": labels.MEDICAL_CONDITION,
-            "SCHOOLS_ATTENDED": labels.SCHOOLS_ATTENDED,
-            "MEMBER_OF": labels.MEMBER_OF,
-            "SIBLING": labels.SIBLING,
-            "ORGANIZES": labels.ORGANIZES,
-            "SPOUSE": labels.SPOUSE
-        }
-
-        super(NerelAnyLabelFormatter, self).__init__(stol=stol)

arekit/contrib/utils/pipelines/sources/nerel_bio/doc_provider.py

@@ -1,29 +0,0 @@
-from arekit.common.data.doc_provider import DocumentProvider
-from arekit.contrib.source.nerelbio.reader import NerelBioDocReader
-from arekit.contrib.source.nerelbio.versions import NerelBioVersions
-
-
-class NERELBioDocProvider(DocumentProvider):
-    """ NEREL-BIO extends the general domain dataset NEREL.
-        NEREL-BIO annotation scheme covers both general and biomedical
-        domains making it suitable for domain transfer experiments.
-        https://github.com/nerel-ds/NEREL-BIO
-    """
-
-    def __init__(self, filename_by_id, version):
-        """ filename_ids: dict
-                Dictionary of {id: filename}, where
-                    - id: int
-                    - filename: str
-            version: NerelBioVersions
-                Specify the appropriate version of the NEREL-BIO collection.
-        """
-        assert(isinstance(filename_by_id, dict))
-        assert(isinstance(version, NerelBioVersions))
-        super(NERELBioDocProvider, self).__init__()
-        self.__filename_by_id = filename_by_id
-        self.__version = version
-        self.__doc_reader = NerelBioDocReader(version)
-
-    def by_id(self, doc_id):
-        return self.__doc_reader.read_document(doc_id=doc_id, filename=self.__filename_by_id[doc_id])

arekit/contrib/utils/pipelines/sources/nerel_bio/extrat_text_relations.py

@@ -1,64 +0,0 @@
-from arekit.common.data.doc_provider import DocumentProvider
-from arekit.common.experiment.data_type import DataType
-from arekit.contrib.source.nerelbio.io_utils import NerelBioIOUtils
-from arekit.contrib.source.nerelbio.versions import NerelBioVersions
-from arekit.contrib.utils.pipelines.sources.nerel_bio.doc_provider import NERELBioDocProvider
-from arekit.contrib.utils.pipelines.sources.nerel_bio.labels_fmt import NerelBioAnyLabelFormatter
-from arekit.contrib.utils.pipelines.text_opinion.annot.predefined import PredefinedTextOpinionAnnotator
-from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
-from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
-
-
-def create_text_relation_extraction_pipeline(nerel_bio_version,
-                                             text_parser,
-                                             label_formatter=NerelBioAnyLabelFormatter(),
-                                             terms_per_context=50,
-                                             doc_ops=None,
-                                             docs_limit=None,
-                                             custom_text_opinion_filters=None):
-    assert(isinstance(nerel_bio_version, NerelBioVersions))
-    assert(isinstance(doc_ops, DocumentProvider) or doc_ops is None)
-    assert(isinstance(custom_text_opinion_filters, list) or custom_text_opinion_filters is None)
-
-    data_folding = None
-
-    if doc_ops is None:
-        # Default Initialization.
-        filenames_by_ids, data_folding = NerelBioIOUtils.read_dataset_split(version=nerel_bio_version,
-                                                                            docs_limit=docs_limit)
-        doc_ops = NERELBioDocProvider(filename_by_id=filenames_by_ids, version=nerel_bio_version)
-
-    text_opinion_filters = [
-        DistanceLimitedTextOpinionFilter(terms_per_context)
-    ]
-
-    # Append with the custom filters afterwards.
-    if custom_text_opinion_filters is not None:
-        text_opinion_filters += custom_text_opinion_filters
-
-    predefined_annot = PredefinedTextOpinionAnnotator(doc_ops, label_formatter)
-
-    pipelines = {
-        DataType.Train: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                         get_doc_by_id_func=doc_ops.by_id,
-                                                         annotators=[predefined_annot],
-                                                         entity_index_func=lambda brat_entity: brat_entity.ID,
-                                                         text_opinion_filters=text_opinion_filters),
-        DataType.Test: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                        get_doc_by_id_func=doc_ops.by_id,
-                                                        annotators=[predefined_annot],
-                                                        entity_index_func=lambda brat_entity: brat_entity.ID,
-                                                        text_opinion_filters=text_opinion_filters),
-        DataType.Dev: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                       get_doc_by_id_func=doc_ops.by_id,
-                                                       annotators=[predefined_annot],
-                                                       entity_index_func=lambda brat_entity: brat_entity.ID,
-                                                       text_opinion_filters=text_opinion_filters),
-    }
-
-    # In the case when we setup a default data-folding.
-    # There is a need to provide it, due to the needs in further.
-    if data_folding is not None:
-        return pipelines, data_folding
-
-    return pipelines

arekit/contrib/utils/pipelines/sources/nerel_bio/labels_fmt.py

@@ -1,79 +0,0 @@
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.contrib.source.nerelbio import labels
-
-
-class NerelBioAnyLabelFormatter(StringLabelsFormatter):
-
-    def __init__(self):
-
-        stol = {
-            "ABBREVIATION": labels.ABBREVIATION,
-            "ALTERNATIVE_NAME": labels.ALTERNATIVE_NAME,
-            "KNOWS": labels.KNOWS,
-            "AGE_IS": labels.AGE_IS,
-            "AGE_DIED_AT": labels.AGE_DIED_AT,
-            "AWARDED_WITH": labels.AWARDED_WITH,
-            "PLACE_OF_BIRTH": labels.PLACE_OF_BIRTH,
-            "DATE_DEFUNCT_IN": labels.DATE_DEFUNCT_IN,
-            "DATE_FOUNDED_IN": labels.DATE_FOUNDED_IN,
-            "DATE_OF_BIRTH": labels.DATE_OF_BIRTH,
-            "DATE_OF_CREATION": labels.DATE_OF_CREATION,
-            "DATE_OF_DEATH": labels.DATE_OF_DEATH,
-            "POINT_IN_TIME": labels.POINT_IN_TIME,
-            "PLACE_OF_DEATH": labels.PLACE_OF_DEATH,
-            "FOUNDED_BY": labels.FOUNDED_BY,
-            "HEADQUARTERED_IN": labels.HEADQUARTERED_IN,
-            "IDEOLOGY_OF": labels.IDEOLOGY_OF,
-            "SPOUSE": labels.SPOUSE,
-            "MEMBER_OF": labels.MEMBER_OF,
-            "ORGANIZES": labels.ORGANIZES,
-            "OWNER_OF": labels.OWNER_OF,
-            "PARENT_OF": labels.PARENT_OF,
-            "PARTICIPANT_IN": labels.PARTICIPANT_IN,
-            "PLACE_RESIDES_IN": labels.PLACE_RESIDES_IN,
-            "PRICE_OF": labels.PRICE_OF,
-            "PRODUCES": labels.PRODUCES,
-            "RELATIVE": labels.RELATIVE,
-            "RELIGION_OF": labels.RELIGION_OF,
-            "SCHOOLS_ATTENDED": labels.SCHOOLS_ATTENDED,
-            "SIBLING": labels.SIBLING,
-            "SUBEVENT_OF": labels.SUBEVENT_OF,
-            "SUBORDINATE_OF": labels.SUBORDINATE_OF,
-            "TAKES_PLACE_IN": labels.TAKES_PLACE_IN,
-            "WORKPLACE": labels.WORKPLACE,
-            "WORKS_AS": labels.WORKS_AS,
-            "CONVICTED_OF": labels.CONVICTED_OF,
-            "PENALIZED_AS": labels.PENALIZED_AS,
-            "START_TIME": labels.START_TIME,
-            "END_TIME": labels.END_TIME,
-            "EXPENDITURE": labels.EXPENDITURE,
-            "AGENT": labels.AGENT,
-            "INANIMATE_INVOLVED": labels.INANIMATE_INVOLVED,
-            "INCOME": labels.INCOME,
-            "SUBCLASS_OF": labels.SUBCLASS_OF,
-            "PART_OF": labels.PART_OF,
-            "LOCATED_IN": labels.LOCATED_IN,
-            "TREATED_USING": labels.TREATED_USING,
-            "ORIGINS_FROM": labels.ORIGINS_FROM,
-            "TO_DETECT_OR_STUDY": labels.TO_DETECT_OR_STUDY,
-            "AFFECTS": labels.AFFECTS,
-            "HAS_CAUSE": labels.HAS_CAUSE,
-            "APPLIED_TO": labels.APPLIED_TO,
-            "USED_IN": labels.USED_IN,
-            "ASSOCIATED_WITH": labels.ASSOCIATED_WITH,
-            "HAS_ADMINISTRATION_ROUTE": labels.HAS_ADMINISTRATION_ROUTE,
-            "HAS_STRENGTH": labels.HAS_STRENGTH,
-            "DURATION_OF": labels.DURATION_OF,
-            "VALUE_IS": labels.VALUE_IS,
-            "PHYSIOLOGY_OF": labels.PHYSIOLOGY_OF,
-            "PROCEDURE_PERFORMED": labels.PROCEDURE_PERFORMED,
-            "MENTAL_PROCESS_OF": labels.MENTAL_PROCESS_OF,
-            "MEDICAL_CONDITION": labels.MEDICAL_CONDITION,
-            "DOSE_IS": labels.DOSE_IS,
-            "FINDING_OF": labels.FINDING_OF,
-            "CAUSE_OF_DEATH": labels.CAUSE_OF_DEATH,
-            "CONSUME": labels.CONSUME,
-        }
-
-        super(NerelBioAnyLabelFormatter, self).__init__(stol=stol)
-

arekit/contrib/utils/pipelines/sources/ruattitudes/doc_provider.py

@@ -1,56 +0,0 @@
-from arekit.common.utils import progress_bar_iter
-from arekit.contrib.source.ruattitudes.collection import RuAttitudesCollection
-from arekit.contrib.source.ruattitudes.io_utils import RuAttitudesVersions
-from arekit.contrib.source.ruattitudes.doc import RuAttitudesDocument
-from arekit.contrib.source.ruattitudes.doc_brat import RuAttitudesDocumentsConverter
-from arekit.contrib.utils.data.doc_provider.dict_based import DictionaryBasedDocumentProvider
-
-
-class RuAttitudesDocumentProvider(DictionaryBasedDocumentProvider):
-
-    def __init__(self, version, keep_doc_ids_only, doc_id_func, limit):
-        d = self.read_ruattitudes_to_brat_in_memory(version=version,
-                                                    keep_doc_ids_only=keep_doc_ids_only,
-                                                    doc_id_func=doc_id_func,
-                                                    limit=limit)
-        super(RuAttitudesDocumentProvider, self).__init__(d)
-
-    @staticmethod
-    def read_ruattitudes_to_brat_in_memory(version, keep_doc_ids_only, doc_id_func, limit=None):
-        """ Performs reading of RuAttitude formatted documents and
-            selection according to 'doc_ids_set' parameter.
-        """
-        assert (isinstance(version, RuAttitudesVersions))
-        assert (isinstance(keep_doc_ids_only, bool))
-        assert (callable(doc_id_func))
-
-        it = RuAttitudesCollection.iter_docs(version=version,
-                                             get_doc_index_func=doc_id_func,
-                                             return_inds_only=keep_doc_ids_only)
-
-        it_formatted_and_logged = progress_bar_iter(
-            iterable=RuAttitudesDocumentProvider.__iter_id_with_doc(
-                docs_it=it, keep_doc_ids_only=keep_doc_ids_only),
-            desc="Loading RuAttitudes Collection [{}]".format("doc ids only" if keep_doc_ids_only else "fully"),
-            unit='docs')
-
-        d = {}
-        docs_read = 0
-        for doc_id, doc in it_formatted_and_logged:
-            assert(isinstance(doc, RuAttitudesDocument) or doc is None)
-            d[doc_id] = RuAttitudesDocumentsConverter.to_brat_doc(doc) if doc is not None else None
-            docs_read += 1
-            if limit is not None and docs_read >= limit:
-                break
-
-        return d
-
-    @staticmethod
-    def __iter_id_with_doc(docs_it, keep_doc_ids_only):
-        if keep_doc_ids_only:
-            for doc_id in docs_it:
-                yield doc_id, None
-        else:
-            for doc in docs_it:
-                assert (isinstance(doc, RuAttitudesDocument))
-                yield doc.ID, doc

arekit/contrib/utils/pipelines/sources/ruattitudes/entity_filter.py

@@ -1,20 +0,0 @@
-from arekit.common.entities.types import OpinionEntityType
-from arekit.contrib.utils.entities.filter import EntityFilter
-
-
-class RuAttitudesEntityFilter(EntityFilter):
-    """ This is a task-specific filter, which is applicable of entity types proposed
-        by the OntoNotesV5 resource: https://catalog.ldc.upenn.edu/LDC2013T19
-        We consider only a short list related to the sentiment attitude extraction task.
-    """
-
-    supported = ["GPE", "PERSON", "LOCAL", "GEO", "ORG"]
-
-    def is_ignored(self, entity, e_type):
-
-        if e_type == OpinionEntityType.Subject:
-            return entity.Type not in RuAttitudesEntityFilter.supported
-        if e_type == OpinionEntityType.Object:
-            return entity.Type not in RuAttitudesEntityFilter.supported
-        else:
-            return True

arekit/contrib/utils/pipelines/sources/ruattitudes/extract_text_opinions.py

@@ -1,65 +0,0 @@
-from arekit.common.labels.scaler.base import BaseLabelScaler
-from arekit.contrib.source.ruattitudes.io_utils import RuAttitudesVersions
-from arekit.contrib.source.ruattitudes.labels_fmt import RuAttitudesLabelFormatter
-from arekit.contrib.utils.pipelines.sources.ruattitudes.doc_provider import RuAttitudesDocumentProvider
-from arekit.contrib.utils.pipelines.sources.ruattitudes.entity_filter import RuAttitudesEntityFilter
-from arekit.contrib.utils.pipelines.text_opinion.annot.predefined import PredefinedTextOpinionAnnotator
-from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
-from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
-from arekit.contrib.utils.pipelines.text_opinion.filters.entity_based import EntityBasedTextOpinionFilter
-
-
-def create_text_opinion_extraction_pipeline(text_parser,
-                                            label_scaler,
-                                            custom_text_opinion_filters=None,
-                                            version=RuAttitudesVersions.V20Large,
-                                            terms_per_context=50,
-                                            limit=None):
-    """ Processing pipeline for RuAttitudes.
-        This pipeline is based on the in-memory RuAttitudes storage.
-
-        Original collection paper:  www.aclweb.org/anthology/r19-1118/
-        Github repository:          https://github.com/nicolay-r/RuAttitudes
-
-        version: enum
-            Version of the RuAttitudes collection.
-            NOTE: we consider to support a variations of the 2.0 versions.
-        label_scaler:
-            Scaler that allows to perform conversion from integer labels (RuAttitudes) to
-            the actual `Label` instances, required in further for text_opinions instances.
-        terms_per_context: int
-            Amount of terms that we consider in between the Object and Subject.
-        limit: int or None
-            Limit of documents to consider.
-    """
-    assert(isinstance(label_scaler, BaseLabelScaler))
-    assert(isinstance(version, RuAttitudesVersions))
-    assert(isinstance(custom_text_opinion_filters, list) or custom_text_opinion_filters is None)
-    assert(version in [RuAttitudesVersions.V20Large, RuAttitudesVersions.V20Base,
-                       RuAttitudesVersions.V20BaseNeut, RuAttitudesVersions.V20LargeNeut])
-
-    doc_provider = RuAttitudesDocumentProvider(version=version,
-                                               keep_doc_ids_only=False,
-                                               doc_id_func=lambda doc_id: doc_id,
-                                               limit=limit)
-
-    text_opinion_filters = [
-        EntityBasedTextOpinionFilter(entity_filter=RuAttitudesEntityFilter()),
-        DistanceLimitedTextOpinionFilter(terms_per_context)
-    ]
-
-    # Append with the custom filters afterwards.
-    if custom_text_opinion_filters is not None:
-        text_opinion_filters += custom_text_opinion_filters
-
-    pipeline = text_opinion_extraction_pipeline(
-        annotators=[
-            PredefinedTextOpinionAnnotator(doc_provider=doc_provider,
-                                           label_formatter=RuAttitudesLabelFormatter(label_scaler))
-        ],
-        text_opinion_filters=custom_text_opinion_filters,
-        get_doc_by_id_func=doc_provider.by_id,
-        entity_index_func=lambda brat_entity: brat_entity.ID,
-        text_parser=text_parser)
-
-    return pipeline

arekit/contrib/utils/pipelines/sources/rusentrel/doc_provider.py

@@ -1,21 +0,0 @@
-from arekit.common.data.doc_provider import DocumentProvider
-from arekit.common.synonyms.base import SynonymsCollection
-from arekit.contrib.source.rusentrel.io_utils import RuSentRelVersions
-from arekit.contrib.source.rusentrel.docs_reader import RuSentRelDocumentsReader
-
-
-class RuSentrelDocumentProvider(DocumentProvider):
-    """ Limitations: Supported only train/test collections format
-    """
-
-    def __init__(self, version, synonyms):
-        assert(isinstance(version, RuSentRelVersions))
-        assert(isinstance(synonyms, SynonymsCollection))
-        super(RuSentrelDocumentProvider, self).__init__()
-        self.__version = version
-        self.__synonyms = synonyms
-
-    def by_id(self, doc_id):
-        assert (isinstance(doc_id, int))
-        return RuSentRelDocumentsReader.read_document(doc_id=doc_id, synonyms=self.__synonyms, version=self.__version)
-