arekit 0.24.0__py3-none-any.whl

arekit/contrib/utils/pipelines/sources/nerel/doc_provider.py

@@ -0,0 +1,27 @@
+from arekit.common.data.doc_provider import DocumentProvider
+from arekit.contrib.source.nerel.reader import NerelDocReader
+from arekit.contrib.source.nerel.versions import NerelVersions
+
+
+class NERELDocProvider(DocumentProvider):
+    """ A Russian dataset with nested named entities, relations, events and linked entities.
+        https://github.com/nerel-ds/NEREL
+    """
+
+    def __init__(self, filename_by_id, version):
+        """ filename_ids: dict
+                Dictionary of {id: filename}, where
+                    - id: int
+                    - filename: str
+            version: NerelVersions
+                Specify the appropriate version of teh NEREL collection.
+        """
+        assert(isinstance(filename_by_id, dict))
+        assert(isinstance(version, NerelVersions))
+        super(NERELDocProvider, self).__init__()
+        self.__filename_by_id = filename_by_id
+        self.__version = version
+        self.__doc_reader = NerelDocReader(version)
+
+    def by_id(self, doc_id):
+        return self.__doc_reader.read_document(doc_id=doc_id, filename=self.__filename_by_id[doc_id])

arekit/contrib/utils/pipelines/sources/nerel/extract_text_relations.py

@@ -0,0 +1,65 @@
+from arekit.common.data.doc_provider import DocumentProvider
+from arekit.common.experiment.data_type import DataType
+from arekit.contrib.source.nerel.io_utils import NerelIOUtils
+from arekit.contrib.source.nerel.versions import NerelVersions
+from arekit.contrib.utils.pipelines.sources.nerel.doc_provider import NERELDocProvider
+from arekit.contrib.utils.pipelines.sources.nerel.labels_fmt import NerelAnyLabelFormatter
+from arekit.contrib.utils.pipelines.text_opinion.annot.predefined import PredefinedTextOpinionAnnotator
+from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
+from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
+
+
+def create_text_relation_extraction_pipeline(nerel_version,
+                                             text_parser,
+                                             label_formatter=NerelAnyLabelFormatter(),
+                                             terms_per_context=50,
+                                             doc_ops=None,
+                                             docs_limit=None,
+                                             custom_text_opinion_filters=None):
+    assert(isinstance(nerel_version, NerelVersions))
+    assert(isinstance(doc_ops, DocumentProvider) or doc_ops is None)
+    assert(isinstance(custom_text_opinion_filters, list) or custom_text_opinion_filters is None)
+
+    data_folding = None
+
+    if doc_ops is None:
+        # Default Initialization.
+        filenames_by_ids, data_folding = NerelIOUtils.read_dataset_split(version=nerel_version,
+                                                                         docs_limit=docs_limit)
+        doc_ops = NERELDocProvider(filename_by_id=filenames_by_ids, version=nerel_version)
+
+    # Default text opinion filters.
+    text_opinion_filters = [
+        DistanceLimitedTextOpinionFilter(terms_per_context)
+    ]
+
+    # Append with the custom filters afterwards.
+    if custom_text_opinion_filters is not None:
+        text_opinion_filters += custom_text_opinion_filters
+
+    predefined_annot = PredefinedTextOpinionAnnotator(doc_ops, label_formatter)
+
+    pipelines = {
+        DataType.Train: text_opinion_extraction_pipeline(text_parser=text_parser,
+                                                         get_doc_by_id_func=doc_ops.by_id,
+                                                         annotators=[predefined_annot],
+                                                         entity_index_func=lambda brat_entity: brat_entity.ID,
+                                                         text_opinion_filters=text_opinion_filters),
+        DataType.Test: text_opinion_extraction_pipeline(text_parser=text_parser,
+                                                        get_doc_by_id_func=doc_ops.by_id,
+                                                        annotators=[predefined_annot],
+                                                        entity_index_func=lambda brat_entity: brat_entity.ID,
+                                                        text_opinion_filters=text_opinion_filters),
+        DataType.Dev: text_opinion_extraction_pipeline(text_parser=text_parser,
+                                                       get_doc_by_id_func=doc_ops.by_id,
+                                                       annotators=[predefined_annot],
+                                                       entity_index_func=lambda brat_entity: brat_entity.ID,
+                                                       text_opinion_filters=text_opinion_filters),
+    }
+
+    # In the case when we setup a default data-folding.
+    # There is a need to provide it, due to the needs in further.
+    if data_folding is not None:
+        return pipelines, data_folding
+
+    return pipelines

arekit/contrib/utils/pipelines/sources/nerel/labels_fmt.py

@@ -0,0 +1,60 @@
+from arekit.common.labels.str_fmt import StringLabelsFormatter
+from arekit.contrib.source.nerel import labels
+
+
+class NerelAnyLabelFormatter(StringLabelsFormatter):
+
+    def __init__(self):
+
+        stol = {
+            "OPINION_BELONGS_TO": labels.OpinionBelongsTo,
+            "OPINION_RELATES_TO": labels.OpinionRelatesTo,
+            "NEG_EFFECT_FROM": labels.NegEffectFrom,
+            "POS_EFFECT_FROM": labels.PosEffectFrom,
+            "NEG_STATE_FROM": labels.NegStateFrom,
+            "POS_STATE_FROM": labels.PosStateFrom,
+            "NEGATIVE_TO": labels.NegativeTo,
+            "POSITIVE_TO": labels.PositiveTo,
+            "STATE_BELONGS_TO": labels.STATE_BELONGS_TO,
+            "POS_AUTHOR_FROM": labels.PosAuthorFrom,
+            "NEG_AUTHOR_FROM": labels.NegAuthorFrom,
+            "ALTERNATIVE_NAME": labels.ALTERNATIVE_NAME,
+            "ORIGINS_FROM": labels.ORIGINS_FROM,
+            "START_TIME": labels.START_TIME,
+            "OWNER_OF": labels.OWNER_OF,
+            "SUBEVENT_OF": labels.SUBEVENT_OF,
+            "PARENT_OF": labels.PARENT_OF,
+            "SUBORDINATE_OF": labels.SUBORDINATE_OF,
+            "PART_OF": labels.PART_OF,
+            "TAKES_PLACE_IN": labels.TAKES_PLACE_IN,
+            "PARTICIPANT_IN": labels.PARTICIPANT_IN,
+            "WORKPLACE": labels.WORKPLACE,
+            "PENALIZED_AS": labels.PENALIZED_AS,
+            "WORKS_AS": labels.WORKS_AS,
+            "PLACE_OF_DEATH": labels.PLACE_OF_DEATH,
+            "PLACE_OF_BIRTH": labels.PLACE_OF_BIRTH,
+            "HAS_CAUSE": labels.HAS_CAUSE,
+            "AWARDED_WITH": labels.AWARDED_WITH,
+            "CAUSE_OF_DEATH": labels.CAUSE_OF_DEATH,
+            "CONVICTED_OF": labels.CONVICTED_OF,
+            "DATE_DEFUNCT_IN": labels.DATE_DEFUNCT_IN,
+            "DATE_FOUNDED_IN": labels.DATE_FOUNDED_IN,
+            "DATE_OF_BIRTH": labels.DATE_OF_BIRTH,
+            "DATE_OF_CREATION": labels.DATE_OF_CREATION,
+            "DATE_OF_DEATH": labels.DATE_OF_DEATH,
+            "END_TIME": labels.END_TIME,
+            "EXPENDITURE": labels.EXPENDITURE,
+            "FOUNDED_BY": labels.FOUNDED_BY,
+            "KNOWS": labels.KNOWS,
+            "RELATIVE": labels.RELATIVE,
+            "LOCATED_IN": labels.LOCATED_IN,
+            "RELIGION_OF": labels.RELIGION_OF,
+            "MEDICAL_CONDITION": labels.MEDICAL_CONDITION,
+            "SCHOOLS_ATTENDED": labels.SCHOOLS_ATTENDED,
+            "MEMBER_OF": labels.MEMBER_OF,
+            "SIBLING": labels.SIBLING,
+            "ORGANIZES": labels.ORGANIZES,
+            "SPOUSE": labels.SPOUSE
+        }
+
+        super(NerelAnyLabelFormatter, self).__init__(stol=stol)

arekit/contrib/utils/pipelines/sources/nerel_bio/doc_provider.py

@@ -0,0 +1,29 @@
+from arekit.common.data.doc_provider import DocumentProvider
+from arekit.contrib.source.nerelbio.reader import NerelBioDocReader
+from arekit.contrib.source.nerelbio.versions import NerelBioVersions
+
+
+class NERELBioDocProvider(DocumentProvider):
+    """ NEREL-BIO extends the general domain dataset NEREL.
+        NEREL-BIO annotation scheme covers both general and biomedical
+        domains making it suitable for domain transfer experiments.
+        https://github.com/nerel-ds/NEREL-BIO
+    """
+
+    def __init__(self, filename_by_id, version):
+        """ filename_ids: dict
+                Dictionary of {id: filename}, where
+                    - id: int
+                    - filename: str
+            version: NerelBioVersions
+                Specify the appropriate version of the NEREL-BIO collection.
+        """
+        assert(isinstance(filename_by_id, dict))
+        assert(isinstance(version, NerelBioVersions))
+        super(NERELBioDocProvider, self).__init__()
+        self.__filename_by_id = filename_by_id
+        self.__version = version
+        self.__doc_reader = NerelBioDocReader(version)
+
+    def by_id(self, doc_id):
+        return self.__doc_reader.read_document(doc_id=doc_id, filename=self.__filename_by_id[doc_id])

arekit/contrib/utils/pipelines/sources/nerel_bio/extrat_text_relations.py

@@ -0,0 +1,64 @@
+from arekit.common.data.doc_provider import DocumentProvider
+from arekit.common.experiment.data_type import DataType
+from arekit.contrib.source.nerelbio.io_utils import NerelBioIOUtils
+from arekit.contrib.source.nerelbio.versions import NerelBioVersions
+from arekit.contrib.utils.pipelines.sources.nerel_bio.doc_provider import NERELBioDocProvider
+from arekit.contrib.utils.pipelines.sources.nerel_bio.labels_fmt import NerelBioAnyLabelFormatter
+from arekit.contrib.utils.pipelines.text_opinion.annot.predefined import PredefinedTextOpinionAnnotator
+from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
+from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
+
+
+def create_text_relation_extraction_pipeline(nerel_bio_version,
+                                             text_parser,
+                                             label_formatter=NerelBioAnyLabelFormatter(),
+                                             terms_per_context=50,
+                                             doc_ops=None,
+                                             docs_limit=None,
+                                             custom_text_opinion_filters=None):
+    assert(isinstance(nerel_bio_version, NerelBioVersions))
+    assert(isinstance(doc_ops, DocumentProvider) or doc_ops is None)
+    assert(isinstance(custom_text_opinion_filters, list) or custom_text_opinion_filters is None)
+
+    data_folding = None
+
+    if doc_ops is None:
+        # Default Initialization.
+        filenames_by_ids, data_folding = NerelBioIOUtils.read_dataset_split(version=nerel_bio_version,
+                                                                            docs_limit=docs_limit)
+        doc_ops = NERELBioDocProvider(filename_by_id=filenames_by_ids, version=nerel_bio_version)
+
+    text_opinion_filters = [
+        DistanceLimitedTextOpinionFilter(terms_per_context)
+    ]
+
+    # Append with the custom filters afterwards.
+    if custom_text_opinion_filters is not None:
+        text_opinion_filters += custom_text_opinion_filters
+
+    predefined_annot = PredefinedTextOpinionAnnotator(doc_ops, label_formatter)
+
+    pipelines = {
+        DataType.Train: text_opinion_extraction_pipeline(text_parser=text_parser,
+                                                         get_doc_by_id_func=doc_ops.by_id,
+                                                         annotators=[predefined_annot],
+                                                         entity_index_func=lambda brat_entity: brat_entity.ID,
+                                                         text_opinion_filters=text_opinion_filters),
+        DataType.Test: text_opinion_extraction_pipeline(text_parser=text_parser,
+                                                        get_doc_by_id_func=doc_ops.by_id,
+                                                        annotators=[predefined_annot],
+                                                        entity_index_func=lambda brat_entity: brat_entity.ID,
+                                                        text_opinion_filters=text_opinion_filters),
+        DataType.Dev: text_opinion_extraction_pipeline(text_parser=text_parser,
+                                                       get_doc_by_id_func=doc_ops.by_id,
+                                                       annotators=[predefined_annot],
+                                                       entity_index_func=lambda brat_entity: brat_entity.ID,
+                                                       text_opinion_filters=text_opinion_filters),
+    }
+
+    # In the case when we setup a default data-folding.
+    # There is a need to provide it, due to the needs in further.
+    if data_folding is not None:
+        return pipelines, data_folding
+
+    return pipelines

arekit/contrib/utils/pipelines/sources/nerel_bio/labels_fmt.py

@@ -0,0 +1,79 @@
+from arekit.common.labels.str_fmt import StringLabelsFormatter
+from arekit.contrib.source.nerelbio import labels
+
+
+class NerelBioAnyLabelFormatter(StringLabelsFormatter):
+
+    def __init__(self):
+
+        stol = {
+            "ABBREVIATION": labels.ABBREVIATION,
+            "ALTERNATIVE_NAME": labels.ALTERNATIVE_NAME,
+            "KNOWS": labels.KNOWS,
+            "AGE_IS": labels.AGE_IS,
+            "AGE_DIED_AT": labels.AGE_DIED_AT,
+            "AWARDED_WITH": labels.AWARDED_WITH,
+            "PLACE_OF_BIRTH": labels.PLACE_OF_BIRTH,
+            "DATE_DEFUNCT_IN": labels.DATE_DEFUNCT_IN,
+            "DATE_FOUNDED_IN": labels.DATE_FOUNDED_IN,
+            "DATE_OF_BIRTH": labels.DATE_OF_BIRTH,
+            "DATE_OF_CREATION": labels.DATE_OF_CREATION,
+            "DATE_OF_DEATH": labels.DATE_OF_DEATH,
+            "POINT_IN_TIME": labels.POINT_IN_TIME,
+            "PLACE_OF_DEATH": labels.PLACE_OF_DEATH,
+            "FOUNDED_BY": labels.FOUNDED_BY,
+            "HEADQUARTERED_IN": labels.HEADQUARTERED_IN,
+            "IDEOLOGY_OF": labels.IDEOLOGY_OF,
+            "SPOUSE": labels.SPOUSE,
+            "MEMBER_OF": labels.MEMBER_OF,
+            "ORGANIZES": labels.ORGANIZES,
+            "OWNER_OF": labels.OWNER_OF,
+            "PARENT_OF": labels.PARENT_OF,
+            "PARTICIPANT_IN": labels.PARTICIPANT_IN,
+            "PLACE_RESIDES_IN": labels.PLACE_RESIDES_IN,
+            "PRICE_OF": labels.PRICE_OF,
+            "PRODUCES": labels.PRODUCES,
+            "RELATIVE": labels.RELATIVE,
+            "RELIGION_OF": labels.RELIGION_OF,
+            "SCHOOLS_ATTENDED": labels.SCHOOLS_ATTENDED,
+            "SIBLING": labels.SIBLING,
+            "SUBEVENT_OF": labels.SUBEVENT_OF,
+            "SUBORDINATE_OF": labels.SUBORDINATE_OF,
+            "TAKES_PLACE_IN": labels.TAKES_PLACE_IN,
+            "WORKPLACE": labels.WORKPLACE,
+            "WORKS_AS": labels.WORKS_AS,
+            "CONVICTED_OF": labels.CONVICTED_OF,
+            "PENALIZED_AS": labels.PENALIZED_AS,
+            "START_TIME": labels.START_TIME,
+            "END_TIME": labels.END_TIME,
+            "EXPENDITURE": labels.EXPENDITURE,
+            "AGENT": labels.AGENT,
+            "INANIMATE_INVOLVED": labels.INANIMATE_INVOLVED,
+            "INCOME": labels.INCOME,
+            "SUBCLASS_OF": labels.SUBCLASS_OF,
+            "PART_OF": labels.PART_OF,
+            "LOCATED_IN": labels.LOCATED_IN,
+            "TREATED_USING": labels.TREATED_USING,
+            "ORIGINS_FROM": labels.ORIGINS_FROM,
+            "TO_DETECT_OR_STUDY": labels.TO_DETECT_OR_STUDY,
+            "AFFECTS": labels.AFFECTS,
+            "HAS_CAUSE": labels.HAS_CAUSE,
+            "APPLIED_TO": labels.APPLIED_TO,
+            "USED_IN": labels.USED_IN,
+            "ASSOCIATED_WITH": labels.ASSOCIATED_WITH,
+            "HAS_ADMINISTRATION_ROUTE": labels.HAS_ADMINISTRATION_ROUTE,
+            "HAS_STRENGTH": labels.HAS_STRENGTH,
+            "DURATION_OF": labels.DURATION_OF,
+            "VALUE_IS": labels.VALUE_IS,
+            "PHYSIOLOGY_OF": labels.PHYSIOLOGY_OF,
+            "PROCEDURE_PERFORMED": labels.PROCEDURE_PERFORMED,
+            "MENTAL_PROCESS_OF": labels.MENTAL_PROCESS_OF,
+            "MEDICAL_CONDITION": labels.MEDICAL_CONDITION,
+            "DOSE_IS": labels.DOSE_IS,
+            "FINDING_OF": labels.FINDING_OF,
+            "CAUSE_OF_DEATH": labels.CAUSE_OF_DEATH,
+            "CONSUME": labels.CONSUME,
+        }
+
+        super(NerelBioAnyLabelFormatter, self).__init__(stol=stol)
+

arekit/contrib/utils/pipelines/sources/ruattitudes/doc_provider.py

@@ -0,0 +1,56 @@
+from arekit.common.utils import progress_bar_iter
+from arekit.contrib.source.ruattitudes.collection import RuAttitudesCollection
+from arekit.contrib.source.ruattitudes.io_utils import RuAttitudesVersions
+from arekit.contrib.source.ruattitudes.doc import RuAttitudesDocument
+from arekit.contrib.source.ruattitudes.doc_brat import RuAttitudesDocumentsConverter
+from arekit.contrib.utils.data.doc_provider.dict_based import DictionaryBasedDocumentProvider
+
+
+class RuAttitudesDocumentProvider(DictionaryBasedDocumentProvider):
+
+    def __init__(self, version, keep_doc_ids_only, doc_id_func, limit):
+        d = self.read_ruattitudes_to_brat_in_memory(version=version,
+                                                    keep_doc_ids_only=keep_doc_ids_only,
+                                                    doc_id_func=doc_id_func,
+                                                    limit=limit)
+        super(RuAttitudesDocumentProvider, self).__init__(d)
+
+    @staticmethod
+    def read_ruattitudes_to_brat_in_memory(version, keep_doc_ids_only, doc_id_func, limit=None):
+        """ Performs reading of RuAttitude formatted documents and
+            selection according to 'doc_ids_set' parameter.
+        """
+        assert (isinstance(version, RuAttitudesVersions))
+        assert (isinstance(keep_doc_ids_only, bool))
+        assert (callable(doc_id_func))
+
+        it = RuAttitudesCollection.iter_docs(version=version,
+                                             get_doc_index_func=doc_id_func,
+                                             return_inds_only=keep_doc_ids_only)
+
+        it_formatted_and_logged = progress_bar_iter(
+            iterable=RuAttitudesDocumentProvider.__iter_id_with_doc(
+                docs_it=it, keep_doc_ids_only=keep_doc_ids_only),
+            desc="Loading RuAttitudes Collection [{}]".format("doc ids only" if keep_doc_ids_only else "fully"),
+            unit='docs')
+
+        d = {}
+        docs_read = 0
+        for doc_id, doc in it_formatted_and_logged:
+            assert(isinstance(doc, RuAttitudesDocument) or doc is None)
+            d[doc_id] = RuAttitudesDocumentsConverter.to_brat_doc(doc) if doc is not None else None
+            docs_read += 1
+            if limit is not None and docs_read >= limit:
+                break
+
+        return d
+
+    @staticmethod
+    def __iter_id_with_doc(docs_it, keep_doc_ids_only):
+        if keep_doc_ids_only:
+            for doc_id in docs_it:
+                yield doc_id, None
+        else:
+            for doc in docs_it:
+                assert (isinstance(doc, RuAttitudesDocument))
+                yield doc.ID, doc

arekit/contrib/utils/pipelines/sources/ruattitudes/entity_filter.py

@@ -0,0 +1,20 @@
+from arekit.common.entities.types import OpinionEntityType
+from arekit.contrib.utils.entities.filter import EntityFilter
+
+
+class RuAttitudesEntityFilter(EntityFilter):
+    """ This is a task-specific filter, which is applicable of entity types proposed
+        by the OntoNotesV5 resource: https://catalog.ldc.upenn.edu/LDC2013T19
+        We consider only a short list related to the sentiment attitude extraction task.
+    """
+
+    supported = ["GPE", "PERSON", "LOCAL", "GEO", "ORG"]
+
+    def is_ignored(self, entity, e_type):
+
+        if e_type == OpinionEntityType.Subject:
+            return entity.Type not in RuAttitudesEntityFilter.supported
+        if e_type == OpinionEntityType.Object:
+            return entity.Type not in RuAttitudesEntityFilter.supported
+        else:
+            return True

arekit/contrib/utils/pipelines/sources/ruattitudes/extract_text_opinions.py

@@ -0,0 +1,65 @@
+from arekit.common.labels.scaler.base import BaseLabelScaler
+from arekit.contrib.source.ruattitudes.io_utils import RuAttitudesVersions
+from arekit.contrib.source.ruattitudes.labels_fmt import RuAttitudesLabelFormatter
+from arekit.contrib.utils.pipelines.sources.ruattitudes.doc_provider import RuAttitudesDocumentProvider
+from arekit.contrib.utils.pipelines.sources.ruattitudes.entity_filter import RuAttitudesEntityFilter
+from arekit.contrib.utils.pipelines.text_opinion.annot.predefined import PredefinedTextOpinionAnnotator
+from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
+from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
+from arekit.contrib.utils.pipelines.text_opinion.filters.entity_based import EntityBasedTextOpinionFilter
+
+
+def create_text_opinion_extraction_pipeline(text_parser,
+                                            label_scaler,
+                                            custom_text_opinion_filters=None,
+                                            version=RuAttitudesVersions.V20Large,
+                                            terms_per_context=50,
+                                            limit=None):
+    """ Processing pipeline for RuAttitudes.
+        This pipeline is based on the in-memory RuAttitudes storage.
+
+        Original collection paper:  www.aclweb.org/anthology/r19-1118/
+        Github repository:          https://github.com/nicolay-r/RuAttitudes
+
+        version: enum
+            Version of the RuAttitudes collection.
+            NOTE: we consider to support a variations of the 2.0 versions.
+        label_scaler:
+            Scaler that allows to perform conversion from integer labels (RuAttitudes) to
+            the actual `Label` instances, required in further for text_opinions instances.
+        terms_per_context: int
+            Amount of terms that we consider in between the Object and Subject.
+        limit: int or None
+            Limit of documents to consider.
+    """
+    assert(isinstance(label_scaler, BaseLabelScaler))
+    assert(isinstance(version, RuAttitudesVersions))
+    assert(isinstance(custom_text_opinion_filters, list) or custom_text_opinion_filters is None)
+    assert(version in [RuAttitudesVersions.V20Large, RuAttitudesVersions.V20Base,
+                       RuAttitudesVersions.V20BaseNeut, RuAttitudesVersions.V20LargeNeut])
+
+    doc_provider = RuAttitudesDocumentProvider(version=version,
+                                               keep_doc_ids_only=False,
+                                               doc_id_func=lambda doc_id: doc_id,
+                                               limit=limit)
+
+    text_opinion_filters = [
+        EntityBasedTextOpinionFilter(entity_filter=RuAttitudesEntityFilter()),
+        DistanceLimitedTextOpinionFilter(terms_per_context)
+    ]
+
+    # Append with the custom filters afterwards.
+    if custom_text_opinion_filters is not None:
+        text_opinion_filters += custom_text_opinion_filters
+
+    pipeline = text_opinion_extraction_pipeline(
+        annotators=[
+            PredefinedTextOpinionAnnotator(doc_provider=doc_provider,
+                                           label_formatter=RuAttitudesLabelFormatter(label_scaler))
+        ],
+        text_opinion_filters=custom_text_opinion_filters,
+        get_doc_by_id_func=doc_provider.by_id,
+        entity_index_func=lambda brat_entity: brat_entity.ID,
+        text_parser=text_parser)
+
+    return pipeline

arekit/contrib/utils/pipelines/sources/rusentrel/doc_provider.py

@@ -0,0 +1,21 @@
+from arekit.common.data.doc_provider import DocumentProvider
+from arekit.common.synonyms.base import SynonymsCollection
+from arekit.contrib.source.rusentrel.io_utils import RuSentRelVersions
+from arekit.contrib.source.rusentrel.docs_reader import RuSentRelDocumentsReader
+
+
+class RuSentrelDocumentProvider(DocumentProvider):
+    """ Limitations: Supported only train/test collections format
+    """
+
+    def __init__(self, version, synonyms):
+        assert(isinstance(version, RuSentRelVersions))
+        assert(isinstance(synonyms, SynonymsCollection))
+        super(RuSentrelDocumentProvider, self).__init__()
+        self.__version = version
+        self.__synonyms = synonyms
+
+    def by_id(self, doc_id):
+        assert (isinstance(doc_id, int))
+        return RuSentRelDocumentsReader.read_document(doc_id=doc_id, synonyms=self.__synonyms, version=self.__version)
+

arekit/contrib/utils/pipelines/sources/rusentrel/extract_text_opinions.py

@@ -0,0 +1,107 @@
+from arekit.common.labels.base import NoLabel
+from arekit.common.labels.provider.constant import ConstantLabelProvider
+from arekit.common.opinions.annot.algo.pair_based import PairBasedOpinionAnnotationAlgorithm
+from arekit.common.opinions.annot.algo.predefined import PredefinedOpinionAnnotationAlgorithm
+from arekit.common.opinions.collection import OpinionCollection
+from arekit.common.synonyms.grouping import SynonymsCollectionValuesGroupingProviders
+from arekit.contrib.source.rusentrel.labels_fmt import RuSentRelLabelsFormatter
+from arekit.contrib.source.rusentrel.opinions.collection import RuSentRelOpinions
+from arekit.contrib.source.rusentrel.synonyms import RuSentRelSynonymsCollectionHelper
+from arekit.contrib.utils.pipelines.sources.rusentrel.doc_provider import RuSentrelDocumentProvider
+from arekit.contrib.utils.pipelines.text_opinion.annot.algo_based import AlgorithmBasedTextOpinionAnnotator
+from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
+from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
+from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
+from arekit.contrib.utils.synonyms.stemmer_based import StemmerBasedSynonymCollection
+
+
+def create_text_opinion_extraction_pipeline(rusentrel_version,
+                                            text_parser,
+                                            labels_fmt,
+                                            custom_text_opinion_filters=None,
+                                            no_label=NoLabel(),
+                                            terms_per_context=50,
+                                            dist_in_sentences=0):
+    """ Processing pipeline for RuSentRel, which combines:
+            - predefined document-level annotation (sentiment labels)
+            - automatic annotation of opinions between mentioned named entities (no-label)
+
+        Original collection paper: arxiv.org/abs/1808.08932
+
+        version: enum
+            Version of the RuSentRel collection.
+        terms_per_context: int
+            Amount of terms that we consider in between the Object and Subject.
+        dist_in_sentences: int
+            considering amount of sentences that could be in between Object and Subject.
+    """
+    assert(isinstance(labels_fmt, RuSentRelLabelsFormatter))
+    assert(isinstance(custom_text_opinion_filters, list) or custom_text_opinion_filters is None)
+
+    synonyms = StemmerBasedSynonymCollection(
+        iter_group_values_lists=RuSentRelSynonymsCollectionHelper.iter_groups(rusentrel_version),
+        stemmer=MystemWrapper(),
+        is_read_only=False)
+
+    doc_provider = RuSentrelDocumentProvider(version=rusentrel_version, synonyms=synonyms)
+
+    text_opinion_filters = [
+        DistanceLimitedTextOpinionFilter(terms_per_context)
+    ]
+
+    # Append with the custom filters afterwards.
+    if custom_text_opinion_filters is not None:
+        text_opinion_filters += custom_text_opinion_filters
+
+    pipeline = text_opinion_extraction_pipeline(
+        annotators=[
+            predefined_annotator(synonyms=synonyms, labels_fmt=labels_fmt),
+            nolabel_annotator(synonyms=synonyms, terms_per_context=terms_per_context,
+                              dist_in_sentences=dist_in_sentences, no_label=no_label)
+        ],
+        text_opinion_filters=text_opinion_filters,
+        get_doc_by_id_func=doc_provider.by_id,
+        entity_index_func=lambda brat_entity: brat_entity.ID,
+        text_parser=text_parser)
+
+    return pipeline
+
+
+def nolabel_annotator(synonyms, terms_per_context, dist_in_sentences=0, no_label=NoLabel()):
+    """ This is a default annotator, utilized to annotate `neutral`-like attitudes.
+        Neutral means that we adopt no-label parameter, and this label might be customized
+        to the one required in your studies.
+    """
+    return AlgorithmBasedTextOpinionAnnotator(
+        annot_algo=PairBasedOpinionAnnotationAlgorithm(dist_in_sents=dist_in_sentences,
+                                                       dist_in_terms_bound=terms_per_context,
+                                                       label_provider=ConstantLabelProvider(no_label),
+                                                       entity_index_func=lambda brat_entity: brat_entity.ID),
+        create_empty_collection_func=lambda: OpinionCollection(
+            synonyms=synonyms, error_on_duplicates=True, error_on_synonym_end_missed=False),
+        value_to_group_id_func=lambda value:
+        SynonymsCollectionValuesGroupingProviders.provide_existed_value(synonyms=synonyms, value=value))
+
+
+def predefined_annotator(synonyms, labels_fmt):
+    """ This is a annotator-converter of the predefined Document-Level opinions onto text-level one
+    """
+    return AlgorithmBasedTextOpinionAnnotator(
+        annot_algo=PredefinedOpinionAnnotationAlgorithm(
+            lambda doc_id: __get_document_opinions(doc_id=doc_id, synonyms=synonyms, labels_fmt=labels_fmt)),
+        create_empty_collection_func=lambda: OpinionCollection(
+            synonyms=synonyms, error_on_duplicates=True, error_on_synonym_end_missed=False),
+        value_to_group_id_func=lambda value:
+        SynonymsCollectionValuesGroupingProviders.provide_existed_value(synonyms=synonyms, value=value))
+
+
+def __get_document_opinions(doc_id, synonyms, labels_fmt):
+    """ RuSentRel provides a pre-defined list of Document-Level Opinions.
+        Within this function we create the related OpinionCollection by a given doc_id.
+    """
+    return OpinionCollection(
+        opinions=RuSentRelOpinions.iter_from_doc(
+            doc_id=doc_id, labels_fmt=labels_fmt),
+        synonyms=synonyms,
+        error_on_synonym_end_missed=True,
+        error_on_duplicates=True)

arekit/contrib/utils/pipelines/sources/sentinerel/doc_provider.py

@@ -0,0 +1,29 @@
+from arekit.common.data.doc_provider import DocumentProvider
+from arekit.contrib.source.sentinerel.io_utils import SentiNerelVersions
+from arekit.contrib.source.sentinerel.reader import SentiNerelDocReader
+
+
+class SentiNERELDocProvider(DocumentProvider):
+    """ Document reader for the collection of the RuSentNE competition 2023.
+        For more details please follow the following repository:
+        github: https://github.com/dialogue-evaluation/RuSentNE-evaluation
+    """
+
+    def __init__(self, filename_by_id, version):
+        """ filename_ids: dict
+                Dictionary of {id: filename}, where
+                    - id: int
+                    - filename: str
+            version: SentiNerelVersions
+                Specify the appropriate version of teh SentiNEREL collection.
+        """
+        assert(isinstance(filename_by_id, dict))
+        assert(isinstance(version, SentiNerelVersions))
+        super(SentiNERELDocProvider, self).__init__()
+        self.__filename_by_id = filename_by_id
+        self.__version = version
+
+    def by_id(self, doc_id):
+        return SentiNerelDocReader.read_document(doc_id=doc_id,
+                                                 version=self.__version,
+                                                 filename=self.__filename_by_id[doc_id])