arekit 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/utils/io_utils/samples.py

@@ -1,78 +0,0 @@
-import logging
-from os.path import join
-
-from arekit.contrib.utils.data.ext import create_writer_extension, create_reader_extension
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.contrib.utils.data.writers.base import BaseWriter
-from arekit.contrib.utils.io_utils.utils import filename_template, check_targets_existence
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class SamplesIO(BaseSamplesIO):
-    """ Samples default IO utils for samples.
-            Sample is a text part which include pair of attitude participants.
-            This class allows to provide saver and loader for such entries, bubbed as samples.
-            Samples required for machine learning training/inferring.
-    """
-
-    def __init__(self, target_dir, writer=None, reader=None, prefix="sample", target_extension=None):
-        assert(isinstance(target_dir, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(writer, BaseWriter) or writer is None)
-        assert(isinstance(reader, BaseReader) or reader is None)
-        assert(isinstance(target_extension, str) or target_extension is None)
-        self.__target_dir = target_dir
-        self.__prefix = prefix
-        self.__writer = writer
-        self.__reader = reader
-        self.__target_extension = target_extension
-
-        if target_extension is None:
-            if writer is not None:
-                self.__target_extension = create_writer_extension(writer)
-            elif reader is not None:
-                self.__target_extension = create_reader_extension(reader)
-
-    # region public methods
-
-    @property
-    def Reader(self):
-        return self.__reader
-
-    @property
-    def Writer(self):
-        return self.__writer
-
-    def create_target(self, data_type, data_folding):
-        return self.__get_input_sample_target(data_type, data_folding=data_folding)
-
-    def check_targets_existed(self, data_types_iter, data_folding):
-        for data_type in data_types_iter:
-
-            targets = [
-                self.__get_input_sample_target(data_type=data_type, data_folding=data_folding),
-            ]
-
-            if not check_targets_existence(targets=targets):
-                return False
-        return True
-
-    # endregion
-
-    def __get_input_sample_target(self, data_type, data_folding):
-        template = filename_template(data_type=data_type, data_folding=data_folding)
-        return self.__get_filepath(out_dir=self.__target_dir,
-                                   template=template,
-                                   prefix=self.__prefix,
-                                   extension=self.__target_extension)
-
-    @staticmethod
-    def __get_filepath(out_dir, template, prefix, extension):
-        assert(isinstance(template, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(extension, str))
-        return join(out_dir, "{prefix}-{template}{extension}".format(
-            prefix=prefix, template=template, extension=extension))

arekit/contrib/utils/lexicons/lexicon.py

@@ -1,43 +0,0 @@
-import pandas as pd
-from arekit.contrib.utils.data.readers.csv_pd import PandasCsvReader
-
-
-class Lexicon(object):
-
-    @property
-    def ToneKey(self):
-        return 'tone'
-
-    @property
-    def TermKey(self):
-        return 'term'
-
-    def __init__(self, dataframe):
-        assert(isinstance(dataframe, pd.DataFrame))
-        self.__lexicon_df = dataframe
-
-    @classmethod
-    def load(cls, filepath, separator=','):
-        reader = PandasCsvReader(compression=None, sep=separator)
-        return cls(reader.read(filepath))
-
-    def get_score(self, lemma):
-        assert(type(lemma) == str)
-        s = self.__lexicon_df[lemma.encode('utf-8') == self.__lexicon_df[self.TermKey]]
-        return s[self.ToneKey].values[0] if len(s) > 0 else 0
-
-    def has_term(self, term):
-        assert(type(term) == str)
-        s = self.__lexicon_df[term.encode('utf-8') == self.__lexicon_df[self.TermKey]]
-        return len(s) > 0
-
-    def __iter__(self):
-        for term in self.__lexicon_df[self.TermKey]:
-            yield term
-
-    def __contains__(self, item):
-        assert(isinstance(item, str))
-        result = self.__lexicon_df[self.__lexicon_df[self.TermKey] == item.encode('utf-8')]
-        return len(result) > 0
-
-

arekit/contrib/utils/lexicons/relation.py

@@ -1,45 +0,0 @@
-import pandas as pd
-
-from arekit.contrib.utils.data.readers.csv_pd import PandasCsvReader
-
-
-class RelationLexicon(object):
-
-    def __init__(self, dataframe):
-        assert(isinstance(dataframe, pd.DataFrame))
-        self.__check(dataframe)
-        self.__lexicon = dataframe
-
-    @classmethod
-    def load(cls, filepath, separator=','):
-        reader = PandasCsvReader(compression=None, sep=separator)
-        return cls(reader.read(filepath))
-
-    @staticmethod
-    def __check(df):
-        for index in df.index:
-            relation = df.loc[index][0]
-            assert(len(relation.split('<->')) == 2)
-
-    @staticmethod
-    def __create_key(l, r):
-        assert(type(l) == str)
-        assert(type(r) == str)
-        return '<->'.join([l, r])
-
-    def get_score(self, left, right):
-        assert(type(left) == str)
-        assert(type(right) == str)
-
-        lr_key = self.__create_key(left, right)
-        rl_key = self.__create_key(right, left)
-
-        lr_score = self.__lexicon[lr_key == self.__lexicon['relation']]
-        rl_score = self.__lexicon[rl_key == self.__lexicon['relation']]
-
-        if len(lr_score) > 0:
-            return lr_score['tone'].values[0]
-        if len(rl_score) > 0:
-            return rl_score['tone'].values[0]
-
-        return None

arekit/contrib/utils/lexicons/rusentilex.py

@@ -1,34 +0,0 @@
-import zipfile
-from os import path
-
-import pandas as pd
-
-from arekit.contrib.source.zip_utils import ZipArchiveUtils
-from arekit.contrib.utils.lexicons.lexicon import Lexicon
-
-
-class RuSentiLexLexicon(Lexicon):
-    """
-    RuSentiLex Lexicon wrapper for csv file stored in /data folder.
-    """
-
-    __INNER_PATH = 'rusentilex.csv'
-
-    @property
-    def ToneKey(self):
-        return 'tone'
-
-    @property
-    def TermKey(self):
-        return 'term'
-
-    @staticmethod
-    def __get_archive_filepath():
-        return path.join(ZipArchiveUtils.get_data_root(), "rusentilex.zip")
-
-    @classmethod
-    def from_zip(cls):
-        with zipfile.ZipFile(cls.__get_archive_filepath(), "r") as zip_ref:
-            with zip_ref.open(cls.__INNER_PATH, mode='r') as csv_file:
-                df = pd.read_csv(csv_file, sep=',')
-                return cls(df)

arekit/contrib/utils/nn/rows.py

@@ -1,83 +0,0 @@
-import collections
-
-from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
-from arekit.common.data.input.terms_mapper import OpinionContainingTextTermsMapper
-from arekit.common.entities.str_fmt import StringEntitiesFormatter
-from arekit.contrib.networks.input.ctx_serialization import NetworkSerializationContext
-from arekit.contrib.networks.input.formatters.pos_mapper import PosTermsMapper
-from arekit.contrib.networks.input.providers.sample import NetworkSampleRowProvider
-from arekit.contrib.networks.input.providers.text import NetworkSingleTextProvider
-from arekit.contrib.networks.input.term_types import TermTypes
-from arekit.contrib.networks.input.terms_mapping import VectorizedNetworkTermMapping
-from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
-from arekit.contrib.utils.resources import load_embedding_news_mystem_skipgram_1000_20_2015
-from arekit.contrib.utils.vectorizers.bpe import BPEVectorizer
-from arekit.contrib.utils.vectorizers.random_norm import RandomNormalVectorizer
-
-
-def __add_term_embedding(dict_data, term, emb_vector):
-    if term in dict_data:
-        return
-    dict_data[term] = emb_vector
-
-
-def create_rows_provider(str_entity_fmt, ctx, vectorizers="default"):
-    """ This method is corresponds to the default initialization of
-        the rows provider for data sampling pipeline.
-
-        vectorizers:
-            NONE: no need to vectorize, just provide text (using SingleTextProvider).
-            DEFAULT: we consider an application of stemmer for Russian Language.
-            DICT: in which for every type there is an assigned Vectorizer
-                vectorization of term types.
-                {
-                    TermType.Word: Vectorizer,
-                    TermType.Entity: Vectorizer,
-                    ...
-                }
-    """
-    assert(isinstance(str_entity_fmt, StringEntitiesFormatter))
-    assert(isinstance(ctx, NetworkSerializationContext))
-    assert(isinstance(vectorizers, dict) or vectorizers == "default" or vectorizers is None)
-
-    term_embedding_pairs = None
-
-    if vectorizers is not None:
-
-        if vectorizers == "default":
-            # initialize default vectorizer for Russian language.
-            embedding = load_embedding_news_mystem_skipgram_1000_20_2015(stemmer=MystemWrapper(), auto_download=True)
-            bpe_vectorizer = BPEVectorizer(embedding=embedding, max_part_size=3)
-            norm_vectorizer = RandomNormalVectorizer(vector_size=embedding.VectorSize,
-                                                     token_offset=12345)
-            vectorizers = {
-                TermTypes.WORD: bpe_vectorizer,
-                TermTypes.ENTITY: bpe_vectorizer,
-                TermTypes.FRAME: bpe_vectorizer,
-                TermTypes.TOKEN: norm_vectorizer
-            }
-
-        # Setup term-embedding pairs collection instance.
-        term_embedding_pairs = collections.OrderedDict()
-
-        # Use text provider with vectorizers.
-        text_provider = NetworkSingleTextProvider(
-            text_terms_mapper=VectorizedNetworkTermMapping(
-                vectorizers=vectorizers,
-                string_entities_formatter=str_entity_fmt),
-            pair_handling_func=lambda pair: __add_term_embedding(
-                dict_data=term_embedding_pairs,
-                term=pair[0],
-                emb_vector=pair[1]))
-    else:
-        # Create text provider which without vectorizers.
-        text_provider = BaseSingleTextProvider(
-            text_terms_mapper=OpinionContainingTextTermsMapper(str_entity_fmt))
-
-    return NetworkSampleRowProvider(
-        label_provider=ctx.LabelProvider,
-        text_provider=text_provider,
-        frames_connotation_provider=ctx.FramesConnotationProvider,
-        frame_role_label_scaler=ctx.FrameRolesLabelScaler,
-        pos_terms_mapper=PosTermsMapper(ctx.PosTagger) if ctx.PosTagger is not None else None,
-        term_embedding_pairs=term_embedding_pairs)

arekit/contrib/utils/pipelines/items/sampling/bert.py

@@ -1,5 +0,0 @@
-from arekit.contrib.utils.pipelines.items.sampling.base import BaseSerializerPipelineItem
-
-
-class BertExperimentInputSerializerPipelineItem(BaseSerializerPipelineItem):
-    pass

arekit/contrib/utils/pipelines/items/text/terms_splitter.py

@@ -1,10 +0,0 @@
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.utils import split_by_whitespaces
-
-
-class TermsSplitterParser(BasePipelineItem):
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        return split_by_whitespaces(input_data)

arekit/contrib/utils/pipelines/items/to_output.py

@@ -1,101 +0,0 @@
-from arekit.common.data import const
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.common.experiment.data_type import DataType
-from arekit.common.folding.base import BaseDataFolding
-from arekit.common.labels.scaler.base import BaseLabelScaler
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.common.model.labeling.modes import LabelCalculationMode
-from arekit.common.opinions.writer import OpinionCollectionWriter
-from arekit.common.pipeline.base import BasePipeline
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.pipeline.items.handle import HandleIterPipelineItem
-from arekit.contrib.utils.data.views.linkages.multilabel import MultilableOpinionLinkagesView
-from arekit.contrib.utils.data.views.opinions import BaseOpinionStorageView
-from arekit.contrib.utils.io_utils.opinions import OpinionsIO
-from arekit.contrib.utils.utils_folding import folding_iter_states, experiment_iter_index
-from arekit.contrib.utils.pipelines.opinion_collections import \
-    text_opinion_linkages_to_opinion_collections_pipeline_part
-
-
-class TextOpinionLinkagesToOpinionConverterPipelineItem(BasePipelineItem):
-
-    def __init__(self, opinions_io, create_opinion_collection_func,
-                 opinion_collection_writer, label_scaler, labels_formatter):
-        """ create_opinion_collection_func: func
-                func () -> OpinionCollection (empty)
-        """
-        assert(isinstance(opinions_io, OpinionsIO))
-        assert(callable(create_opinion_collection_func))
-        assert(isinstance(label_scaler, BaseLabelScaler))
-        assert(isinstance(labels_formatter, StringLabelsFormatter))
-        assert(isinstance(opinion_collection_writer, OpinionCollectionWriter))
-        super(TextOpinionLinkagesToOpinionConverterPipelineItem, self).__init__()
-
-        self.__opinions_io = opinions_io
-        self.__labels_formatter = labels_formatter
-        self.__label_scaler = label_scaler
-        self.__create_opinion_collection_func = create_opinion_collection_func
-        self.__opinion_collection_writer = opinion_collection_writer
-
-    def __convert(self, data_folding, output_storage, target_func, data_type, pipeline_ctx):
-        """ From `output_storage` to `target` conversion.
-            output_storage: BaseRowsStorage
-            target_func: func(doc_id) -- considered to provide a target for the particular document.
-        """
-        assert(isinstance(data_folding, BaseDataFolding))
-        assert(isinstance(output_storage, BaseRowsStorage))
-        assert(isinstance(data_type, DataType))
-        assert(callable(target_func))
-
-        # We utilize google bert format, where every row
-        # consist of label probabilities per every class
-        linkages_view = MultilableOpinionLinkagesView(labels_scaler=self.__label_scaler,
-                                                      storage=output_storage)
-        target = self.__opinions_io.create_target(data_type=data_type, data_folding=data_folding)
-        storage = self.__opinions_io.Reader.read(target)
-
-        converter_part = text_opinion_linkages_to_opinion_collections_pipeline_part(
-            iter_opinion_linkages_func=lambda doc_id: linkages_view.iter_opinion_linkages(
-                doc_id=doc_id, opinions_view=BaseOpinionStorageView(storage)),
-            doc_ids_set=set(data_folding.fold_doc_ids_set()[data_type]),
-            create_opinion_collection_func=self.__create_opinion_collection_func,
-            labels_scaler=self.__label_scaler,
-            label_calc_mode=LabelCalculationMode.AVERAGE)
-
-        pipeline = BasePipeline(
-            converter_part +
-            [HandleIterPipelineItem(lambda data: self.__opinion_collection_writer.serialize(
-                collection=data[1],
-                encoding='utf-8',
-                labels_formatter=self.__labels_formatter,
-                error_on_non_supported=True,
-                target=target_func(data[0])))
-             ])
-
-        input_data = set(output_storage.iter_column_values(column_name=const.DOC_ID))
-
-        # iterate over the result.
-        for _ in pipeline.run(input_data, parent_ctx=pipeline_ctx):
-            pass
-
-    def _iter_output_and_target_pairs(self, iter_index, data_type):
-        raise NotImplementedError()
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        assert("data_folding" in pipeline_ctx)
-        assert("data_type" in pipeline_ctx)
-
-        data_folding = pipeline_ctx.provide("data_folding")
-        data_type = pipeline_ctx.provide("data_type")
-
-        for _ in folding_iter_states(data_folding):
-            iter_index = experiment_iter_index(data_folding)
-            pairs_it = self._iter_output_and_target_pairs(iter_index=iter_index, data_type=data_type)
-            for output_storage, target in pairs_it:
-                self.__convert(output_storage=output_storage,
-                               target_func=target,
-                               data_type=data_type,
-                               data_folding=data_folding,
-                               pipeline_ctx=pipeline_ctx)

arekit/contrib/utils/pipelines/sources/nerel/doc_ops.py

@@ -1,27 +0,0 @@
-from arekit.common.experiment.api.ops_doc import DocumentOperations
-from arekit.contrib.source.nerel.reader import NerelDocReader
-from arekit.contrib.source.nerel.versions import NerelVersions
-
-
-class NERELDocOperation(DocumentOperations):
-    """ A Russian dataset with nested named entities, relations, events and linked entities.
-        https://github.com/nerel-ds/NEREL
-    """
-
-    def __init__(self, filename_by_id, version):
-        """ filename_ids: dict
-                Dictionary of {id: filename}, where
-                    - id: int
-                    - filename: str
-            version: NerelVersions
-                Specify the appropriate version of teh NEREL collection.
-        """
-        assert(isinstance(filename_by_id, dict))
-        assert(isinstance(version, NerelVersions))
-        super(NERELDocOperation, self).__init__()
-        self.__filename_by_id = filename_by_id
-        self.__version = version
-        self.__doc_reader = NerelDocReader(version)
-
-    def by_id(self, doc_id):
-        return self.__doc_reader.read_document(doc_id=doc_id, filename=self.__filename_by_id[doc_id])

arekit/contrib/utils/pipelines/sources/nerel/extract_text_relations.py

@@ -1,59 +0,0 @@
-from arekit.common.experiment.api.ops_doc import DocumentOperations
-from arekit.common.experiment.data_type import DataType
-from arekit.contrib.source.nerel.io_utils import NerelIOUtils
-from arekit.contrib.source.nerel.versions import NerelVersions
-from arekit.contrib.utils.pipelines.sources.nerel.doc_ops import NERELDocOperation
-from arekit.contrib.utils.pipelines.sources.nerel.labels_fmt import NerelAnyLabelFormatter
-from arekit.contrib.utils.pipelines.text_opinion.annot.predefined import PredefinedTextOpinionAnnotator
-from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
-from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
-from arekit.contrib.utils.pipelines.text_opinion.filters.entity_based import EntityBasedTextOpinionFilter
-
-
-def create_text_relation_extraction_pipeline(nerel_version,
-                                             text_parser,
-                                             label_formatter=NerelAnyLabelFormatter(),
-                                             terms_per_context=50,
-                                             doc_ops=None,
-                                             docs_limit=None,
-                                             entity_filter=None):
-    assert(isinstance(nerel_version, NerelVersions))
-    assert(isinstance(doc_ops, DocumentOperations) or doc_ops is None)
-
-    data_folding = None
-
-    if doc_ops is None:
-        # Default Initialization.
-        filenames_by_ids, data_folding = NerelIOUtils.read_dataset_split(version=nerel_version,
-                                                                         docs_limit=docs_limit)
-        doc_ops = NERELDocOperation(filename_by_id=filenames_by_ids,
-                                    version=nerel_version)
-
-    text_opinion_filters = [
-        EntityBasedTextOpinionFilter(entity_filter=entity_filter),
-        DistanceLimitedTextOpinionFilter(terms_per_context)
-    ]
-
-    predefined_annot = PredefinedTextOpinionAnnotator(doc_ops, label_formatter)
-
-    pipelines = {
-        DataType.Train: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                         get_doc_by_id_func=doc_ops.by_id,
-                                                         annotators=[predefined_annot],
-                                                         text_opinion_filters=text_opinion_filters),
-        DataType.Test: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                        get_doc_by_id_func=doc_ops.by_id,
-                                                        annotators=[predefined_annot],
-                                                        text_opinion_filters=text_opinion_filters),
-        DataType.Dev: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                       get_doc_by_id_func=doc_ops.by_id,
-                                                       annotators=[predefined_annot],
-                                                       text_opinion_filters=text_opinion_filters),
-    }
-
-    # In the case when we setup a default data-folding.
-    # There is a need to provide it, due to the needs in further.
-    if data_folding is not None:
-        return pipelines, data_folding
-
-    return pipelines

arekit/contrib/utils/pipelines/sources/nerel/labels_fmt.py

@@ -1,60 +0,0 @@
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.contrib.source.nerel import labels
-
-
-class NerelAnyLabelFormatter(StringLabelsFormatter):
-
-    def __init__(self):
-
-        stol = {
-            "OPINION_BELONGS_TO": labels.OpinionBelongsTo,
-            "OPINION_RELATES_TO": labels.OpinionRelatesTo,
-            "NEG_EFFECT_FROM": labels.NegEffectFrom,
-            "POS_EFFECT_FROM": labels.PosEffectFrom,
-            "NEG_STATE_FROM": labels.NegStateFrom,
-            "POS_STATE_FROM": labels.PosStateFrom,
-            "NEGATIVE_TO": labels.NegativeTo,
-            "POSITIVE_TO": labels.PositiveTo,
-            "STATE_BELONGS_TO": labels.STATE_BELONGS_TO,
-            "POS_AUTHOR_FROM": labels.PosAuthorFrom,
-            "NEG_AUTHOR_FROM": labels.NegAuthorFrom,
-            "ALTERNATIVE_NAME": labels.ALTERNATIVE_NAME,
-            "ORIGINS_FROM": labels.ORIGINS_FROM,
-            "START_TIME": labels.START_TIME,
-            "OWNER_OF": labels.OWNER_OF,
-            "SUBEVENT_OF": labels.SUBEVENT_OF,
-            "PARENT_OF": labels.PARENT_OF,
-            "SUBORDINATE_OF": labels.SUBORDINATE_OF,
-            "PART_OF": labels.PART_OF,
-            "TAKES_PLACE_IN": labels.TAKES_PLACE_IN,
-            "PARTICIPANT_IN": labels.PARTICIPANT_IN,
-            "WORKPLACE": labels.WORKPLACE,
-            "PENALIZED_AS": labels.PENALIZED_AS,
-            "WORKS_AS": labels.WORKS_AS,
-            "PLACE_OF_DEATH": labels.PLACE_OF_DEATH,
-            "PLACE_OF_BIRTH": labels.PLACE_OF_BIRTH,
-            "HAS_CAUSE": labels.HAS_CAUSE,
-            "AWARDED_WITH": labels.AWARDED_WITH,
-            "CAUSE_OF_DEATH": labels.CAUSE_OF_DEATH,
-            "CONVICTED_OF": labels.CONVICTED_OF,
-            "DATE_DEFUNCT_IN": labels.DATE_DEFUNCT_IN,
-            "DATE_FOUNDED_IN": labels.DATE_FOUNDED_IN,
-            "DATE_OF_BIRTH": labels.DATE_OF_BIRTH,
-            "DATE_OF_CREATION": labels.DATE_OF_CREATION,
-            "DATE_OF_DEATH": labels.DATE_OF_DEATH,
-            "END_TIME": labels.END_TIME,
-            "EXPENDITURE": labels.EXPENDITURE,
-            "FOUNDED_BY": labels.FOUNDED_BY,
-            "KNOWS": labels.KNOWS,
-            "RELATIVE": labels.RELATIVE,
-            "LOCATED_IN": labels.LOCATED_IN,
-            "RELIGION_OF": labels.RELIGION_OF,
-            "MEDICAL_CONDITION": labels.MEDICAL_CONDITION,
-            "SCHOOLS_ATTENDED": labels.SCHOOLS_ATTENDED,
-            "MEMBER_OF": labels.MEMBER_OF,
-            "SIBLING": labels.SIBLING,
-            "ORGANIZES": labels.ORGANIZES,
-            "SPOUSE": labels.SPOUSE
-        }
-
-        super(NerelAnyLabelFormatter, self).__init__(stol=stol)

arekit/contrib/utils/pipelines/sources/nerel_bio/doc_ops.py

@@ -1,29 +0,0 @@
-from arekit.common.experiment.api.ops_doc import DocumentOperations
-from arekit.contrib.source.nerelbio.reader import NerelBioDocReader
-from arekit.contrib.source.nerelbio.versions import NerelBioVersions
-
-
-class NERELBioDocOperation(DocumentOperations):
-    """ NEREL-BIO extends the general domain dataset NEREL.
-        NEREL-BIO annotation scheme covers both general and biomedical
-        domains making it suitable for domain transfer experiments.
-        https://github.com/nerel-ds/NEREL-BIO
-    """
-
-    def __init__(self, filename_by_id, version):
-        """ filename_ids: dict
-                Dictionary of {id: filename}, where
-                    - id: int
-                    - filename: str
-            version: NerelBioVersions
-                Specify the appropriate version of the NEREL-BIO collection.
-        """
-        assert(isinstance(filename_by_id, dict))
-        assert(isinstance(version, NerelBioVersions))
-        super(NERELBioDocOperation, self).__init__()
-        self.__filename_by_id = filename_by_id
-        self.__version = version
-        self.__doc_reader = NerelBioDocReader(version)
-
-    def by_id(self, doc_id):
-        return self.__doc_reader.read_document(doc_id=doc_id, filename=self.__filename_by_id[doc_id])

arekit/contrib/utils/pipelines/sources/nerel_bio/extrat_text_relations.py

@@ -1,59 +0,0 @@
-from arekit.common.experiment.api.ops_doc import DocumentOperations
-from arekit.common.experiment.data_type import DataType
-from arekit.contrib.source.nerelbio.io_utils import NerelBioIOUtils
-from arekit.contrib.source.nerelbio.versions import NerelBioVersions
-from arekit.contrib.utils.pipelines.sources.nerel_bio.doc_ops import NERELBioDocOperation
-from arekit.contrib.utils.pipelines.sources.nerel_bio.labels_fmt import NerelBioAnyLabelFormatter
-from arekit.contrib.utils.pipelines.text_opinion.annot.predefined import PredefinedTextOpinionAnnotator
-from arekit.contrib.utils.pipelines.text_opinion.extraction import text_opinion_extraction_pipeline
-from arekit.contrib.utils.pipelines.text_opinion.filters.distance_based import DistanceLimitedTextOpinionFilter
-from arekit.contrib.utils.pipelines.text_opinion.filters.entity_based import EntityBasedTextOpinionFilter
-
-
-def create_text_relation_extraction_pipeline(nerel_bio_version,
-                                             text_parser,
-                                             label_formatter=NerelBioAnyLabelFormatter(),
-                                             terms_per_context=50,
-                                             doc_ops=None,
-                                             docs_limit=None,
-                                             entity_filter=None):
-    assert(isinstance(nerel_bio_version, NerelBioVersions))
-    assert(isinstance(doc_ops, DocumentOperations) or doc_ops is None)
-
-    data_folding = None
-
-    if doc_ops is None:
-        # Default Initialization.
-        filenames_by_ids, data_folding = NerelBioIOUtils.read_dataset_split(version=nerel_bio_version,
-                                                                            docs_limit=docs_limit)
-        doc_ops = NERELBioDocOperation(filename_by_id=filenames_by_ids,
-                                       version=nerel_bio_version)
-
-    text_opinion_filters = [
-        EntityBasedTextOpinionFilter(entity_filter=entity_filter),
-        DistanceLimitedTextOpinionFilter(terms_per_context)
-    ]
-
-    predefined_annot = PredefinedTextOpinionAnnotator(doc_ops, label_formatter)
-
-    pipelines = {
-        DataType.Train: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                         get_doc_by_id_func=doc_ops.by_id,
-                                                         annotators=[predefined_annot],
-                                                         text_opinion_filters=text_opinion_filters),
-        DataType.Test: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                        get_doc_by_id_func=doc_ops.by_id,
-                                                        annotators=[predefined_annot],
-                                                        text_opinion_filters=text_opinion_filters),
-        DataType.Dev: text_opinion_extraction_pipeline(text_parser=text_parser,
-                                                       get_doc_by_id_func=doc_ops.by_id,
-                                                       annotators=[predefined_annot],
-                                                       text_opinion_filters=text_opinion_filters),
-    }
-
-    # In the case when we setup a default data-folding.
-    # There is a need to provide it, due to the needs in further.
-    if data_folding is not None:
-        return pipelines, data_folding
-
-    return pipelines