arekit 0.24.0__py3-none-any.whl → 0.25.1__py3-none-any.whl

arekit/contrib/utils/io_utils/opinions.py

@@ -1,37 +0,0 @@
-from os.path import join
-
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.contrib.utils.io_utils.utils import filename_template
-
-
-class OpinionsIO(BaseSamplesIO):
-
-    def __init__(self, target_dir, reader=None, prefix="opinion"):
-        assert(isinstance(reader, BaseReader))
-        self.__target_dir = target_dir
-        self.__prefix = prefix
-        self.__reader = reader
-        self.__target_extension = reader.extension()
-
-    @property
-    def Reader(self):
-        return self.__reader
-
-    def create_target(self, data_type):
-        return self.__get_input_opinions_target(data_type)
-
-    def __get_input_opinions_target(self, data_type):
-        template = filename_template(data_type=data_type)
-        return self.__get_filepath(out_dir=self.__target_dir,
-                                   template=template,
-                                   prefix=self.__prefix,
-                                   extension=self.__target_extension)
-
-    @staticmethod
-    def __get_filepath(out_dir, template, prefix, extension):
-        assert(isinstance(template, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(extension, str))
-        return join(out_dir, "{prefix}-{template}{extension}".format(
-            prefix=prefix, template=template, extension=extension))

arekit/contrib/utils/io_utils/samples.py

@@ -1,79 +0,0 @@
-import logging
-from os.path import join
-
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.contrib.utils.data.writers.base import BaseWriter
-from arekit.contrib.utils.io_utils.utils import filename_template, check_targets_existence
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class SamplesIO(BaseSamplesIO):
-    """ Samples default IO utils for samples.
-            Sample is a text part which include pair of attitude participants.
-            This class allows to provide saver and loader for such entries, bubbed as samples.
-            Samples required for machine learning training/inferring.
-    """
-
-    def __init__(self, target_dir, writer=None, reader=None, prefix="sample"):
-        assert(isinstance(target_dir, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(writer, BaseWriter) or writer is None)
-        assert(isinstance(reader, BaseReader) or reader is None)
-        self.__target_dir = target_dir
-        self.__prefix = prefix
-        self.__writer = writer
-        self.__reader = reader
-
-        self.__target_extension = None
-        if writer is not None:
-            self.__target_extension = writer.extension()
-        elif reader is not None:
-            self.__target_extension = reader.extension()
-
-    # region public methods
-
-    @property
-    def Prefix(self):
-        return self.__prefix
-
-    @property
-    def Reader(self):
-        return self.__reader
-
-    @property
-    def Writer(self):
-        return self.__writer
-
-    def create_target(self, data_type):
-        return self.__get_input_sample_target(data_type)
-
-    def check_targets_existed(self, data_types_iter):
-        for data_type in data_types_iter:
-
-            targets = [
-                self.__get_input_sample_target(data_type=data_type),
-            ]
-
-            if not check_targets_existence(targets=targets):
-                return False
-        return True
-
-    # endregion
-
-    def __get_input_sample_target(self, data_type):
-        template = filename_template(data_type=data_type)
-        return self.__get_filepath(out_dir=self.__target_dir,
-                                   template=template,
-                                   prefix=self.__prefix,
-                                   extension=self.__target_extension)
-
-    @staticmethod
-    def __get_filepath(out_dir, template, prefix, extension):
-        assert(isinstance(template, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(extension, str))
-        return join(out_dir, "{prefix}-{template}{extension}".format(
-            prefix=prefix, template=template, extension=extension))

arekit/contrib/utils/lexicons/lexicon.py

@@ -1,41 +0,0 @@
-from arekit.contrib.utils.data.readers.csv_pd import PandasCsvReader
-
-
-class Lexicon(object):
-
-    @property
-    def ToneKey(self):
-        return 'tone'
-
-    @property
-    def TermKey(self):
-        return 'term'
-
-    def __init__(self, dataframe):
-        self.__lexicon_df = dataframe
-
-    @classmethod
-    def load(cls, filepath, separator=','):
-        reader = PandasCsvReader(compression=None, sep=separator)
-        return cls(reader.read(filepath))
-
-    def get_score(self, lemma):
-        assert(type(lemma) == str)
-        s = self.__lexicon_df[lemma.encode('utf-8') == self.__lexicon_df[self.TermKey]]
-        return s[self.ToneKey].values[0] if len(s) > 0 else 0
-
-    def has_term(self, term):
-        assert(type(term) == str)
-        s = self.__lexicon_df[term.encode('utf-8') == self.__lexicon_df[self.TermKey]]
-        return len(s) > 0
-
-    def __iter__(self):
-        for term in self.__lexicon_df[self.TermKey]:
-            yield term
-
-    def __contains__(self, item):
-        assert(isinstance(item, str))
-        result = self.__lexicon_df[self.__lexicon_df[self.TermKey] == item.encode('utf-8')]
-        return len(result) > 0
-
-

arekit/contrib/utils/lexicons/relation.py

@@ -1,42 +0,0 @@
-from arekit.contrib.utils.data.readers.csv_pd import PandasCsvReader
-
-
-class RelationLexicon(object):
-
-    def __init__(self, dataframe):
-        self.__check(dataframe)
-        self.__lexicon = dataframe
-
-    @classmethod
-    def load(cls, filepath, separator=','):
-        reader = PandasCsvReader(compression=None, sep=separator)
-        return cls(reader.read(filepath))
-
-    @staticmethod
-    def __check(df):
-        for index in df.index:
-            relation = df.loc[index][0]
-            assert(len(relation.split('<->')) == 2)
-
-    @staticmethod
-    def __create_key(l, r):
-        assert(type(l) == str)
-        assert(type(r) == str)
-        return '<->'.join([l, r])
-
-    def get_score(self, left, right):
-        assert(type(left) == str)
-        assert(type(right) == str)
-
-        lr_key = self.__create_key(left, right)
-        rl_key = self.__create_key(right, left)
-
-        lr_score = self.__lexicon[lr_key == self.__lexicon['relation']]
-        rl_score = self.__lexicon[rl_key == self.__lexicon['relation']]
-
-        if len(lr_score) > 0:
-            return lr_score['tone'].values[0]
-        if len(rl_score) > 0:
-            return rl_score['tone'].values[0]
-
-        return None

arekit/contrib/utils/lexicons/rusentilex.py

@@ -1,37 +0,0 @@
-import importlib
-import zipfile
-from os import path
-
-
-from arekit.contrib.source.zip_utils import ZipArchiveUtils
-from arekit.contrib.utils.lexicons.lexicon import Lexicon
-
-
-class RuSentiLexLexicon(Lexicon):
-    """
-    RuSentiLex Lexicon wrapper for csv file stored in /data folder.
-    """
-
-    __INNER_PATH = 'rusentilex.csv'
-
-    @property
-    def ToneKey(self):
-        return 'tone'
-
-    @property
-    def TermKey(self):
-        return 'term'
-
-    @staticmethod
-    def __get_archive_filepath():
-        return path.join(ZipArchiveUtils.get_data_root(), "rusentilex.zip")
-
-    @classmethod
-    def from_zip(cls):
-        """ Using Pandas API to read lexicon.
-        """
-        pd = importlib.import_module("pandas")
-        with zipfile.ZipFile(cls.__get_archive_filepath(), "r") as zip_ref:
-            with zip_ref.open(cls.__INNER_PATH, mode='r') as csv_file:
-                df = pd.read_csv(csv_file, sep=',')
-                return cls(df)

arekit/contrib/utils/nn/rows.py

@@ -1,83 +0,0 @@
-import collections
-
-from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
-from arekit.common.data.input.terms_mapper import OpinionContainingTextTermsMapper
-from arekit.common.entities.str_fmt import StringEntitiesFormatter
-from arekit.contrib.networks.input.ctx_serialization import NetworkSerializationContext
-from arekit.contrib.networks.input.formatters.pos_mapper import PosTermsMapper
-from arekit.contrib.networks.input.providers.sample import NetworkSampleRowProvider
-from arekit.contrib.networks.input.providers.text import NetworkSingleTextProvider
-from arekit.contrib.networks.input.term_types import TermTypes
-from arekit.contrib.networks.input.terms_mapping import VectorizedNetworkTermMapping
-from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
-from arekit.contrib.utils.resources import load_embedding_news_mystem_skipgram_1000_20_2015
-from arekit.contrib.utils.vectorizers.bpe import BPEVectorizer
-from arekit.contrib.utils.vectorizers.random_norm import RandomNormalVectorizer
-
-
-def __add_term_embedding(dict_data, term, emb_vector):
-    if term in dict_data:
-        return
-    dict_data[term] = emb_vector
-
-
-def create_rows_provider(str_entity_fmt, ctx, vectorizers="default"):
-    """ This method is corresponds to the default initialization of
-        the rows provider for data sampling pipeline.
-
-        vectorizers:
-            NONE: no need to vectorize, just provide text (using SingleTextProvider).
-            DEFAULT: we consider an application of stemmer for Russian Language.
-            DICT: in which for every type there is an assigned Vectorizer
-                vectorization of term types.
-                {
-                    TermType.Word: Vectorizer,
-                    TermType.Entity: Vectorizer,
-                    ...
-                }
-    """
-    assert(isinstance(str_entity_fmt, StringEntitiesFormatter))
-    assert(isinstance(ctx, NetworkSerializationContext))
-    assert(isinstance(vectorizers, dict) or vectorizers == "default" or vectorizers is None)
-
-    term_embedding_pairs = None
-
-    if vectorizers is not None:
-
-        if vectorizers == "default":
-            # initialize default vectorizer for Russian language.
-            embedding = load_embedding_news_mystem_skipgram_1000_20_2015(stemmer=MystemWrapper(), auto_download=True)
-            bpe_vectorizer = BPEVectorizer(embedding=embedding, max_part_size=3)
-            norm_vectorizer = RandomNormalVectorizer(vector_size=embedding.VectorSize,
-                                                     token_offset=12345)
-            vectorizers = {
-                TermTypes.WORD: bpe_vectorizer,
-                TermTypes.ENTITY: bpe_vectorizer,
-                TermTypes.FRAME: bpe_vectorizer,
-                TermTypes.TOKEN: norm_vectorizer
-            }
-
-        # Setup term-embedding pairs collection instance.
-        term_embedding_pairs = collections.OrderedDict()
-
-        # Use text provider with vectorizers.
-        text_provider = NetworkSingleTextProvider(
-            text_terms_mapper=VectorizedNetworkTermMapping(
-                vectorizers=vectorizers,
-                string_entities_formatter=str_entity_fmt),
-            pair_handling_func=lambda pair: __add_term_embedding(
-                dict_data=term_embedding_pairs,
-                term=pair[0],
-                emb_vector=pair[1]))
-    else:
-        # Create text provider which without vectorizers.
-        text_provider = BaseSingleTextProvider(
-            text_terms_mapper=OpinionContainingTextTermsMapper(str_entity_fmt))
-
-    return NetworkSampleRowProvider(
-        label_provider=ctx.LabelProvider,
-        text_provider=text_provider,
-        frames_connotation_provider=ctx.FramesConnotationProvider,
-        frame_role_label_scaler=ctx.FrameRolesLabelScaler,
-        pos_terms_mapper=PosTermsMapper(ctx.PosTagger) if ctx.PosTagger is not None else None,
-        term_embedding_pairs=term_embedding_pairs)

arekit/contrib/utils/np_utils/embedding.py

@@ -1,22 +0,0 @@
-import logging
-
-from arekit.contrib.utils.np_utils.npz_utils import NpzRepositoryUtils
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class NpzEmbeddingHelper:
-
-    @staticmethod
-    def save_embedding(data, target):
-        NpzRepositoryUtils.save(data=data, target=target)
-        logger.info("Saving embedding [size={shape}]: {filepath}".format(shape=data.shape,
-                                                                         filepath=target))
-
-    @staticmethod
-    def load_embedding(source):
-        embedding = NpzRepositoryUtils.load(source)
-        logger.info("Embedding read [size={size}]: {filepath}".format(size=embedding.shape,
-                                                                      filepath=source))
-        return embedding

arekit/contrib/utils/np_utils/npz_utils.py

@@ -1,13 +0,0 @@
-import numpy as np
-
-
-class NpzRepositoryUtils(object):
-
-    @staticmethod
-    def save(data, target):
-        np.savez(target, data)
-
-    @staticmethod
-    def load(source):
-        data = np.load(source)
-        return data['arr_0']

arekit/contrib/utils/np_utils/vocab.py

@@ -1,20 +0,0 @@
-import logging
-
-import numpy as np
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class VocabRepositoryUtils(object):
-
-    @staticmethod
-    def save(data, target):
-        logger.info("Saving vocabulary [size={size}]: {filepath}".format(size=len(data), filepath=target))
-        np.savetxt(target, data, fmt='%s')
-
-    @staticmethod
-    def load(source):
-        vocab = np.loadtxt(source, dtype=str, comments=None)
-        logger.info("Loading vocabulary [size={size}]: {filepath}".format(size=len(vocab), filepath=source))
-        return vocab

arekit/contrib/utils/pipelines/items/sampling/base.py

@@ -1,99 +0,0 @@
-from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.common.experiment.data_type import DataType
-from arekit.common.pipeline.base import BasePipeline
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.contrib.utils.serializer import InputDataSerializationHelper
-
-
-class BaseSerializerPipelineItem(BasePipelineItem):
-
-    def __init__(self, rows_provider, samples_io, save_labels_func, storage):
-        """ sample_rows_formatter:
-                how we format input texts for a BERT model, for example:
-                    - single text
-                    - two sequences, separated by [SEP] token
-
-            save_labels_func: function
-                data_type -> bool
-        """
-        assert(isinstance(rows_provider, BaseSampleRowProvider))
-        assert(isinstance(samples_io, BaseSamplesIO))
-        assert(callable(save_labels_func))
-        assert(isinstance(storage, BaseRowsStorage))
-
-        self._rows_provider = rows_provider
-        self._samples_io = samples_io
-        self._save_labels_func = save_labels_func
-        self._storage = storage
-
-    def _serialize_iteration(self, data_type, pipeline, data_folding, doc_ids):
-        assert(isinstance(data_type, DataType))
-        assert(isinstance(pipeline, BasePipeline))
-        assert(isinstance(data_folding, dict) or data_folding is None)
-        assert(isinstance(doc_ids, list) or doc_ids is None)
-        assert(doc_ids is not None or data_folding is not None)
-
-        repos = {
-            "sample": InputDataSerializationHelper.create_samples_repo(
-                keep_labels=self._save_labels_func(data_type),
-                rows_provider=self._rows_provider,
-                storage=self._storage),
-        }
-
-        writer_and_targets = {
-            "sample": (self._samples_io.Writer,
-                       self._samples_io.create_target(data_type=data_type)),
-        }
-
-        for description, repo in repos.items():
-
-            if data_folding is None:
-                # Consider only the predefined doc_ids.
-                doc_ids_iter = doc_ids
-            else:
-                # Take particular data_type.
-                doc_ids_iter = data_folding[data_type]
-                # Consider only predefined doc_ids.
-                if doc_ids is not None:
-                    doc_ids_iter = set(doc_ids_iter).intersection(doc_ids)
-
-            InputDataSerializationHelper.fill_and_write(
-                repo=repo,
-                pipeline=pipeline,
-                doc_ids_iter=doc_ids_iter,
-                desc="{desc} [{data_type}]".format(desc=description, data_type=data_type),
-                writer=writer_and_targets[description][0],
-                target=writer_and_targets[description][1])
-
-    def _handle_iteration(self, data_type_pipelines, data_folding, doc_ids):
-        """ Performing data serialization for a particular iteration
-        """
-        assert(isinstance(data_type_pipelines, dict))
-        for data_type, pipeline in data_type_pipelines.items():
-            self._serialize_iteration(data_type=data_type, pipeline=pipeline, data_folding=data_folding,
-                                      doc_ids=doc_ids)
-
-    def apply_core(self, input_data, pipeline_ctx):
-        """
-            data_type_pipelines: dict of, for example:
-                {
-                    DataType.Train: BasePipeline,
-                    DataType.Test: BasePipeline
-                }
-
-                data_type_pipelines: doc_id -> parsed_doc -> annot -> opinion linkages
-                    for example, function: sentiment_attitude_extraction_default_pipeline
-                doc_ids: optional
-                    this parameter allows to limit amount of documents considered for sampling
-        """
-        assert(isinstance(input_data, PipelineContext))
-        assert("data_type_pipelines" in input_data)
-
-        data_folding = input_data.provide_or_none("data_folding")
-
-        self._handle_iteration(data_type_pipelines=input_data.provide("data_type_pipelines"),
-                               doc_ids=input_data.provide_or_none("doc_ids"),
-                               data_folding=data_folding)

arekit/contrib/utils/pipelines/items/sampling/networks.py

@@ -1,54 +0,0 @@
-from arekit.contrib.networks.input.embedding.matrix import create_term_embedding_matrix
-from arekit.contrib.networks.input.embedding.offsets import TermsEmbeddingOffsets
-from arekit.contrib.networks.embedding import Embedding
-from arekit.contrib.networks.input.providers.sample import NetworkSampleRowProvider
-from arekit.contrib.utils.io_utils.embedding import NpEmbeddingIO
-from arekit.contrib.utils.pipelines.items.sampling.base import BaseSerializerPipelineItem
-
-
-class NetworksInputSerializerPipelineItem(BaseSerializerPipelineItem):
-
-    def __init__(self, save_labels_func, rows_provider, samples_io, emb_io, storage, save_embedding=True):
-        """ This pipeline item allows to perform a data preparation for neural network models.
-
-            considering a list of the whole data_types with the related pipelines,
-            which are supported and required in a handler. It is necessary to know
-            data_types in advance as it allows to create a complete vocabulary of input terms,
-            with the related embeddings.
-        """
-        assert(isinstance(emb_io, NpEmbeddingIO))
-        assert(isinstance(rows_provider, NetworkSampleRowProvider))
-        assert(isinstance(save_embedding, bool))
-        super(NetworksInputSerializerPipelineItem, self).__init__(
-            rows_provider=rows_provider,
-            samples_io=samples_io,
-            save_labels_func=save_labels_func,
-            storage=storage)
-
-        self.__emb_io = emb_io
-        self.__save_embedding = save_embedding
-
-    def _handle_iteration(self, data_type_pipelines, data_folding, doc_ids):
-        """ Performing data serialization for a particular iteration
-        """
-        assert(isinstance(data_type_pipelines, dict))
-
-        # Prepare for the present iteration.
-        self._rows_provider.clear_embedding_pairs()
-
-        super(NetworksInputSerializerPipelineItem, self)._handle_iteration(
-            data_type_pipelines=data_type_pipelines, data_folding=data_folding, doc_ids=doc_ids)
-
-        if not (self.__save_embedding and self._rows_provider.HasEmbeddingPairs):
-            return
-
-        # Save embedding information additionally.
-        term_embedding = Embedding.from_word_embedding_pairs_iter(self._rows_provider.iter_term_embedding_pairs())
-        embedding_matrix = create_term_embedding_matrix(term_embedding=term_embedding)
-        vocab = list(TermsEmbeddingOffsets.extract_vocab(words_embedding=term_embedding))
-
-        # Save embedding matrix
-        self.__emb_io.save_embedding(data=embedding_matrix)
-        self.__emb_io.save_vocab(data=vocab)
-
-        del embedding_matrix

arekit/contrib/utils/pipelines/items/text/frames_lemmatized.py

@@ -1,36 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.contrib.utils.pipelines.items.text.frames import FrameVariantsParser
-from arekit.contrib.utils.processing.languages.ru.mods import RussianLanguageMods
-
-
-class LemmasBasedFrameVariantsParser(FrameVariantsParser):
-
-    def __init__(self, frame_variants, stemmer, locale_mods=RussianLanguageMods, save_lemmas=False):
-        assert(isinstance(stemmer, Stemmer))
-        assert(isinstance(save_lemmas, bool))
-        super(LemmasBasedFrameVariantsParser, self).__init__(frame_variants=frame_variants)
-
-        self.__frame_variants = frame_variants
-        self.__stemmer = stemmer
-        self.__save_lemmas = save_lemmas
-        self.__max_variant_len = max([len(variant) for _, variant in frame_variants.iter_variants()])
-        self.__locale_mods = locale_mods
-
-    def __lemmatize_term(self, term):
-        # we first split onto words for lemmatization and then join all of them.
-        lemma = "".join(self.__stemmer.lemmatize_to_list(term))
-        # then we replace certain chars according to the locale restrictions.
-        return self.__locale_mods.replace_specific_word_chars(lemma)
-
-    def __provide_lemmatized_terms(self, terms):
-        """
-        Compose a list of lemmatized versions of parsed_doc
-        PS: Might be significantly slow, depending on stemmer were used.
-        """
-        assert(isinstance(terms, list))
-        return [self.__lemmatize_term(term) if isinstance(term, str) else term for term in terms]
-
-    def apply_core(self, input_data, pipeline_ctx):
-        lemmas = self.__provide_lemmatized_terms(input_data)
-        processed_it = self._iter_processed(terms=lemmas, origin=lemmas if self.__save_lemmas else input_data)
-        return list(processed_it)

arekit/contrib/utils/pipelines/items/text/frames_negation.py

@@ -1,32 +0,0 @@
-from arekit.common.frames.text_variant import TextFrameVariant
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.contrib.utils.processing.languages.mods import BaseLanguageMods
-from arekit.contrib.utils.processing.languages.ru.mods import RussianLanguageMods
-
-
-class FrameVariantsSentimentNegation(BasePipelineItem):
-
-    def __init__(self, locale_mods=RussianLanguageMods):
-        assert(issubclass(locale_mods, BaseLanguageMods))
-        self._locale_mods = locale_mods
-
-    @staticmethod
-    def __get_preposition(terms, index):
-        return terms[index-1] if index > 0 else None
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(input_data, list))
-        assert(isinstance(pipeline_ctx, PipelineContext))
-
-        for curr_ind, term in enumerate(input_data):
-
-            if not isinstance(term, TextFrameVariant):
-                continue
-
-            prep_term = self.__get_preposition(terms=input_data, index=curr_ind)
-            is_negated = self._locale_mods.is_negation_word(prep_term) if prep_term is not None else False
-
-            term.set_is_negated(is_negated)
-
-        return input_data

arekit/contrib/utils/pipelines/items/text/terms_splitter.py

@@ -1,10 +0,0 @@
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.utils import split_by_whitespaces
-
-
-class TermsSplitterParser(BasePipelineItem):
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        return split_by_whitespaces(input_data)