arekit 0.25.0__py3-none-any.whl → 0.25.1__py3-none-any.whl

arekit/contrib/utils/np_utils/vocab.py

@@ -1,20 +0,0 @@
-import logging
-
-import numpy as np
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class VocabRepositoryUtils(object):
-
-    @staticmethod
-    def save(data, target):
-        logger.info("Saving vocabulary [size={size}]: {filepath}".format(size=len(data), filepath=target))
-        np.savetxt(target, data, fmt='%s')
-
-    @staticmethod
-    def load(source):
-        vocab = np.loadtxt(source, dtype=str, comments=None)
-        logger.info("Loading vocabulary [size={size}]: {filepath}".format(size=len(vocab), filepath=source))
-        return vocab

arekit/contrib/utils/pipelines/items/sampling/base.py

@@ -1,94 +0,0 @@
-from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.common.experiment.data_type import DataType
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.contrib.utils.serializer import InputDataSerializationHelper
-
-
-class BaseSerializerPipelineItem(BasePipelineItem):
-
-    def __init__(self, rows_provider, samples_io, save_labels_func, storage, **kwargs):
-        """ sample_rows_formatter:
-                how we format input texts for a BERT model, for example:
-                    - single text
-                    - two sequences, separated by [SEP] token
-
-            save_labels_func: function
-                data_type -> bool
-        """
-        assert(isinstance(rows_provider, BaseSampleRowProvider))
-        assert(isinstance(samples_io, BaseSamplesIO))
-        assert(callable(save_labels_func))
-        assert(isinstance(storage, BaseRowsStorage))
-        super(BaseSerializerPipelineItem, self).__init__(**kwargs)
-
-        self._rows_provider = rows_provider
-        self._samples_io = samples_io
-        self._save_labels_func = save_labels_func
-        self._storage = storage
-
-    def _serialize_iteration(self, data_type, pipeline, data_folding, doc_ids):
-        assert(isinstance(data_type, DataType))
-        assert(isinstance(pipeline, list))
-        assert(isinstance(data_folding, dict) or data_folding is None)
-        assert(isinstance(doc_ids, list) or doc_ids is None)
-        assert(doc_ids is not None or data_folding is not None)
-
-        repos = {
-            "sample": InputDataSerializationHelper.create_samples_repo(
-                keep_labels=self._save_labels_func(data_type),
-                rows_provider=self._rows_provider,
-                storage=self._storage),
-        }
-
-        writer_and_targets = {
-            "sample": (self._samples_io.Writer,
-                       self._samples_io.create_target(data_type=data_type)),
-        }
-
-        for description, repo in repos.items():
-
-            if data_folding is None:
-                # Consider only the predefined doc_ids.
-                doc_ids_iter = doc_ids
-            else:
-                # Take particular data_type.
-                doc_ids_iter = data_folding[data_type]
-                # Consider only predefined doc_ids.
-                if doc_ids is not None:
-                    doc_ids_iter = set(doc_ids_iter).intersection(doc_ids)
-
-            InputDataSerializationHelper.fill_and_write(
-                repo=repo,
-                pipeline=pipeline,
-                doc_ids_iter=doc_ids_iter,
-                desc="{desc} [{data_type}]".format(desc=description, data_type=data_type),
-                writer=writer_and_targets[description][0],
-                target=writer_and_targets[description][1])
-
-    def _handle_iteration(self, data_type_pipelines, data_folding, doc_ids):
-        """ Performing data serialization for a particular iteration
-        """
-        assert(isinstance(data_type_pipelines, dict))
-        for data_type, pipeline in data_type_pipelines.items():
-            self._serialize_iteration(data_type=data_type, pipeline=pipeline, data_folding=data_folding,
-                                      doc_ids=doc_ids)
-
-    def apply_core(self, input_data, pipeline_ctx):
-        """
-            data_type_pipelines: dict of, for example:
-                {
-                    DataType.Train: BasePipeline,
-                    DataType.Test: BasePipeline
-                }
-
-                data_type_pipelines: doc_id -> parsed_doc -> annot -> opinion linkages
-                    for example, function: sentiment_attitude_extraction_default_pipeline
-                doc_ids: optional
-                    this parameter allows to limit amount of documents considered for sampling
-        """
-        assert("data_type_pipelines" in pipeline_ctx)
-        self._handle_iteration(data_type_pipelines=pipeline_ctx.provide("data_type_pipelines"),
-                               doc_ids=pipeline_ctx.provide_or_none("doc_ids"),
-                               data_folding=pipeline_ctx.provide_or_none("data_folding"))

arekit/contrib/utils/pipelines/items/sampling/networks.py

@@ -1,55 +0,0 @@
-from arekit.contrib.networks.input.embedding.matrix import create_term_embedding_matrix
-from arekit.contrib.networks.input.embedding.offsets import TermsEmbeddingOffsets
-from arekit.contrib.networks.embedding import Embedding
-from arekit.contrib.networks.input.providers.sample import NetworkSampleRowProvider
-from arekit.contrib.utils.io_utils.embedding import NpEmbeddingIO
-from arekit.contrib.utils.pipelines.items.sampling.base import BaseSerializerPipelineItem
-
-
-class NetworksInputSerializerPipelineItem(BaseSerializerPipelineItem):
-
-    def __init__(self, save_labels_func, rows_provider, samples_io, emb_io, storage, save_embedding=True, **kwargs):
-        """ This pipeline item allows to perform a data preparation for neural network models.
-
-            considering a list of the whole data_types with the related pipelines,
-            which are supported and required in a handler. It is necessary to know
-            data_types in advance as it allows to create a complete vocabulary of input terms,
-            with the related embeddings.
-        """
-        assert(isinstance(emb_io, NpEmbeddingIO))
-        assert(isinstance(rows_provider, NetworkSampleRowProvider))
-        assert(isinstance(save_embedding, bool))
-        super(NetworksInputSerializerPipelineItem, self).__init__(
-            rows_provider=rows_provider,
-            samples_io=samples_io,
-            save_labels_func=save_labels_func,
-            storage=storage,
-            **kwargs)
-
-        self.__emb_io = emb_io
-        self.__save_embedding = save_embedding
-
-    def _handle_iteration(self, data_type_pipelines, data_folding, doc_ids):
-        """ Performing data serialization for a particular iteration
-        """
-        assert(isinstance(data_type_pipelines, dict))
-
-        # Prepare for the present iteration.
-        self._rows_provider.clear_embedding_pairs()
-
-        super(NetworksInputSerializerPipelineItem, self)._handle_iteration(
-            data_type_pipelines=data_type_pipelines, data_folding=data_folding, doc_ids=doc_ids)
-
-        if not (self.__save_embedding and self._rows_provider.HasEmbeddingPairs):
-            return
-
-        # Save embedding information additionally.
-        term_embedding = Embedding.from_word_embedding_pairs_iter(self._rows_provider.iter_term_embedding_pairs())
-        embedding_matrix = create_term_embedding_matrix(term_embedding=term_embedding)
-        vocab = list(TermsEmbeddingOffsets.extract_vocab(words_embedding=term_embedding))
-
-        # Save embedding matrix
-        self.__emb_io.save_embedding(data=embedding_matrix)
-        self.__emb_io.save_vocab(data=vocab)
-
-        del embedding_matrix

arekit/contrib/utils/pipelines/items/text/frames_lemmatized.py

@@ -1,36 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.contrib.utils.pipelines.items.text.frames import FrameVariantsParser
-from arekit.contrib.utils.processing.languages.ru.mods import RussianLanguageMods
-
-
-class LemmasBasedFrameVariantsParser(FrameVariantsParser):
-
-    def __init__(self, frame_variants, stemmer, locale_mods=RussianLanguageMods, save_lemmas=False, **kwargs):
-        assert(isinstance(stemmer, Stemmer))
-        assert(isinstance(save_lemmas, bool))
-        super(LemmasBasedFrameVariantsParser, self).__init__(frame_variants=frame_variants, **kwargs)
-
-        self.__frame_variants = frame_variants
-        self.__stemmer = stemmer
-        self.__save_lemmas = save_lemmas
-        self.__max_variant_len = max([len(variant) for _, variant in frame_variants.iter_variants()])
-        self.__locale_mods = locale_mods
-
-    def __lemmatize_term(self, term):
-        # we first split onto words for lemmatization and then join all of them.
-        lemma = "".join(self.__stemmer.lemmatize_to_list(term))
-        # then we replace certain chars according to the locale restrictions.
-        return self.__locale_mods.replace_specific_word_chars(lemma)
-
-    def __provide_lemmatized_terms(self, terms):
-        """
-        Compose a list of lemmatized versions of parsed_doc
-        PS: Might be significantly slow, depending on stemmer were used.
-        """
-        assert(isinstance(terms, list))
-        return [self.__lemmatize_term(term) if isinstance(term, str) else term for term in terms]
-
-    def apply_core(self, input_data, pipeline_ctx):
-        lemmas = self.__provide_lemmatized_terms(input_data)
-        processed_it = self._iter_processed(terms=lemmas, origin=lemmas if self.__save_lemmas else input_data)
-        return list(processed_it)

arekit/contrib/utils/pipelines/items/text/frames_negation.py

@@ -1,33 +0,0 @@
-from arekit.common.frames.text_variant import TextFrameVariant
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.contrib.utils.processing.languages.mods import BaseLanguageMods
-from arekit.contrib.utils.processing.languages.ru.mods import RussianLanguageMods
-
-
-class FrameVariantsSentimentNegation(BasePipelineItem):
-
-    def __init__(self, locale_mods=RussianLanguageMods, **kwargs):
-        assert(issubclass(locale_mods, BaseLanguageMods))
-        super(FrameVariantsSentimentNegation, self).__init__(**kwargs)
-        self._locale_mods = locale_mods
-
-    @staticmethod
-    def __get_preposition(terms, index):
-        return terms[index-1] if index > 0 else None
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(input_data, list))
-        assert(isinstance(pipeline_ctx, PipelineContext))
-
-        for curr_ind, term in enumerate(input_data):
-
-            if not isinstance(term, TextFrameVariant):
-                continue
-
-            prep_term = self.__get_preposition(terms=input_data, index=curr_ind)
-            is_negated = self._locale_mods.is_negation_word(prep_term) if prep_term is not None else False
-
-            term.set_is_negated(is_negated)
-
-        return input_data

arekit/contrib/utils/pipelines/items/text/tokenizer.py

@@ -1,105 +0,0 @@
-import logging
-
-from arekit.common.context.token import Token
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.utils import split_by_whitespaces
-from arekit.contrib.utils.processing.text.tokens import Tokens
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-
-class DefaultTextTokenizer(BasePipelineItem):
-    """ Default parser implementation.
-    """
-
-    def __init__(self, keep_tokens=True, **kwargs):
-        super(DefaultTextTokenizer, self).__init__(**kwargs)
-        self.__keep_tokens = keep_tokens
-
-    # region protected methods
-
-    def apply_core(self, input_data, pipeline_ctx):
-        output_data = self.__process_parts(input_data)
-        if not self.__keep_tokens:
-            output_data = [word for word in output_data if not isinstance(word, Token)]
-        return output_data
-
-    # endregion
-
-    # region private static methods
-
-    def __process_parts(self, parts):
-        assert(isinstance(parts, list))
-
-        parsed = []
-        for part in parts:
-
-            if part is None:
-                continue
-
-            # Keep non str words as it is and try to parse str-based words.
-            processed = [part] if not isinstance(part, str) else \
-                self.__iter_processed_part(part=part)
-
-            parsed.extend(processed)
-
-        return parsed
-
-    def __iter_processed_part(self, part):
-        for word in split_by_whitespaces(part):
-            for term in self.__process_word(word):
-                yield term
-
-    def __process_word(self, word):
-        assert(isinstance(word, str))
-        return self.__split_tokens(word)
-
-    @staticmethod
-    def __split_tokens(term):
-        """
-        Splitting off tokens from parsed_doc ending, i.e. for example:
-            term: "сказать,-" -> "(term: "сказать", ["COMMA_TOKEN", "DASH_TOKEN"])
-        return: (unicode or None, list)
-            modified term and list of extracted tokens.
-        """
-
-        url = Tokens.try_create_url(term)
-        if url is not None:
-            return [url]
-
-        l = 0
-        words_and_tokens = []
-        while l < len(term):
-
-            # Token.
-            token = Tokens.try_create(term[l])
-            if token is not None:
-                if token.get_token_value() != Tokens.NEW_LINE:
-                    words_and_tokens.append(token)
-                l += 1
-
-            # Number.
-            elif str.isdigit(term[l]):
-                k = l + 1
-                while k < len(term) and str.isdigit(term[k]):
-                    k += 1
-                token = Tokens.try_create_number(term[l:k])
-                assert(token is not None)
-                words_and_tokens.append(token)
-                l = k
-
-            # Term.
-            else:
-                k = l + 1
-                while k < len(term):
-                    token = Tokens.try_create(term[k])
-                    if token is not None and token.get_token_value() != Tokens.DASH:
-                        break
-                    k += 1
-                words_and_tokens.append(term[l:k])
-                l = k
-
-        return words_and_tokens
-
-    # endregion

arekit/contrib/utils/pipelines/items/text/translator.py

@@ -1,136 +0,0 @@
-from arekit.common.data.input.providers.const import IDLE_MODE
-from arekit.common.pipeline.conts import PARENT_CTX
-from arekit.common.entities.base import Entity
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-
-
-class MLTextTranslatorPipelineItem(BasePipelineItem):
-    """ Machine learning based translator pipeline item.
-    """
-
-    def __init__(self, batch_translate_model, do_translate_entity=True, **kwargs):
-        """ Model, which is based on translation of the text,
-            represented as a list of words.
-        """
-        super(MLTextTranslatorPipelineItem, self).__init__(**kwargs)
-        self.__do_translate_entity = do_translate_entity
-        self.__translate = batch_translate_model
-
-    def fast_most_accurate_approach(self, input_data, entity_placeholder_template="<entityTag={}/>"):
-        """ This approach assumes that the translation won't corrupt the original
-            meta-annotation for entities and objects mentioned in text.
-        """
-
-        def __optionally_register(prts):
-            if len(prts) > 0:
-                content.append(" ".join(prts))
-            parts_to_join.clear()
-
-        content = []
-        origin_entities = []
-        parts_to_join = []
-
-        for part in input_data:
-            if isinstance(part, str) and part.strip():
-                parts_to_join.append(part)
-            elif isinstance(part, Entity):
-                entity_index = len(origin_entities)
-                parts_to_join.append(entity_placeholder_template.format(entity_index))
-                # Register entities information for further restoration.
-                origin_entities.append(part)
-
-        # Register original text with masked named entities.
-        __optionally_register(parts_to_join)
-        # Register all named entities in order of their appearance in text.
-        content.extend([e.Value for e in origin_entities])
-
-        # Compose text parts.
-        translated_parts = self.__translate(content)
-
-        if len(translated_parts) == 0:
-            return None
-
-        # Take the original text.
-        text = translated_parts[0]
-        for entity_index in range(len(origin_entities)):
-            if entity_placeholder_template.format(entity_index) not in text:
-                return None
-
-        # Enumerate entities.
-        from_ind = 0
-        text_parts = []
-        for entity_index, translated_value in enumerate(translated_parts[1:]):
-            entity_placeholder_instance = entity_placeholder_template.format(entity_index)
-            # Cropping text part.
-            to_ind = text.index(entity_placeholder_instance)
-
-            if self.__do_translate_entity:
-                origin_entities[entity_index].set_display_value(translated_value.strip())
-
-            # Register entities.
-            text_parts.append(text[from_ind:to_ind])
-            text_parts.append(origin_entities[entity_index])
-            # Update from index.
-            from_ind = to_ind + len(entity_placeholder_instance)
-
-        # Consider the remaining part.
-        text_parts.append(text[from_ind:])
-        return text_parts
-
-    def default_pre_part_splitting_approach(self, input_data):
-        """ This is the original strategy, based on the manually cropped named entities
-            before the actual translation call.
-        """
-
-        def __optionally_register(prts):
-            if len(prts) > 0:
-                content.append(" ".join(prts))
-            parts_to_join.clear()
-
-        content = []
-        origin_entities = []
-        origin_entity_ind = []
-        parts_to_join = []
-
-        for _, part in enumerate(input_data):
-            if isinstance(part, str) and part.strip():
-                parts_to_join.append(part)
-            elif isinstance(part, Entity):
-                # Register first the prior parts were merged.
-                __optionally_register(parts_to_join)
-                # Register entities information for further restoration.
-                origin_entity_ind.append(len(content))
-                origin_entities.append(part)
-                content.append(part.Value)
-
-        __optionally_register(parts_to_join)
-
-        # Compose text parts.
-        translated_parts = self.__translate(content)
-
-        for entity_ind, entity_part_ind in enumerate(origin_entity_ind):
-            entity = origin_entities[entity_ind]
-            if self.__do_translate_entity:
-                entity.set_display_value(translated_parts[entity_part_ind].strip())
-            translated_parts[entity_part_ind] = entity
-
-        return translated_parts
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        assert(isinstance(input_data, list))
-
-        # Check the pipeline state whether is an idle mode or not.
-        parent_ctx = pipeline_ctx.provide(PARENT_CTX)
-        idle_mode = parent_ctx.provide(IDLE_MODE)
-
-        # When pipeline utilized only for the assessing the expected amount
-        # of rows (common case of idle_mode), there is no need to perform
-        # translation.
-        if idle_mode:
-            return
-
-        fast_accurate = self.fast_most_accurate_approach(input_data)
-        return self.default_pre_part_splitting_approach(input_data) \
-            if fast_accurate is None else fast_accurate

arekit/contrib/utils/processing/languages/mods.py

@@ -1,12 +0,0 @@
-class BaseLanguageMods(object):
-    """
-    Represents a class with static methods that provides text/word modifies.
-    """
-
-    @staticmethod
-    def replace_specific_word_chars(word):
-        pass
-
-    @staticmethod
-    def is_negation_word(word):
-        pass

arekit/contrib/utils/processing/languages/pos.py

@@ -1,23 +0,0 @@
-from enum import IntEnum
-
-
-class PartOfSpeechType(IntEnum):
-
-    NOUN = 1
-    ADV = 2
-    ADVPRO = 3
-    ANUM = 4
-    APRO = 5
-    COM = 6
-    CONJ = 7
-    INTJ = 8
-    NUM = 9
-    PART = 10
-    PR = 11
-    ADJ = 12
-    SPRO = 13
-    VERB = 14
-
-    Unknown = 15
-
-    Empty = 16

arekit/contrib/utils/processing/languages/ru/cases.py

@@ -1,78 +0,0 @@
-from enum import Enum
-
-
-class RussianCases(Enum):
-    """ Падежи русского языка
-    """
-
-    """ не определено
-    """
-    UNKN = 10
-
-    """ именительный
-    """
-    NOM = 1
-
-    """ родительный
-    """
-    GEN = 2
-
-    """ дательный
-    """
-    DAT = 3
-
-    """ винительный
-    """
-    ACC = 4
-
-    """ творительный
-    """
-    INS = 5
-
-    """ предложный
-    """
-    ABL = 6
-
-    """ партитив
-    """
-    PART = 7
-
-    """ местный 
-    """
-    LOC = 8
-
-    """ звательный
-    """
-    VOC = 9
-
-
-class RussianCasesService(object):
-
-    __english = {
-        'nom': RussianCases.NOM,
-        'gen': RussianCases.GEN,
-        'dat': RussianCases.DAT,
-        'acc': RussianCases.ACC,
-        'ins': RussianCases.INS,
-        'abl': RussianCases.ABL,
-        'part': RussianCases.PART,
-        'loc': RussianCases.LOC,
-        'voc': RussianCases.VOC,
-    }
-
-    __mystem_russian = {
-        'им': RussianCases.NOM,
-        'род': RussianCases.GEN,
-        'дат': RussianCases.DAT,
-        'вин': RussianCases.ACC,
-        'твор': RussianCases.INS,
-        'пр': RussianCases.ABL,
-        'парт': RussianCases.PART,
-        'местн': RussianCases.LOC,
-        'зват': RussianCases.VOC,
-    }
-
-    @staticmethod
-    def iter_rus_mystem_tags():
-        for key, value in RussianCasesService.__mystem_russian.items():
-            yield key, value

arekit/contrib/utils/processing/languages/ru/constants.py

@@ -1,6 +0,0 @@
-class RussianConstants:
-
-    PrepositionSet = {'к', 'на', 'по', 'с', 'до', 'в', 'во', "у", "а"}
-
-    def __init__(self):
-        pass

arekit/contrib/utils/processing/languages/ru/mods.py

@@ -1,13 +0,0 @@
-from arekit.contrib.utils.processing.languages.mods import BaseLanguageMods
-
-
-class RussianLanguageMods(BaseLanguageMods):
-
-    @staticmethod
-    def replace_specific_word_chars(word):
-        assert(isinstance(word, str))
-        return word.replace('ё', 'e')
-
-    @staticmethod
-    def is_negation_word(word):
-        return word == 'не'

arekit/contrib/utils/processing/languages/ru/number.py

@@ -1,23 +0,0 @@
-from enum import Enum
-
-
-class RussianNumberType(Enum):
-
-    UNKN = 3
-
-    Plural = 1
-
-    Single = 2
-
-
-class RussianNumberTypeService(object):
-
-    __russian = {
-        'ед': RussianNumberType.Single,
-        'мн': RussianNumberType.Plural
-    }
-
-    @staticmethod
-    def iter_rus_mystem_tags():
-        for key, value in RussianNumberTypeService.__russian.items():
-            yield key, value