arekit 0.25.0__py3-none-any.whl → 0.25.1__py3-none-any.whl

arekit/contrib/utils/processing/languages/ru/pos_service.py

@@ -1,36 +0,0 @@
-from arekit.contrib.utils.processing.languages.pos import PartOfSpeechType
-
-
-class PartOfSpeechTypesService(object):
-
-    __pos_names = {
-        "S": PartOfSpeechType.NOUN,
-        "ADV": PartOfSpeechType.ADV,
-        "ADVPRO": PartOfSpeechType.ADVPRO,
-        "ANUM": PartOfSpeechType.ANUM,
-        "APRO": PartOfSpeechType.APRO,
-        "COM": PartOfSpeechType.COM,
-        "CONJ": PartOfSpeechType.CONJ,
-        "INTJ": PartOfSpeechType.INTJ,
-        "NUM": PartOfSpeechType.NUM,
-        "PART": PartOfSpeechType.PART,
-        "PR": PartOfSpeechType.PR,
-        "A": PartOfSpeechType.ADJ,
-        "SPRO": PartOfSpeechType.SPRO,
-        "V": PartOfSpeechType.VERB,
-        "UNKN": PartOfSpeechType.Unknown,
-        "EMPTY": PartOfSpeechType.Empty}
-
-    @staticmethod
-    def iter_mystem_tags():
-        for key, value in PartOfSpeechTypesService.__pos_names.items():
-            yield key, value
-
-    @staticmethod
-    def get_mystem_from_string(value):
-        return PartOfSpeechTypesService.__pos_names[value]
-
-    @staticmethod
-    def get_mystem_pos_count():
-        return len(PartOfSpeechTypesService.__pos_names)
-

arekit/contrib/utils/processing/lemmatization/mystem.py

@@ -1,51 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.common.utils import filter_whitespaces
-from pymystem3 import Mystem
-
-
-class MystemWrapper(Stemmer):
-    """ Yandex MyStem wrapper
-
-        part of speech description:
-        https://tech.yandex.ru/mystem/doc/grammemes-values-docpage/
-    """
-
-    def __init__(self, entire_input=False):
-        """
-        entire_input: bool
-            Mystem parameter that allows to keep all information from input (true) or
-            remove garbage characters
-        """
-        self.__mystem = Mystem(entire_input=entire_input)
-
-    # region properties
-
-    @property
-    def MystemInstance(self):
-        return self.__mystem
-
-    # endregion
-
-    # region public methods
-
-    def lemmatize_to_list(self, text):
-        return self.__lemmatize_core(text)
-
-    def lemmatize_to_str(self, text):
-        result = " ".join(self.__lemmatize_core(text))
-        return result if len(result) != 0 else self.__process_original_text(text)
-
-    # endregion
-
-    # region private methods
-
-    def __lemmatize_core(self, text):
-        assert(isinstance(text, str))
-        result_list = self.__mystem.lemmatize(self.__process_original_text(text))
-        return filter_whitespaces(result_list)
-
-    @staticmethod
-    def __process_original_text(text):
-        return text.lower()
-
-    # endregion

arekit/contrib/utils/processing/pos/base.py

@@ -1,12 +0,0 @@
-# TODO. Move to base class.
-class POSTagger:
-
-    def get_term_pos(self, term):
-        raise NotImplementedError()
-
-    def get_term_number(self, term):
-        raise NotImplementedError()
-
-    def pos_to_int(self, pos):
-        raise NotImplementedError()
-

arekit/contrib/utils/processing/pos/mystem_wrap.py

@@ -1,134 +0,0 @@
-from pymystem3 import Mystem
-
-from arekit.contrib.utils.processing.languages.pos import PartOfSpeechType
-from arekit.contrib.utils.processing.languages.ru.cases import RussianCases, RussianCasesService
-from arekit.contrib.utils.processing.languages.ru.number import RussianNumberType, RussianNumberTypeService
-from arekit.contrib.utils.processing.languages.ru.pos_service import PartOfSpeechTypesService
-from arekit.contrib.utils.processing.pos.russian import RussianPOSTagger
-
-
-class POSMystemWrapper(RussianPOSTagger):
-
-    _ArgsSeparator = ','
-    _GrammarKey = 'gr'
-
-    def __init__(self, mystem):
-        assert(isinstance(mystem, Mystem))
-        self.__mystem = mystem
-
-    # region private methods
-
-    @staticmethod
-    def __extract_from_analysis(analysis, func):
-        """
-        part of speech description:
-            https://tech.yandex.ru/mystem/doc/grammemes-values-docpage/
-        func: f(args) -> out
-        returns: str or None
-        """
-        assert(callable(func))
-
-        if 'analysis' not in analysis:
-            return func(None)
-
-        info = analysis['analysis']
-        if len(info) == 0:
-            return func(None)
-
-        return func(info[0])
-
-    @staticmethod
-    def __get_pos(arguments):
-        if arguments is None:
-            return PartOfSpeechType.Unknown
-
-        pos = arguments[POSMystemWrapper._GrammarKey].split(POSMystemWrapper._ArgsSeparator)[0]
-        if '=' in pos:
-            pos = pos.split('=')[0]
-
-        return PartOfSpeechTypesService.get_mystem_from_string(pos)
-
-    @staticmethod
-    def __get_russian_case(arguments):
-        if arguments is None:
-            return RussianCases.UNKN
-
-        all_params = set(POSMystemWrapper.__iter_params(arguments))
-
-        for key, case in RussianCasesService.iter_rus_mystem_tags():
-            if key in all_params:
-                return case
-
-        return RussianCases.UNKN
-
-    @staticmethod
-    def __get_number(arguments):
-        if arguments is None:
-            return RussianNumberType.UNKN
-
-        all_params = set(POSMystemWrapper.__iter_params(arguments))
-
-        for key, case in RussianNumberTypeService.iter_rus_mystem_tags():
-            if key in all_params:
-                return case
-
-        return RussianNumberType.UNKN
-
-    @staticmethod
-    def __iter_params(arguments):
-        params = arguments[POSMystemWrapper._GrammarKey].split(POSMystemWrapper._ArgsSeparator)
-        for optionally_combined in params:
-            for param in optionally_combined.split('='):
-                yield param
-
-    # endregion
-
-    def get_term_pos(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_pos) \
-            if len(analyzed) > 0 else PartOfSpeechType.Unknown
-
-    def get_term_case(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_russian_case) \
-            if len(analyzed) > 0 else RussianCases.UNKN
-
-    def get_term_number(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_number) \
-            if len(analyzed) > 0 else RussianNumberType.UNKN
-
-    def get_terms_russian_cases(self, text):
-        """ list of part of speech according to the certain word in text
-        """
-        assert(isinstance(text, str))
-        cases = []
-
-        analyzed = self.__mystem.analyze(text)
-        for a in analyzed:
-            pos = self.__extract_from_analysis(a, self.__get_russian_case) if len(analyzed) > 0 else RussianCases.UNKN
-            cases.append(pos)
-
-        return cases
-
-    def pos_to_int(self, pos):
-        assert(isinstance(pos, PartOfSpeechType))
-        return int(pos)
-
-    @staticmethod
-    def is_adjective(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.ADJ
-
-    @staticmethod
-    def is_noun(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.NOUN
-
-    @staticmethod
-    def is_verb(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.VERB

arekit/contrib/utils/processing/pos/russian.py

@@ -1,10 +0,0 @@
-from arekit.contrib.utils.processing.pos.base import POSTagger
-
-
-class RussianPOSTagger(POSTagger):
-    """ Provides cases support ('падежи')
-    """
-
-    def get_term_case(self, term):
-        raise NotImplementedError()
-

arekit/contrib/utils/processing/text/tokens.py

@@ -1,127 +0,0 @@
-from urllib.parse import urlparse
-from arekit.common.context.token import Token
-
-
-# TODO. Provide the base (BaseTokens) type.
-# TODO. With the related API at BaseTokens.
-class Tokens:
-    """
-    Tokens used to describe a non-word text units, such as punctuation,
-    uknown words/chars, smiles, etc.
-    """
-
-    _wrapper = "<[{}]>"
-    COMMA = _wrapper.format(',')
-    SEMICOLON = _wrapper.format(';')
-    COLON = _wrapper.format(':')
-    QUOTE = _wrapper.format('QUOTE')
-    DASH = _wrapper.format('-')
-    LONG_DASH = _wrapper.format('long_dash')
-    DOT = _wrapper.format('.')
-    TRIPLE_DOTS = _wrapper.format('…')
-    EXC_SIGN = _wrapper.format('!')
-    QUESTION_SIGN = _wrapper.format('?')
-    OPEN_BRACKET = _wrapper.format('OPEN_BRACKET')
-    CLOSED_BRACKET = _wrapper.format('CLOSED_BRACKET')
-    NUMBER = _wrapper.format('NUMBER')
-    NEW_LINE = _wrapper.format("NEW_LINE")
-    UNKNOWN_CHAR = _wrapper.format('UNKNOWN_CHAR')
-    UNKNOWN_WORD = _wrapper.format('UNKNOWN_WORD')
-    URL = _wrapper.format("URL")
-
-    __token_mapping = {
-        ',': COMMA,
-        '.': DOT,
-        '…': TRIPLE_DOTS,
-        ':': COLON,
-        ';': SEMICOLON,
-        '-': DASH,
-        '—': LONG_DASH,
-        '?': QUESTION_SIGN,
-        '!': EXC_SIGN,
-        '(': OPEN_BRACKET,
-        ')': CLOSED_BRACKET,
-        '{': OPEN_BRACKET,
-        '}': CLOSED_BRACKET,
-        '[': OPEN_BRACKET,
-        ']': CLOSED_BRACKET,
-        '\n': NEW_LINE,
-        '«': QUOTE,
-        '»': QUOTE,
-        '"': QUOTE,
-    }
-
-    __supported_tokens = {
-        COMMA,
-        SEMICOLON,
-        COLON,
-        QUOTE,
-        DASH,
-        DOT,
-        LONG_DASH,
-        TRIPLE_DOTS,
-        EXC_SIGN,
-        QUESTION_SIGN,
-        OPEN_BRACKET,
-        CLOSED_BRACKET,
-        NUMBER,
-        URL,
-        NEW_LINE,
-        UNKNOWN_CHAR,
-        UNKNOWN_WORD}
-
-    @staticmethod
-    def try_create(subterm):
-        """
-        Trying create a token by given 'term' parameter
-        subterm: unicode
-           I.e. term ending, so means a part of original term
-        """
-        assert(isinstance(subterm, str))
-        if subterm not in Tokens.__token_mapping:
-            return None
-        return Token(term=subterm, token_value=Tokens.__token_mapping[subterm])
-
-    @staticmethod
-    def try_parse(term):
-        assert(isinstance(term, str))
-        for origin, token_value in Tokens.__token_mapping.items():
-            if term == token_value:
-                return Token(term=origin, token_value=token_value)
-
-    @staticmethod
-    def try_create_number(term):
-        assert(isinstance(term, str))
-        if not term.isdigit():
-            return None
-        return Token(term=term, token_value=Tokens.NUMBER)
-
-    @staticmethod
-    def try_create_url(term):
-        assert(isinstance(term, str))
-        result = urlparse(term)
-        is_correct = result.scheme and result.netloc and result.path
-        if not is_correct:
-            return None
-        return Token(term=term, token_value=Tokens.URL)
-
-    @staticmethod
-    def is_token(term):
-        assert(isinstance(term, str))
-        return term in Tokens.__supported_tokens
-
-    @staticmethod
-    def iter_chars_by_token(term):
-        """
-        Iterate through charts that is related to term
-        token: char
-        """
-        assert(isinstance(term, str))
-        for char, token in Tokens.__token_mapping.items():
-            if term == token:
-                yield char
-
-    @staticmethod
-    def iter_supported_tokens():
-        for token in Tokens.__supported_tokens:
-            yield token

arekit/contrib/utils/serializer.py

@@ -1,42 +0,0 @@
-import logging
-
-from collections.abc import Iterable
-
-from arekit.common.data.input.providers.columns.sample import SampleColumnsProvider
-from arekit.common.data.input.providers.rows.base import BaseRowProvider
-from arekit.common.data.input.repositories.base import BaseInputRepository
-from arekit.common.data.input.repositories.sample import BaseInputSamplesRepository
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.contents.opinions import InputTextOpinionProvider
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class InputDataSerializationHelper(object):
-
-    @staticmethod
-    def create_samples_repo(keep_labels, rows_provider, storage):
-        assert(isinstance(rows_provider, BaseRowProvider))
-        assert(isinstance(keep_labels, bool))
-        assert(isinstance(storage, BaseRowsStorage))
-        return BaseInputSamplesRepository(
-            columns_provider=SampleColumnsProvider(store_labels=keep_labels),
-            rows_provider=rows_provider,
-            storage=storage)
-
-    @staticmethod
-    def fill_and_write(pipeline, repo, target, writer, doc_ids_iter, desc=""):
-        assert(isinstance(pipeline, list))
-        assert(isinstance(doc_ids_iter, Iterable))
-        assert(isinstance(repo, BaseInputRepository))
-
-        doc_ids = list(doc_ids_iter)
-
-        repo.populate(contents_provider=InputTextOpinionProvider(pipeline),
-                      doc_ids=doc_ids,
-                      desc=desc,
-                      writer=writer,
-                      target=target)
-
-        repo.push(writer=writer, target=target)

arekit/contrib/utils/vectorizers/bpe.py

@@ -1,93 +0,0 @@
-import numpy as np
-
-from arekit.common.log_utils import logger
-from arekit.contrib.networks.embedding import Embedding
-from arekit.contrib.networks.vectorizer import BaseVectorizer
-
-
-class BPEVectorizer(BaseVectorizer):
-    """ Embedding algorithm based on parts (trigrams originally)
-    """
-
-    def __init__(self, embedding, max_part_size=3, word_separator=' '):
-        assert(isinstance(embedding, Embedding))
-        assert(isinstance(max_part_size, int))
-        self.__embedding = embedding
-        self.__max_part_size = max_part_size
-        self.__word_separator = word_separator
-
-    def create_term_embedding(self, term):
-        """ Note: returns modified term value in a form of the `word` returning parameter.
-        """
-        assert(isinstance(term, str))
-
-        word, word_embedding = self.__get_from_embedding(term=term) \
-            if term in self.__embedding else self.__compose_from_parts(term=term)
-
-        # In order to prevent a problem of the further separations during reading process.
-        # it is necessary to replace the separators with the other chars.
-        word = word.replace(self.__word_separator, '-')
-
-        return word, word_embedding
-
-    def __compose_from_parts(self, term, do_lowercase=True):
-        # remove empty spaces before and after.
-        term = term.strip()
-
-        # perform lowercasing
-        if do_lowercase:
-            term = term.lower()
-
-        # Calculating vector from term parts
-        count = 0
-        vector = np.zeros(self.__embedding.VectorSize)
-        for word in term.split(self.__word_separator):
-            v, c = self.__create_embedding_for_word(word=word, embedding=self.__embedding)
-            count += c
-            vector = vector + v
-
-        return term, vector / count if count > 0 else vector
-
-    def __get_from_embedding(self, term):
-        return self.__embedding.try_get_related_word(term), self.__embedding[term]
-
-    def __create_embedding_for_word(self, word, embedding):
-        assert(isinstance(word, str))
-        assert(isinstance(embedding, Embedding))
-
-        if word in embedding:
-            return embedding[word], 1
-
-        c_i = 0
-        c_l = self.__max_part_size
-        count = 0
-        vector = np.zeros(embedding.VectorSize)
-        missings = []
-
-        while c_i < len(word):
-
-            if c_l == 0:
-                missings.append(c_i)
-                c_i += 1
-                c_l = self.__max_part_size
-                continue
-
-            right_b = min(len(word), c_i + c_l)
-
-            s_i = embedding.try_find_index_by_plain_word(word=word[c_i:right_b])
-
-            if s_i is None:
-                c_l -= 1
-                continue
-
-            vector += embedding.get_vector_by_index(s_i)
-            c_i += c_l
-            count += 1
-
-        debug = False
-        if debug:
-            w_debug = ''.join(['?' if i in missings else ch
-                               for i, ch in enumerate(word)])
-            logger.debug('Embedded: {}'.format(w_debug).encode('utf-8'))
-
-        return vector, count

arekit/contrib/utils/vectorizers/random_norm.py

@@ -1,39 +0,0 @@
-import numpy as np
-
-from arekit.contrib.networks.vectorizer import BaseVectorizer
-
-
-class RandomNormalVectorizer(BaseVectorizer):
-
-    def __init__(self, vector_size, token_offset=12345, max_tokens_count=100):
-        assert(isinstance(vector_size, int))
-        self.__vector_size = vector_size
-        self.__seed_token_offset = token_offset
-        self.__max_tokens_count = max_tokens_count
-
-    def create_term_embedding(self, term):
-        """ term: is its index.
-        """
-        embedding = self.__get_random_normal_distribution(
-            vector_size=self.__vector_size,
-            seed=(self.__string_to_int(term) % self.__max_tokens_count) + self.__seed_token_offset,
-            loc=0.05,
-            scale=0.025)
-        return term, embedding
-
-    # region private methods
-
-    def __string_to_int(self, s):
-        # Originally taken from here:
-        # https://stackoverflow.com/questions/2511058/persistent-hashing-of-strings-in-python
-        ord3 = lambda x: '%.3d' % ord(x)
-        return int(''.join(map(ord3, s)))
-
-    @staticmethod
-    def __get_random_normal_distribution(vector_size, seed, loc, scale):
-        assert (isinstance(vector_size, int))
-        assert (isinstance(seed, int))
-        np.random.seed(seed)
-        return np.random.normal(loc=loc, scale=scale, size=vector_size)
-
-    # endregion