arekit 0.25.0__py3-none-any.whl → 0.25.2__py3-none-any.whl

arekit/contrib/utils/processing/text/tokens.py

@@ -1,127 +0,0 @@
-from urllib.parse import urlparse
-from arekit.common.context.token import Token
-
-
-# TODO. Provide the base (BaseTokens) type.
-# TODO. With the related API at BaseTokens.
-class Tokens:
-    """
-    Tokens used to describe a non-word text units, such as punctuation,
-    uknown words/chars, smiles, etc.
-    """
-
-    _wrapper = "<[{}]>"
-    COMMA = _wrapper.format(',')
-    SEMICOLON = _wrapper.format(';')
-    COLON = _wrapper.format(':')
-    QUOTE = _wrapper.format('QUOTE')
-    DASH = _wrapper.format('-')
-    LONG_DASH = _wrapper.format('long_dash')
-    DOT = _wrapper.format('.')
-    TRIPLE_DOTS = _wrapper.format('…')
-    EXC_SIGN = _wrapper.format('!')
-    QUESTION_SIGN = _wrapper.format('?')
-    OPEN_BRACKET = _wrapper.format('OPEN_BRACKET')
-    CLOSED_BRACKET = _wrapper.format('CLOSED_BRACKET')
-    NUMBER = _wrapper.format('NUMBER')
-    NEW_LINE = _wrapper.format("NEW_LINE")
-    UNKNOWN_CHAR = _wrapper.format('UNKNOWN_CHAR')
-    UNKNOWN_WORD = _wrapper.format('UNKNOWN_WORD')
-    URL = _wrapper.format("URL")
-
-    __token_mapping = {
-        ',': COMMA,
-        '.': DOT,
-        '…': TRIPLE_DOTS,
-        ':': COLON,
-        ';': SEMICOLON,
-        '-': DASH,
-        '—': LONG_DASH,
-        '?': QUESTION_SIGN,
-        '!': EXC_SIGN,
-        '(': OPEN_BRACKET,
-        ')': CLOSED_BRACKET,
-        '{': OPEN_BRACKET,
-        '}': CLOSED_BRACKET,
-        '[': OPEN_BRACKET,
-        ']': CLOSED_BRACKET,
-        '\n': NEW_LINE,
-        '«': QUOTE,
-        '»': QUOTE,
-        '"': QUOTE,
-    }
-
-    __supported_tokens = {
-        COMMA,
-        SEMICOLON,
-        COLON,
-        QUOTE,
-        DASH,
-        DOT,
-        LONG_DASH,
-        TRIPLE_DOTS,
-        EXC_SIGN,
-        QUESTION_SIGN,
-        OPEN_BRACKET,
-        CLOSED_BRACKET,
-        NUMBER,
-        URL,
-        NEW_LINE,
-        UNKNOWN_CHAR,
-        UNKNOWN_WORD}
-
-    @staticmethod
-    def try_create(subterm):
-        """
-        Trying create a token by given 'term' parameter
-        subterm: unicode
-           I.e. term ending, so means a part of original term
-        """
-        assert(isinstance(subterm, str))
-        if subterm not in Tokens.__token_mapping:
-            return None
-        return Token(term=subterm, token_value=Tokens.__token_mapping[subterm])
-
-    @staticmethod
-    def try_parse(term):
-        assert(isinstance(term, str))
-        for origin, token_value in Tokens.__token_mapping.items():
-            if term == token_value:
-                return Token(term=origin, token_value=token_value)
-
-    @staticmethod
-    def try_create_number(term):
-        assert(isinstance(term, str))
-        if not term.isdigit():
-            return None
-        return Token(term=term, token_value=Tokens.NUMBER)
-
-    @staticmethod
-    def try_create_url(term):
-        assert(isinstance(term, str))
-        result = urlparse(term)
-        is_correct = result.scheme and result.netloc and result.path
-        if not is_correct:
-            return None
-        return Token(term=term, token_value=Tokens.URL)
-
-    @staticmethod
-    def is_token(term):
-        assert(isinstance(term, str))
-        return term in Tokens.__supported_tokens
-
-    @staticmethod
-    def iter_chars_by_token(term):
-        """
-        Iterate through charts that is related to term
-        token: char
-        """
-        assert(isinstance(term, str))
-        for char, token in Tokens.__token_mapping.items():
-            if term == token:
-                yield char
-
-    @staticmethod
-    def iter_supported_tokens():
-        for token in Tokens.__supported_tokens:
-            yield token

arekit/contrib/utils/serializer.py

@@ -1,42 +0,0 @@
-import logging
-
-from collections.abc import Iterable
-
-from arekit.common.data.input.providers.columns.sample import SampleColumnsProvider
-from arekit.common.data.input.providers.rows.base import BaseRowProvider
-from arekit.common.data.input.repositories.base import BaseInputRepository
-from arekit.common.data.input.repositories.sample import BaseInputSamplesRepository
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.contents.opinions import InputTextOpinionProvider
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class InputDataSerializationHelper(object):
-
-    @staticmethod
-    def create_samples_repo(keep_labels, rows_provider, storage):
-        assert(isinstance(rows_provider, BaseRowProvider))
-        assert(isinstance(keep_labels, bool))
-        assert(isinstance(storage, BaseRowsStorage))
-        return BaseInputSamplesRepository(
-            columns_provider=SampleColumnsProvider(store_labels=keep_labels),
-            rows_provider=rows_provider,
-            storage=storage)
-
-    @staticmethod
-    def fill_and_write(pipeline, repo, target, writer, doc_ids_iter, desc=""):
-        assert(isinstance(pipeline, list))
-        assert(isinstance(doc_ids_iter, Iterable))
-        assert(isinstance(repo, BaseInputRepository))
-
-        doc_ids = list(doc_ids_iter)
-
-        repo.populate(contents_provider=InputTextOpinionProvider(pipeline),
-                      doc_ids=doc_ids,
-                      desc=desc,
-                      writer=writer,
-                      target=target)
-
-        repo.push(writer=writer, target=target)

arekit/contrib/utils/vectorizers/bpe.py

@@ -1,93 +0,0 @@
-import numpy as np
-
-from arekit.common.log_utils import logger
-from arekit.contrib.networks.embedding import Embedding
-from arekit.contrib.networks.vectorizer import BaseVectorizer
-
-
-class BPEVectorizer(BaseVectorizer):
-    """ Embedding algorithm based on parts (trigrams originally)
-    """
-
-    def __init__(self, embedding, max_part_size=3, word_separator=' '):
-        assert(isinstance(embedding, Embedding))
-        assert(isinstance(max_part_size, int))
-        self.__embedding = embedding
-        self.__max_part_size = max_part_size
-        self.__word_separator = word_separator
-
-    def create_term_embedding(self, term):
-        """ Note: returns modified term value in a form of the `word` returning parameter.
-        """
-        assert(isinstance(term, str))
-
-        word, word_embedding = self.__get_from_embedding(term=term) \
-            if term in self.__embedding else self.__compose_from_parts(term=term)
-
-        # In order to prevent a problem of the further separations during reading process.
-        # it is necessary to replace the separators with the other chars.
-        word = word.replace(self.__word_separator, '-')
-
-        return word, word_embedding
-
-    def __compose_from_parts(self, term, do_lowercase=True):
-        # remove empty spaces before and after.
-        term = term.strip()
-
-        # perform lowercasing
-        if do_lowercase:
-            term = term.lower()
-
-        # Calculating vector from term parts
-        count = 0
-        vector = np.zeros(self.__embedding.VectorSize)
-        for word in term.split(self.__word_separator):
-            v, c = self.__create_embedding_for_word(word=word, embedding=self.__embedding)
-            count += c
-            vector = vector + v
-
-        return term, vector / count if count > 0 else vector
-
-    def __get_from_embedding(self, term):
-        return self.__embedding.try_get_related_word(term), self.__embedding[term]
-
-    def __create_embedding_for_word(self, word, embedding):
-        assert(isinstance(word, str))
-        assert(isinstance(embedding, Embedding))
-
-        if word in embedding:
-            return embedding[word], 1
-
-        c_i = 0
-        c_l = self.__max_part_size
-        count = 0
-        vector = np.zeros(embedding.VectorSize)
-        missings = []
-
-        while c_i < len(word):
-
-            if c_l == 0:
-                missings.append(c_i)
-                c_i += 1
-                c_l = self.__max_part_size
-                continue
-
-            right_b = min(len(word), c_i + c_l)
-
-            s_i = embedding.try_find_index_by_plain_word(word=word[c_i:right_b])
-
-            if s_i is None:
-                c_l -= 1
-                continue
-
-            vector += embedding.get_vector_by_index(s_i)
-            c_i += c_l
-            count += 1
-
-        debug = False
-        if debug:
-            w_debug = ''.join(['?' if i in missings else ch
-                               for i, ch in enumerate(word)])
-            logger.debug('Embedded: {}'.format(w_debug).encode('utf-8'))
-
-        return vector, count

arekit/contrib/utils/vectorizers/random_norm.py

@@ -1,39 +0,0 @@
-import numpy as np
-
-from arekit.contrib.networks.vectorizer import BaseVectorizer
-
-
-class RandomNormalVectorizer(BaseVectorizer):
-
-    def __init__(self, vector_size, token_offset=12345, max_tokens_count=100):
-        assert(isinstance(vector_size, int))
-        self.__vector_size = vector_size
-        self.__seed_token_offset = token_offset
-        self.__max_tokens_count = max_tokens_count
-
-    def create_term_embedding(self, term):
-        """ term: is its index.
-        """
-        embedding = self.__get_random_normal_distribution(
-            vector_size=self.__vector_size,
-            seed=(self.__string_to_int(term) % self.__max_tokens_count) + self.__seed_token_offset,
-            loc=0.05,
-            scale=0.025)
-        return term, embedding
-
-    # region private methods
-
-    def __string_to_int(self, s):
-        # Originally taken from here:
-        # https://stackoverflow.com/questions/2511058/persistent-hashing-of-strings-in-python
-        ord3 = lambda x: '%.3d' % ord(x)
-        return int(''.join(map(ord3, s)))
-
-    @staticmethod
-    def __get_random_normal_distribution(vector_size, seed, loc, scale):
-        assert (isinstance(vector_size, int))
-        assert (isinstance(seed, int))
-        np.random.seed(seed)
-        return np.random.normal(loc=loc, scale=scale, size=vector_size)
-
-    # endregion