arekit 0.24.0__py3-none-any.whl → 0.25.1__py3-none-any.whl

arekit/contrib/utils/processing/languages/ru/cases.py

@@ -1,78 +0,0 @@
-from enum import Enum
-
-
-class RussianCases(Enum):
-    """ Падежи русского языка
-    """
-
-    """ не определено
-    """
-    UNKN = 10
-
-    """ именительный
-    """
-    NOM = 1
-
-    """ родительный
-    """
-    GEN = 2
-
-    """ дательный
-    """
-    DAT = 3
-
-    """ винительный
-    """
-    ACC = 4
-
-    """ творительный
-    """
-    INS = 5
-
-    """ предложный
-    """
-    ABL = 6
-
-    """ партитив
-    """
-    PART = 7
-
-    """ местный 
-    """
-    LOC = 8
-
-    """ звательный
-    """
-    VOC = 9
-
-
-class RussianCasesService(object):
-
-    __english = {
-        'nom': RussianCases.NOM,
-        'gen': RussianCases.GEN,
-        'dat': RussianCases.DAT,
-        'acc': RussianCases.ACC,
-        'ins': RussianCases.INS,
-        'abl': RussianCases.ABL,
-        'part': RussianCases.PART,
-        'loc': RussianCases.LOC,
-        'voc': RussianCases.VOC,
-    }
-
-    __mystem_russian = {
-        'им': RussianCases.NOM,
-        'род': RussianCases.GEN,
-        'дат': RussianCases.DAT,
-        'вин': RussianCases.ACC,
-        'твор': RussianCases.INS,
-        'пр': RussianCases.ABL,
-        'парт': RussianCases.PART,
-        'местн': RussianCases.LOC,
-        'зват': RussianCases.VOC,
-    }
-
-    @staticmethod
-    def iter_rus_mystem_tags():
-        for key, value in RussianCasesService.__mystem_russian.items():
-            yield key, value

arekit/contrib/utils/processing/languages/ru/constants.py

@@ -1,6 +0,0 @@
-class RussianConstants:
-
-    PrepositionSet = {'к', 'на', 'по', 'с', 'до', 'в', 'во', "у", "а"}
-
-    def __init__(self):
-        pass

arekit/contrib/utils/processing/languages/ru/mods.py

@@ -1,13 +0,0 @@
-from arekit.contrib.utils.processing.languages.mods import BaseLanguageMods
-
-
-class RussianLanguageMods(BaseLanguageMods):
-
-    @staticmethod
-    def replace_specific_word_chars(word):
-        assert(isinstance(word, str))
-        return word.replace('ё', 'e')
-
-    @staticmethod
-    def is_negation_word(word):
-        return word == 'не'

arekit/contrib/utils/processing/languages/ru/number.py

@@ -1,23 +0,0 @@
-from enum import Enum
-
-
-class RussianNumberType(Enum):
-
-    UNKN = 3
-
-    Plural = 1
-
-    Single = 2
-
-
-class RussianNumberTypeService(object):
-
-    __russian = {
-        'ед': RussianNumberType.Single,
-        'мн': RussianNumberType.Plural
-    }
-
-    @staticmethod
-    def iter_rus_mystem_tags():
-        for key, value in RussianNumberTypeService.__russian.items():
-            yield key, value

arekit/contrib/utils/processing/languages/ru/pos_service.py

@@ -1,36 +0,0 @@
-from arekit.contrib.utils.processing.languages.pos import PartOfSpeechType
-
-
-class PartOfSpeechTypesService(object):
-
-    __pos_names = {
-        "S": PartOfSpeechType.NOUN,
-        "ADV": PartOfSpeechType.ADV,
-        "ADVPRO": PartOfSpeechType.ADVPRO,
-        "ANUM": PartOfSpeechType.ANUM,
-        "APRO": PartOfSpeechType.APRO,
-        "COM": PartOfSpeechType.COM,
-        "CONJ": PartOfSpeechType.CONJ,
-        "INTJ": PartOfSpeechType.INTJ,
-        "NUM": PartOfSpeechType.NUM,
-        "PART": PartOfSpeechType.PART,
-        "PR": PartOfSpeechType.PR,
-        "A": PartOfSpeechType.ADJ,
-        "SPRO": PartOfSpeechType.SPRO,
-        "V": PartOfSpeechType.VERB,
-        "UNKN": PartOfSpeechType.Unknown,
-        "EMPTY": PartOfSpeechType.Empty}
-
-    @staticmethod
-    def iter_mystem_tags():
-        for key, value in PartOfSpeechTypesService.__pos_names.items():
-            yield key, value
-
-    @staticmethod
-    def get_mystem_from_string(value):
-        return PartOfSpeechTypesService.__pos_names[value]
-
-    @staticmethod
-    def get_mystem_pos_count():
-        return len(PartOfSpeechTypesService.__pos_names)
-

arekit/contrib/utils/processing/lemmatization/mystem.py

@@ -1,51 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.common.utils import filter_whitespaces
-from pymystem3 import Mystem
-
-
-class MystemWrapper(Stemmer):
-    """ Yandex MyStem wrapper
-
-        part of speech description:
-        https://tech.yandex.ru/mystem/doc/grammemes-values-docpage/
-    """
-
-    def __init__(self, entire_input=False):
-        """
-        entire_input: bool
-            Mystem parameter that allows to keep all information from input (true) or
-            remove garbage characters
-        """
-        self.__mystem = Mystem(entire_input=entire_input)
-
-    # region properties
-
-    @property
-    def MystemInstance(self):
-        return self.__mystem
-
-    # endregion
-
-    # region public methods
-
-    def lemmatize_to_list(self, text):
-        return self.__lemmatize_core(text)
-
-    def lemmatize_to_str(self, text):
-        result = " ".join(self.__lemmatize_core(text))
-        return result if len(result) != 0 else self.__process_original_text(text)
-
-    # endregion
-
-    # region private methods
-
-    def __lemmatize_core(self, text):
-        assert(isinstance(text, str))
-        result_list = self.__mystem.lemmatize(self.__process_original_text(text))
-        return filter_whitespaces(result_list)
-
-    @staticmethod
-    def __process_original_text(text):
-        return text.lower()
-
-    # endregion

arekit/contrib/utils/processing/pos/base.py

@@ -1,12 +0,0 @@
-# TODO. Move to base class.
-class POSTagger:
-
-    def get_term_pos(self, term):
-        raise NotImplementedError()
-
-    def get_term_number(self, term):
-        raise NotImplementedError()
-
-    def pos_to_int(self, pos):
-        raise NotImplementedError()
-

arekit/contrib/utils/processing/pos/mystem_wrap.py

@@ -1,134 +0,0 @@
-from pymystem3 import Mystem
-
-from arekit.contrib.utils.processing.languages.pos import PartOfSpeechType
-from arekit.contrib.utils.processing.languages.ru.cases import RussianCases, RussianCasesService
-from arekit.contrib.utils.processing.languages.ru.number import RussianNumberType, RussianNumberTypeService
-from arekit.contrib.utils.processing.languages.ru.pos_service import PartOfSpeechTypesService
-from arekit.contrib.utils.processing.pos.russian import RussianPOSTagger
-
-
-class POSMystemWrapper(RussianPOSTagger):
-
-    _ArgsSeparator = ','
-    _GrammarKey = 'gr'
-
-    def __init__(self, mystem):
-        assert(isinstance(mystem, Mystem))
-        self.__mystem = mystem
-
-    # region private methods
-
-    @staticmethod
-    def __extract_from_analysis(analysis, func):
-        """
-        part of speech description:
-            https://tech.yandex.ru/mystem/doc/grammemes-values-docpage/
-        func: f(args) -> out
-        returns: str or None
-        """
-        assert(callable(func))
-
-        if 'analysis' not in analysis:
-            return func(None)
-
-        info = analysis['analysis']
-        if len(info) == 0:
-            return func(None)
-
-        return func(info[0])
-
-    @staticmethod
-    def __get_pos(arguments):
-        if arguments is None:
-            return PartOfSpeechType.Unknown
-
-        pos = arguments[POSMystemWrapper._GrammarKey].split(POSMystemWrapper._ArgsSeparator)[0]
-        if '=' in pos:
-            pos = pos.split('=')[0]
-
-        return PartOfSpeechTypesService.get_mystem_from_string(pos)
-
-    @staticmethod
-    def __get_russian_case(arguments):
-        if arguments is None:
-            return RussianCases.UNKN
-
-        all_params = set(POSMystemWrapper.__iter_params(arguments))
-
-        for key, case in RussianCasesService.iter_rus_mystem_tags():
-            if key in all_params:
-                return case
-
-        return RussianCases.UNKN
-
-    @staticmethod
-    def __get_number(arguments):
-        if arguments is None:
-            return RussianNumberType.UNKN
-
-        all_params = set(POSMystemWrapper.__iter_params(arguments))
-
-        for key, case in RussianNumberTypeService.iter_rus_mystem_tags():
-            if key in all_params:
-                return case
-
-        return RussianNumberType.UNKN
-
-    @staticmethod
-    def __iter_params(arguments):
-        params = arguments[POSMystemWrapper._GrammarKey].split(POSMystemWrapper._ArgsSeparator)
-        for optionally_combined in params:
-            for param in optionally_combined.split('='):
-                yield param
-
-    # endregion
-
-    def get_term_pos(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_pos) \
-            if len(analyzed) > 0 else PartOfSpeechType.Unknown
-
-    def get_term_case(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_russian_case) \
-            if len(analyzed) > 0 else RussianCases.UNKN
-
-    def get_term_number(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_number) \
-            if len(analyzed) > 0 else RussianNumberType.UNKN
-
-    def get_terms_russian_cases(self, text):
-        """ list of part of speech according to the certain word in text
-        """
-        assert(isinstance(text, str))
-        cases = []
-
-        analyzed = self.__mystem.analyze(text)
-        for a in analyzed:
-            pos = self.__extract_from_analysis(a, self.__get_russian_case) if len(analyzed) > 0 else RussianCases.UNKN
-            cases.append(pos)
-
-        return cases
-
-    def pos_to_int(self, pos):
-        assert(isinstance(pos, PartOfSpeechType))
-        return int(pos)
-
-    @staticmethod
-    def is_adjective(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.ADJ
-
-    @staticmethod
-    def is_noun(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.NOUN
-
-    @staticmethod
-    def is_verb(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.VERB

arekit/contrib/utils/processing/pos/russian.py

@@ -1,10 +0,0 @@
-from arekit.contrib.utils.processing.pos.base import POSTagger
-
-
-class RussianPOSTagger(POSTagger):
-    """ Provides cases support ('падежи')
-    """
-
-    def get_term_case(self, term):
-        raise NotImplementedError()
-

arekit/contrib/utils/processing/text/tokens.py

@@ -1,127 +0,0 @@
-from urllib.parse import urlparse
-from arekit.common.context.token import Token
-
-
-# TODO. Provide the base (BaseTokens) type.
-# TODO. With the related API at BaseTokens.
-class Tokens:
-    """
-    Tokens used to describe a non-word text units, such as punctuation,
-    uknown words/chars, smiles, etc.
-    """
-
-    _wrapper = "<[{}]>"
-    COMMA = _wrapper.format(',')
-    SEMICOLON = _wrapper.format(';')
-    COLON = _wrapper.format(':')
-    QUOTE = _wrapper.format('QUOTE')
-    DASH = _wrapper.format('-')
-    LONG_DASH = _wrapper.format('long_dash')
-    DOT = _wrapper.format('.')
-    TRIPLE_DOTS = _wrapper.format('…')
-    EXC_SIGN = _wrapper.format('!')
-    QUESTION_SIGN = _wrapper.format('?')
-    OPEN_BRACKET = _wrapper.format('OPEN_BRACKET')
-    CLOSED_BRACKET = _wrapper.format('CLOSED_BRACKET')
-    NUMBER = _wrapper.format('NUMBER')
-    NEW_LINE = _wrapper.format("NEW_LINE")
-    UNKNOWN_CHAR = _wrapper.format('UNKNOWN_CHAR')
-    UNKNOWN_WORD = _wrapper.format('UNKNOWN_WORD')
-    URL = _wrapper.format("URL")
-
-    __token_mapping = {
-        ',': COMMA,
-        '.': DOT,
-        '…': TRIPLE_DOTS,
-        ':': COLON,
-        ';': SEMICOLON,
-        '-': DASH,
-        '—': LONG_DASH,
-        '?': QUESTION_SIGN,
-        '!': EXC_SIGN,
-        '(': OPEN_BRACKET,
-        ')': CLOSED_BRACKET,
-        '{': OPEN_BRACKET,
-        '}': CLOSED_BRACKET,
-        '[': OPEN_BRACKET,
-        ']': CLOSED_BRACKET,
-        '\n': NEW_LINE,
-        '«': QUOTE,
-        '»': QUOTE,
-        '"': QUOTE,
-    }
-
-    __supported_tokens = {
-        COMMA,
-        SEMICOLON,
-        COLON,
-        QUOTE,
-        DASH,
-        DOT,
-        LONG_DASH,
-        TRIPLE_DOTS,
-        EXC_SIGN,
-        QUESTION_SIGN,
-        OPEN_BRACKET,
-        CLOSED_BRACKET,
-        NUMBER,
-        URL,
-        NEW_LINE,
-        UNKNOWN_CHAR,
-        UNKNOWN_WORD}
-
-    @staticmethod
-    def try_create(subterm):
-        """
-        Trying create a token by given 'term' parameter
-        subterm: unicode
-           I.e. term ending, so means a part of original term
-        """
-        assert(isinstance(subterm, str))
-        if subterm not in Tokens.__token_mapping:
-            return None
-        return Token(term=subterm, token_value=Tokens.__token_mapping[subterm])
-
-    @staticmethod
-    def try_parse(term):
-        assert(isinstance(term, str))
-        for origin, token_value in Tokens.__token_mapping.items():
-            if term == token_value:
-                return Token(term=origin, token_value=token_value)
-
-    @staticmethod
-    def try_create_number(term):
-        assert(isinstance(term, str))
-        if not term.isdigit():
-            return None
-        return Token(term=term, token_value=Tokens.NUMBER)
-
-    @staticmethod
-    def try_create_url(term):
-        assert(isinstance(term, str))
-        result = urlparse(term)
-        is_correct = result.scheme and result.netloc and result.path
-        if not is_correct:
-            return None
-        return Token(term=term, token_value=Tokens.URL)
-
-    @staticmethod
-    def is_token(term):
-        assert(isinstance(term, str))
-        return term in Tokens.__supported_tokens
-
-    @staticmethod
-    def iter_chars_by_token(term):
-        """
-        Iterate through charts that is related to term
-        token: char
-        """
-        assert(isinstance(term, str))
-        for char, token in Tokens.__token_mapping.items():
-            if term == token:
-                yield char
-
-    @staticmethod
-    def iter_supported_tokens():
-        for token in Tokens.__supported_tokens:
-            yield token

arekit/contrib/utils/resources.py

@@ -1,25 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.contrib.utils.download import NEWS_MYSTEM_SKIPGRAM_1000_20_2015, load_embedding_and_vocab
-from arekit.contrib.utils.embeddings.rusvectores import RusvectoresEmbedding
-
-
-def load_embedding_news_mystem_skipgram_1000_20_2015(stemmer, auto_download=False):
-    """ Embedding from https://rusvectores.org/ru/models/
-        Description:        Russian news, from 2013 till the october 2015
-        Corpora size:       2.5 milliard words
-        Vocabulary volume:  147 358
-        Frequency bound:    200
-        Tagset:             Mystem
-        Algorithm:          Continuous Skip-Gram
-        Vector size:        1000
-
-        stemmer: Stemmer
-            It is expected to adopt MystemWrapper.
-        auto_download: bool
-            Whether try to download if the resource was missed.
-    """
-    assert(isinstance(stemmer, Stemmer) or stemmer is None)
-    embedding, vocab = load_embedding_and_vocab(local_name=NEWS_MYSTEM_SKIPGRAM_1000_20_2015, check_existance=True,
-                                                download_if_missed=auto_download)
-    embedding = RusvectoresEmbedding(matrix=embedding, words=vocab, stemmer=stemmer)
-    return embedding

arekit/contrib/utils/serializer.py

@@ -1,43 +0,0 @@
-import logging
-
-from collections.abc import Iterable
-
-from arekit.common.data.input.providers.columns.sample import SampleColumnsProvider
-from arekit.common.data.input.providers.rows.base import BaseRowProvider
-from arekit.common.data.input.repositories.base import BaseInputRepository
-from arekit.common.data.input.repositories.sample import BaseInputSamplesRepository
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.common.pipeline.base import BasePipeline
-from arekit.contrib.utils.data.contents.opinions import InputTextOpinionProvider
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class InputDataSerializationHelper(object):
-
-    @staticmethod
-    def create_samples_repo(keep_labels, rows_provider, storage):
-        assert(isinstance(rows_provider, BaseRowProvider))
-        assert(isinstance(keep_labels, bool))
-        assert(isinstance(storage, BaseRowsStorage))
-        return BaseInputSamplesRepository(
-            columns_provider=SampleColumnsProvider(store_labels=keep_labels),
-            rows_provider=rows_provider,
-            storage=storage)
-
-    @staticmethod
-    def fill_and_write(pipeline, repo, target, writer, doc_ids_iter, desc=""):
-        assert(isinstance(pipeline, BasePipeline))
-        assert(isinstance(doc_ids_iter, Iterable))
-        assert(isinstance(repo, BaseInputRepository))
-
-        doc_ids = list(doc_ids_iter)
-
-        repo.populate(contents_provider=InputTextOpinionProvider(pipeline),
-                      doc_ids=doc_ids,
-                      desc=desc,
-                      writer=writer,
-                      target=target)
-
-        repo.push(writer=writer, target=target)

arekit/contrib/utils/sources/sentinerel/text_opinion/prof_per_org_filter.py

@@ -1,63 +0,0 @@
-from arekit.common.entities.base import Entity
-from arekit.common.docs.parsed.base import ParsedDocument
-from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider
-from arekit.common.docs.parsed.term_position import TermPositionTypes, TermPosition
-from arekit.common.text.enums import TermFormat
-from arekit.common.text.parsed import BaseParsedText
-from arekit.common.text_opinions.base import TextOpinion
-from arekit.contrib.utils.pipelines.text_opinion.filters.base import TextOpinionFilter
-
-
-class ProfessionAsCharacteristicSentimentTextOpinionFilter(TextOpinionFilter):
-    """ This is a filter, based on the PROFESSION type prefixed entity for
-        the SentiNEREL collection.
-
-        In this case, profession acts as a characteristics of the Person, and
-        therefore there is no need to consider these attitudes in annotation.
-
-        For a greater details, see:
-        https://github.com/nicolay-r/AREkit/issues/404
-    """
-
-    def __init__(self, char_type="PROFESSION"):
-        self.__char_type = char_type
-        self.__next_entity_types = ["PERSON"]
-
-    def filter(self, text_opinion, parsed_doc, entity_service_provider):
-        assert(isinstance(text_opinion, TextOpinion))
-        assert(isinstance(parsed_doc, ParsedDocument))
-        assert(isinstance(entity_service_provider, EntityServiceProvider))
-
-        # Picking up entity.
-        target_entity = entity_service_provider._doc_entities[text_opinion.TargetId]
-        assert(isinstance(target_entity, Entity))
-
-        if target_entity.Type != self.__char_type:
-            # This is not our case.
-            return True
-
-        # Picking up the related target entity position.
-        target_pos = entity_service_provider.get_entity_position(text_opinion.TargetId)
-        assert(isinstance(target_pos, TermPosition))
-
-        # Picking up the related sentence of target.
-        t_sent = target_pos.get_index(TermPositionTypes.SentenceIndex)
-        sentence = parsed_doc.get_sentence(t_sent)
-        assert(isinstance(sentence, BaseParsedText))
-
-        # Picking up the entity position in sentence.
-        target_term_ind = target_pos.get_index(TermPositionTypes.IndexInSentence)
-
-        # We pick up the next term within the parsed sentece.
-        next_term = sentence.get_term(target_term_ind + 1, term_format=TermFormat.Raw) \
-            if len(sentence) > target_term_ind + 1 else None
-
-        if next_term is None:
-            # This is not our case.
-            return True
-
-        if isinstance(next_term, Entity) and next_term.Type in self.__next_entity_types:
-            # We reject this opinion from the annotation, since this is not expected to be a sentiment one.
-            return False
-
-        return True