arekit 0.25.0__py3-none-any.whl → 0.25.2__py3-none-any.whl

arekit/contrib/utils/pipelines/items/text/tokenizer.py

@@ -1,105 +0,0 @@
-import logging
-
-from arekit.common.context.token import Token
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.utils import split_by_whitespaces
-from arekit.contrib.utils.processing.text.tokens import Tokens
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-
-class DefaultTextTokenizer(BasePipelineItem):
-    """ Default parser implementation.
-    """
-
-    def __init__(self, keep_tokens=True, **kwargs):
-        super(DefaultTextTokenizer, self).__init__(**kwargs)
-        self.__keep_tokens = keep_tokens
-
-    # region protected methods
-
-    def apply_core(self, input_data, pipeline_ctx):
-        output_data = self.__process_parts(input_data)
-        if not self.__keep_tokens:
-            output_data = [word for word in output_data if not isinstance(word, Token)]
-        return output_data
-
-    # endregion
-
-    # region private static methods
-
-    def __process_parts(self, parts):
-        assert(isinstance(parts, list))
-
-        parsed = []
-        for part in parts:
-
-            if part is None:
-                continue
-
-            # Keep non str words as it is and try to parse str-based words.
-            processed = [part] if not isinstance(part, str) else \
-                self.__iter_processed_part(part=part)
-
-            parsed.extend(processed)
-
-        return parsed
-
-    def __iter_processed_part(self, part):
-        for word in split_by_whitespaces(part):
-            for term in self.__process_word(word):
-                yield term
-
-    def __process_word(self, word):
-        assert(isinstance(word, str))
-        return self.__split_tokens(word)
-
-    @staticmethod
-    def __split_tokens(term):
-        """
-        Splitting off tokens from parsed_doc ending, i.e. for example:
-            term: "сказать,-" -> "(term: "сказать", ["COMMA_TOKEN", "DASH_TOKEN"])
-        return: (unicode or None, list)
-            modified term and list of extracted tokens.
-        """
-
-        url = Tokens.try_create_url(term)
-        if url is not None:
-            return [url]
-
-        l = 0
-        words_and_tokens = []
-        while l < len(term):
-
-            # Token.
-            token = Tokens.try_create(term[l])
-            if token is not None:
-                if token.get_token_value() != Tokens.NEW_LINE:
-                    words_and_tokens.append(token)
-                l += 1
-
-            # Number.
-            elif str.isdigit(term[l]):
-                k = l + 1
-                while k < len(term) and str.isdigit(term[k]):
-                    k += 1
-                token = Tokens.try_create_number(term[l:k])
-                assert(token is not None)
-                words_and_tokens.append(token)
-                l = k
-
-            # Term.
-            else:
-                k = l + 1
-                while k < len(term):
-                    token = Tokens.try_create(term[k])
-                    if token is not None and token.get_token_value() != Tokens.DASH:
-                        break
-                    k += 1
-                words_and_tokens.append(term[l:k])
-                l = k
-
-        return words_and_tokens
-
-    # endregion

arekit/contrib/utils/pipelines/items/text/translator.py

@@ -1,136 +0,0 @@
-from arekit.common.data.input.providers.const import IDLE_MODE
-from arekit.common.pipeline.conts import PARENT_CTX
-from arekit.common.entities.base import Entity
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-
-
-class MLTextTranslatorPipelineItem(BasePipelineItem):
-    """ Machine learning based translator pipeline item.
-    """
-
-    def __init__(self, batch_translate_model, do_translate_entity=True, **kwargs):
-        """ Model, which is based on translation of the text,
-            represented as a list of words.
-        """
-        super(MLTextTranslatorPipelineItem, self).__init__(**kwargs)
-        self.__do_translate_entity = do_translate_entity
-        self.__translate = batch_translate_model
-
-    def fast_most_accurate_approach(self, input_data, entity_placeholder_template="<entityTag={}/>"):
-        """ This approach assumes that the translation won't corrupt the original
-            meta-annotation for entities and objects mentioned in text.
-        """
-
-        def __optionally_register(prts):
-            if len(prts) > 0:
-                content.append(" ".join(prts))
-            parts_to_join.clear()
-
-        content = []
-        origin_entities = []
-        parts_to_join = []
-
-        for part in input_data:
-            if isinstance(part, str) and part.strip():
-                parts_to_join.append(part)
-            elif isinstance(part, Entity):
-                entity_index = len(origin_entities)
-                parts_to_join.append(entity_placeholder_template.format(entity_index))
-                # Register entities information for further restoration.
-                origin_entities.append(part)
-
-        # Register original text with masked named entities.
-        __optionally_register(parts_to_join)
-        # Register all named entities in order of their appearance in text.
-        content.extend([e.Value for e in origin_entities])
-
-        # Compose text parts.
-        translated_parts = self.__translate(content)
-
-        if len(translated_parts) == 0:
-            return None
-
-        # Take the original text.
-        text = translated_parts[0]
-        for entity_index in range(len(origin_entities)):
-            if entity_placeholder_template.format(entity_index) not in text:
-                return None
-
-        # Enumerate entities.
-        from_ind = 0
-        text_parts = []
-        for entity_index, translated_value in enumerate(translated_parts[1:]):
-            entity_placeholder_instance = entity_placeholder_template.format(entity_index)
-            # Cropping text part.
-            to_ind = text.index(entity_placeholder_instance)
-
-            if self.__do_translate_entity:
-                origin_entities[entity_index].set_display_value(translated_value.strip())
-
-            # Register entities.
-            text_parts.append(text[from_ind:to_ind])
-            text_parts.append(origin_entities[entity_index])
-            # Update from index.
-            from_ind = to_ind + len(entity_placeholder_instance)
-
-        # Consider the remaining part.
-        text_parts.append(text[from_ind:])
-        return text_parts
-
-    def default_pre_part_splitting_approach(self, input_data):
-        """ This is the original strategy, based on the manually cropped named entities
-            before the actual translation call.
-        """
-
-        def __optionally_register(prts):
-            if len(prts) > 0:
-                content.append(" ".join(prts))
-            parts_to_join.clear()
-
-        content = []
-        origin_entities = []
-        origin_entity_ind = []
-        parts_to_join = []
-
-        for _, part in enumerate(input_data):
-            if isinstance(part, str) and part.strip():
-                parts_to_join.append(part)
-            elif isinstance(part, Entity):
-                # Register first the prior parts were merged.
-                __optionally_register(parts_to_join)
-                # Register entities information for further restoration.
-                origin_entity_ind.append(len(content))
-                origin_entities.append(part)
-                content.append(part.Value)
-
-        __optionally_register(parts_to_join)
-
-        # Compose text parts.
-        translated_parts = self.__translate(content)
-
-        for entity_ind, entity_part_ind in enumerate(origin_entity_ind):
-            entity = origin_entities[entity_ind]
-            if self.__do_translate_entity:
-                entity.set_display_value(translated_parts[entity_part_ind].strip())
-            translated_parts[entity_part_ind] = entity
-
-        return translated_parts
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        assert(isinstance(input_data, list))
-
-        # Check the pipeline state whether is an idle mode or not.
-        parent_ctx = pipeline_ctx.provide(PARENT_CTX)
-        idle_mode = parent_ctx.provide(IDLE_MODE)
-
-        # When pipeline utilized only for the assessing the expected amount
-        # of rows (common case of idle_mode), there is no need to perform
-        # translation.
-        if idle_mode:
-            return
-
-        fast_accurate = self.fast_most_accurate_approach(input_data)
-        return self.default_pre_part_splitting_approach(input_data) \
-            if fast_accurate is None else fast_accurate

arekit/contrib/utils/processing/languages/mods.py

@@ -1,12 +0,0 @@
-class BaseLanguageMods(object):
-    """
-    Represents a class with static methods that provides text/word modifies.
-    """
-
-    @staticmethod
-    def replace_specific_word_chars(word):
-        pass
-
-    @staticmethod
-    def is_negation_word(word):
-        pass

arekit/contrib/utils/processing/languages/pos.py

@@ -1,23 +0,0 @@
-from enum import IntEnum
-
-
-class PartOfSpeechType(IntEnum):
-
-    NOUN = 1
-    ADV = 2
-    ADVPRO = 3
-    ANUM = 4
-    APRO = 5
-    COM = 6
-    CONJ = 7
-    INTJ = 8
-    NUM = 9
-    PART = 10
-    PR = 11
-    ADJ = 12
-    SPRO = 13
-    VERB = 14
-
-    Unknown = 15
-
-    Empty = 16

arekit/contrib/utils/processing/languages/ru/cases.py

@@ -1,78 +0,0 @@
-from enum import Enum
-
-
-class RussianCases(Enum):
-    """ Падежи русского языка
-    """
-
-    """ не определено
-    """
-    UNKN = 10
-
-    """ именительный
-    """
-    NOM = 1
-
-    """ родительный
-    """
-    GEN = 2
-
-    """ дательный
-    """
-    DAT = 3
-
-    """ винительный
-    """
-    ACC = 4
-
-    """ творительный
-    """
-    INS = 5
-
-    """ предложный
-    """
-    ABL = 6
-
-    """ партитив
-    """
-    PART = 7
-
-    """ местный 
-    """
-    LOC = 8
-
-    """ звательный
-    """
-    VOC = 9
-
-
-class RussianCasesService(object):
-
-    __english = {
-        'nom': RussianCases.NOM,
-        'gen': RussianCases.GEN,
-        'dat': RussianCases.DAT,
-        'acc': RussianCases.ACC,
-        'ins': RussianCases.INS,
-        'abl': RussianCases.ABL,
-        'part': RussianCases.PART,
-        'loc': RussianCases.LOC,
-        'voc': RussianCases.VOC,
-    }
-
-    __mystem_russian = {
-        'им': RussianCases.NOM,
-        'род': RussianCases.GEN,
-        'дат': RussianCases.DAT,
-        'вин': RussianCases.ACC,
-        'твор': RussianCases.INS,
-        'пр': RussianCases.ABL,
-        'парт': RussianCases.PART,
-        'местн': RussianCases.LOC,
-        'зват': RussianCases.VOC,
-    }
-
-    @staticmethod
-    def iter_rus_mystem_tags():
-        for key, value in RussianCasesService.__mystem_russian.items():
-            yield key, value

arekit/contrib/utils/processing/languages/ru/constants.py

@@ -1,6 +0,0 @@
-class RussianConstants:
-
-    PrepositionSet = {'к', 'на', 'по', 'с', 'до', 'в', 'во', "у", "а"}
-
-    def __init__(self):
-        pass

arekit/contrib/utils/processing/languages/ru/mods.py

@@ -1,13 +0,0 @@
-from arekit.contrib.utils.processing.languages.mods import BaseLanguageMods
-
-
-class RussianLanguageMods(BaseLanguageMods):
-
-    @staticmethod
-    def replace_specific_word_chars(word):
-        assert(isinstance(word, str))
-        return word.replace('ё', 'e')
-
-    @staticmethod
-    def is_negation_word(word):
-        return word == 'не'

arekit/contrib/utils/processing/languages/ru/number.py

@@ -1,23 +0,0 @@
-from enum import Enum
-
-
-class RussianNumberType(Enum):
-
-    UNKN = 3
-
-    Plural = 1
-
-    Single = 2
-
-
-class RussianNumberTypeService(object):
-
-    __russian = {
-        'ед': RussianNumberType.Single,
-        'мн': RussianNumberType.Plural
-    }
-
-    @staticmethod
-    def iter_rus_mystem_tags():
-        for key, value in RussianNumberTypeService.__russian.items():
-            yield key, value

arekit/contrib/utils/processing/languages/ru/pos_service.py

@@ -1,36 +0,0 @@
-from arekit.contrib.utils.processing.languages.pos import PartOfSpeechType
-
-
-class PartOfSpeechTypesService(object):
-
-    __pos_names = {
-        "S": PartOfSpeechType.NOUN,
-        "ADV": PartOfSpeechType.ADV,
-        "ADVPRO": PartOfSpeechType.ADVPRO,
-        "ANUM": PartOfSpeechType.ANUM,
-        "APRO": PartOfSpeechType.APRO,
-        "COM": PartOfSpeechType.COM,
-        "CONJ": PartOfSpeechType.CONJ,
-        "INTJ": PartOfSpeechType.INTJ,
-        "NUM": PartOfSpeechType.NUM,
-        "PART": PartOfSpeechType.PART,
-        "PR": PartOfSpeechType.PR,
-        "A": PartOfSpeechType.ADJ,
-        "SPRO": PartOfSpeechType.SPRO,
-        "V": PartOfSpeechType.VERB,
-        "UNKN": PartOfSpeechType.Unknown,
-        "EMPTY": PartOfSpeechType.Empty}
-
-    @staticmethod
-    def iter_mystem_tags():
-        for key, value in PartOfSpeechTypesService.__pos_names.items():
-            yield key, value
-
-    @staticmethod
-    def get_mystem_from_string(value):
-        return PartOfSpeechTypesService.__pos_names[value]
-
-    @staticmethod
-    def get_mystem_pos_count():
-        return len(PartOfSpeechTypesService.__pos_names)
-

arekit/contrib/utils/processing/lemmatization/mystem.py

@@ -1,51 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.common.utils import filter_whitespaces
-from pymystem3 import Mystem
-
-
-class MystemWrapper(Stemmer):
-    """ Yandex MyStem wrapper
-
-        part of speech description:
-        https://tech.yandex.ru/mystem/doc/grammemes-values-docpage/
-    """
-
-    def __init__(self, entire_input=False):
-        """
-        entire_input: bool
-            Mystem parameter that allows to keep all information from input (true) or
-            remove garbage characters
-        """
-        self.__mystem = Mystem(entire_input=entire_input)
-
-    # region properties
-
-    @property
-    def MystemInstance(self):
-        return self.__mystem
-
-    # endregion
-
-    # region public methods
-
-    def lemmatize_to_list(self, text):
-        return self.__lemmatize_core(text)
-
-    def lemmatize_to_str(self, text):
-        result = " ".join(self.__lemmatize_core(text))
-        return result if len(result) != 0 else self.__process_original_text(text)
-
-    # endregion
-
-    # region private methods
-
-    def __lemmatize_core(self, text):
-        assert(isinstance(text, str))
-        result_list = self.__mystem.lemmatize(self.__process_original_text(text))
-        return filter_whitespaces(result_list)
-
-    @staticmethod
-    def __process_original_text(text):
-        return text.lower()
-
-    # endregion

arekit/contrib/utils/processing/pos/base.py

@@ -1,12 +0,0 @@
-# TODO. Move to base class.
-class POSTagger:
-
-    def get_term_pos(self, term):
-        raise NotImplementedError()
-
-    def get_term_number(self, term):
-        raise NotImplementedError()
-
-    def pos_to_int(self, pos):
-        raise NotImplementedError()
-

arekit/contrib/utils/processing/pos/mystem_wrap.py

@@ -1,134 +0,0 @@
-from pymystem3 import Mystem
-
-from arekit.contrib.utils.processing.languages.pos import PartOfSpeechType
-from arekit.contrib.utils.processing.languages.ru.cases import RussianCases, RussianCasesService
-from arekit.contrib.utils.processing.languages.ru.number import RussianNumberType, RussianNumberTypeService
-from arekit.contrib.utils.processing.languages.ru.pos_service import PartOfSpeechTypesService
-from arekit.contrib.utils.processing.pos.russian import RussianPOSTagger
-
-
-class POSMystemWrapper(RussianPOSTagger):
-
-    _ArgsSeparator = ','
-    _GrammarKey = 'gr'
-
-    def __init__(self, mystem):
-        assert(isinstance(mystem, Mystem))
-        self.__mystem = mystem
-
-    # region private methods
-
-    @staticmethod
-    def __extract_from_analysis(analysis, func):
-        """
-        part of speech description:
-            https://tech.yandex.ru/mystem/doc/grammemes-values-docpage/
-        func: f(args) -> out
-        returns: str or None
-        """
-        assert(callable(func))
-
-        if 'analysis' not in analysis:
-            return func(None)
-
-        info = analysis['analysis']
-        if len(info) == 0:
-            return func(None)
-
-        return func(info[0])
-
-    @staticmethod
-    def __get_pos(arguments):
-        if arguments is None:
-            return PartOfSpeechType.Unknown
-
-        pos = arguments[POSMystemWrapper._GrammarKey].split(POSMystemWrapper._ArgsSeparator)[0]
-        if '=' in pos:
-            pos = pos.split('=')[0]
-
-        return PartOfSpeechTypesService.get_mystem_from_string(pos)
-
-    @staticmethod
-    def __get_russian_case(arguments):
-        if arguments is None:
-            return RussianCases.UNKN
-
-        all_params = set(POSMystemWrapper.__iter_params(arguments))
-
-        for key, case in RussianCasesService.iter_rus_mystem_tags():
-            if key in all_params:
-                return case
-
-        return RussianCases.UNKN
-
-    @staticmethod
-    def __get_number(arguments):
-        if arguments is None:
-            return RussianNumberType.UNKN
-
-        all_params = set(POSMystemWrapper.__iter_params(arguments))
-
-        for key, case in RussianNumberTypeService.iter_rus_mystem_tags():
-            if key in all_params:
-                return case
-
-        return RussianNumberType.UNKN
-
-    @staticmethod
-    def __iter_params(arguments):
-        params = arguments[POSMystemWrapper._GrammarKey].split(POSMystemWrapper._ArgsSeparator)
-        for optionally_combined in params:
-            for param in optionally_combined.split('='):
-                yield param
-
-    # endregion
-
-    def get_term_pos(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_pos) \
-            if len(analyzed) > 0 else PartOfSpeechType.Unknown
-
-    def get_term_case(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_russian_case) \
-            if len(analyzed) > 0 else RussianCases.UNKN
-
-    def get_term_number(self, term):
-        assert(isinstance(term, str))
-        analyzed = self.__mystem.analyze(term)
-        return self.__extract_from_analysis(analyzed[0], self.__get_number) \
-            if len(analyzed) > 0 else RussianNumberType.UNKN
-
-    def get_terms_russian_cases(self, text):
-        """ list of part of speech according to the certain word in text
-        """
-        assert(isinstance(text, str))
-        cases = []
-
-        analyzed = self.__mystem.analyze(text)
-        for a in analyzed:
-            pos = self.__extract_from_analysis(a, self.__get_russian_case) if len(analyzed) > 0 else RussianCases.UNKN
-            cases.append(pos)
-
-        return cases
-
-    def pos_to_int(self, pos):
-        assert(isinstance(pos, PartOfSpeechType))
-        return int(pos)
-
-    @staticmethod
-    def is_adjective(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.ADJ
-
-    @staticmethod
-    def is_noun(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.NOUN
-
-    @staticmethod
-    def is_verb(pos_type):
-        assert(isinstance(pos_type, PartOfSpeechType))
-        return pos_type == PartOfSpeechType.VERB

arekit/contrib/utils/processing/pos/russian.py

@@ -1,10 +0,0 @@
-from arekit.contrib.utils.processing.pos.base import POSTagger
-
-
-class RussianPOSTagger(POSTagger):
-    """ Provides cases support ('падежи')
-    """
-
-    def get_term_case(self, term):
-        raise NotImplementedError()
-