arekit 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/source/ruattitudes/io_utils.py

@@ -1,56 +0,0 @@
-from os import path
-
-from enum import Enum
-
-from arekit.contrib.source.zip_utils import ZipArchiveUtils
-
-
-class RuAttitudesVersions(Enum):
-    Debug = "dbg"
-    V10 = "v1_0"
-    V11 = "v1_1"
-    V20Base = 'v2_0_base'
-    V20Large = 'v2_0_large'
-    V20BaseNeut = 'v2_0_base_neut'
-    V20LargeNeut = 'v2_0_large_neut'
-
-
-class RuAttitudesVersionsService:
-
-    @staticmethod
-    def __iter_type_and_names():
-        for version_type in RuAttitudesVersions:
-            yield version_type, version_type.value
-
-    @staticmethod
-    def find_by_name(name):
-        for version_type, related_name in RuAttitudesVersionsService.__iter_type_and_names():
-            if name == related_name:
-                return version_type
-        raise Exception("Version `{}` does not supported".format(name))
-
-    @staticmethod
-    def iter_supported_names():
-        for _, name in RuAttitudesVersionsService.__iter_type_and_names():
-            yield name
-
-
-class RuAttitudesIOUtils(ZipArchiveUtils):
-
-    # region internal methods
-
-    @staticmethod
-    def get_archive_filepath(version):
-        assert(isinstance(version, str))
-        return path.join(RuAttitudesIOUtils.get_data_root(),
-                         "ruattitudes-{version}.zip".format(version=version))
-
-    @staticmethod
-    def get_collection_filepath():
-        return "collection.txt"
-
-    @classmethod
-    def get_synonyms_innerpath(cls):
-        return "synonyms.txt"
-
-    # endregion

arekit/contrib/source/ruattitudes/labels_fmt.py

@@ -1,12 +0,0 @@
-from arekit.common.labels.scaler.base import BaseLabelScaler
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-
-
-class RuAttitudesLabelFormatter(StringLabelsFormatter):
-
-    def __init__(self, label_scaler):
-        assert(isinstance(label_scaler, BaseLabelScaler))
-        stol = {}
-        for int_label in [-1, 0, 1]:
-            stol[str(int_label)] = type(label_scaler.int_to_label(int_label))
-        super(RuAttitudesLabelFormatter, self).__init__(stol=stol)

arekit/contrib/source/ruattitudes/news.py

@@ -1,51 +0,0 @@
-from arekit.contrib.source.ruattitudes.sentence import RuAttitudesSentence
-
-
-class RuAttitudesNews(object):
-
-    def __init__(self, sentences, news_index):
-        assert(len(sentences) > 0)
-
-        self.__sentences = sentences
-        self.__objects_before_sentence = self.__cache_objects_declared_before()
-        self.__news_index = news_index
-
-        self.__set_owners()
-
-    # region properties
-
-    @property
-    def ID(self):
-        return self.__news_index
-
-    @property
-    def Title(self):
-        return self.__sentences[0]
-
-    # endregion
-
-    # region private methods
-
-    def __set_owners(self):
-        for sentence in self.__sentences:
-            assert(isinstance(sentence, RuAttitudesSentence))
-            sentence.set_owner(self)
-
-    def __cache_objects_declared_before(self):
-        d = {}
-        before = 0
-        for s in self.__sentences:
-            assert(isinstance(s, RuAttitudesSentence))
-            d[s.SentenceIndex] = before
-            before += s.ObjectsCount
-
-        return d
-
-    # endregion
-
-    def get_objects_declared_before(self, sentence_index):
-        return self.__objects_before_sentence[sentence_index]
-
-    def iter_sentences(self):
-        for sentence in self.__sentences:
-            yield sentence

arekit/contrib/source/ruattitudes/news_brat.py

@@ -1,44 +0,0 @@
-from arekit.contrib.source.brat.news import BratNews
-from arekit.contrib.source.brat.sentence import BratSentence
-from arekit.contrib.source.ruattitudes.news import RuAttitudesNews
-from arekit.contrib.source.ruattitudes.opinions.base import SentenceOpinion
-from arekit.contrib.source.ruattitudes.opinions.converter import RuAttitudesSentenceOpinionConverter
-from arekit.contrib.source.ruattitudes.sentence import RuAttitudesSentence
-from arekit.common.utils import split_by_whitespaces
-
-
-class RuAttitudesNewsConverter(object):
-    """ Performs conversion to a brat-based representation.
-        The latter allows then allows to adopt pipelines for TextOpnion extraction.
-    """
-
-    @staticmethod
-    def to_brat_news(news):
-        assert(isinstance(news, RuAttitudesNews))
-        text_opinions = RuAttitudesNewsConverter.__iter_text_opinions(news=news)
-        brat_sentences = RuAttitudesNewsConverter.__to_brat_sentences(news.iter_sentences())
-        return BratNews(doc_id=news.ID,
-                        sentences=brat_sentences,
-                        text_relations=list(text_opinions))
-
-    @staticmethod
-    def __to_brat_sentences(sentences_iter):
-        sentences = []
-        for s in sentences_iter:
-            assert(isinstance(s, RuAttitudesSentence))
-            assert(s.Owner is not None)
-            brat_entities = [obj.to_entity(s.get_doc_level_text_object_id) for obj in s.iter_objects()]
-            brat_sentence = BratSentence(text=split_by_whitespaces(s.Text), index_begin=0, entities=brat_entities)
-            sentences.append(brat_sentence)
-        return sentences
-
-    @staticmethod
-    def __iter_text_opinions(news):
-        assert(isinstance(news, RuAttitudesNews))
-        for sentence in news.iter_sentences():
-            assert(isinstance(sentence, RuAttitudesSentence))
-            for sentence_opinion in sentence.iter_sentence_opins():
-                assert(isinstance(sentence_opinion, SentenceOpinion))
-                yield RuAttitudesSentenceOpinionConverter.to_brat_relation(
-                    sentence_opinion=sentence_opinion,
-                    end_to_doc_id_func=sentence.get_doc_level_text_object_id)

arekit/contrib/source/ruattitudes/opinions/base.py

@@ -1,28 +0,0 @@
-class SentenceOpinion(object):
-    """ Provides an opinion within a sentence.
-        Specific for RuAttitudes collection, as the latter provides
-        connections within a sentence.
-    """
-
-    def __init__(self, source_id, target_id, label_int, tag):
-        assert(isinstance(label_int, int))
-        self.__label_int = label_int
-        self.__source_id = source_id
-        self.__target_id = target_id
-        self.__tag = tag
-
-    @property
-    def SourceID(self):
-        return self.__source_id
-
-    @property
-    def TargetID(self):
-        return self.__target_id
-
-    @property
-    def Label(self):
-        return self.__label_int
-
-    @property
-    def Tag(self):
-        return self.__tag

arekit/contrib/source/ruattitudes/opinions/converter.py

@@ -1,37 +0,0 @@
-from arekit.common.labels.scaler.base import BaseLabelScaler
-from arekit.common.opinions.base import Opinion
-from arekit.contrib.source.brat.relation import BratRelation
-from arekit.contrib.source.ruattitudes.opinions.base import SentenceOpinion
-
-
-class RuAttitudesSentenceOpinionConverter:
-
-    @staticmethod
-    def to_brat_relation(sentence_opinion, end_to_doc_id_func):
-        """ Converts opinion into brat-related relation.
-            NOTE: for rel_type we just call str() over int-based value.
-        """
-        assert(isinstance(sentence_opinion, SentenceOpinion))
-        return BratRelation(id_in_doc="0",
-                            source_id=end_to_doc_id_func(sentence_opinion.SourceID),
-                            target_id=end_to_doc_id_func(sentence_opinion.TargetID),
-                            rel_type=str(sentence_opinion.Label))
-
-    @staticmethod
-    def to_opinion(sentence_opinion, source_value, target_value, label_scaler):
-        """
-        Converts onto document, non referenced opinion
-        (non bounded to the text).
-        """
-        assert(isinstance(sentence_opinion, SentenceOpinion))
-        assert(isinstance(label_scaler, BaseLabelScaler))
-
-        opinion = Opinion(source_value=source_value,
-                          target_value=target_value,
-                          sentiment=label_scaler.int_to_label(sentence_opinion.Label))
-
-        # Using this tag allows to perform a revert operation,
-        # i.e. to find opinion_ref by opinion.
-        opinion.set_tag(sentence_opinion.Tag)
-
-        return opinion

arekit/contrib/source/ruattitudes/reader.py

@@ -1,268 +0,0 @@
-from arekit.common.utils import split_by_whitespaces
-from arekit.contrib.source.ruattitudes.news import RuAttitudesNews
-from arekit.contrib.source.ruattitudes.opinions.base import SentenceOpinion
-from arekit.contrib.source.ruattitudes.sentence import RuAttitudesSentence
-from arekit.contrib.source.ruattitudes.text_object import TextObject
-
-
-class RuAttitudesFormatReader(object):
-
-    DOC_SEP_KEY = '--------'
-    FILE_KEY = "File:"
-    OBJ_KEY = "Object:"
-    TITLE_KEY = "Title:"
-    SINDEX_KEY = "Sentence:"
-    OPINION_KEY = "Attitude:"
-    STEXT_KEY = "Text:"
-    TERMS_IN_TITLE = "TermsInTitle:"
-    TERMS_IN_TEXT = "TermsInText:"
-    FRAMEVAR_TITLE = "FrameVariant:"
-
-    AUTH_LABEL = '<AUTH>'
-
-    def __iter__(self):
-        pass
-
-    # region private methods
-
-    @staticmethod
-    def iter_news_inds(input_file, get_news_index_func):
-        assert(callable(get_news_index_func))
-
-        title = None
-        local_news_ind = 0
-        has_sentences = False
-
-        for line in RuAttitudesFormatReader.__iter_lines(input_file):
-
-            if RuAttitudesFormatReader.__check_is_title(line):
-                # We use a placeholder, there is no need in actual value out there.
-                title = "title"
-                has_sentences = True
-
-            if RuAttitudesFormatReader.__check_is_news_sep(line=line, title=title):
-                yield RuAttitudesFormatReader.__assign_news_index(news_index_func=get_news_index_func,
-                                                                  local_index=local_news_ind)
-                local_news_ind += 1
-                title = None
-
-        if has_sentences:
-            yield RuAttitudesFormatReader.__assign_news_index(news_index_func=get_news_index_func,
-                                                              local_index=local_news_ind)
-
-    @staticmethod
-    def iter_news(input_file, get_news_index_func):
-        assert(callable(get_news_index_func))
-
-        reset = False
-        title = None
-        title_terms_count = None
-        text_terms_count = None
-        sentences = []
-        opinions_list = []
-        objects_list = []
-        s_index = 0
-        objects_in_prior_sentences_count = 0
-        local_news_ind = 0
-
-        for line in RuAttitudesFormatReader.__iter_lines(input_file):
-
-            if RuAttitudesFormatReader.FILE_KEY in line:
-                pass
-
-            if RuAttitudesFormatReader.OBJ_KEY in line:
-                object = RuAttitudesFormatReader.__parse_object(line)
-                objects_list.append(object)
-
-            if RuAttitudesFormatReader.OPINION_KEY in line:
-                sentence_opin = RuAttitudesFormatReader.__parse_sentence_opin(line)
-                opinions_list.append(sentence_opin)
-
-            if RuAttitudesFormatReader.FRAMEVAR_TITLE in line:
-                # TODO. This information is ommited now.
-                pass
-
-            if RuAttitudesFormatReader.TERMS_IN_TITLE in line:
-                title_terms_count = RuAttitudesFormatReader.__parse_terms_in_title_count(line)
-
-            if RuAttitudesFormatReader.SINDEX_KEY in line:
-                s_index = RuAttitudesFormatReader.__parse_sentence_index(line)
-
-            if RuAttitudesFormatReader.__check_is_title(line):
-                title = RuAttitudesSentence(is_title=True,
-                                            text=RuAttitudesFormatReader.__parse_sentence(line, True),
-                                            sentence_opins=opinions_list,
-                                            objects_list=objects_list,
-                                            sentence_index=-1)
-                sentences.append(title)
-                t_len = RuAttitudesFormatReader.__calculate_terms_in_line(line)
-                assert(title_terms_count == t_len or title_terms_count is None)
-                reset = True
-
-            if RuAttitudesFormatReader.STEXT_KEY in line and line.index(RuAttitudesFormatReader.STEXT_KEY) == 0:
-                sentence = RuAttitudesSentence(is_title=False,
-                                               text=RuAttitudesFormatReader.__parse_sentence(line, False),
-                                               sentence_opins=opinions_list,
-                                               objects_list=objects_list,
-                                               sentence_index=s_index)
-                sentences.append(sentence)
-                objects_in_prior_sentences_count += len(objects_list)
-                t_len = RuAttitudesFormatReader.__calculate_terms_in_line(line)
-                assert(text_terms_count == t_len or text_terms_count is None)
-                reset = True
-
-            if RuAttitudesFormatReader.__check_is_news_sep(line=line, title=title):
-                news_index = RuAttitudesFormatReader.__assign_news_index(news_index_func=get_news_index_func,
-                                                                         local_index=local_news_ind)
-                yield RuAttitudesNews(sentences=sentences,
-                                      news_index=news_index)
-                local_news_ind += 1
-                sentences = []
-                reset = True
-
-            if RuAttitudesFormatReader.TERMS_IN_TEXT in line:
-                text_terms_count = RuAttitudesFormatReader.__parse_terms_in_text_count(line)
-
-            if reset:
-                opinions_list = []
-                objects_list = []
-                title_terms_count = None
-                reset = False
-
-        if len(sentences) > 0:
-            news_index = RuAttitudesFormatReader.__assign_news_index(news_index_func=get_news_index_func,
-                                                                     local_index=local_news_ind)
-            yield RuAttitudesNews(sentences=sentences,
-                                  news_index=news_index)
-            sentences = []
-
-        assert(len(sentences) == 0)
-
-    @staticmethod
-    def __assign_news_index(news_index_func, local_index):
-        assert(callable(news_index_func))
-        return news_index_func(local_index)
-
-    @staticmethod
-    def __check_is_news_sep(line, title):
-        return RuAttitudesFormatReader.DOC_SEP_KEY in line and title is not None
-
-    @staticmethod
-    def __check_is_title(line):
-        return RuAttitudesFormatReader.TITLE_KEY in line
-
-    @staticmethod
-    def __iter_lines(input_file):
-        for line in input_file.readlines():
-            yield line.decode('utf-8')
-
-    @staticmethod
-    def __calculate_terms_in_line(line):
-        assert(isinstance(line, str))
-        return len(split_by_whitespaces(line))
-
-    @staticmethod
-    def __parse_sentence(line, is_title):
-        assert(isinstance(is_title, bool))
-
-        key = RuAttitudesFormatReader.STEXT_KEY if not is_title else RuAttitudesFormatReader.TITLE_KEY
-        text = line[len(key):]
-        return text.strip()
-
-    @staticmethod
-    def __parse_sentence_opin(line):
-        line = line[len(RuAttitudesFormatReader.OPINION_KEY):]
-
-        s_from = line.index('b:(')
-        s_to = line.index(')', s_from)
-        label = int(line[s_from + 3:s_to])
-
-        o_from = line.index('oi:[')
-        o_to = line.index(']', o_from)
-        source_object_id_in_sentence, target_object_id_in_sentence = line[o_from + 4:o_to].split(',')
-
-        source_object_id_in_sentence = int(source_object_id_in_sentence)
-        target_object_id_in_sentence = int(target_object_id_in_sentence)
-
-        s_from = line.index('si:{')
-        s_to = line.index('}', s_from)
-        opninion_key = line[s_from+4:s_to]
-
-        sentence_opin = SentenceOpinion(source_id=source_object_id_in_sentence,
-                                        target_id=target_object_id_in_sentence,
-                                        label_int=label,
-                                        tag=opninion_key)
-
-        return sentence_opin
-
-    @staticmethod
-    def __parse_object(line):
-        assert(isinstance(line, str))
-
-        line = line[len(RuAttitudesFormatReader.OBJ_KEY):]
-
-        obj_ind_begin = line.index('oi:[', 0)
-        obj_ind_end = line.index(']', obj_ind_begin + 1)
-
-        o_begin = line.index("'", 0)
-        o_end = line.index("'", o_begin + 1)
-
-        b_from = line.index('b:(')
-        b_to = line.index(')', b_from)
-
-        id_in_sentence = int(line[obj_ind_begin + 4:obj_ind_end])
-        term_index, length = line[b_from+3:b_to].split(',')
-        value = line[o_begin + 1:o_end]
-
-        obj_type = RuAttitudesFormatReader.__try_get_type(line)
-
-        sg_from = line.index('si:{')
-        sg_to = line.index('}', sg_from)
-        group_index = int(line[sg_from+4:sg_to])
-
-        is_auth = '<AUTH>' in line
-
-        text_object = TextObject(id_in_sentence=id_in_sentence,
-                                 value=value,
-                                 obj_type=obj_type,
-                                 position=int(term_index),
-                                 terms_count=int(length),
-                                 syn_group_index=group_index,
-                                 is_auth=is_auth)
-
-        return text_object
-
-    @staticmethod
-    def __parse_terms_in_title_count(line):
-        line = line[len(RuAttitudesFormatReader.TERMS_IN_TITLE):]
-        return int(line)
-
-    @staticmethod
-    def __parse_terms_in_text_count(line):
-        line = line[len(RuAttitudesFormatReader.TERMS_IN_TEXT):]
-        return int(line)
-
-    @staticmethod
-    def __parse_sentence_index(line):
-        line = line[len(RuAttitudesFormatReader.SINDEX_KEY):]
-        return int(line)
-
-    @staticmethod
-    def __try_get_type(line):
-
-        # Tag, utilized in RuAttitudes-2.0 format.
-        template = 'type:'
-        if template in line:
-            is_auth = RuAttitudesFormatReader.AUTH_LABEL in line
-            t_from = line.index(template)
-            t_to = line.index(RuAttitudesFormatReader.AUTH_LABEL[0], t_from) if is_auth else len(line)
-            return line[t_from + len(template):t_to].strip()
-
-        # Tag, utilized in RuAttitudes-1.* format.
-        template = 't:['
-        if template in line:
-            t_from = line.index(template)
-            t_to = line.index(']', t_from)
-            return line[t_from + len(template):t_to].strip()
-
-    # endregion

arekit/contrib/source/ruattitudes/sentence.py

@@ -1,73 +0,0 @@
-from arekit.common.news.sentence import BaseNewsSentence
-from arekit.contrib.source.ruattitudes.opinions.base import SentenceOpinion
-
-
-class RuAttitudesSentence(BaseNewsSentence):
-
-    def __init__(self, is_title, text, sentence_opins, objects_list, sentence_index):
-        assert(isinstance(is_title, bool))
-        assert(isinstance(sentence_opins, list))
-        assert(isinstance(objects_list, list))
-        assert(isinstance(sentence_index, int))
-        super(RuAttitudesSentence, self).__init__(text)
-
-        self.__is_title = is_title
-        self.__sentence_opins = sentence_opins
-        self.__objects = objects_list
-        self.__sentence_index = sentence_index
-        self.__owner = None
-
-    # region properties
-
-    @property
-    def SentenceIndex(self):
-        return self.__sentence_index
-
-    @property
-    def IsTitle(self):
-        return self.__is_title
-
-    @property
-    def Owner(self):
-        return self.__owner
-
-    @property
-    def ObjectsCount(self):
-        return len(self.__objects)
-
-    # endregion
-
-    # region public methods
-
-    def set_owner(self, owner):
-        if self.__owner is not None:
-            raise Exception("Owner is already declared")
-        self.__owner = owner
-
-    def get_objects(self, sentence_opin):
-        assert(isinstance(sentence_opin, SentenceOpinion))
-        source_obj = self.__objects[sentence_opin.SourceID]
-        target_obj = self.__objects[sentence_opin.TargetID]
-        return source_obj, target_obj
-
-    def get_doc_level_text_object_id(self, text_object_ind):
-        return text_object_ind + self.__owner.get_objects_declared_before(self.SentenceIndex)
-
-    def iter_objects(self):
-        for object in self.__objects:
-            yield object
-
-    def find_sentence_opin_by_key(self, key):
-        assert(key is not None)
-
-        for opinion in self.__sentence_opins:
-            if opinion.Tag == key:
-                return opinion
-
-        return None
-
-    def iter_sentence_opins(self):
-        for opinion in self.__sentence_opins:
-            yield opinion
-
-    # endregion

arekit/contrib/source/ruattitudes/synonyms.py

@@ -1,17 +0,0 @@
-from arekit.contrib.source.ruattitudes.io_utils import RuAttitudesIOUtils
-from arekit.contrib.source.synonyms.utils import iter_synonym_groups
-
-
-class RuAttitudesSynonymsCollectionHelper(object):
-
-    @staticmethod
-    def iter_groups(version):
-        it = RuAttitudesIOUtils.iter_from_zip(
-            inner_path=RuAttitudesIOUtils.get_synonyms_innerpath(),
-            process_func=lambda input_file: iter_synonym_groups(
-                input_file,
-                desc="Loading RuAttitudes SynonymsCollection"),
-            version=version)
-
-        for group in it:
-            yield group

arekit/contrib/source/ruattitudes/text_object.py

@@ -1,57 +0,0 @@
-from arekit.common.bound import Bound
-from arekit.contrib.source.brat.entities.entity import BratEntity
-
-
-class TextObject(object):
-    """
-    Considering any part of text, labeled by 'position', and 'type'
-    The latter is used to emphasize the entity type.
-    """
-
-    def __init__(self, id_in_sentence, value, obj_type, position, terms_count, syn_group_index, is_auth):
-        assert(isinstance(id_in_sentence, int))
-        assert(isinstance(value, str))
-        assert(isinstance(position, int))
-        assert(isinstance(terms_count, int) and terms_count > 0)
-        assert(isinstance(obj_type, str) or obj_type is None)
-        assert(isinstance(syn_group_index, int))
-        assert(isinstance(is_auth, bool))
-        self.__value = value
-        self.__type = obj_type
-        self.__id_in_sentence = id_in_sentence
-        self.__syn_group_index = syn_group_index
-        self.__is_auth = is_auth
-        self.__bound = Bound(pos=position, length=terms_count)
-
-    def to_entity(self, to_doc_id_func):
-        assert(callable(to_doc_id_func))
-        return BratEntity(id_in_doc=to_doc_id_func(self.__id_in_sentence),
-                          value=self.__value if len(self.__value) > 0 else '[empty]',
-                          e_type=self.__type,
-                          index_begin=self.__bound.Position,
-                          index_end=self.__bound.Position + self.__bound.Length,
-                          group_index=self.__syn_group_index)
-
-    # region properties
-
-    @property
-    def Value(self):
-        return self.__value
-
-    @property
-    def Type(self):
-        return self.__type
-
-    @property
-    def IdInSentence(self):
-        return self.__id_in_sentence
-
-    @property
-    def Bound(self):
-        return self.__bound
-
-    @property
-    def IsAuthorized(self):
-        return self.__is_auth
-
-    # endregion