arekit 0.24.0__py3-none-any.whl

arekit/contrib/source/ruattitudes/reader.py

@@ -0,0 +1,268 @@
+from arekit.common.utils import split_by_whitespaces
+from arekit.contrib.source.ruattitudes.doc import RuAttitudesDocument
+from arekit.contrib.source.ruattitudes.opinions.base import SentenceOpinion
+from arekit.contrib.source.ruattitudes.sentence import RuAttitudesSentence
+from arekit.contrib.source.ruattitudes.text_object import TextObject
+
+
+class RuAttitudesFormatReader(object):
+
+    DOC_SEP_KEY = '--------'
+    FILE_KEY = "File:"
+    OBJ_KEY = "Object:"
+    TITLE_KEY = "Title:"
+    SINDEX_KEY = "Sentence:"
+    OPINION_KEY = "Attitude:"
+    STEXT_KEY = "Text:"
+    TERMS_IN_TITLE = "TermsInTitle:"
+    TERMS_IN_TEXT = "TermsInText:"
+    FRAMEVAR_TITLE = "FrameVariant:"
+
+    AUTH_LABEL = '<AUTH>'
+
+    def __iter__(self):
+        pass
+
+    # region private methods
+
+    @staticmethod
+    def iter_docs_inds(input_file, get_doc_index_func):
+        assert(callable(get_doc_index_func))
+
+        title = None
+        local_doc_ind = 0
+        has_sentences = False
+
+        for line in RuAttitudesFormatReader.__iter_lines(input_file):
+
+            if RuAttitudesFormatReader.__check_is_title(line):
+                # We use a placeholder, there is no need in actual value out there.
+                title = "title"
+                has_sentences = True
+
+            if RuAttitudesFormatReader.__check_is_doc_sep(line=line, title=title):
+                yield RuAttitudesFormatReader.__assign_doc_index(doc_index_func=get_doc_index_func,
+                                                                 local_index=local_doc_ind)
+                local_doc_ind += 1
+                title = None
+
+        if has_sentences:
+            yield RuAttitudesFormatReader.__assign_doc_index(doc_index_func=get_doc_index_func,
+                                                             local_index=local_doc_ind)
+
+    @staticmethod
+    def iter_docs(input_file, get_doc_index_func):
+        assert(callable(get_doc_index_func))
+
+        reset = False
+        title = None
+        title_terms_count = None
+        text_terms_count = None
+        sentences = []
+        opinions_list = []
+        objects_list = []
+        s_index = 0
+        objects_in_prior_sentences_count = 0
+        local_doc_ind = 0
+
+        for line in RuAttitudesFormatReader.__iter_lines(input_file):
+
+            if RuAttitudesFormatReader.FILE_KEY in line:
+                pass
+
+            if RuAttitudesFormatReader.OBJ_KEY in line:
+                object = RuAttitudesFormatReader.__parse_object(line)
+                objects_list.append(object)
+
+            if RuAttitudesFormatReader.OPINION_KEY in line:
+                sentence_opin = RuAttitudesFormatReader.__parse_sentence_opin(line)
+                opinions_list.append(sentence_opin)
+
+            if RuAttitudesFormatReader.FRAMEVAR_TITLE in line:
+                # TODO. This information is ommited now.
+                pass
+
+            if RuAttitudesFormatReader.TERMS_IN_TITLE in line:
+                title_terms_count = RuAttitudesFormatReader.__parse_terms_in_title_count(line)
+
+            if RuAttitudesFormatReader.SINDEX_KEY in line:
+                s_index = RuAttitudesFormatReader.__parse_sentence_index(line)
+
+            if RuAttitudesFormatReader.__check_is_title(line):
+                title = RuAttitudesSentence(is_title=True,
+                                            text=RuAttitudesFormatReader.__parse_sentence(line, True),
+                                            sentence_opins=opinions_list,
+                                            objects_list=objects_list,
+                                            sentence_index=-1)
+                sentences.append(title)
+                t_len = RuAttitudesFormatReader.__calculate_terms_in_line(line)
+                assert(title_terms_count == t_len or title_terms_count is None)
+                reset = True
+
+            if RuAttitudesFormatReader.STEXT_KEY in line and line.index(RuAttitudesFormatReader.STEXT_KEY) == 0:
+                sentence = RuAttitudesSentence(is_title=False,
+                                               text=RuAttitudesFormatReader.__parse_sentence(line, False),
+                                               sentence_opins=opinions_list,
+                                               objects_list=objects_list,
+                                               sentence_index=s_index)
+                sentences.append(sentence)
+                objects_in_prior_sentences_count += len(objects_list)
+                t_len = RuAttitudesFormatReader.__calculate_terms_in_line(line)
+                assert(text_terms_count == t_len or text_terms_count is None)
+                reset = True
+
+            if RuAttitudesFormatReader.__check_is_doc_sep(line=line, title=title):
+                doc_index = RuAttitudesFormatReader.__assign_doc_index(doc_index_func=get_doc_index_func,
+                                                                       local_index=local_doc_ind)
+                yield RuAttitudesDocument(sentences=sentences,
+                                          doc_index=doc_index)
+                local_doc_ind += 1
+                sentences = []
+                reset = True
+
+            if RuAttitudesFormatReader.TERMS_IN_TEXT in line:
+                text_terms_count = RuAttitudesFormatReader.__parse_terms_in_text_count(line)
+
+            if reset:
+                opinions_list = []
+                objects_list = []
+                title_terms_count = None
+                reset = False
+
+        if len(sentences) > 0:
+            doc_index = RuAttitudesFormatReader.__assign_doc_index(doc_index_func=get_doc_index_func,
+                                                                   local_index=local_doc_ind)
+            yield RuAttitudesDocument(sentences=sentences,
+                                      doc_index=doc_index)
+            sentences = []
+
+        assert(len(sentences) == 0)
+
+    @staticmethod
+    def __assign_doc_index(doc_index_func, local_index):
+        assert(callable(doc_index_func))
+        return doc_index_func(local_index)
+
+    @staticmethod
+    def __check_is_doc_sep(line, title):
+        return RuAttitudesFormatReader.DOC_SEP_KEY in line and title is not None
+
+    @staticmethod
+    def __check_is_title(line):
+        return RuAttitudesFormatReader.TITLE_KEY in line
+
+    @staticmethod
+    def __iter_lines(input_file):
+        for line in input_file.readlines():
+            yield line.decode('utf-8')
+
+    @staticmethod
+    def __calculate_terms_in_line(line):
+        assert(isinstance(line, str))
+        return len(split_by_whitespaces(line))
+
+    @staticmethod
+    def __parse_sentence(line, is_title):
+        assert(isinstance(is_title, bool))
+
+        key = RuAttitudesFormatReader.STEXT_KEY if not is_title else RuAttitudesFormatReader.TITLE_KEY
+        text = line[len(key):]
+        return text.strip()
+
+    @staticmethod
+    def __parse_sentence_opin(line):
+        line = line[len(RuAttitudesFormatReader.OPINION_KEY):]
+
+        s_from = line.index('b:(')
+        s_to = line.index(')', s_from)
+        label = int(line[s_from + 3:s_to])
+
+        o_from = line.index('oi:[')
+        o_to = line.index(']', o_from)
+        source_object_id_in_sentence, target_object_id_in_sentence = line[o_from + 4:o_to].split(',')
+
+        source_object_id_in_sentence = int(source_object_id_in_sentence)
+        target_object_id_in_sentence = int(target_object_id_in_sentence)
+
+        s_from = line.index('si:{')
+        s_to = line.index('}', s_from)
+        opninion_key = line[s_from+4:s_to]
+
+        sentence_opin = SentenceOpinion(source_id=source_object_id_in_sentence,
+                                        target_id=target_object_id_in_sentence,
+                                        label_int=label,
+                                        tag=opninion_key)
+
+        return sentence_opin
+
+    @staticmethod
+    def __parse_object(line):
+        assert(isinstance(line, str))
+
+        line = line[len(RuAttitudesFormatReader.OBJ_KEY):]
+
+        obj_ind_begin = line.index('oi:[', 0)
+        obj_ind_end = line.index(']', obj_ind_begin + 1)
+
+        o_begin = line.index("'", 0)
+        o_end = line.index("'", o_begin + 1)
+
+        b_from = line.index('b:(')
+        b_to = line.index(')', b_from)
+
+        id_in_sentence = int(line[obj_ind_begin + 4:obj_ind_end])
+        term_index, length = line[b_from+3:b_to].split(',')
+        value = line[o_begin + 1:o_end]
+
+        obj_type = RuAttitudesFormatReader.__try_get_type(line)
+
+        sg_from = line.index('si:{')
+        sg_to = line.index('}', sg_from)
+        group_index = int(line[sg_from+4:sg_to])
+
+        is_auth = '<AUTH>' in line
+
+        text_object = TextObject(id_in_sentence=id_in_sentence,
+                                 value=value,
+                                 obj_type=obj_type,
+                                 position=int(term_index),
+                                 terms_count=int(length),
+                                 syn_group_index=group_index,
+                                 is_auth=is_auth)
+
+        return text_object
+
+    @staticmethod
+    def __parse_terms_in_title_count(line):
+        line = line[len(RuAttitudesFormatReader.TERMS_IN_TITLE):]
+        return int(line)
+
+    @staticmethod
+    def __parse_terms_in_text_count(line):
+        line = line[len(RuAttitudesFormatReader.TERMS_IN_TEXT):]
+        return int(line)
+
+    @staticmethod
+    def __parse_sentence_index(line):
+        line = line[len(RuAttitudesFormatReader.SINDEX_KEY):]
+        return int(line)
+
+    @staticmethod
+    def __try_get_type(line):
+
+        # Tag, utilized in RuAttitudes-2.0 format.
+        template = 'type:'
+        if template in line:
+            is_auth = RuAttitudesFormatReader.AUTH_LABEL in line
+            t_from = line.index(template)
+            t_to = line.index(RuAttitudesFormatReader.AUTH_LABEL[0], t_from) if is_auth else len(line)
+            return line[t_from + len(template):t_to].strip()
+
+        # Tag, utilized in RuAttitudes-1.* format.
+        template = 't:['
+        if template in line:
+            t_from = line.index(template)
+            t_to = line.index(']', t_from)
+            return line[t_from + len(template):t_to].strip()
+
+    # endregion

arekit/contrib/source/ruattitudes/sentence.py

@@ -0,0 +1,73 @@
+from arekit.common.docs.sentence import BaseDocumentSentence
+from arekit.contrib.source.ruattitudes.opinions.base import SentenceOpinion
+
+
+class RuAttitudesSentence(BaseDocumentSentence):
+
+    def __init__(self, is_title, text, sentence_opins, objects_list, sentence_index):
+        assert(isinstance(is_title, bool))
+        assert(isinstance(sentence_opins, list))
+        assert(isinstance(objects_list, list))
+        assert(isinstance(sentence_index, int))
+        super(RuAttitudesSentence, self).__init__(text)
+
+        self.__is_title = is_title
+        self.__sentence_opins = sentence_opins
+        self.__objects = objects_list
+        self.__sentence_index = sentence_index
+        self.__owner = None
+
+    # region properties
+
+    @property
+    def SentenceIndex(self):
+        return self.__sentence_index
+
+    @property
+    def IsTitle(self):
+        return self.__is_title
+
+    @property
+    def Owner(self):
+        return self.__owner
+
+    @property
+    def ObjectsCount(self):
+        return len(self.__objects)
+
+    # endregion
+
+    # region public methods
+
+    def set_owner(self, owner):
+        if self.__owner is not None:
+            raise Exception("Owner is already declared")
+        self.__owner = owner
+
+    def get_objects(self, sentence_opin):
+        assert(isinstance(sentence_opin, SentenceOpinion))
+        source_obj = self.__objects[sentence_opin.SourceID]
+        target_obj = self.__objects[sentence_opin.TargetID]
+        return source_obj, target_obj
+
+    def get_doc_level_text_object_id(self, text_object_ind):
+        return text_object_ind + self.__owner.get_objects_declared_before(self.SentenceIndex)
+
+    def iter_objects(self):
+        for object in self.__objects:
+            yield object
+
+    def find_sentence_opin_by_key(self, key):
+        assert(key is not None)
+
+        for opinion in self.__sentence_opins:
+            if opinion.Tag == key:
+                return opinion
+
+        return None
+
+    def iter_sentence_opins(self):
+        for opinion in self.__sentence_opins:
+            yield opinion
+
+    # endregion

arekit/contrib/source/ruattitudes/synonyms.py

@@ -0,0 +1,17 @@
+from arekit.contrib.source.ruattitudes.io_utils import RuAttitudesIOUtils
+from arekit.contrib.source.synonyms.utils import iter_synonym_groups
+
+
+class RuAttitudesSynonymsCollectionHelper(object):
+
+    @staticmethod
+    def iter_groups(version):
+        it = RuAttitudesIOUtils.iter_from_zip(
+            inner_path=RuAttitudesIOUtils.get_synonyms_innerpath(),
+            process_func=lambda input_file: iter_synonym_groups(
+                input_file,
+                desc="Loading RuAttitudes SynonymsCollection"),
+            version=version)
+
+        for group in it:
+            yield group

arekit/contrib/source/ruattitudes/text_object.py

@@ -0,0 +1,59 @@
+from arekit.common.bound import Bound
+from arekit.contrib.source.brat.entities.entity import BratEntity
+
+
+class TextObject(object):
+    """
+    Considering any part of text, labeled by 'position', and 'type'
+    The latter is used to emphasize the entity type.
+    """
+
+    def __init__(self, id_in_sentence, value, obj_type, position, terms_count, syn_group_index, is_auth):
+        assert(isinstance(id_in_sentence, int))
+        assert(isinstance(value, str))
+        assert(isinstance(position, int))
+        assert(isinstance(terms_count, int) and terms_count > 0)
+        assert(isinstance(obj_type, str) or obj_type is None)
+        assert(isinstance(syn_group_index, int))
+        assert(isinstance(is_auth, bool))
+        self.__value = value
+        self.__type = obj_type
+        self.__id_in_sentence = id_in_sentence
+        self.__syn_group_index = syn_group_index
+        self.__is_auth = is_auth
+        self.__bound = Bound(pos=position, length=terms_count)
+
+    def to_entity(self, to_doc_id_func):
+        assert(callable(to_doc_id_func))
+        return BratEntity(id_in_doc=to_doc_id_func(self.__id_in_sentence),
+                          value=self.__value if len(self.__value) > 0 else '[empty]',
+                          e_type=self.__type,
+                          index_begin=self.__bound.Position,
+                          index_end=self.__bound.Position + self.__bound.Length,
+                          group_index=self.__syn_group_index,
+                          # In the case of RuAttitudes collection we do not support childs.
+                          childs=None)
+
+    # region properties
+
+    @property
+    def Value(self):
+        return self.__value
+
+    @property
+    def Type(self):
+        return self.__type
+
+    @property
+    def IdInSentence(self):
+        return self.__id_in_sentence
+
+    @property
+    def Bound(self):
+        return self.__bound
+
+    @property
+    def IsAuthorized(self):
+        return self.__is_auth
+
+    # endregion

arekit/contrib/source/rusentiframes/collection.py

@@ -0,0 +1,157 @@
+import json
+
+from arekit.common.labels.str_fmt import StringLabelsFormatter
+from arekit.contrib.source.rusentiframes.effect import FrameEffect
+from arekit.contrib.source.rusentiframes.io_utils import RuSentiFramesIOUtils
+from arekit.contrib.source.rusentiframes.types import RuSentiFramesVersions
+from arekit.contrib.source.rusentiframes.labels_fmt import RuSentiFramesLabelsFormatter, \
+    RuSentiFramesEffectLabelsFormatter
+from arekit.contrib.source.rusentiframes.polarity import RuSentiFramesFramePolarity
+from arekit.contrib.source.rusentiframes.role import FrameRole
+from arekit.contrib.source.rusentiframes.state import FrameState
+
+
+class RuSentiFramesCollection(object):
+
+    __frames_key = "frames"
+    __polarity_key = "polarity"
+    __state_key = "state"
+    __effect_key = "effect"
+    __variants_key = "variants"
+
+    def __init__(self, data, labels_fmt, effect_labels_fmt, lowercase_variants=True):
+        """ data: dict
+                Has the following structure of the frame contents:
+                {
+                    "frame_id": [ ... variants string list ... ]
+                    ...
+                }
+            lowercase_variants: bool
+                If 'True', forcely treat frame-variants as case-insensitive (lowercased)
+                or avoiding lowercasing operation in case of 'False'.
+        """
+        assert(isinstance(data, dict))
+        assert(isinstance(labels_fmt, StringLabelsFormatter))
+        assert(isinstance(effect_labels_fmt, StringLabelsFormatter))
+        self.__labels_fmt = labels_fmt
+        self.__effect_labels_fmt = effect_labels_fmt
+        self.__data = data
+
+        if lowercase_variants:
+            for frame_id, frame in self.__data.items():
+                frame[self.__variants_key] = [variant.lower() for variant in frame[self.__variants_key]]
+
+    # region classmethods
+
+    @classmethod
+    def read(cls, version, labels_fmt, effect_labels_fmt):
+        assert(isinstance(version, RuSentiFramesVersions))
+        assert(isinstance(labels_fmt, RuSentiFramesLabelsFormatter))
+        assert(isinstance(effect_labels_fmt, RuSentiFramesEffectLabelsFormatter))
+
+        return RuSentiFramesIOUtils.read_from_zip(
+            inner_path=RuSentiFramesIOUtils.get_collection_filepath(),
+            process_func=lambda input_file: cls.__from_json(
+                input_file=input_file,
+                labels_fmt=labels_fmt,
+                effect_labels_fmt=effect_labels_fmt),
+            version=version)
+
+    @classmethod
+    def __from_json(cls, input_file, labels_fmt, effect_labels_fmt):
+        data = json.load(input_file)
+        return cls(data=data,
+                   labels_fmt=labels_fmt,
+                   effect_labels_fmt=effect_labels_fmt)
+
+    # endregion
+
+    # region public 'try get' methods
+
+    def try_get_frame_polarity(self, frame_id, role_src, role_dest):
+        assert(isinstance(role_src, str))
+        assert(isinstance(role_dest, str))
+
+        if not self.__check_has_frame_polarity_key(frame_id):
+            return None
+
+        for args in self.__data[frame_id][self.__frames_key][self.__polarity_key]:
+            if args[0] == role_src and args[1] == role_dest:
+                return self.__frame_polarity_from_args(args)
+        return None
+
+    # endregion
+
+    # region public 'get' methods
+
+    def get_frame_roles(self, frame_id):
+        assert(isinstance(frame_id, str))
+        return [FrameRole(source=key, description=value)
+                for key, value in self.__data[frame_id]["roles"].items()]
+
+    def get_frame_polarities(self, frame_id):
+        assert(isinstance(frame_id, str))
+
+        if not self.__check_has_frame_polarity_key(frame_id):
+            return []
+
+        return [self.__frame_polarity_from_args(args)
+                for args in self.__data[frame_id][self.__frames_key][self.__polarity_key]]
+
+    def get_frame_states(self, frame_id):
+        assert(isinstance(frame_id, str))
+
+        if self.__state_key not in self.__data[frame_id][self.__frames_key]:
+            return []
+
+        return [FrameState(role=args[0], label=self.__labels_fmt.str_to_label(args[1]), prob=args[2])
+                for args in self.__data[frame_id][self.__frames_key][self.__state_key]]
+
+    def get_frame_titles(self, frame_id):
+        assert(isinstance(frame_id, str))
+        return self.__data[frame_id]["title"]
+
+    def get_frame_variants(self, frame_id):
+        return self.__data[frame_id][self.__variants_key]
+
+    def get_frame_values(self, frame_id):
+        assert(isinstance(frame_id, str))
+        # TODO. Not implemented yet.
+        pass
+
+    def get_frame_effects(self, frame_id):
+        assert(isinstance(frame_id, str))
+
+        if self.__effect_key not in self.__data[frame_id][self.__frames_key]:
+            return []
+
+        return [FrameEffect(role=args[0], label=self.__effect_labels_fmt.str_to_label(args[1]), prob=args[2])
+                for args in self.__data[frame_id][self.__frames_key][self.__effect_key]]
+
+    # endregion
+
+    # region public 'iter' methods
+
+    def iter_frames_ids(self):
+        for frame_id in self.__data.keys():
+            yield frame_id
+
+    def iter_frame_id_and_variants(self):
+        for id, frame in self.__data.items():
+            for variant in frame[self.__variants_key]:
+                yield id, variant
+
+    # endregion
+
+    # region private methods
+
+    def __check_has_frame_polarity_key(self, frame_id):
+        return self.__polarity_key in self.__data[frame_id][self.__frames_key]
+
+    def __frame_polarity_from_args(self, args):
+        return RuSentiFramesFramePolarity(role_src=args[0],
+                                          role_dest=args[1],
+                                          label=self.__labels_fmt.str_to_label(args[2]),
+                                          prob=args[3])
+
+    # endregion

arekit/contrib/source/rusentiframes/effect.py

@@ -0,0 +1,24 @@
+from arekit.common.labels.base import Label
+
+
+class FrameEffect(object):
+
+    def __init__(self, role, label, prob):
+        assert(isinstance(role, str))
+        assert(isinstance(label, Label))
+        assert(isinstance(prob, float))
+        self.__role = role
+        self.__label = label
+        self.__prob = prob
+
+    @property
+    def Role(self):
+        return self.__role
+
+    @property
+    def Label(self):
+        return self.__label
+
+    @property
+    def Prob(self):
+        return self.__prob

arekit/contrib/source/rusentiframes/io_utils.py

@@ -0,0 +1,19 @@
+from os import path
+
+from arekit.contrib.source.zip_utils import ZipArchiveUtils
+
+
+class RuSentiFramesIOUtils(ZipArchiveUtils):
+
+    # region internal methods
+
+    @staticmethod
+    def get_archive_filepath(version):
+        assert(isinstance(version, str))
+        return path.join(RuSentiFramesIOUtils.get_data_root(), "rusentiframes-{version}.zip".format(version=version))
+
+    @staticmethod
+    def get_collection_filepath():
+        return "frames.json"
+
+    # endregion

arekit/contrib/source/rusentiframes/labels_fmt.py

@@ -0,0 +1,22 @@
+from arekit.common.labels.base import Label
+from arekit.common.labels.str_fmt import StringLabelsFormatter
+
+
+class RuSentiFramesLabelsFormatter(StringLabelsFormatter):
+
+    def __init__(self, pos_label_type, neg_label_type):
+        assert(issubclass(pos_label_type, Label))
+        assert(issubclass(neg_label_type, Label))
+        stol = {'neg': neg_label_type, 'pos': pos_label_type}
+        super(RuSentiFramesLabelsFormatter, self).__init__(stol=stol)
+
+
+class RuSentiFramesEffectLabelsFormatter(StringLabelsFormatter):
+    """ Effect formatter utilizes '-' and '+' signs.
+    """
+
+    def __init__(self, pos_label_type, neg_label_type):
+        assert(issubclass(pos_label_type, Label))
+        assert(issubclass(neg_label_type, Label))
+        stol = {'-': neg_label_type, '+': pos_label_type}
+        super(RuSentiFramesEffectLabelsFormatter, self).__init__(stol=stol)

arekit/contrib/source/rusentiframes/polarity.py

@@ -0,0 +1,35 @@
+from arekit.common.frames.connotations.descriptor import FrameConnotationDescriptor
+from arekit.common.labels.base import Label
+
+
+class RuSentiFramesFramePolarity(FrameConnotationDescriptor):
+    """
+    Polarity description between source (Agent) towards dest (Theme)
+    The latter are related to roles of frame polarity.
+    """
+
+    def __init__(self, role_src, role_dest, label, prob):
+        assert(isinstance(role_src, str))
+        assert(isinstance(role_dest, str))
+        assert(isinstance(label, Label))
+        assert(isinstance(prob, float))
+        self.__role_src = role_src
+        self.__role_dest = role_dest
+        self.__label = label
+        self.__prob = prob
+
+    @property
+    def Source(self):
+        return self.__role_src
+
+    @property
+    def Destination(self):
+        return self.__role_dest
+
+    @property
+    def Label(self):
+        return self.__label
+
+    @property
+    def Prob(self):
+        return self.__prob