arekit 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/common/text/partitioning/terms.py

@@ -1,35 +0,0 @@
-from collections.abc import Iterable
-
-from arekit.common.bound import Bound
-from arekit.common.text.partitioning.base import BasePartitioning
-
-
-class TermsPartitioning(BasePartitioning):
-    """ NOTE: considering that provided parts
-        has no intersections between each other
-    """
-
-    def provide(self, text, parts_it):
-        assert(isinstance(text, list))
-        assert(isinstance(parts_it, Iterable))
-
-        start = 0
-        parts = []
-        for value, bound in parts_it:
-            assert(isinstance(bound, Bound))
-            assert(bound.Position >= start)
-
-            # Release everythig till the current value position.
-            part = text[start:bound.Position]
-
-            parts.extend(part)
-
-            # Release the entity value.
-            parts.extend([value])
-
-            start = bound.Position + bound.Length
-
-        # Release everything after the last entity.
-        parts.extend(text[start:len(text)])
-
-        return parts

arekit/contrib/source/brat/annot.py

@@ -1,84 +0,0 @@
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.brat.relation import BratRelation
-
-
-class BratAnnotationParser:
-
-    ENTITIES = "entities"
-    RELATIONS = "relations"
-
-    @staticmethod
-    def __non_prefixed_id(value):
-        assert (isinstance(value, str))
-        return value[1:]
-
-    @staticmethod
-    def handle_entity(args):
-        """ T2	Location 10 23	South America
-            T1	Location 0 5;16 23	North America
-        """
-        assert(len(args) == 3)
-
-        e_id = int(BratAnnotationParser.__non_prefixed_id(args[0]))
-        entity_params = args[1].split()
-
-        if len(entity_params) != 3:
-            # We do not support the case of a non-continuous entity mentions.
-            return None
-
-        e_str_type, e_begin, e_end = entity_params
-
-        return BratEntity(id_in_doc=e_id,
-                          e_type=e_str_type,
-                          index_begin=int(e_begin),
-                          index_end=int(e_end),
-                          childs=None,
-                          value=args[2].strip())
-
-    @staticmethod
-    def handle_relation(args):
-        """ Example:
-            R1	Origin Arg1:T3 Arg2:T4
-        """
-
-        # Parse identifier index.
-        e_id = args[0][1:]
-
-        # Parse relation arguments.
-        rel_type, source, target = args[1].split()
-
-        source_id = source.split(':')[1]
-        target_id = target.split(':')[1]
-
-        return BratRelation(id_in_doc=e_id,
-                            source_id=int(BratAnnotationParser.__non_prefixed_id(source_id)),
-                            target_id=int(BratAnnotationParser.__non_prefixed_id(target_id)),
-                            rel_type=rel_type)
-
-    @staticmethod
-    def parse_annotations(input_file, encoding='utf-8'):
-        """ Read annotation collection from file
-        """
-        entities = []
-        relations = []
-
-        for line in input_file.readlines():
-            line = line.decode(encoding)
-
-            args = line.split('\t')
-
-            record_type = args[0][0]
-
-            # Entities (objects) are prefixed with `T`
-            if record_type == "T":
-                entity = BratAnnotationParser.handle_entity(args)
-                if entity is not None:
-                    entities.append(entity)
-
-            elif record_type == "R":
-                relations.append(BratAnnotationParser.handle_relation(args))
-
-        return {
-            BratAnnotationParser.ENTITIES: entities,
-            BratAnnotationParser.RELATIONS: relations
-        }

arekit/contrib/source/brat/doc.py

@@ -1,28 +0,0 @@
-from arekit.common.docs.base import Document
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.brat.sentence import BratSentence
-
-
-class BratDocument(Document):
-
-    def __init__(self, doc_id, sentences, text_relations):
-        assert(isinstance(text_relations, list) or text_relations is None)
-        super(BratDocument, self).__init__(doc_id=doc_id, sentences=sentences)
-        self.__text_relations = text_relations
-        self.__entity_by_id = {}
-        for sentence in sentences:
-            assert(isinstance(sentence, BratSentence))
-            for brat_entity, _ in sentence.iter_entity_with_local_bounds():
-                assert(isinstance(brat_entity, BratEntity))
-                self.__entity_by_id[brat_entity.ID] = brat_entity
-
-    @property
-    def Relations(self):
-        for brat_relation in self.__text_relations:
-            yield brat_relation
-
-    def contains_entity(self, entity_id):
-        return entity_id in self.__entity_by_id
-
-    def get_entity_by_id(self, entity_id):
-        return self.__entity_by_id[entity_id]

arekit/contrib/source/brat/entities/compound.py

@@ -1,13 +0,0 @@
-from arekit.contrib.source.brat.entities.entity import BratEntity
-
-
-class BratCompoundEntity(BratEntity):
-    """ Entity which contains the hierarchy of the other entities.
-    """
-
-    @classmethod
-    def from_list(cls, root, childs):
-        assert(isinstance(root, BratEntity))
-        assert(isinstance(childs, list) and len(childs) > 0)
-        return cls(id_in_doc=root.ID, value=root.Value, e_type=root.Type, childs=childs,
-                   index_begin=root.IndexBegin, index_end=root.IndexEnd)

arekit/contrib/source/brat/entities/entity.py

@@ -1,42 +0,0 @@
-from arekit.common.entities.base import Entity
-
-
-class BratEntity(Entity):
-    """ Annotated entity in Brat-based collection corpus.
-        Provides bounds, i.e. char indices in related sentence.
-    """
-
-    def __init__(self, id_in_doc, e_type, index_begin, index_end, value, childs, display_value=None, group_index=None):
-        """ index_begin: int
-                - char index (in case of string type of `text`)
-                - term index (in case of list type of `text`)
-            index_end: int
-                - char index (in case of string type of `text`)
-                - term index (in case of list type of `text`)
-        """
-        assert(isinstance(e_type, str))
-        assert(isinstance(index_begin, int))
-        assert(isinstance(index_end, int))
-        super(BratEntity, self).__init__(value=value, e_type=e_type, childs=childs,
-                                         display_value=display_value, group_index=group_index)
-
-        self.__e_type = e_type
-        self.__begin = index_begin
-        self.__end = index_end
-        self.__id = id_in_doc
-
-    @property
-    def IndexBegin(self):
-        return self.__begin
-
-    @property
-    def IndexEnd(self):
-        return self.__end
-
-    @property
-    def Type(self):
-        return self.__e_type
-
-    @property
-    def ID(self):
-        return self.__id

arekit/contrib/source/brat/entities/parser.py

@@ -1,53 +0,0 @@
-from arekit.common.docs.objects_parser import SentenceObjectsParserPipelineItem
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.text.partitioning.str import StringPartitioning
-from arekit.common.text.partitioning.terms import TermsPartitioning
-from arekit.contrib.source.brat.sentence import BratSentence
-
-
-class BratTextEntitiesParser(SentenceObjectsParserPipelineItem):
-
-    KEY = "sentence"
-
-    ################################
-    # NOTE: Supported partitionings.
-    ################################
-    # By default, BRAT annotation proposes to adopt entities annotation
-    # based on string input, which means that entity ends described as
-    # `char-ind-begin` and `char-ind-end`. However, the latter could be
-    # expanded to list of terms, which means that we deal with `ind-begin`
-    # and `ind-end` list indices.
-    __supported_partitionings = {
-        "string": StringPartitioning(),
-        "terms": TermsPartitioning()
-    }
-
-    def __init__(self, partitioning="string"):
-        assert(isinstance(partitioning, str))
-        super(BratTextEntitiesParser, self).__init__(self.__supported_partitionings[partitioning])
-
-    # region protected methods
-
-    def _get_text(self, pipeline_ctx):
-        sentence = self.__get_sentence(pipeline_ctx)
-        return sentence.Text
-
-    def _get_parts_provider_func(self, input_data, pipeline_ctx):
-        sentence = self.__get_sentence(pipeline_ctx)
-        return self.__iter_subs_values_with_bounds(sentence)
-
-    # endregion
-
-    # region private methods
-
-    def __get_sentence(self, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        assert(self.KEY in pipeline_ctx)
-        return pipeline_ctx.provide(self.KEY)
-
-    @staticmethod
-    def __iter_subs_values_with_bounds(sentence):
-        assert(isinstance(sentence, BratSentence))
-        return sentence.iter_entity_with_local_bounds()
-
-    # endregion

arekit/contrib/source/brat/opinions/converter.py

@@ -1,19 +0,0 @@
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.common.text_opinions.base import TextOpinion
-from arekit.contrib.source.brat.relation import BratRelation
-
-
-class BratRelationConverter(object):
-
-    @staticmethod
-    def to_text_opinion(brat_relation, doc_id, label_formatter):
-        """ Converts opinion into document-level referenced opinion
-        """
-        assert (isinstance(brat_relation, BratRelation))
-        assert(isinstance(label_formatter, StringLabelsFormatter))
-
-        return TextOpinion(doc_id=doc_id,
-                           text_opinion_id=int(brat_relation.ID),
-                           source_id=brat_relation.SourceID,
-                           target_id=brat_relation.TargetID,
-                           label=label_formatter.str_to_label(brat_relation.Type))

arekit/contrib/source/brat/relation.py

@@ -1,32 +0,0 @@
-class BratRelation(object):
-
-    def __init__(self, id_in_doc, source_id, target_id, rel_type):
-        assert(isinstance(id_in_doc, str))
-        assert(isinstance(source_id, int))
-        assert(isinstance(target_id, int))
-        assert(isinstance(rel_type, str))
-
-        self.__id = id_in_doc
-        self.__rel_type = rel_type
-        self.__source_id = source_id
-        self.__target_id = target_id
-
-    @property
-    def ID(self):
-        return self.__id
-
-    @property
-    def Type(self):
-        return self.__rel_type
-
-    @property
-    def SourceID(self):
-        """ Arg0.
-        """
-        return self.__source_id
-
-    @property
-    def TargetID(self):
-        """ Arg1.
-        """
-        return self.__target_id

arekit/contrib/source/brat/sentence.py

@@ -1,69 +0,0 @@
-from functools import cmp_to_key
-
-from arekit.common.bound import Bound
-from arekit.common.docs.sentence import BaseDocumentSentence
-from arekit.contrib.source.brat.entities.compound import BratCompoundEntity
-from arekit.contrib.source.brat.entities.entity import BratEntity
-
-
-class BratSentence(BaseDocumentSentence):
-    """ Represent a raw sentence of BRAT.
-        Provides text could be used to parse then.
-        Provides API to store entities.
-    """
-
-    def __init__(self, text, index_begin, entities):
-        """ entities: list of BratEntities
-            index_begin: int
-                - char index (in case of string type of `text`)
-                - term index (in case of list type of `text`)
-        """
-        assert(isinstance(text, str) or isinstance(text, list))
-        assert(isinstance(index_begin, int))
-        assert(isinstance(entities, list))
-        super(BratSentence, self).__init__(text=text)
-        self.__index_begin = index_begin
-        self.__entities = entities
-
-    @staticmethod
-    def cmp_entities(a, b):
-        assert(isinstance(a, BratEntity))
-        assert(isinstance(b, BratEntity))
-        if a.IndexBegin != b.IndexBegin:
-            # Ordered by appearance
-            return a.IndexBegin - b.IndexBegin
-        else:
-            # Ordered by length first
-            b_length = b.IndexEnd - b.IndexBegin
-            a_length = a.IndexEnd - a.IndexBegin
-            return b_length - a_length
-
-    def iter_entity_with_local_bounds(self):
-        self.__entities.sort(key=cmp_to_key(lambda a, b: self.cmp_entities(a, b)))
-
-        bounds_and_entities = []
-
-        # Merging nested entities.
-        for entity in self.__entities:
-            start = entity.IndexBegin - self.__index_begin
-            end = entity.IndexEnd - self.__index_begin
-            bound = Bound(pos=start, length=end - start)
-
-            updated = False
-            if len(bounds_and_entities) > 0:
-                last_bound, last_entities = bounds_and_entities[-1]
-                if bound.itersects_with(last_bound):
-                    # Update.
-                    last_entities.append(entity)
-                    bounds_and_entities[-1] = (bound.intersect(last_bound), last_entities)
-                    updated = True
-
-            if not updated:
-                bounds_and_entities.append((bound, [entity]))
-
-        # Returning result.
-        for item in bounds_and_entities:
-            bound, entities = item
-            entity = entities[0] if len(entities) == 1 else \
-                BratCompoundEntity.from_list(root=entities[0], childs=entities[1:])
-            yield entity, bound

arekit/contrib/source/brat/sentences_reader.py

@@ -1,128 +0,0 @@
-from arekit.common.entities.collection import EntityCollection
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.brat.sentence import BratSentence
-
-
-class BratDocumentSentencesReader(object):
-
-    @staticmethod
-    def from_file(input_file, entities, line_handler=None, skip_entity_func=None):
-        assert(isinstance(entities, EntityCollection))
-        assert(callable(skip_entity_func) or skip_entity_func is None)
-
-        sentences_data = BratDocumentSentencesReader._parse_sentences(
-            input_file=input_file, line_handler=line_handler)
-
-        sentence_entities = BratDocumentSentencesReader._parse_entities(
-            sentences_data=sentences_data,
-            entities=entities,
-            skip_entity_func=skip_entity_func)
-
-        # Convert all the content to brat sentences.
-        brat_sentences = []
-        for s_ind, s_dict in enumerate(sentences_data):
-            brat_sentence = BratSentence(text=s_dict["text"],
-                                         index_begin=s_dict["ind_begin"],
-                                         entities=sentence_entities[s_ind])
-            brat_sentences.append(brat_sentence)
-
-        return brat_sentences
-
-    @staticmethod
-    def from_sentences_data(entities, sentences_data, skip_entity_func=None):
-        assert(isinstance(entities, EntityCollection))
-
-        sentence_entities = BratDocumentSentencesReader._parse_entities(
-            sentences_data=sentences_data,
-            entities=entities,
-            skip_entity_func=skip_entity_func)
-
-        # Convert all the content to brat sentences.
-        brat_sentences = []
-        for s_ind, s_dict in enumerate(sentences_data):
-            brat_sentence = BratSentence(text=s_dict["text"],
-                                         index_begin=s_dict["ind_begin"],
-                                         entities=sentence_entities[s_ind])
-            brat_sentences.append(brat_sentence)
-
-        return brat_sentences
-
-    @staticmethod
-    def __is_sentence_contains(sentence_data, entity):
-        assert(isinstance(sentence_data, dict))
-        assert(isinstance(entity, BratEntity))
-        return entity.IndexBegin >= sentence_data["ind_begin"] and \
-               entity.IndexEnd <= sentence_data["ind_end"]
-
-    @staticmethod
-    def _parse_entities(sentences_data, entities, skip_entity_func):
-        """ Sentences is a list of json-like data (dictionaries).
-        """
-        assert(isinstance(sentences_data, list))
-        assert(isinstance(entities, EntityCollection))
-
-        entities_in_sentences = [[] for _ in range(len(sentences_data))]
-
-        s_ind = 0
-        e_ind = 0
-
-        while s_ind < len(sentences_data) and e_ind < len(entities):
-            e = entities.get_entity_by_index(e_ind)
-            assert (isinstance(e, BratEntity))
-
-            s = sentences_data[s_ind]
-            entities_in_sentence = entities_in_sentences[s_ind]
-
-            # If entity goes after the current sentence.
-            if e.IndexBegin > s["ind_end"]:
-                s_ind += 1
-                continue
-
-            if skip_entity_func is not None and skip_entity_func(e):
-                e_ind += 1
-                continue
-
-            if BratDocumentSentencesReader.__is_sentence_contains(sentence_data=s, entity=e):
-                entities_in_sentence.append(e)
-                e_ind += 1
-                continue
-
-            if e.IndexEnd > s["ind_end"]:
-                # Intersects with the right border of sentence
-                s_ind += 1
-                continue
-
-            if e.IndexBegin < s["ind_begin"]:
-                # Intersects with the left border of sentence
-                e_ind += 1
-                continue
-
-            raise Exception("e_i:{} e:('{}',{},{}), s_i:{}, s_b: [{} {}]".format(
-                e_ind,
-                e.Value, e.IndexBegin, e.IndexEnd,
-                s_ind,
-                s["ind_begin"], s["ind_end"]))
-
-        return entities_in_sentences
-
-    @staticmethod
-    def _parse_sentences(input_file, line_handler):
-        assert(callable(line_handler) or line_handler is None)
-        sentences = []
-        line_start = 0
-
-        for line in input_file.readlines():
-
-            line = line.decode('utf-8')
-            handled_line = line_handler(line) if line_handler is not None else line
-
-            assert(len(line) == len(handled_line))
-
-            line_end = line_start + len(handled_line) - 1
-
-            if handled_line != str('\r\n'):
-                sentences.append({"text": handled_line, "ind_begin": line_start, "ind_end": line_end})
-
-            line_start = line_end + 1
-
-        return sentences

arekit/contrib/source/download.py

@@ -1,41 +0,0 @@
-from os.path import join
-
-from arekit.common import utils
-
-
-def download():
-    root_dir = utils.get_default_download_dir()
-
-    data = {
-        # RuSentiLex
-        "rusentilex.zip": "https://www.dropbox.com/s/bdsl3kney30y45z/rusentilex.zip?dl=1",
-        # RuSentRel-v1.1
-        "rusentrel-v1_1.zip": "https://www.dropbox.com/s/6aw5jv84jf5hrl2/rusentrel-v1_1.zip?dl=1",
-        # RuSentiFrames
-        "rusentiframes-v1_0.zip": "https://www.dropbox.com/s/zvkis77li3f40bm/rusentiframes-v1_0.zip?dl=1",
-        "rusentiframes-v2_0.zip": "https://www.dropbox.com/s/slbyma7eudmmugp/rusentiframes-v2_0.zip?dl=1",
-        # RuAttitudes-v1.0 (Many variations)
-        "ruattitudes-dbg.zip": "https://www.dropbox.com/s/5lmqw9kyb4tfm94/ruattitudes-dbg.zip?dl=1",
-        "ruattitudes-v1_0.zip": "https://www.dropbox.com/s/wg6oa447msdytj3/ruattitudes-v1_0.zip?dl=1",
-        "ruattitudes-v1_1.zip": "https://www.dropbox.com/s/e3menx5iqyush19/ruattitudes-v1_1.zip?dl=1",
-        # RuAttitudes-v2.0 Base
-        "ruattitudes-v2_0_base.zip": "https://www.dropbox.com/s/y39vqzzjumqhce1/ruattitudes_20_base.zip?dl=1",
-        "ruattitudes-v2_0_base_neut.zip": "https://www.dropbox.com/s/3xh7gd004oyuwx5/ruattitudes_20_base_neut.zip?dl=1",
-        # RuAttitudes-v2.0 Large
-        "ruattitudes-v2_0_large.zip": "https://www.dropbox.com/s/43iqoxlyh38qk8u/ruattitudes_20_large.zip?dl=1",
-        "ruattitudes-v2_0_large_neut.zip": "https://www.dropbox.com/s/6edqsxehtus4c61/ruattitudes_20_large_neut.zip?dl=1",
-        # SentiNEREL
-        "sentinerel-v1_0.zip": "https://www.dropbox.com/s/<HIDDEN>/sentinerel-v1_0.zip?dl=1",
-        "sentinerel-v2_0.zip": "https://www.dropbox.com/s/<HIDDEN>/sentinerel-v2_0.zip?dl=1",
-        "sentinerel-v2_1.zip": "https://www.dropbox.com/s/<HIDDEN>/sentinerel-v2_1.zip?dl=1",
-        # NEREL
-        "nerel-v1_0.zip": "https://www.dropbox.com/scl/fi/vegk0aczjdm9km410loqv/nerel-v1_0.zip?rlkey=wv0ut86n3x5ao6xabsaxd7lh7&dl=1",
-        "nerel-v1_1.zip": "https://www.dropbox.com/scl/fi/oaytj0rvx7vhdxjk98x7g/nerel-v1_1.zip?rlkey=klrq0l5rpn10cf7e2swkay6r4&dl=1",
-        # NEREL-BIO
-        "nerel-bio-v1_0.zip": "https://www.dropbox.com/scl/fi/nltuulfixbkhg3raczash/nerel-bio-v1_0.zip?rlkey=86uizq1hbkgkx302c5p5znpp6&dl=1"
-    }
-
-    # Perform downloading ...
-    for local_name, url_link in data.items():
-        utils.download(dest_file_path=join(root_dir, local_name),
-                       source_url=url_link)

arekit/contrib/source/nerel/entities.py

@@ -1,55 +0,0 @@
-from arekit.common.entities.collection import EntityCollection
-from arekit.common.synonyms.grouping import SynonymsCollectionValuesGroupingProviders
-from arekit.contrib.source.brat.annot import BratAnnotationParser
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.nerel.io_utils import NerelIOUtils
-from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
-from arekit.contrib.utils.synonyms.stemmer_based import StemmerBasedSynonymCollection
-
-
-class NerelEntityCollection(EntityCollection):
-
-    def __init__(self, contents, value_to_group_id_func, entities_to_ignore=None):
-        """
-            entities_to_ignore: list or None
-                this parameter is required because of the simplified implementation of
-                the nested objects of the BRAT annotation.
-        """
-        assert(isinstance(contents, dict))
-        assert(BratAnnotationParser.ENTITIES in contents)
-        assert(isinstance(entities_to_ignore, list) or entities_to_ignore is None)
-
-        self.__discard_entities = set([] if entities_to_ignore is None else entities_to_ignore)
-        contents[BratAnnotationParser.ENTITIES] = [e for e in contents[BratAnnotationParser.ENTITIES]
-                                                   if self.__keep_entity(e)]
-
-        super(NerelEntityCollection, self).__init__(
-            entities=contents[BratAnnotationParser.ENTITIES],
-            value_to_group_id_func=value_to_group_id_func)
-
-        self._sort_entities(key=lambda entity: entity.IndexBegin)
-
-    def __keep_entity(self, entity):
-        assert(isinstance(entity, BratEntity))
-        return entity.Type not in self.__discard_entities
-
-    @classmethod
-    def read_collection(cls, filename, version, io_utils, entities_to_ignore=None):
-        assert(isinstance(io_utils, NerelIOUtils))
-        assert(isinstance(filename, str))
-
-        # Since this dataset does not provide the synonyms collection by default,
-        # it is necessary to declare an empty collection to populate so in further.
-        synonyms = StemmerBasedSynonymCollection(stemmer=MystemWrapper(), is_read_only=False)
-
-        doc_fold = io_utils.map_doc_to_fold_type(version)
-
-        return io_utils.read_from_zip(
-            inner_path=io_utils.get_annotation_innerpath(folding_data_type=doc_fold[filename], filename=filename),
-            process_func=lambda input_file: cls(
-                contents=BratAnnotationParser.parse_annotations(input_file=input_file, encoding='utf-8-sig'),
-                entities_to_ignore=entities_to_ignore,
-                value_to_group_id_func=lambda value:
-                SynonymsCollectionValuesGroupingProviders.provide_existed_or_register_missed_value(
-                    synonyms, value)),
-            version=version)