arekit 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/source/brat/entities/parser.py

@@ -1,53 +0,0 @@
-from arekit.common.news.objects_parser import SentenceObjectsParserPipelineItem
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.text.partitioning.str import StringPartitioning
-from arekit.common.text.partitioning.terms import TermsPartitioning
-from arekit.contrib.source.brat.sentence import BratSentence
-
-
-class BratTextEntitiesParser(SentenceObjectsParserPipelineItem):
-
-    KEY = "sentence"
-
-    ################################
-    # NOTE: Supported partitionings.
-    ################################
-    # By default, BRAT annotation proposes to adopt entities annotation
-    # based on string input, which means that entity ends described as
-    # `char-ind-begin` and `char-ind-end`. However, the latter could be
-    # expanded to list of terms, which means that we deal with `ind-begin`
-    # and `ind-end` list indices.
-    __supported_partitionings = {
-        "string": StringPartitioning(),
-        "terms": TermsPartitioning()
-    }
-
-    def __init__(self, partitioning="string"):
-        assert(isinstance(partitioning, str))
-        super(BratTextEntitiesParser, self).__init__(self.__supported_partitionings[partitioning])
-
-    # region protected methods
-
-    def _get_text(self, pipeline_ctx):
-        sentence = self.__get_sentence(pipeline_ctx)
-        return sentence.Text
-
-    def _get_parts_provider_func(self, input_data, pipeline_ctx):
-        sentence = self.__get_sentence(pipeline_ctx)
-        return self.__iter_subs_values_with_bounds(sentence)
-
-    # endregion
-
-    # region private methods
-
-    def __get_sentence(self, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        assert(self.KEY in pipeline_ctx)
-        return pipeline_ctx.provide(self.KEY)
-
-    @staticmethod
-    def __iter_subs_values_with_bounds(sentence):
-        assert(isinstance(sentence, BratSentence))
-        return sentence.iter_entity_with_local_bounds()
-
-    # endregion

arekit/contrib/source/brat/news.py

@@ -1,28 +0,0 @@
-from arekit.common.news.base import News
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.brat.sentence import BratSentence
-
-
-class BratNews(News):
-
-    def __init__(self, doc_id, sentences, text_relations):
-        assert(isinstance(text_relations, list) or text_relations is None)
-        super(BratNews, self).__init__(doc_id=doc_id, sentences=sentences)
-        self.__text_relations = text_relations
-        self.__entity_by_id = {}
-        for sentence in sentences:
-            assert(isinstance(sentence, BratSentence))
-            for brat_entity, _ in sentence.iter_entity_with_local_bounds():
-                assert(isinstance(brat_entity, BratEntity))
-                self.__entity_by_id[brat_entity.ID] = brat_entity
-
-    @property
-    def Relations(self):
-        for brat_relation in self.__text_relations:
-            yield brat_relation
-
-    def contains_entity(self, entity_id):
-        return entity_id in self.__entity_by_id
-
-    def get_entity_by_id(self, entity_id):
-        return self.__entity_by_id[entity_id]

arekit/contrib/source/brat/opinions/converter.py

@@ -1,19 +0,0 @@
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.common.text_opinions.base import TextOpinion
-from arekit.contrib.source.brat.relation import BratRelation
-
-
-class BratRelationConverter(object):
-
-    @staticmethod
-    def to_text_opinion(brat_relation, doc_id, label_formatter):
-        """ Converts opinion into document-level referenced opinion
-        """
-        assert (isinstance(brat_relation, BratRelation))
-        assert(isinstance(label_formatter, StringLabelsFormatter))
-
-        return TextOpinion(doc_id=doc_id,
-                           text_opinion_id=int(brat_relation.ID),
-                           source_id=brat_relation.SourceID,
-                           target_id=brat_relation.TargetID,
-                           label=label_formatter.str_to_label(brat_relation.Type))

arekit/contrib/source/brat/relation.py

@@ -1,32 +0,0 @@
-class BratRelation(object):
-
-    def __init__(self, id_in_doc, source_id, target_id, rel_type):
-        assert(isinstance(id_in_doc, str))
-        assert(isinstance(source_id, int))
-        assert(isinstance(target_id, int))
-        assert(isinstance(rel_type, str))
-
-        self.__id = id_in_doc
-        self.__rel_type = rel_type
-        self.__source_id = source_id
-        self.__target_id = target_id
-
-    @property
-    def ID(self):
-        return self.__id
-
-    @property
-    def Type(self):
-        return self.__rel_type
-
-    @property
-    def SourceID(self):
-        """ Arg0.
-        """
-        return self.__source_id
-
-    @property
-    def TargetID(self):
-        """ Arg1.
-        """
-        return self.__target_id

arekit/contrib/source/brat/sentence.py

@@ -1,69 +0,0 @@
-from functools import cmp_to_key
-
-from arekit.common.bound import Bound
-from arekit.common.news.sentence import BaseNewsSentence
-from arekit.contrib.source.brat.entities.compound import BratCompoundEntity
-from arekit.contrib.source.brat.entities.entity import BratEntity
-
-
-class BratSentence(BaseNewsSentence):
-    """ Represent a raw sentence of BRAT.
-        Provides text could be used to parse then.
-        Provides API to store entities.
-    """
-
-    def __init__(self, text, index_begin, entities):
-        """ entities: list of BratEntities
-            index_begin: int
-                - char index (in case of string type of `text`)
-                - term index (in case of list type of `text`)
-        """
-        assert(isinstance(text, str) or isinstance(text, list))
-        assert(isinstance(index_begin, int))
-        assert(isinstance(entities, list))
-        super(BratSentence, self).__init__(text=text)
-        self.__index_begin = index_begin
-        self.__entities = entities
-
-    @staticmethod
-    def cmp_entities(a, b):
-        assert(isinstance(a, BratEntity))
-        assert(isinstance(b, BratEntity))
-        if a.IndexBegin != b.IndexBegin:
-            # Ordered by appearance
-            return a.IndexBegin - b.IndexBegin
-        else:
-            # Ordered by length first
-            b_length = b.IndexEnd - b.IndexBegin
-            a_length = a.IndexEnd - a.IndexBegin
-            return b_length - a_length
-
-    def iter_entity_with_local_bounds(self):
-        self.__entities.sort(key=cmp_to_key(lambda a, b: self.cmp_entities(a, b)))
-
-        bounds_and_entities = []
-
-        # Merging nested entities.
-        for entity in self.__entities:
-            start = entity.IndexBegin - self.__index_begin
-            end = entity.IndexEnd - self.__index_begin
-            bound = Bound(pos=start, length=end - start)
-
-            updated = False
-            if len(bounds_and_entities) > 0:
-                last_bound, last_entities = bounds_and_entities[-1]
-                if bound.itersects_with(last_bound):
-                    # Update.
-                    last_entities.append(entity)
-                    bounds_and_entities[-1] = (bound.intersect(last_bound), last_entities)
-                    updated = True
-
-            if not updated:
-                bounds_and_entities.append((bound, [entity]))
-
-        # Returning result.
-        for item in bounds_and_entities:
-            bound, entities = item
-            entity = entities[0] if len(entities) == 1 else \
-                BratCompoundEntity.from_list(root=entities[0], childs=entities[1:])
-            yield entity, bound

arekit/contrib/source/brat/sentences_reader.py

@@ -1,128 +0,0 @@
-from arekit.common.entities.collection import EntityCollection
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.brat.sentence import BratSentence
-
-
-class BratDocumentSentencesReader(object):
-
-    @staticmethod
-    def from_file(input_file, entities, line_handler=None, skip_entity_func=None):
-        assert(isinstance(entities, EntityCollection))
-        assert(callable(skip_entity_func) or skip_entity_func is None)
-
-        sentences_data = BratDocumentSentencesReader._parse_sentences(
-            input_file=input_file, line_handler=line_handler)
-
-        sentence_entities = BratDocumentSentencesReader._parse_entities(
-            sentences_data=sentences_data,
-            entities=entities,
-            skip_entity_func=skip_entity_func)
-
-        # Convert all the content to brat sentences.
-        brat_sentences = []
-        for s_ind, s_dict in enumerate(sentences_data):
-            brat_sentence = BratSentence(text=s_dict["text"],
-                                         index_begin=s_dict["ind_begin"],
-                                         entities=sentence_entities[s_ind])
-            brat_sentences.append(brat_sentence)
-
-        return brat_sentences
-
-    @staticmethod
-    def from_sentences_data(entities, sentences_data, skip_entity_func=None):
-        assert(isinstance(entities, EntityCollection))
-
-        sentence_entities = BratDocumentSentencesReader._parse_entities(
-            sentences_data=sentences_data,
-            entities=entities,
-            skip_entity_func=skip_entity_func)
-
-        # Convert all the content to brat sentences.
-        brat_sentences = []
-        for s_ind, s_dict in enumerate(sentences_data):
-            brat_sentence = BratSentence(text=s_dict["text"],
-                                         index_begin=s_dict["ind_begin"],
-                                         entities=sentence_entities[s_ind])
-            brat_sentences.append(brat_sentence)
-
-        return brat_sentences
-
-    @staticmethod
-    def __is_sentence_contains(sentence_data, entity):
-        assert(isinstance(sentence_data, dict))
-        assert(isinstance(entity, BratEntity))
-        return entity.IndexBegin >= sentence_data["ind_begin"] and \
-               entity.IndexEnd <= sentence_data["ind_end"]
-
-    @staticmethod
-    def _parse_entities(sentences_data, entities, skip_entity_func):
-        """ Sentences is a list of json-like data (dictionaries).
-        """
-        assert(isinstance(sentences_data, list))
-        assert(isinstance(entities, EntityCollection))
-
-        entities_in_sentences = [[] for _ in range(len(sentences_data))]
-
-        s_ind = 0
-        e_ind = 0
-
-        while s_ind < len(sentences_data) and e_ind < len(entities):
-            e = entities.get_entity_by_index(e_ind)
-            assert (isinstance(e, BratEntity))
-
-            s = sentences_data[s_ind]
-            entities_in_sentence = entities_in_sentences[s_ind]
-
-            # If entity goes after the current sentence.
-            if e.IndexBegin > s["ind_end"]:
-                s_ind += 1
-                continue
-
-            if skip_entity_func is not None and skip_entity_func(e):
-                e_ind += 1
-                continue
-
-            if BratDocumentSentencesReader.__is_sentence_contains(sentence_data=s, entity=e):
-                entities_in_sentence.append(e)
-                e_ind += 1
-                continue
-
-            if e.IndexEnd > s["ind_end"]:
-                # Intersects with the right border of sentence
-                s_ind += 1
-                continue
-
-            if e.IndexBegin < s["ind_begin"]:
-                # Intersects with the left border of sentence
-                e_ind += 1
-                continue
-
-            raise Exception("e_i:{} e:('{}',{},{}), s_i:{}, s_b: [{} {}]".format(
-                e_ind,
-                e.Value, e.IndexBegin, e.IndexEnd,
-                s_ind,
-                s["ind_begin"], s["ind_end"]))
-
-        return entities_in_sentences
-
-    @staticmethod
-    def _parse_sentences(input_file, line_handler):
-        assert(callable(line_handler) or line_handler is None)
-        sentences = []
-        line_start = 0
-
-        for line in input_file.readlines():
-
-            line = line.decode('utf-8')
-            handled_line = line_handler(line) if line_handler is not None else line
-
-            assert(len(line) == len(handled_line))
-
-            line_end = line_start + len(handled_line) - 1
-
-            if handled_line != str('\r\n'):
-                sentences.append({"text": handled_line, "ind_begin": line_start, "ind_end": line_end})
-
-            line_start = line_end + 1
-
-        return sentences

arekit/contrib/source/download.py

@@ -1,41 +0,0 @@
-from os.path import join
-
-from arekit.common import utils
-
-
-def download():
-    root_dir = utils.get_default_download_dir()
-
-    data = {
-        # RuSentiLex
-        "rusentilex.zip": "https://www.dropbox.com/s/bdsl3kney30y45z/rusentilex.zip?dl=1",
-        # RuSentRel-v1.1
-        "rusentrel-v1_1.zip": "https://www.dropbox.com/s/6aw5jv84jf5hrl2/rusentrel-v1_1.zip?dl=1",
-        # RuSentiFrames
-        "rusentiframes-v1_0.zip": "https://www.dropbox.com/s/zvkis77li3f40bm/rusentiframes-v1_0.zip?dl=1",
-        "rusentiframes-v2_0.zip": "https://www.dropbox.com/s/slbyma7eudmmugp/rusentiframes-v2_0.zip?dl=1",
-        # RuAttitudes-v1.0 (Many variations)
-        "ruattitudes-dbg.zip": "https://www.dropbox.com/s/5lmqw9kyb4tfm94/ruattitudes-dbg.zip?dl=1",
-        "ruattitudes-v1_0.zip": "https://www.dropbox.com/s/wg6oa447msdytj3/ruattitudes-v1_0.zip?dl=1",
-        "ruattitudes-v1_1.zip": "https://www.dropbox.com/s/e3menx5iqyush19/ruattitudes-v1_1.zip?dl=1",
-        # RuAttitudes-v2.0 Base
-        "ruattitudes-v2_0_base.zip": "https://www.dropbox.com/s/y39vqzzjumqhce1/ruattitudes_20_base.zip?dl=1",
-        "ruattitudes-v2_0_base_neut.zip": "https://www.dropbox.com/s/3xh7gd004oyuwx5/ruattitudes_20_base_neut.zip?dl=1",
-        # RuAttitudes-v2.0 Large
-        "ruattitudes-v2_0_large.zip": "https://www.dropbox.com/s/43iqoxlyh38qk8u/ruattitudes_20_large.zip?dl=1",
-        "ruattitudes-v2_0_large_neut.zip": "https://www.dropbox.com/s/6edqsxehtus4c61/ruattitudes_20_large_neut.zip?dl=1",
-        # SentiNEREL
-        "sentinerel-v1_0.zip": "https://www.dropbox.com/s/<HIDDEN>/sentinerel-v1_0.zip?dl=1",
-        "sentinerel-v2_0.zip": "https://www.dropbox.com/s/<HIDDEN>/sentinerel-v2_0.zip?dl=1",
-        "sentinerel-v2_1.zip": "https://www.dropbox.com/s/<HIDDEN>/sentinerel-v2_1.zip?dl=1",
-        # NEREL
-        "nerel-v1_0.zip": "https://www.dropbox.com/scl/fi/vegk0aczjdm9km410loqv/nerel-v1_0.zip?rlkey=wv0ut86n3x5ao6xabsaxd7lh7&dl=1",
-        "nerel-v1_1.zip": "https://www.dropbox.com/scl/fi/oaytj0rvx7vhdxjk98x7g/nerel-v1_1.zip?rlkey=klrq0l5rpn10cf7e2swkay6r4&dl=1",
-        # NEREL-BIO
-        "nerel-bio-v1_0.zip": "https://www.dropbox.com/scl/fi/nltuulfixbkhg3raczash/nerel-bio-v1_0.zip?rlkey=86uizq1hbkgkx302c5p5znpp6&dl=1"
-    }
-
-    # Perform downloading ...
-    for local_name, url_link in data.items():
-        utils.download(dest_file_path=join(root_dir, local_name),
-                       source_url=url_link)

arekit/contrib/source/nerel/entities.py

@@ -1,55 +0,0 @@
-from arekit.common.entities.collection import EntityCollection
-from arekit.common.synonyms.grouping import SynonymsCollectionValuesGroupingProviders
-from arekit.contrib.source.brat.annot import BratAnnotationParser
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.nerel.io_utils import NerelIOUtils
-from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
-from arekit.contrib.utils.synonyms.stemmer_based import StemmerBasedSynonymCollection
-
-
-class NerelEntityCollection(EntityCollection):
-
-    def __init__(self, contents, value_to_group_id_func, entities_to_ignore=None):
-        """
-            entities_to_ignore: list or None
-                this parameter is required because of the simplified implementation of
-                the nested objects of the BRAT annotation.
-        """
-        assert(isinstance(contents, dict))
-        assert(BratAnnotationParser.ENTITIES in contents)
-        assert(isinstance(entities_to_ignore, list) or entities_to_ignore is None)
-
-        self.__discard_entities = set([] if entities_to_ignore is None else entities_to_ignore)
-        contents[BratAnnotationParser.ENTITIES] = [e for e in contents[BratAnnotationParser.ENTITIES]
-                                                   if self.__keep_entity(e)]
-
-        super(NerelEntityCollection, self).__init__(
-            entities=contents[BratAnnotationParser.ENTITIES],
-            value_to_group_id_func=value_to_group_id_func)
-
-        self._sort_entities(key=lambda entity: entity.IndexBegin)
-
-    def __keep_entity(self, entity):
-        assert(isinstance(entity, BratEntity))
-        return entity.Type not in self.__discard_entities
-
-    @classmethod
-    def read_collection(cls, filename, version, io_utils, entities_to_ignore=None):
-        assert(isinstance(io_utils, NerelIOUtils))
-        assert(isinstance(filename, str))
-
-        # Since this dataset does not provide the synonyms collection by default,
-        # it is necessary to declare an empty collection to populate so in further.
-        synonyms = StemmerBasedSynonymCollection(stemmer=MystemWrapper(), is_read_only=False)
-
-        doc_fold = io_utils.map_doc_to_fold_type(version)
-
-        return io_utils.read_from_zip(
-            inner_path=io_utils.get_annotation_innerpath(folding_data_type=doc_fold[filename], filename=filename),
-            process_func=lambda input_file: cls(
-                contents=BratAnnotationParser.parse_annotations(input_file=input_file, encoding='utf-8-sig'),
-                entities_to_ignore=entities_to_ignore,
-                value_to_group_id_func=lambda value:
-                SynonymsCollectionValuesGroupingProviders.provide_existed_or_register_missed_value(
-                    synonyms, value)),
-            version=version)

arekit/contrib/source/nerel/folding/fixed.py

@@ -1,75 +0,0 @@
-from collections import OrderedDict
-
-from arekit.common.experiment.data_type import DataType
-from arekit.common.folding.fixed import FixedFolding
-
-
-def create_fixed_folding(train_filenames, dev_filenames, test_filenames, limit=None):
-    """ Create fixed data-folding based on the predefined list of filenames,
-        written in file.
-    """
-    assert(isinstance(train_filenames, list))
-    assert(isinstance(dev_filenames, list))
-    assert(isinstance(test_filenames, list))
-
-    filenames_by_ids = create_filenames_by_ids(filenames=train_filenames + dev_filenames + test_filenames)
-
-    ids_by_filenames = {}
-    for doc_id, filename in filenames_by_ids.items():
-        ids_by_filenames[filename] = doc_id
-
-    train_filenames = train_filenames if limit is None else train_filenames[:limit]
-    test_filenames = test_filenames if limit is None else test_filenames[:limit]
-    dev_filenames = dev_filenames if limit is None else dev_filenames[:limit]
-
-    fixed_folding = FixedFolding.from_parts({
-        DataType.Train: [ids_by_filenames[filename] for filename in train_filenames],
-        DataType.Test: [ids_by_filenames[filename] for filename in test_filenames],
-        DataType.Dev: [ids_by_filenames[filename] for filename in dev_filenames]
-    })
-
-    return filenames_by_ids, fixed_folding
-
-
-def create_filenames_by_ids(filenames):
-    """ Indexing filenames
-    """
-
-    def __create_new_id(default_id):
-        new_id = default_id
-        while new_id in filenames_by_ids:
-            new_id += 1
-        return new_id
-
-    default_id = 0
-
-    filenames_by_ids = OrderedDict()
-    for fname in filenames:
-
-        doc_id = number_from_string(fname)
-
-        if doc_id is None:
-            doc_id = __create_new_id(default_id)
-            default_id = doc_id
-
-        assert(doc_id not in filenames_by_ids)
-        filenames_by_ids[doc_id] = fname
-
-    return filenames_by_ids
-
-
-def number_from_string(s):
-    assert(isinstance(s, str))
-
-    digit_chars_prefix = []
-
-    for chr in s:
-        if chr.isdigit():
-            digit_chars_prefix.append(chr)
-        else:
-            break
-
-    if len(digit_chars_prefix) == 0:
-        return None
-
-    return int("".join(digit_chars_prefix))

arekit/contrib/source/nerel/io_utils.py

@@ -1,62 +0,0 @@
-from os import path
-
-from arekit.common.experiment.data_type import DataType
-from arekit.contrib.source.nerel.folding.fixed import create_fixed_folding
-from arekit.contrib.source.nerel.utils import iter_filename_and_splittype
-from arekit.contrib.source.zip_utils import ZipArchiveUtils
-
-
-class NerelIOUtils(ZipArchiveUtils):
-
-    splits = {
-        DataType.Train: "train",
-        DataType.Dev: "dev",
-        DataType.Test: "test"
-    }
-
-    @staticmethod
-    def get_archive_filepath(version):
-        return path.join(NerelIOUtils.get_data_root(), "nerel-{}.zip".format(version))
-
-    @staticmethod
-    def get_annotation_innerpath(folding_data_type, filename):
-        assert(isinstance(filename, str))
-        return path.join(NerelIOUtils.splits[folding_data_type], "{}.ann".format(filename))
-
-    @staticmethod
-    def get_news_innerpath(folding_data_type, filename):
-        assert(isinstance(filename, str))
-        return path.join(NerelIOUtils.splits[folding_data_type], "{}.txt".format(filename))
-
-    @staticmethod
-    def map_doc_to_fold_type(version):
-
-        it = iter_filename_and_splittype(
-            filenames_it=NerelIOUtils.iter_filenames_from_zip(version),
-            splits=NerelIOUtils.splits.items())
-
-        d2f = {}
-        for filename, split_type in it:
-            d2f[filename] = split_type
-
-        return d2f
-
-    @staticmethod
-    def read_dataset_split(version, docs_limit=None):
-
-        it = iter_filename_and_splittype(
-            filenames_it=NerelIOUtils.iter_filenames_from_zip(version),
-            splits=NerelIOUtils.splits.items())
-
-        f2d = {}
-        for filename, split_type in it:
-            if split_type not in f2d:
-                f2d[split_type] = []
-            f2d[split_type].append(filename)
-
-        filenames_by_ids, data_folding = create_fixed_folding(train_filenames=f2d[DataType.Train],
-                                                              test_filenames=f2d[DataType.Test],
-                                                              dev_filenames=f2d[DataType.Dev],
-                                                              limit=docs_limit)
-
-        return filenames_by_ids, data_folding