arekit 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/source/rusentiframes/role.py

@@ -1,15 +0,0 @@
-class FrameRole(object):
-
-    def __init__(self, source, description):
-        assert(isinstance(source, str))
-        assert(isinstance(description, str))
-        self.__source = source
-        self.__description = description
-
-    @property
-    def Source(self):
-        return self.__source
-
-    @property
-    def Description(self):
-        return self.__description

arekit/contrib/source/rusentiframes/state.py

@@ -1,24 +0,0 @@
-from arekit.common.labels.base import Label
-
-
-class FrameState(object):
-
-    def __init__(self, role, label, prob):
-        assert(isinstance(role, str))
-        assert(isinstance(label, Label))
-        assert(isinstance(prob, float))
-        self.__role = role
-        self.__label = label
-        self.__prob = prob
-
-    @property
-    def Role(self):
-        return self.__role
-
-    @property
-    def Label(self):
-        return self.__label
-
-    @property
-    def Prob(self):
-        return self.__prob

arekit/contrib/source/rusentiframes/types.py

@@ -1,42 +0,0 @@
-from enum import Enum
-
-
-class RuSentiFramesVersions(Enum):
-
-    # Papers for description:
-    # Distant Supervision for Sentiment Attitude Extraction (RANLP-2019)
-    # Nicolay Rusnachenko, Natalia Loukachevitch, Elena Tutubalina
-    # https://www.aclweb.org/anthology/R19-1118/
-    # https://github.com/nicolay-r/RuSentiFrames/tree/v1.0
-    V10 = "v1_0"
-
-    # Papers for description:
-    # Sentiment Frames for Attitude Extraction in Russian (DIALOG-2020)
-    # Natalia Loukachevitch, Nicolay Rusnachenko
-    # https://github.com/nicolay-r/RuSentiFrames/tree/v2.0
-    V20 = "v2_0"
-
-
-class RuSentiFramesVersionsService:
-
-    @staticmethod
-    def __iter_supported_types():
-        return iter(RuSentiFramesVersions)
-
-    @staticmethod
-    def get_name_by_type(version_type):
-        assert(isinstance(version_type, RuSentiFramesVersions))
-        return version_type.value
-
-    @staticmethod
-    def get_type_by_name(name):
-        for version_type in RuSentiFramesVersionsService.__iter_supported_types():
-            if version_type.value == name:
-                return version_type
-
-        raise Exception("RuSentiFrames version by name `{}` was hot found!".format(name))
-
-    @staticmethod
-    def iter_supported_names():
-        for version_type in RuSentiFramesVersionsService.__iter_supported_types():
-            yield version_type.value

arekit/contrib/source/rusentiframes/value.py

@@ -1,2 +0,0 @@
-class FrameValue(object):
-    pass

arekit/contrib/source/rusentrel/const.py

@@ -1,3 +0,0 @@
-# Defaul label formattings.
-POS_LABEL_STR = "pos"
-NEG_LABEL_STR = "neg"

arekit/contrib/source/rusentrel/docs_reader.py

@@ -1,51 +0,0 @@
-from arekit.common.synonyms.base import SynonymsCollection
-from arekit.contrib.source.brat.doc import BratDocument
-from arekit.contrib.source.brat.sentences_reader import BratDocumentSentencesReader
-from arekit.contrib.source.rusentrel.entities import RuSentRelDocumentEntityCollection
-from arekit.contrib.source.rusentrel.io_utils import RuSentRelVersions, RuSentRelIOUtils
-
-
-class RuSentRelDocumentsReader(object):
-
-    # region class methods
-
-    @staticmethod
-    def hide_first_entry(line, entry, hide_with=" "):
-
-        index = line.find(entry)
-
-        if index >= 0:
-            pad = hide_with * len(entry)
-            before = line[0:index]
-            after = line[index+len(entry):]
-            line = "".join([before, pad, after])
-
-        return line
-
-    @staticmethod
-    def read_document(doc_id, synonyms, version=RuSentRelVersions.V11, target_doc_id=None):
-        assert(isinstance(synonyms, SynonymsCollection))
-        assert(isinstance(version, RuSentRelVersions))
-        assert(isinstance(target_doc_id, int) or target_doc_id is None)
-
-        def file_to_doc(input_file):
-
-            sentences = BratDocumentSentencesReader.from_file(
-                input_file=input_file,
-                entities=entities,
-                line_handler=lambda line: RuSentRelDocumentsReader.hide_first_entry(line, entry="{Author, Unknown}"),
-                skip_entity_func=lambda entity: entity.Value in ['author', 'unknown'])
-
-            return BratDocument(doc_id=target_doc_id if target_doc_id is not None else doc_id,
-                                sentences=sentences,
-                                text_relations=[])
-
-        entities = RuSentRelDocumentEntityCollection.read_collection(
-            doc_id=doc_id,
-            synonyms=synonyms,
-            version=version)
-
-        return RuSentRelIOUtils.read_from_zip(
-            inner_path=RuSentRelIOUtils.get_doc_innerpath(index=doc_id, version=version),
-            process_func=file_to_doc,
-            version=version)

arekit/contrib/source/rusentrel/entities.py

@@ -1,26 +0,0 @@
-from arekit.common.entities.collection import EntityCollection
-from arekit.common.synonyms.base import SynonymsCollection
-from arekit.contrib.source.brat.annot import BratAnnotationParser
-from arekit.contrib.source.rusentrel.io_utils import RuSentRelVersions, RuSentRelIOUtils
-
-
-class RuSentRelDocumentEntityCollection(EntityCollection):
-
-    def __init__(self, entities, value_to_group_id_func):
-        super(RuSentRelDocumentEntityCollection, self).__init__(
-            entities=entities,
-            value_to_group_id_func=value_to_group_id_func)
-
-        self._sort_entities(key=lambda entity: entity.IndexBegin)
-
-    @classmethod
-    def read_collection(cls, doc_id, synonyms, version=RuSentRelVersions.V11):
-        assert (isinstance(synonyms, SynonymsCollection))
-        assert (isinstance(doc_id, int))
-
-        return RuSentRelIOUtils.read_from_zip(
-            inner_path=RuSentRelIOUtils.get_entity_innerpath(index=doc_id, version=version),
-            process_func=lambda input_file: cls(
-                entities=BratAnnotationParser.parse_annotations(input_file)["entities"],
-                value_to_group_id_func=synonyms.get_synonym_group_index),
-            version=version)

arekit/contrib/source/rusentrel/io_utils.py

@@ -1,125 +0,0 @@
-from os import path
-
-from enum import Enum
-
-from arekit.contrib.source.zip_utils import ZipArchiveUtils
-
-
-class RuSentRelVersions(Enum):
-    """ Original collection repository: https://github.com/nicolay-r/RuSentRel
-        Paper: https://arxiv.org/abs/1808.08932
-    """
-    V11 = "v1_1"
-
-
-class RuSentRelIOUtils(ZipArchiveUtils):
-
-    TEST_FOLDER = "test"
-    TRAIN_FOLDER = "train"
-    ETALON_FOLDER = "etalon"
-
-    @staticmethod
-    def get_archive_filepath(version):
-        assert(version, str)
-        return path.join(RuSentRelIOUtils.get_data_root(), "rusentrel-{}.zip".format(version))
-
-    # region internal methods
-
-    @staticmethod
-    def get_sentiment_opin_filepath(index, version, prefix='art'):
-        root = RuSentRelIOUtils.__get_root_by_index(index, version=version, keep_etalon=True)
-        return path.join(root, "{prefix}{index}.opin.txt".format(prefix=prefix, index=index))
-
-    @staticmethod
-    def get_entity_innerpath(index, version):
-        assert(isinstance(index, int))
-        assert(isinstance(version, RuSentRelVersions))
-        inner_root = RuSentRelIOUtils.__get_root_by_index(doc_id=index, version=version)
-        return path.join(inner_root, "art{}.ann".format(index))
-
-    @staticmethod
-    def get_doc_innerpath(index, version):
-        assert(isinstance(index, int))
-        assert(isinstance(version, RuSentRelVersions))
-        inner_root = RuSentRelIOUtils.__get_root_by_index(doc_id=index, version=version)
-        return path.join(inner_root, "art{}.txt".format(index))
-
-    @staticmethod
-    def get_synonyms_innerpath():
-        return "synonyms.txt"
-
-    # endregion
-
-    @staticmethod
-    def __get_root_by_index(doc_id, version, keep_etalon=False):
-        assert(RuSentRelIOUtils.__is_supported(version))
-        assert(isinstance(version, RuSentRelVersions))
-        assert(isinstance(doc_id, int))
-        other_dir = RuSentRelIOUtils.ETALON_FOLDER if keep_etalon else RuSentRelIOUtils.TEST_FOLDER
-        test_indices = set(RuSentRelIOUtils.__iter_indicies_from_dataset(version, RuSentRelIOUtils.TEST_FOLDER))
-        return other_dir if doc_id in test_indices else RuSentRelIOUtils.TRAIN_FOLDER
-
-    @staticmethod
-    def __is_supported(version):
-        assert(isinstance(version, RuSentRelVersions))
-        return version == RuSentRelVersions.V11
-
-    @staticmethod
-    def __number_from_string(s):
-        digit_chars = [chr for chr in s if chr.isdigit()]
-
-        if len(digit_chars) == 0:
-            return None
-
-        return int("".join(digit_chars))
-
-    @staticmethod
-    def __iter_indicies_from_dataset(version, folder_name):
-        assert(isinstance(folder_name, str))
-        assert(RuSentRelIOUtils.__is_supported(version))
-
-        used = set()
-
-        for filename in RuSentRelIOUtils.iter_filenames_from_zip(version):
-            if not folder_name in filename:
-                continue
-
-            index = RuSentRelIOUtils.__number_from_string(filename)
-
-            if index is None:
-                continue
-
-            if index in used:
-                continue
-
-            used.add(index)
-
-            yield index
-
-    # region public methods
-
-    @staticmethod
-    def iter_test_indices(version):
-        assert(RuSentRelIOUtils.__is_supported(version))
-        indices_iter = RuSentRelIOUtils.__iter_indicies_from_dataset(
-            version=version, folder_name="{}/".format(RuSentRelIOUtils.TEST_FOLDER))
-        for index in indices_iter:
-            yield index
-
-    @staticmethod
-    def iter_train_indices(version):
-        assert(RuSentRelIOUtils.__is_supported(version))
-        indices_iter = RuSentRelIOUtils.__iter_indicies_from_dataset(
-            version=version, folder_name="{}/".format(RuSentRelIOUtils.TRAIN_FOLDER))
-        for index in indices_iter:
-            yield index
-
-    @staticmethod
-    def iter_collection_indices(version):
-        assert(RuSentRelIOUtils.__is_supported(version))
-        for index in RuSentRelIOUtils.iter_train_indices(version):
-            yield index
-        for index in RuSentRelIOUtils.iter_test_indices(version):
-            yield index
-
-    # endregion

arekit/contrib/source/rusentrel/labels_fmt.py

@@ -1,12 +0,0 @@
-from arekit.common.labels.base import Label
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.contrib.source.rusentrel.const import NEG_LABEL_STR, POS_LABEL_STR
-
-
-class RuSentRelLabelsFormatter(StringLabelsFormatter):
-
-    def __init__(self, pos_label_type, neg_label_type):
-        assert(issubclass(pos_label_type, Label))
-        assert(issubclass(neg_label_type, Label))
-        stol = {NEG_LABEL_STR: neg_label_type, POS_LABEL_STR: pos_label_type}
-        super(RuSentRelLabelsFormatter, self).__init__(stol=stol)

arekit/contrib/source/rusentrel/opinions/collection.py

@@ -1,30 +0,0 @@
-from arekit.contrib.source.rusentrel.const import POS_LABEL_STR, NEG_LABEL_STR
-from arekit.contrib.source.rusentrel.io_utils import RuSentRelIOUtils, RuSentRelVersions
-from arekit.contrib.source.rusentrel.labels_fmt import RuSentRelLabelsFormatter
-from arekit.contrib.source.rusentrel.opinions.provider import RuSentRelOpinionCollectionProvider
-
-
-class RuSentRelOpinions:
-    """
-    Collection of sentiment opinions between entities
-    """
-
-    @staticmethod
-    def iter_from_doc(doc_id, labels_fmt, version=RuSentRelVersions.V11):
-        """ doc_id:
-            synonyms: None or SynonymsCollection
-                None corresponds to the related synonym collection from RuSentRel collection.
-            version: RuSentrelVersions enum
-        """
-        assert(isinstance(version, RuSentRelVersions))
-        assert(isinstance(labels_fmt, RuSentRelLabelsFormatter))
-        assert(labels_fmt.supports_value(POS_LABEL_STR))
-        assert(labels_fmt.supports_value(NEG_LABEL_STR))
-
-        return RuSentRelIOUtils.iter_from_zip(
-            inner_path=RuSentRelIOUtils.get_sentiment_opin_filepath(index=doc_id, version=version),
-            process_func=lambda input_file: RuSentRelOpinionCollectionProvider._iter_opinions_from_file(
-                input_file=input_file,
-                labels_formatter=labels_fmt,
-                error_on_non_supported=True),
-            version=version)

arekit/contrib/source/rusentrel/opinions/converter.py

@@ -1,40 +0,0 @@
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.common.opinions.base import Opinion
-
-
-class OpinionConverter(object):
-    """ Opinion type <-> string Converter.
-    """
-
-    @staticmethod
-    def try_from_string(line, labels_formatter):
-        assert(isinstance(line, str))
-
-        args = line.strip().split(',')
-        assert (len(args) >= 3)
-
-        source_value = args[0].strip()
-        target_value = args[1].strip()
-        str_label = args[2].strip()
-
-        if not labels_formatter.supports_value(str_label):
-            return None
-
-        return Opinion(source_value=source_value,
-                       target_value=target_value,
-                       label=labels_formatter.str_to_label(str_label))
-
-    @staticmethod
-    def try_to_string(opinion, labels_formatter):
-        assert(isinstance(opinion, Opinion))
-        assert(isinstance(labels_formatter, StringLabelsFormatter))
-
-        label = opinion.Label
-
-        if not labels_formatter.supports_label(label):
-            return None
-
-        return "{}, {}, {}, current".format(
-            opinion.SourceValue,
-            opinion.TargetValue,
-            labels_formatter.label_to_str(opinion.Label))

arekit/contrib/source/rusentrel/opinions/provider.py

@@ -1,54 +0,0 @@
-from arekit.common.opinions.provider import OpinionCollectionsProvider
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.contrib.source.rusentrel.opinions.converter import OpinionConverter
-
-
-class RuSentRelOpinionCollectionProvider(OpinionCollectionsProvider):
-
-    @staticmethod
-    def _iter_opinions_from_file(input_file, labels_formatter, error_on_non_supported):
-        assert(isinstance(labels_formatter, StringLabelsFormatter))
-        assert(isinstance(error_on_non_supported, bool))
-
-        for line in input_file.readlines():
-
-            # Force perform decoding if needed.
-            if isinstance(line, bytes):
-                line = line.decode()
-
-            if line == '\n':
-                continue
-
-            str_opinion = OpinionConverter.try_from_string(
-                line=line,
-                labels_formatter=labels_formatter)
-
-            if str_opinion is None:
-                if error_on_non_supported:
-                    raise Exception("Line '{line}' has non supported label")
-                else:
-                    continue
-
-            yield str_opinion
-
-    # region public methods
-
-    def iter_opinions(self, source, encoding, labels_formatter, error_on_non_supported=True):
-        """
-        Important: For externally saved collections (using save_to_file method) and related usage
-        """
-        assert(isinstance(source, str))
-        assert(isinstance(labels_formatter, StringLabelsFormatter))
-        assert(isinstance(error_on_non_supported, bool))
-
-        with open(source, 'r', encoding=encoding) as input_file:
-
-            it = RuSentRelOpinionCollectionProvider._iter_opinions_from_file(
-                input_file=input_file,
-                labels_formatter=labels_formatter,
-                error_on_non_supported=error_on_non_supported)
-
-            for opinion in it:
-                yield opinion
-
-    # endregion

arekit/contrib/source/rusentrel/opinions/writer.py

@@ -1,42 +0,0 @@
-import io
-
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.common.opinions.base import Opinion
-from arekit.common.opinions.collection import OpinionCollection
-from arekit.common.opinions.writer import OpinionCollectionWriter
-from arekit.common.utils import create_dir_if_not_exists
-from arekit.contrib.source.rusentrel.opinions.converter import OpinionConverter
-
-
-class RuSentRelOpinionCollectionWriter(OpinionCollectionWriter):
-
-    def serialize(self, collection, target, encoding, labels_formatter, error_on_non_supported=True):
-        assert(isinstance(collection, OpinionCollection))
-        assert(isinstance(target, str))
-        assert(isinstance(labels_formatter, StringLabelsFormatter))
-        assert(isinstance(error_on_non_supported, bool))
-
-        def __opinion_key(opinion):
-            assert (isinstance(opinion, Opinion))
-            return opinion.SourceValue + opinion.TargetValue
-
-        sorted_ops = sorted(collection, key=__opinion_key)
-
-        create_dir_if_not_exists(target)
-
-        with io.open(target, 'w', encoding=encoding) as f:
-            for o in sorted_ops:
-
-                str_value = OpinionConverter.try_to_string(
-                    opinion=o,
-                    labels_formatter=labels_formatter)
-
-                if str_value is None:
-                    if error_on_non_supported:
-                        raise Exception("Opinion label `{label}` is not supported by formatter".format(
-                            label=o.Label))
-                    else:
-                        continue
-
-                f.write(str_value)
-                f.write('\n')

arekit/contrib/source/rusentrel/synonyms.py

@@ -1,17 +0,0 @@
-from arekit.contrib.source.synonyms.utils import iter_synonym_groups
-from arekit.contrib.source.rusentrel.io_utils import RuSentRelIOUtils
-
-
-class RuSentRelSynonymsCollectionHelper(object):
-
-    @staticmethod
-    def iter_groups(version):
-        it = RuSentRelIOUtils.iter_from_zip(
-            inner_path=RuSentRelIOUtils.get_synonyms_innerpath(),
-            process_func=lambda input_file: iter_synonym_groups(
-                input_file,
-                desc="Loading RuSentRel Collection"),
-            version=version)
-
-        for group in it:
-            yield group

arekit/contrib/source/sentinerel/entities.py

@@ -1,52 +0,0 @@
-from arekit.common.entities.collection import EntityCollection
-from arekit.common.synonyms.grouping import SynonymsCollectionValuesGroupingProviders
-from arekit.contrib.source.brat.annot import BratAnnotationParser
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.sentinerel.io_utils import SentiNerelIOUtils
-from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
-from arekit.contrib.utils.synonyms.stemmer_based import StemmerBasedSynonymCollection
-
-
-class SentiNerelEntityCollection(EntityCollection):
-
-    def __init__(self, contents, value_to_group_id_func, entities_to_ignore=None):
-        """
-            entities_to_ignore: list or None
-                this parameter is required because of the simplified implmentation of
-                the nested objects of the BRAT annotation.
-        """
-        assert(isinstance(contents, dict))
-        assert(BratAnnotationParser.ENTITIES in contents)
-        assert(isinstance(entities_to_ignore, list) or entities_to_ignore is None)
-
-        self.__dicard_entities = set([] if entities_to_ignore is None else entities_to_ignore)
-        contents[BratAnnotationParser.ENTITIES] = [e for e in contents[BratAnnotationParser.ENTITIES]
-                                                   if self.__keep_entity(e)]
-
-        super(SentiNerelEntityCollection, self).__init__(
-            entities=contents[BratAnnotationParser.ENTITIES],
-            value_to_group_id_func=value_to_group_id_func)
-
-        self._sort_entities(key=lambda entity: entity.IndexBegin)
-
-    def __keep_entity(self, entity):
-        assert(isinstance(entity, BratEntity))
-        return entity.Type not in self.__dicard_entities
-
-    @classmethod
-    def read_collection(cls, filename, version, entities_to_ignore=None):
-        assert(isinstance(filename, str))
-
-        # Since this dataset does not provide the synonyms collection by default,
-        # it is necessary to declare an empty collection to populate so in further.
-        synonyms = StemmerBasedSynonymCollection(stemmer=MystemWrapper(), is_read_only=False)
-
-        return SentiNerelIOUtils.read_from_zip(
-            inner_path=SentiNerelIOUtils.get_annotation_innerpath(filename),
-            process_func=lambda input_file: cls(
-                contents=BratAnnotationParser.parse_annotations(input_file=input_file, encoding='utf-8-sig'),
-                entities_to_ignore=entities_to_ignore,
-                value_to_group_id_func=lambda value:
-                    SynonymsCollectionValuesGroupingProviders.provide_existed_or_register_missed_value(
-                        synonyms, value)),
-            version=version)

arekit/contrib/source/sentinerel/folding/factory.py

@@ -1,31 +0,0 @@
-from arekit.contrib.source.sentinerel.folding.fixed import create_fixed_folding_doc_ids
-
-
-class SentiNERELFoldingFactory:
-    """ Factory of the variety types of the splits that
-        are considered within the present experiments.
-    """
-
-    @staticmethod
-    def create_fixed_folding(file, limit=None):
-        """ limit: int
-                Allows to limit amount of documents (utilized for testing reasons)
-        """
-
-        train_filenames, test_filenames = SentiNERELFoldingFactory._read_train_test(f=file)
-        if limit is not None:
-            train_filenames = train_filenames[:limit]
-            test_filenames = test_filenames[:limit]
-        filenames_by_ids, data_folding = create_fixed_folding_doc_ids(train_filenames=train_filenames,
-                                                                      test_filenames=test_filenames)
-
-        return filenames_by_ids, data_folding
-
-    @staticmethod
-    def _read_train_test(f):
-        parts = []
-        for line in f.readlines():
-            if isinstance(line, bytes):
-                line = line.decode('utf-8')
-            parts.append(line.strip().split(','))
-        return parts[0], parts[1]

arekit/contrib/source/sentinerel/folding/fixed.py

@@ -1,70 +0,0 @@
-from collections import OrderedDict
-
-from arekit.common.experiment.data_type import DataType
-
-
-def create_fixed_folding_doc_ids(train_filenames, test_filenames):
-    """ Create fixed data-folding based on the predefined list of filenames,
-        written in file.
-    """
-    assert(isinstance(train_filenames, list))
-    assert(isinstance(test_filenames, list))
-
-    filenames_by_ids = __create_filenames_by_ids(filenames=train_filenames + test_filenames)
-
-    ids_by_filenames = {}
-    for doc_id, filename in filenames_by_ids.items():
-        ids_by_filenames[filename] = doc_id
-
-    train_doc_ids = [ids_by_filenames[filename] for filename in train_filenames]
-    test_doc_ids = [ids_by_filenames[filename] for filename in test_filenames]
-
-    return {
-        DataType.Train: train_doc_ids,
-        DataType.Test: test_doc_ids,
-        DataType.Etalon: test_doc_ids,
-        DataType.Dev: test_doc_ids
-    }
-
-
-def __create_filenames_by_ids(filenames):
-    """ Indexing filenames
-    """
-
-    def __create_new_id(default_id):
-        new_id = default_id
-        while new_id in filenames_by_ids:
-            new_id += 1
-        return new_id
-
-    default_id = 0
-
-    filenames_by_ids = OrderedDict()
-    for fname in filenames:
-
-        doc_id = __number_from_string(fname)
-
-        if doc_id is None:
-            doc_id = __create_new_id(default_id)
-            default_id = doc_id
-
-        filenames_by_ids[doc_id] = fname
-
-    return filenames_by_ids
-
-
-def __number_from_string(s):
-    assert(isinstance(s, str))
-
-    digit_chars_prefix = []
-
-    for chr in s:
-        if chr.isdigit():
-            digit_chars_prefix.append(chr)
-        else:
-            break
-
-    if len(digit_chars_prefix) == 0:
-        return None
-
-    return int("".join(digit_chars_prefix))