arekit 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/source/rusentrel/opinions/provider.py

@@ -1,54 +0,0 @@
-from arekit.common.opinions.provider import OpinionCollectionsProvider
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.contrib.source.rusentrel.opinions.converter import OpinionConverter
-
-
-class RuSentRelOpinionCollectionProvider(OpinionCollectionsProvider):
-
-    @staticmethod
-    def _iter_opinions_from_file(input_file, labels_formatter, error_on_non_supported):
-        assert(isinstance(labels_formatter, StringLabelsFormatter))
-        assert(isinstance(error_on_non_supported, bool))
-
-        for line in input_file.readlines():
-
-            # Force perform decoding if needed.
-            if isinstance(line, bytes):
-                line = line.decode()
-
-            if line == '\n':
-                continue
-
-            str_opinion = OpinionConverter.try_from_string(
-                line=line,
-                labels_formatter=labels_formatter)
-
-            if str_opinion is None:
-                if error_on_non_supported:
-                    raise Exception("Line '{line}' has non supported label")
-                else:
-                    continue
-
-            yield str_opinion
-
-    # region public methods
-
-    def iter_opinions(self, source, encoding, labels_formatter, error_on_non_supported=True):
-        """
-        Important: For externally saved collections (using save_to_file method) and related usage
-        """
-        assert(isinstance(source, str))
-        assert(isinstance(labels_formatter, StringLabelsFormatter))
-        assert(isinstance(error_on_non_supported, bool))
-
-        with open(source, 'r', encoding=encoding) as input_file:
-
-            it = RuSentRelOpinionCollectionProvider._iter_opinions_from_file(
-                input_file=input_file,
-                labels_formatter=labels_formatter,
-                error_on_non_supported=error_on_non_supported)
-
-            for opinion in it:
-                yield opinion
-
-    # endregion

arekit/contrib/source/rusentrel/opinions/writer.py

@@ -1,42 +0,0 @@
-import io
-
-from arekit.common.labels.str_fmt import StringLabelsFormatter
-from arekit.common.opinions.base import Opinion
-from arekit.common.opinions.collection import OpinionCollection
-from arekit.common.opinions.writer import OpinionCollectionWriter
-from arekit.common.utils import create_dir_if_not_exists
-from arekit.contrib.source.rusentrel.opinions.converter import OpinionConverter
-
-
-class RuSentRelOpinionCollectionWriter(OpinionCollectionWriter):
-
-    def serialize(self, collection, target, encoding, labels_formatter, error_on_non_supported=True):
-        assert(isinstance(collection, OpinionCollection))
-        assert(isinstance(target, str))
-        assert(isinstance(labels_formatter, StringLabelsFormatter))
-        assert(isinstance(error_on_non_supported, bool))
-
-        def __opinion_key(opinion):
-            assert (isinstance(opinion, Opinion))
-            return opinion.SourceValue + opinion.TargetValue
-
-        sorted_ops = sorted(collection, key=__opinion_key)
-
-        create_dir_if_not_exists(target)
-
-        with io.open(target, 'w', encoding=encoding) as f:
-            for o in sorted_ops:
-
-                str_value = OpinionConverter.try_to_string(
-                    opinion=o,
-                    labels_formatter=labels_formatter)
-
-                if str_value is None:
-                    if error_on_non_supported:
-                        raise Exception("Opinion label `{label}` is not supported by formatter".format(
-                            label=o.Sentiment))
-                    else:
-                        continue
-
-                f.write(str_value)
-                f.write('\n')

arekit/contrib/source/rusentrel/synonyms.py

@@ -1,17 +0,0 @@
-from arekit.contrib.source.synonyms.utils import iter_synonym_groups
-from arekit.contrib.source.rusentrel.io_utils import RuSentRelIOUtils
-
-
-class RuSentRelSynonymsCollectionHelper(object):
-
-    @staticmethod
-    def iter_groups(version):
-        it = RuSentRelIOUtils.iter_from_zip(
-            inner_path=RuSentRelIOUtils.get_synonyms_innerpath(),
-            process_func=lambda input_file: iter_synonym_groups(
-                input_file,
-                desc="Loading RuSentRel Collection"),
-            version=version)
-
-        for group in it:
-            yield group

arekit/contrib/source/sentinerel/entities.py

@@ -1,52 +0,0 @@
-from arekit.common.entities.collection import EntityCollection
-from arekit.common.synonyms.grouping import SynonymsCollectionValuesGroupingProviders
-from arekit.contrib.source.brat.annot import BratAnnotationParser
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.sentinerel.io_utils import SentiNerelIOUtils
-from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
-from arekit.contrib.utils.synonyms.stemmer_based import StemmerBasedSynonymCollection
-
-
-class SentiNerelEntityCollection(EntityCollection):
-
-    def __init__(self, contents, value_to_group_id_func, entities_to_ignore=None):
-        """
-            entities_to_ignore: list or None
-                this parameter is required because of the simplified implmentation of
-                the nested objects of the BRAT annotation.
-        """
-        assert(isinstance(contents, dict))
-        assert(BratAnnotationParser.ENTITIES in contents)
-        assert(isinstance(entities_to_ignore, list) or entities_to_ignore is None)
-
-        self.__dicard_entities = set([] if entities_to_ignore is None else entities_to_ignore)
-        contents[BratAnnotationParser.ENTITIES] = [e for e in contents[BratAnnotationParser.ENTITIES]
-                                                   if self.__keep_entity(e)]
-
-        super(SentiNerelEntityCollection, self).__init__(
-            entities=contents[BratAnnotationParser.ENTITIES],
-            value_to_group_id_func=value_to_group_id_func)
-
-        self._sort_entities(key=lambda entity: entity.IndexBegin)
-
-    def __keep_entity(self, entity):
-        assert(isinstance(entity, BratEntity))
-        return entity.Type not in self.__dicard_entities
-
-    @classmethod
-    def read_collection(cls, filename, version, entities_to_ignore=None):
-        assert(isinstance(filename, str))
-
-        # Since this dataset does not provide the synonyms collection by default,
-        # it is necessary to declare an empty collection to populate so in further.
-        synonyms = StemmerBasedSynonymCollection(stemmer=MystemWrapper(), is_read_only=False)
-
-        return SentiNerelIOUtils.read_from_zip(
-            inner_path=SentiNerelIOUtils.get_annotation_innerpath(filename),
-            process_func=lambda input_file: cls(
-                contents=BratAnnotationParser.parse_annotations(input_file=input_file, encoding='utf-8-sig'),
-                entities_to_ignore=entities_to_ignore,
-                value_to_group_id_func=lambda value:
-                    SynonymsCollectionValuesGroupingProviders.provide_existed_or_register_missed_value(
-                        synonyms, value)),
-            version=version)

arekit/contrib/source/sentinerel/folding/factory.py

@@ -1,32 +0,0 @@
-from arekit.contrib.source.sentinerel.folding.fixed import create_fixed_folding
-
-
-class SentiNERELFoldingFactory:
-    """ Factory of the variety types of the splits that
-        are considered within the present experiments.
-    """
-
-    @staticmethod
-    def create_fixed_folding(file, limit=None):
-        """
-            limit: int
-                Allows to limit amount of documents (utilized for testing reasons)
-        """
-
-        train_filenames, test_filenames = SentiNERELFoldingFactory._read_train_test(f=file)
-        if limit is not None:
-            train_filenames = train_filenames[:limit]
-            test_filenames = test_filenames[:limit]
-        filenames_by_ids, data_folding = create_fixed_folding(train_filenames=train_filenames,
-                                                              test_filenames=test_filenames)
-
-        return filenames_by_ids, data_folding
-
-    @staticmethod
-    def _read_train_test(f):
-        parts = []
-        for line in f.readlines():
-            if isinstance(line, bytes):
-                line = line.decode('utf-8')
-            parts.append(line.strip().split(','))
-        return parts[0], parts[1]

arekit/contrib/source/sentinerel/folding/fixed.py

@@ -1,73 +0,0 @@
-from collections import OrderedDict
-
-from arekit.common.experiment.data_type import DataType
-from arekit.common.folding.fixed import FixedFolding
-
-
-def create_fixed_folding(train_filenames, test_filenames):
-    """ Create fixed data-folding based on the predefined list of filenames,
-        written in file.
-    """
-    assert(isinstance(train_filenames, list))
-    assert(isinstance(test_filenames, list))
-
-    filenames_by_ids = create_filenames_by_ids(filenames=train_filenames + test_filenames)
-
-    ids_by_filenames = {}
-    for doc_id, filename in filenames_by_ids.items():
-        ids_by_filenames[filename] = doc_id
-
-    train_doc_ids = [ids_by_filenames[filename] for filename in train_filenames]
-    test_doc_ids = [ids_by_filenames[filename] for filename in test_filenames]
-
-    fixed_folding = FixedFolding.from_parts({
-        DataType.Train: train_doc_ids,
-        DataType.Test: test_doc_ids,
-        DataType.Etalon: test_doc_ids,
-        DataType.Dev: test_doc_ids
-    })
-
-    return filenames_by_ids, fixed_folding
-
-
-def create_filenames_by_ids(filenames):
-    """ Indexing filenames
-    """
-
-    def __create_new_id(default_id):
-        new_id = default_id
-        while new_id in filenames_by_ids:
-            new_id += 1
-        return new_id
-
-    default_id = 0
-
-    filenames_by_ids = OrderedDict()
-    for fname in filenames:
-
-        doc_id = number_from_string(fname)
-
-        if doc_id is None:
-            doc_id = __create_new_id(default_id)
-            default_id = doc_id
-
-        filenames_by_ids[doc_id] = fname
-
-    return filenames_by_ids
-
-
-def number_from_string(s):
-    assert(isinstance(s, str))
-
-    digit_chars_prefix = []
-
-    for chr in s:
-        if chr.isdigit():
-            digit_chars_prefix.append(chr)
-        else:
-            break
-
-    if len(digit_chars_prefix) == 0:
-        return None
-
-    return int("".join(digit_chars_prefix))

arekit/contrib/source/sentinerel/io_utils.py

@@ -1,87 +0,0 @@
-from enum import Enum
-from os import path
-from os.path import basename, join
-
-import enum
-
-from arekit.contrib.source.sentinerel.folding.factory import SentiNERELFoldingFactory
-from arekit.contrib.source.zip_utils import ZipArchiveUtils
-
-
-class SentiNerelVersions(Enum):
-    """ List of the supported version of this collection
-    """
-
-    # Initial version.
-    V1 = "v1_0"
-    # Updated annotation within the second half of the texts. (September 2022)
-    V2 = "v2_0"
-    # Updated annotation within the first half of the texts.  (October 2022)
-    # Become a source of the RuSentNE-2023 competition.
-    # https://github.com/dialogue-evaluation/RuSentNE-evaluation
-    V21 = "v2_1"
-
-
-DEFAULT_VERSION = SentiNerelVersions.V21
-
-
-class SentiNerelIOUtils(ZipArchiveUtils):
-
-    inner_root = "sentiment_dataset"
-
-    @staticmethod
-    def get_archive_filepath(version):
-        return path.join(SentiNerelIOUtils.get_data_root(), "sentinerel-{}.zip".format(version))
-
-    @staticmethod
-    def get_annotation_innerpath(filename):
-        assert(isinstance(filename, str))
-        return path.join(SentiNerelIOUtils.inner_root, "{}.ann".format(filename))
-
-    @staticmethod
-    def get_news_innerpath(filename):
-        assert(isinstance(filename, str))
-        return path.join(SentiNerelIOUtils.inner_root, "{}.txt".format(filename))
-
-    @staticmethod
-    def __iter_filenames_from_dataset(folder_name, version):
-        assert(isinstance(version, enum.Enum))
-        assert(isinstance(folder_name, str))
-
-        for filename in SentiNerelIOUtils.iter_filenames_from_zip(version):
-
-            extension = filename[-4:]
-
-            # Crop extension.
-            filename = filename[:-4]
-
-            if extension != ".txt":
-                continue
-
-            if not folder_name in filename:
-                continue
-
-            yield basename(filename)
-
-    # region public methods
-
-    @staticmethod
-    def iter_collection_filenames(version=DEFAULT_VERSION):
-        filenames_it = SentiNerelIOUtils.__iter_filenames_from_dataset(
-            folder_name=SentiNerelIOUtils.inner_root, version=version)
-
-        for doc_id, filename in enumerate(filenames_it):
-            yield doc_id, filename
-
-    @staticmethod
-    def read_dataset_split(version=DEFAULT_VERSION, docs_limit=None):
-        """ Provides a fixed split of the dataset onto
-            `test` and `training` part:
-            https://github.com/nicolay-r/SentiNEREL-attitude-extraction
-        """
-        return SentiNerelIOUtils.read_from_zip(
-            inner_path=join(SentiNerelIOUtils.inner_root, "split_fixed.txt"),
-            process_func=lambda f: SentiNERELFoldingFactory.create_fixed_folding(file=f, limit=docs_limit),
-            version=version)
-
-    # endregion

arekit/contrib/source/sentinerel/labels.py

@@ -1,53 +0,0 @@
-from arekit.common.labels.base import Label
-
-
-class OpinionBelongsTo(Label):
-    pass
-
-
-class OpinionRelatesTo(Label):
-    pass
-
-
-class NegEffectFrom(Label):
-    pass
-
-
-class NegStateFrom(Label):
-    pass
-
-
-class PosEffectFrom(Label):
-    pass
-
-
-class PosAuthorFrom(Label):
-    pass
-
-
-class NegAuthorFrom(Label):
-    pass
-
-
-class PosStateFrom(Label):
-    pass
-
-
-class NegativeTo(Label):
-    pass
-
-
-class PositiveTo(Label):
-    pass
-
-
-class AlternativeName(Label):
-    pass
-
-
-class StateBelongsTo(Label):
-    pass
-
-
-class OriginsFrom(Label):
-    pass

arekit/contrib/source/sentinerel/labels_scaler.py

@@ -1,30 +0,0 @@
-from collections import OrderedDict
-
-from arekit.common.labels.scaler.base import BaseLabelScaler
-from arekit.contrib.source.sentinerel import labels
-
-
-class SentiNerelLabelScaler(BaseLabelScaler):
-    """ This is a complete label scaler of all the labels supported by NEREL dataset.
-    """
-
-    def __init__(self):
-
-        self.__uint_to_label_dict = OrderedDict([
-            (labels.OpinionBelongsTo(), 0),
-            (labels.OpinionRelatesTo(), 1),
-            (labels.NegEffectFrom(), 2),
-            (labels.PosEffectFrom(), 3),
-            (labels.NegStateFrom(), 4),
-            (labels.PosStateFrom(), 5),
-            (labels.NegativeTo(), 6),
-            (labels.PositiveTo(), 7),
-            (labels.StateBelongsTo(), 8),
-            (labels.PosAuthorFrom(), 9),
-            (labels.NegAuthorFrom(), 10),
-            (labels.AlternativeName(), 11),
-            (labels.OriginsFrom(), 12)
-        ])
-
-        super(SentiNerelLabelScaler, self).__init__(int_dict=self.__uint_to_label_dict,
-                                                    uint_dict=self.__uint_to_label_dict)

arekit/contrib/source/sentinerel/reader.py

@@ -1,42 +0,0 @@
-from arekit.contrib.source.brat.annot import BratAnnotationParser
-from arekit.contrib.source.brat.news import BratNews
-from arekit.contrib.source.brat.sentences_reader import BratDocumentSentencesReader
-from arekit.contrib.source.sentinerel.entities import SentiNerelEntityCollection
-from arekit.contrib.source.sentinerel.io_utils import SentiNerelIOUtils, DEFAULT_VERSION
-
-
-class SentiNerelDocReader(object):
-
-    @staticmethod
-    def read_text_relations(filename, version):
-        assert(isinstance(filename, str))
-
-        return SentiNerelIOUtils.read_from_zip(
-            inner_path=SentiNerelIOUtils.get_annotation_innerpath(filename),
-            process_func=lambda input_file: [
-                relation for relation in BratAnnotationParser.parse_annotations(
-                    input_file=input_file, encoding='utf-8-sig')["relations"]],
-            version=version)
-
-    @staticmethod
-    def read_document(filename, doc_id, version=DEFAULT_VERSION, entities_to_ignore=None):
-        assert(isinstance(filename, str))
-        assert(isinstance(doc_id, int))
-
-        def file_to_doc(input_file):
-            sentences = BratDocumentSentencesReader.from_file(input_file=input_file, entities=entities)
-            return BratNews(doc_id=doc_id, sentences=sentences, text_relations=text_relations)
-
-        # TODO. #398 issue -- in some cases entities might be nested. Therefore we limit the set
-        # TODO. of the potential named entities.
-        eti = ["EFFECT_NEG", "EFFECT_POS", "ARGUMENT_NEG", "ARGUMENT_POS", "EVENT"] \
-            if entities_to_ignore is None else entities_to_ignore
-
-        entities = SentiNerelEntityCollection.read_collection(
-            filename=filename, version=version, entities_to_ignore=eti)
-        text_relations = SentiNerelDocReader.read_text_relations(filename=filename, version=version)
-
-        return SentiNerelIOUtils.read_from_zip(
-            inner_path=SentiNerelIOUtils.get_news_innerpath(filename=filename),
-            process_func=file_to_doc,
-            version=version)

arekit/contrib/source/synonyms/utils.py

@@ -1,19 +0,0 @@
-from arekit.common.utils import progress_bar_defined
-
-
-def iter_synonym_groups(input_file, sep=",", desc=""):
-    """ All the synonyms groups organized in lines, separated by `sep`
-    """
-    lines = input_file.readlines()
-
-    lines_it = progress_bar_defined(lines,
-                                    total=len(lines),
-                                    desc=desc,
-                                    unit="opins")
-
-    for line in lines_it:
-
-        if isinstance(line, bytes):
-            line = line.decode()
-
-        yield line.split(sep)

arekit/contrib/source/zip_utils.py

@@ -1,47 +0,0 @@
-import zipfile
-
-import enum
-
-from arekit.common import utils
-
-
-class ZipArchiveUtils(object):
-
-    @staticmethod
-    def get_archive_filepath(version):
-        raise NotImplementedError()
-
-    @classmethod
-    def read_from_zip(cls, inner_path, process_func, version):
-        """
-        process_func:
-            func which receives a file reader
-        """
-        assert(isinstance(inner_path, str))
-        assert(callable(process_func))
-        assert(isinstance(version, enum.Enum))
-
-        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
-            with zip_ref.open(inner_path, mode='r') as c_file:
-                return process_func(c_file)
-
-    @classmethod
-    def iter_from_zip(cls, inner_path, process_func, version):
-        assert(isinstance(inner_path, str))
-        assert(callable(process_func))
-        assert(isinstance(version, enum.Enum))
-
-        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
-            with zip_ref.open(inner_path, mode='r') as c_file:
-                for result in process_func(c_file):
-                    yield result
-
-    @classmethod
-    def iter_filenames_from_zip(cls, version):
-        assert(isinstance(version, enum.Enum))
-        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
-            return iter(zip_ref.namelist())
-
-    @staticmethod
-    def get_data_root():
-        return utils.get_default_download_dir()

arekit/contrib/utils/bert/text_b_rus.py

@@ -1,18 +0,0 @@
-from enum import Enum
-
-
-class BertTextBRussianPrompts(Enum):
-    """
-    Default, based on COLA, but includes an extra text_b.
-        text_b: Pseudo-sentence w/o S.P (S.P -- sentiment polarity)
-        text_b: Question w/o S.P (S.P -- sentiment polarity)
-
-    Multilabel variant
-
-    Notation were taken from paper:
-    https://www.aclweb.org/anthology/N19-1035.pdf
-    """
-
-    NLI = '{subject} к {object} в контексте : << {context} >>'
-
-    QA = 'Что вы думаете по поводу отношения {subject} к {object} в контексте : << {context} >> ?'

arekit/contrib/utils/connotations/rusentiframes_sentiment.py

@@ -1,23 +0,0 @@
-from arekit.common.frames.connotations.provider import FrameConnotationProvider
-from arekit.contrib.source.rusentiframes.collection import RuSentiFramesCollection
-
-
-class RuSentiFramesConnotationProvider(FrameConnotationProvider):
-    """ This is a provider based on A0->A1 label type of RuSentiFrames collection.
-        For a greater details, checkout the related collection at:
-        https://github.com/nicolay-r/RuSentiFrames
-
-        Papers:
-            [1] Natalia Loukachevitch, Nicolay Rusnachenko: Sentiment Frames
-                for Attitude Extraction in Russian, 2020
-            [2] Distant Supervision for Sentiment Attitude Extraction, 2019
-    """
-
-    def __init__(self, collection):
-        assert(isinstance(collection, RuSentiFramesCollection))
-        self.__collection = collection
-
-    def try_provide(self, frame_id):
-        return self.__collection.try_get_frame_polarity(frame_id=frame_id,
-                                                        role_src='a0',
-                                                        role_dest='a1')

arekit/contrib/utils/cv/doc_stat/base.py

@@ -1,37 +0,0 @@
-class BaseDocumentStatGenerator(object):
-    """
-    Provides statistic on certain document.
-    Abstract, considered a specific implementation for document processing operation.
-    """
-
-    def __init__(self, doc_reader_func):
-        """
-        news_parser_func: func -> news
-            assumes to provide a news by a certain doc_id
-        """
-        assert(callable(doc_reader_func))
-        self.__doc_reader_func = doc_reader_func
-
-    # region abstract protected methods
-
-    def _calc(self, news):
-        """ Abstract method that provides quantitative statistic
-            for a particular news
-        """
-        raise NotImplementedError()
-
-    # endregion
-
-    # region public methods
-
-    def calculate(self, doc_ids_iter):
-        docs_info = []
-
-        for doc_id in doc_ids_iter:
-            doc = self.__doc_reader_func(doc_id)
-            s_count = self._calc(doc)
-            docs_info.append((doc_id, s_count))
-
-        return docs_info
-
-    # endregion

arekit/contrib/utils/cv/doc_stat/sentence.py

@@ -1,12 +0,0 @@
-from arekit.common.news.base import News
-from arekit.contrib.utils.cv.doc_stat.base import BaseDocumentStatGenerator
-
-
-class SentenceBasedDocumentStatGenerator(BaseDocumentStatGenerator):
-
-    def __init__(self, doc_reader_func):
-        super(SentenceBasedDocumentStatGenerator, self).__init__(doc_reader_func)
-
-    def _calc(self, news):
-        assert(isinstance(news, News))
-        return news.SentencesCount

arekit/contrib/utils/cv/splitters/base.py

@@ -1,4 +0,0 @@
-class CrossValidationSplitter(object):
-
-    def items_to_cv_pairs(self, doc_ids, cv_count):
-        pass