arekit 0.24.0__py3-none-any.whl

arekit/contrib/source/sentinerel/io_utils.py

@@ -0,0 +1,87 @@
+from enum import Enum
+from os import path
+from os.path import basename, join
+
+import enum
+
+from arekit.contrib.source.sentinerel.folding.factory import SentiNERELFoldingFactory
+from arekit.contrib.source.zip_utils import ZipArchiveUtils
+
+
+class SentiNerelVersions(Enum):
+    """ List of the supported version of this collection
+    """
+
+    # Initial version.
+    V1 = "v1_0"
+    # Updated annotation within the second half of the texts. (September 2022)
+    V2 = "v2_0"
+    # Updated annotation within the first half of the texts.  (October 2022)
+    # Become a source of the RuSentNE-2023 competition.
+    # https://github.com/dialogue-evaluation/RuSentNE-evaluation
+    V21 = "v2_1"
+
+
+DEFAULT_VERSION = SentiNerelVersions.V21
+
+
+class SentiNerelIOUtils(ZipArchiveUtils):
+
+    inner_root = "sentiment_dataset"
+
+    @staticmethod
+    def get_archive_filepath(version):
+        return path.join(SentiNerelIOUtils.get_data_root(), "sentinerel-{}.zip".format(version))
+
+    @staticmethod
+    def get_annotation_innerpath(filename):
+        assert(isinstance(filename, str))
+        return path.join(SentiNerelIOUtils.inner_root, "{}.ann".format(filename))
+
+    @staticmethod
+    def get_doc_innerpath(filename):
+        assert(isinstance(filename, str))
+        return path.join(SentiNerelIOUtils.inner_root, "{}.txt".format(filename))
+
+    @staticmethod
+    def __iter_filenames_from_dataset(folder_name, version):
+        assert(isinstance(version, enum.Enum))
+        assert(isinstance(folder_name, str))
+
+        for filename in SentiNerelIOUtils.iter_filenames_from_zip(version):
+
+            extension = filename[-4:]
+
+            # Crop extension.
+            filename = filename[:-4]
+
+            if extension != ".txt":
+                continue
+
+            if not folder_name in filename:
+                continue
+
+            yield basename(filename)
+
+    # region public methods
+
+    @staticmethod
+    def iter_collection_filenames(version=DEFAULT_VERSION):
+        filenames_it = SentiNerelIOUtils.__iter_filenames_from_dataset(
+            folder_name=SentiNerelIOUtils.inner_root, version=version)
+
+        for doc_id, filename in enumerate(filenames_it):
+            yield doc_id, filename
+
+    @staticmethod
+    def read_dataset_split(version=DEFAULT_VERSION, docs_limit=None):
+        """ Provides a fixed split of the dataset onto
+            `test` and `training` part:
+            https://github.com/nicolay-r/SentiNEREL-attitude-extraction
+        """
+        return SentiNerelIOUtils.read_from_zip(
+            inner_path=join(SentiNerelIOUtils.inner_root, "split_fixed.txt"),
+            process_func=lambda f: SentiNERELFoldingFactory.create_fixed_folding(file=f, limit=docs_limit),
+            version=version)
+
+    # endregion

arekit/contrib/source/sentinerel/labels.py

@@ -0,0 +1,53 @@
+from arekit.common.labels.base import Label
+
+
+class OpinionBelongsTo(Label):
+    pass
+
+
+class OpinionRelatesTo(Label):
+    pass
+
+
+class NegEffectFrom(Label):
+    pass
+
+
+class NegStateFrom(Label):
+    pass
+
+
+class PosEffectFrom(Label):
+    pass
+
+
+class PosAuthorFrom(Label):
+    pass
+
+
+class NegAuthorFrom(Label):
+    pass
+
+
+class PosStateFrom(Label):
+    pass
+
+
+class NegativeTo(Label):
+    pass
+
+
+class PositiveTo(Label):
+    pass
+
+
+class AlternativeName(Label):
+    pass
+
+
+class StateBelongsTo(Label):
+    pass
+
+
+class OriginsFrom(Label):
+    pass

arekit/contrib/source/sentinerel/labels_scaler.py

@@ -0,0 +1,30 @@
+from collections import OrderedDict
+
+from arekit.common.labels.scaler.base import BaseLabelScaler
+from arekit.contrib.source.sentinerel import labels
+
+
+class SentiNerelLabelScaler(BaseLabelScaler):
+    """ This is a complete label scaler of all the labels supported by NEREL dataset.
+    """
+
+    def __init__(self):
+
+        self.__uint_to_label_dict = OrderedDict([
+            (labels.OpinionBelongsTo(), 0),
+            (labels.OpinionRelatesTo(), 1),
+            (labels.NegEffectFrom(), 2),
+            (labels.PosEffectFrom(), 3),
+            (labels.NegStateFrom(), 4),
+            (labels.PosStateFrom(), 5),
+            (labels.NegativeTo(), 6),
+            (labels.PositiveTo(), 7),
+            (labels.StateBelongsTo(), 8),
+            (labels.PosAuthorFrom(), 9),
+            (labels.NegAuthorFrom(), 10),
+            (labels.AlternativeName(), 11),
+            (labels.OriginsFrom(), 12)
+        ])
+
+        super(SentiNerelLabelScaler, self).__init__(int_dict=self.__uint_to_label_dict,
+                                                    uint_dict=self.__uint_to_label_dict)

arekit/contrib/source/sentinerel/reader.py

@@ -0,0 +1,42 @@
+from arekit.contrib.source.brat.annot import BratAnnotationParser
+from arekit.contrib.source.brat.doc import BratDocument
+from arekit.contrib.source.brat.sentences_reader import BratDocumentSentencesReader
+from arekit.contrib.source.sentinerel.entities import SentiNerelEntityCollection
+from arekit.contrib.source.sentinerel.io_utils import SentiNerelIOUtils, DEFAULT_VERSION
+
+
+class SentiNerelDocReader(object):
+
+    @staticmethod
+    def read_text_relations(filename, version):
+        assert(isinstance(filename, str))
+
+        return SentiNerelIOUtils.read_from_zip(
+            inner_path=SentiNerelIOUtils.get_annotation_innerpath(filename),
+            process_func=lambda input_file: [
+                relation for relation in BratAnnotationParser.parse_annotations(
+                    input_file=input_file, encoding='utf-8-sig')["relations"]],
+            version=version)
+
+    @staticmethod
+    def read_document(filename, doc_id, version=DEFAULT_VERSION, entities_to_ignore=None):
+        assert(isinstance(filename, str))
+        assert(isinstance(doc_id, int))
+
+        def file_to_doc(input_file):
+            sentences = BratDocumentSentencesReader.from_file(input_file=input_file, entities=entities)
+            return BratDocument(doc_id=doc_id, sentences=sentences, text_relations=text_relations)
+
+        # TODO. #398 issue -- in some cases entities might be nested. Therefore we limit the set
+        # TODO. of the potential named entities.
+        eti = ["EFFECT_NEG", "EFFECT_POS", "ARGUMENT_NEG", "ARGUMENT_POS", "EVENT"] \
+            if entities_to_ignore is None else entities_to_ignore
+
+        entities = SentiNerelEntityCollection.read_collection(
+            filename=filename, version=version, entities_to_ignore=eti)
+        text_relations = SentiNerelDocReader.read_text_relations(filename=filename, version=version)
+
+        return SentiNerelIOUtils.read_from_zip(
+            inner_path=SentiNerelIOUtils.get_doc_innerpath(filename=filename),
+            process_func=file_to_doc,
+            version=version)

arekit/contrib/source/synonyms/utils.py

@@ -0,0 +1,19 @@
+from arekit.common.utils import progress_bar_defined
+
+
+def iter_synonym_groups(input_file, sep=",", desc=""):
+    """ All the synonyms groups organized in lines, separated by `sep`
+    """
+    lines = input_file.readlines()
+
+    lines_it = progress_bar_defined(lines,
+                                    total=len(lines),
+                                    desc=desc,
+                                    unit="opins")
+
+    for line in lines_it:
+
+        if isinstance(line, bytes):
+            line = line.decode()
+
+        yield line.split(sep)

arekit/contrib/source/zip_utils.py

@@ -0,0 +1,47 @@
+import zipfile
+
+import enum
+
+from arekit.common import utils
+
+
+class ZipArchiveUtils(object):
+
+    @staticmethod
+    def get_archive_filepath(version):
+        raise NotImplementedError()
+
+    @classmethod
+    def read_from_zip(cls, inner_path, process_func, version):
+        """
+        process_func:
+            func which receives a file reader
+        """
+        assert(isinstance(inner_path, str))
+        assert(callable(process_func))
+        assert(isinstance(version, enum.Enum))
+
+        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
+            with zip_ref.open(inner_path, mode='r') as c_file:
+                return process_func(c_file)
+
+    @classmethod
+    def iter_from_zip(cls, inner_path, process_func, version):
+        assert(isinstance(inner_path, str))
+        assert(callable(process_func))
+        assert(isinstance(version, enum.Enum))
+
+        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
+            with zip_ref.open(inner_path, mode='r') as c_file:
+                for result in process_func(c_file):
+                    yield result
+
+    @classmethod
+    def iter_filenames_from_zip(cls, version):
+        assert(isinstance(version, enum.Enum))
+        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
+            return iter(zip_ref.namelist())
+
+    @staticmethod
+    def get_data_root():
+        return utils.get_default_download_dir()

arekit/contrib/utils/bert/samplers.py

@@ -0,0 +1,17 @@
+from arekit.common.data.input.providers.label.multiple import MultipleLabelProvider
+from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
+from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
+from arekit.common.data.input.terms_mapper import OpinionContainingTextTermsMapper
+from arekit.contrib.bert.input.providers.text_pair import PairTextProvider
+
+
+def create_sample_provider(label_scaler, text_terms_mapper, text_b_prompt=None):
+    assert(isinstance(text_terms_mapper, OpinionContainingTextTermsMapper))
+
+    text_provider = BaseSingleTextProvider(text_terms_mapper=text_terms_mapper) \
+        if text_b_prompt is None else PairTextProvider(text_b_prompt=text_b_prompt,
+                                                       text_terms_mapper=text_terms_mapper)
+
+    label_provider = MultipleLabelProvider(label_scaler=label_scaler)
+
+    return BaseSampleRowProvider(text_provider=text_provider, label_provider=label_provider)

arekit/contrib/utils/connotations/rusentiframes_sentiment.py

@@ -0,0 +1,23 @@
+from arekit.common.frames.connotations.provider import FrameConnotationProvider
+from arekit.contrib.source.rusentiframes.collection import RuSentiFramesCollection
+
+
+class RuSentiFramesConnotationProvider(FrameConnotationProvider):
+    """ This is a provider based on A0->A1 label type of RuSentiFrames collection.
+        For a greater details, checkout the related collection at:
+        https://github.com/nicolay-r/RuSentiFrames
+
+        Papers:
+            [1] Natalia Loukachevitch, Nicolay Rusnachenko: Sentiment Frames
+                for Attitude Extraction in Russian, 2020
+            [2] Distant Supervision for Sentiment Attitude Extraction, 2019
+    """
+
+    def __init__(self, collection):
+        assert(isinstance(collection, RuSentiFramesCollection))
+        self.__collection = collection
+
+    def try_provide(self, frame_id):
+        return self.__collection.try_get_frame_polarity(frame_id=frame_id,
+                                                        role_src='a0',
+                                                        role_dest='a1')

arekit/contrib/utils/data/contents/opinions.py

@@ -0,0 +1,37 @@
+from arekit.common.data.input.providers.const import IDLE_MODE
+from arekit.common.data.input.providers.contents import ContentsProvider
+from arekit.common.linkage.base import LinkedDataWrapper
+from arekit.common.linkage.text_opinions import TextOpinionsLinkage
+from arekit.common.pipeline.base import BasePipeline
+from arekit.common.text_opinions.base import TextOpinion
+
+
+class InputTextOpinionProvider(ContentsProvider):
+
+    def __init__(self, pipeline):
+        """ NOTE: it is important that the output of the pipeline
+            results in a TextOpinionLinkage instances.
+            pipeline: id -> ... -> TextOpinionLinkage[]
+        """
+        assert(isinstance(pipeline, BasePipeline))
+        self.__pipeline = pipeline
+        self.__current_id = None
+
+    # endregion
+
+    def __assign_ids(self, linkage):
+        """ Perform IDs assignation.
+        """
+        assert(isinstance(linkage, TextOpinionsLinkage))
+        for text_opinion in linkage:
+            assert(isinstance(text_opinion, TextOpinion))
+            text_opinion.set_text_opinion_id(self.__current_id)
+            self.__current_id += 1
+
+    def from_doc_ids(self, doc_ids, idle_mode=False):
+        self.__current_id = 0
+        for linkage in self.__pipeline.run(doc_ids, params_dict={IDLE_MODE: idle_mode}):
+            assert(isinstance(linkage, LinkedDataWrapper))
+            if isinstance(linkage, TextOpinionsLinkage):
+                self.__assign_ids(linkage)
+            yield linkage

arekit/contrib/utils/data/doc_provider/dict_based.py

@@ -0,0 +1,13 @@
+from arekit.common.data.doc_provider import DocumentProvider
+
+
+class DictionaryBasedDocumentProvider(DocumentProvider):
+
+    def __init__(self, d):
+        assert(isinstance(d, dict))
+        super(DictionaryBasedDocumentProvider, self).__init__()
+        self.__d = d
+
+    def by_id(self, doc_id):
+        assert(isinstance(doc_id, int))
+        return self.__d[doc_id]

arekit/contrib/utils/data/doc_provider/dir_based.py

@@ -0,0 +1,53 @@
+from os.path import join
+
+from arekit.common.data.doc_provider import DocumentProvider
+from arekit.common.docs.base import Document
+from arekit.common.docs.sentence import BaseDocumentSentence
+
+
+class DirectoryFilesDocProvider(DocumentProvider):
+    """ Document Providers based on the list of provided file paths
+        for the particular directory.
+    """
+
+    def __init__(self, dir_path, file_names=None, sentence_parser=None):
+        """
+            dir_path: str
+                path to the root directory.
+            file_names: list
+                list of file paths related to documents.
+            sentence_splitter: object
+                how data is suppose to be separated onto sentences.
+                str -> list(str)
+        """
+        assert(isinstance(dir_path, str))
+        assert(isinstance(file_names, list) or file_names is None)
+        assert(callable(sentence_parser) or sentence_parser is None)
+
+        self.__dir_path = dir_path
+        self.__file_names = file_names
+
+        # Line-split sentence parser by default.
+        self.__sentence_parser = (lambda text: [t.strip() for t in text.split('\n')]) \
+            if sentence_parser is None else sentence_parser
+
+    def __read_doc(self, doc_id, contents):
+        """ Parse a single document.
+        """
+        # setup input data.
+        sentences = self.__sentence_parser(contents)
+        sentences = list(map(lambda text: BaseDocumentSentence(text), sentences))
+
+        # Parse text.
+        return Document(doc_id=doc_id, sentences=sentences)
+
+    def by_id(self, doc_id):
+        """ Perform reading operation of the document.
+        """
+        file_name = self.__file_names[doc_id]
+        with open(join(self.__dir_path, file_name), "r") as f:
+            contents = f.read()
+            return self.__read_doc(doc_id=file_name, contents=contents)
+
+    def __len__(self):
+        return len(self.__file_names)

arekit/contrib/utils/data/readers/base.py

@@ -0,0 +1,7 @@
+class BaseReader(object):
+
+    def extension(self):
+        raise NotImplementedError()
+
+    def read(self, target):
+        raise NotImplementedError()

arekit/contrib/utils/data/readers/csv_pd.py

@@ -0,0 +1,38 @@
+import importlib
+
+from arekit.contrib.utils.data.readers.base import BaseReader
+from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
+
+
+class PandasCsvReader(BaseReader):
+    """ Represents a CSV-based reader, implmented via pandas API.
+    """
+
+    def __init__(self, sep='\t', header='infer', compression='infer', encoding='utf-8', col_types=None,
+                 custom_extension=None):
+        self.__sep = sep
+        self.__compression = compression
+        self.__encoding = encoding
+        self.__header = header
+        self.__custom_extension = custom_extension
+
+        # Special assignation of types for certain columns.
+        self.__col_types = col_types
+        if self.__col_types is None:
+            self.__col_types = dict()
+
+    def extension(self):
+        return ".tsv.gz" if self.__custom_extension is None else self.__custom_extension
+
+    def __from_csv(self, filepath):
+        pd = importlib.import_module("pandas")
+        return pd.read_csv(filepath,
+                           sep=self.__sep,
+                           encoding=self.__encoding,
+                           compression=self.__compression,
+                           dtype=self.__col_types,
+                           header=self.__header)
+
+    def read(self, target):
+        df = self.__from_csv(filepath=target)
+        return PandasBasedRowsStorage(df)

arekit/contrib/utils/data/readers/jsonl.py

@@ -0,0 +1,15 @@
+from arekit.contrib.utils.data.readers.base import BaseReader
+from arekit.contrib.utils.data.storages.jsonl_based import JsonlBasedRowsStorage
+
+
+class JsonlReader(BaseReader):
+
+    def extension(self):
+        return ".jsonl"
+
+    def read(self, target):
+        rows = []
+        with open(target, "r") as f:
+            for line in f.readlines():
+                rows.append(line)
+        return JsonlBasedRowsStorage(rows)

arekit/contrib/utils/data/service/balance.py

@@ -0,0 +1,50 @@
+import gc
+import importlib
+from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
+
+
+class PandasBasedStorageBalancing(object):
+
+    @staticmethod
+    def create_balanced_from(storage, column_name, free_origin=True):
+        """ Performs oversampled balancing.
+
+            Note: it is quite important to remove previously created storage
+            in order to avoid memory leaking.
+
+            storage: PandasBasedRowsStorage
+                storage contents to be balanced.
+
+            column_name: str
+                column utilized for balancing.
+
+            free_origin: bool
+                indicates whether there is a need to release the resources
+                utilized for the original storage.
+        """
+        assert(isinstance(storage, PandasBasedRowsStorage))
+
+        original_df = storage.DataFrame
+
+        max_size = original_df[column_name].value_counts().max()
+
+        dframes = []
+        for class_index, group in original_df.groupby(column_name):
+            dframes.append(group.sample(max_size - len(group), replace=True))
+
+        # Clear resources.
+        pd = importlib.import_module("pandas")
+        balanced_df = pd.concat(dframes + [original_df])
+
+        # Removing temporary created dataframe.
+        for df in dframes:
+            del df
+
+        # Marking the original dataframe as released
+        # in terms of the allocated memory for it.
+        if free_origin:
+            storage.free()
+
+        gc.collect()
+
+        return PandasBasedRowsStorage(df=balanced_df)

arekit/contrib/utils/data/storages/jsonl_based.py

@@ -0,0 +1,18 @@
+import json
+
+from arekit.common.data.storages.base import BaseRowsStorage
+
+
+class JsonlBasedRowsStorage(BaseRowsStorage):
+
+    def __init__(self, rows):
+        assert(isinstance(rows, list))
+        self.__rows = rows
+
+    def _iter_rows(self):
+        for row_index, row in enumerate(self.__rows):
+            assert(isinstance(row, str))
+            yield row_index, json.loads(row)
+
+    def _get_rows_count(self):
+        return len(self.__rows)