arekit 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/source/sentinerel/io_utils.py

@@ -1,87 +0,0 @@
-from enum import Enum
-from os import path
-from os.path import basename, join
-
-import enum
-
-from arekit.contrib.source.sentinerel.folding.factory import SentiNERELFoldingFactory
-from arekit.contrib.source.zip_utils import ZipArchiveUtils
-
-
-class SentiNerelVersions(Enum):
-    """ List of the supported version of this collection
-    """
-
-    # Initial version.
-    V1 = "v1_0"
-    # Updated annotation within the second half of the texts. (September 2022)
-    V2 = "v2_0"
-    # Updated annotation within the first half of the texts.  (October 2022)
-    # Become a source of the RuSentNE-2023 competition.
-    # https://github.com/dialogue-evaluation/RuSentNE-evaluation
-    V21 = "v2_1"
-
-
-DEFAULT_VERSION = SentiNerelVersions.V21
-
-
-class SentiNerelIOUtils(ZipArchiveUtils):
-
-    inner_root = "sentiment_dataset"
-
-    @staticmethod
-    def get_archive_filepath(version):
-        return path.join(SentiNerelIOUtils.get_data_root(), "sentinerel-{}.zip".format(version))
-
-    @staticmethod
-    def get_annotation_innerpath(filename):
-        assert(isinstance(filename, str))
-        return path.join(SentiNerelIOUtils.inner_root, "{}.ann".format(filename))
-
-    @staticmethod
-    def get_doc_innerpath(filename):
-        assert(isinstance(filename, str))
-        return path.join(SentiNerelIOUtils.inner_root, "{}.txt".format(filename))
-
-    @staticmethod
-    def __iter_filenames_from_dataset(folder_name, version):
-        assert(isinstance(version, enum.Enum))
-        assert(isinstance(folder_name, str))
-
-        for filename in SentiNerelIOUtils.iter_filenames_from_zip(version):
-
-            extension = filename[-4:]
-
-            # Crop extension.
-            filename = filename[:-4]
-
-            if extension != ".txt":
-                continue
-
-            if not folder_name in filename:
-                continue
-
-            yield basename(filename)
-
-    # region public methods
-
-    @staticmethod
-    def iter_collection_filenames(version=DEFAULT_VERSION):
-        filenames_it = SentiNerelIOUtils.__iter_filenames_from_dataset(
-            folder_name=SentiNerelIOUtils.inner_root, version=version)
-
-        for doc_id, filename in enumerate(filenames_it):
-            yield doc_id, filename
-
-    @staticmethod
-    def read_dataset_split(version=DEFAULT_VERSION, docs_limit=None):
-        """ Provides a fixed split of the dataset onto
-            `test` and `training` part:
-            https://github.com/nicolay-r/SentiNEREL-attitude-extraction
-        """
-        return SentiNerelIOUtils.read_from_zip(
-            inner_path=join(SentiNerelIOUtils.inner_root, "split_fixed.txt"),
-            process_func=lambda f: SentiNERELFoldingFactory.create_fixed_folding(file=f, limit=docs_limit),
-            version=version)
-
-    # endregion

arekit/contrib/source/sentinerel/labels.py

@@ -1,53 +0,0 @@
-from arekit.common.labels.base import Label
-
-
-class OpinionBelongsTo(Label):
-    pass
-
-
-class OpinionRelatesTo(Label):
-    pass
-
-
-class NegEffectFrom(Label):
-    pass
-
-
-class NegStateFrom(Label):
-    pass
-
-
-class PosEffectFrom(Label):
-    pass
-
-
-class PosAuthorFrom(Label):
-    pass
-
-
-class NegAuthorFrom(Label):
-    pass
-
-
-class PosStateFrom(Label):
-    pass
-
-
-class NegativeTo(Label):
-    pass
-
-
-class PositiveTo(Label):
-    pass
-
-
-class AlternativeName(Label):
-    pass
-
-
-class StateBelongsTo(Label):
-    pass
-
-
-class OriginsFrom(Label):
-    pass

arekit/contrib/source/sentinerel/labels_scaler.py

@@ -1,30 +0,0 @@
-from collections import OrderedDict
-
-from arekit.common.labels.scaler.base import BaseLabelScaler
-from arekit.contrib.source.sentinerel import labels
-
-
-class SentiNerelLabelScaler(BaseLabelScaler):
-    """ This is a complete label scaler of all the labels supported by NEREL dataset.
-    """
-
-    def __init__(self):
-
-        self.__uint_to_label_dict = OrderedDict([
-            (labels.OpinionBelongsTo(), 0),
-            (labels.OpinionRelatesTo(), 1),
-            (labels.NegEffectFrom(), 2),
-            (labels.PosEffectFrom(), 3),
-            (labels.NegStateFrom(), 4),
-            (labels.PosStateFrom(), 5),
-            (labels.NegativeTo(), 6),
-            (labels.PositiveTo(), 7),
-            (labels.StateBelongsTo(), 8),
-            (labels.PosAuthorFrom(), 9),
-            (labels.NegAuthorFrom(), 10),
-            (labels.AlternativeName(), 11),
-            (labels.OriginsFrom(), 12)
-        ])
-
-        super(SentiNerelLabelScaler, self).__init__(int_dict=self.__uint_to_label_dict,
-                                                    uint_dict=self.__uint_to_label_dict)

arekit/contrib/source/sentinerel/reader.py

@@ -1,42 +0,0 @@
-from arekit.contrib.source.brat.annot import BratAnnotationParser
-from arekit.contrib.source.brat.doc import BratDocument
-from arekit.contrib.source.brat.sentences_reader import BratDocumentSentencesReader
-from arekit.contrib.source.sentinerel.entities import SentiNerelEntityCollection
-from arekit.contrib.source.sentinerel.io_utils import SentiNerelIOUtils, DEFAULT_VERSION
-
-
-class SentiNerelDocReader(object):
-
-    @staticmethod
-    def read_text_relations(filename, version):
-        assert(isinstance(filename, str))
-
-        return SentiNerelIOUtils.read_from_zip(
-            inner_path=SentiNerelIOUtils.get_annotation_innerpath(filename),
-            process_func=lambda input_file: [
-                relation for relation in BratAnnotationParser.parse_annotations(
-                    input_file=input_file, encoding='utf-8-sig')["relations"]],
-            version=version)
-
-    @staticmethod
-    def read_document(filename, doc_id, version=DEFAULT_VERSION, entities_to_ignore=None):
-        assert(isinstance(filename, str))
-        assert(isinstance(doc_id, int))
-
-        def file_to_doc(input_file):
-            sentences = BratDocumentSentencesReader.from_file(input_file=input_file, entities=entities)
-            return BratDocument(doc_id=doc_id, sentences=sentences, text_relations=text_relations)
-
-        # TODO. #398 issue -- in some cases entities might be nested. Therefore we limit the set
-        # TODO. of the potential named entities.
-        eti = ["EFFECT_NEG", "EFFECT_POS", "ARGUMENT_NEG", "ARGUMENT_POS", "EVENT"] \
-            if entities_to_ignore is None else entities_to_ignore
-
-        entities = SentiNerelEntityCollection.read_collection(
-            filename=filename, version=version, entities_to_ignore=eti)
-        text_relations = SentiNerelDocReader.read_text_relations(filename=filename, version=version)
-
-        return SentiNerelIOUtils.read_from_zip(
-            inner_path=SentiNerelIOUtils.get_doc_innerpath(filename=filename),
-            process_func=file_to_doc,
-            version=version)

arekit/contrib/source/synonyms/utils.py

@@ -1,19 +0,0 @@
-from arekit.common.utils import progress_bar_defined
-
-
-def iter_synonym_groups(input_file, sep=",", desc=""):
-    """ All the synonyms groups organized in lines, separated by `sep`
-    """
-    lines = input_file.readlines()
-
-    lines_it = progress_bar_defined(lines,
-                                    total=len(lines),
-                                    desc=desc,
-                                    unit="opins")
-
-    for line in lines_it:
-
-        if isinstance(line, bytes):
-            line = line.decode()
-
-        yield line.split(sep)

arekit/contrib/source/zip_utils.py

@@ -1,47 +0,0 @@
-import zipfile
-
-import enum
-
-from arekit.common import utils
-
-
-class ZipArchiveUtils(object):
-
-    @staticmethod
-    def get_archive_filepath(version):
-        raise NotImplementedError()
-
-    @classmethod
-    def read_from_zip(cls, inner_path, process_func, version):
-        """
-        process_func:
-            func which receives a file reader
-        """
-        assert(isinstance(inner_path, str))
-        assert(callable(process_func))
-        assert(isinstance(version, enum.Enum))
-
-        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
-            with zip_ref.open(inner_path, mode='r') as c_file:
-                return process_func(c_file)
-
-    @classmethod
-    def iter_from_zip(cls, inner_path, process_func, version):
-        assert(isinstance(inner_path, str))
-        assert(callable(process_func))
-        assert(isinstance(version, enum.Enum))
-
-        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
-            with zip_ref.open(inner_path, mode='r') as c_file:
-                for result in process_func(c_file):
-                    yield result
-
-    @classmethod
-    def iter_filenames_from_zip(cls, version):
-        assert(isinstance(version, enum.Enum))
-        with zipfile.ZipFile(cls.get_archive_filepath(version.value), "r") as zip_ref:
-            return iter(zip_ref.namelist())
-
-    @staticmethod
-    def get_data_root():
-        return utils.get_default_download_dir()

arekit/contrib/utils/connotations/rusentiframes_sentiment.py

@@ -1,23 +0,0 @@
-from arekit.common.frames.connotations.provider import FrameConnotationProvider
-from arekit.contrib.source.rusentiframes.collection import RuSentiFramesCollection
-
-
-class RuSentiFramesConnotationProvider(FrameConnotationProvider):
-    """ This is a provider based on A0->A1 label type of RuSentiFrames collection.
-        For a greater details, checkout the related collection at:
-        https://github.com/nicolay-r/RuSentiFrames
-
-        Papers:
-            [1] Natalia Loukachevitch, Nicolay Rusnachenko: Sentiment Frames
-                for Attitude Extraction in Russian, 2020
-            [2] Distant Supervision for Sentiment Attitude Extraction, 2019
-    """
-
-    def __init__(self, collection):
-        assert(isinstance(collection, RuSentiFramesCollection))
-        self.__collection = collection
-
-    def try_provide(self, frame_id):
-        return self.__collection.try_get_frame_polarity(frame_id=frame_id,
-                                                        role_src='a0',
-                                                        role_dest='a1')

arekit/contrib/utils/download.py

@@ -1,77 +0,0 @@
-import os
-import tarfile
-from os.path import join, exists
-
-from arekit.common import utils
-from arekit.contrib.utils.np_utils.embedding import NpzEmbeddingHelper
-from arekit.contrib.utils.np_utils.vocab import VocabRepositoryUtils
-
-NEWS_MYSTEM_SKIPGRAM_1000_20_2015 = "news_mystem_skipgram_1000_20_2015.tar.gz"
-
-
-def __get_resource(local_name, check_existance=False, download_if_missed=False):
-    assert (isinstance(local_name, str))
-    filepath = join(utils.get_default_download_dir(), local_name)
-
-    if check_existance and not exists(filepath):
-        if download_if_missed:
-            download()
-            # We try to ger the resource again but won't attempt to download it again.
-            __get_resource(local_name, check_existance=check_existance, download_if_missed=False)
-        else:
-            raise Exception("Resource could not be found: {}".format(filepath))
-
-    return filepath
-
-
-def __get_embedding_dir(filepath):
-    return filepath.replace(".tar.gz", "")
-
-
-def load_embedding_and_vocab(local_name, check_existance=False, download_if_missed=False):
-    tar_gz_archive = __get_resource(local_name, check_existance=check_existance,
-                                    download_if_missed=download_if_missed)
-    target_dir = __get_embedding_dir(tar_gz_archive)
-    embedding = NpzEmbeddingHelper.load_embedding(os.path.join(target_dir, "embedding.npz"))
-    vocab = VocabRepositoryUtils.load(os.path.join(target_dir, "vocab.txt"))
-    return embedding, vocab
-
-
-def download():
-    data = {
-        NEWS_MYSTEM_SKIPGRAM_1000_20_2015: "https://www.dropbox.com/s/0omnlgzgnjhxlmf/{filename}?dl=1".format(
-            filename=NEWS_MYSTEM_SKIPGRAM_1000_20_2015),
-    }
-
-    # Perform downloading ...
-    for local_name, url_link in data.items():
-        utils.download(dest_file_path=__get_resource(local_name),
-                       source_url=url_link)
-
-    # Untar files ...
-    for local_name in data.keys():
-
-        if ".tar.gz" not in local_name:
-            continue
-
-        target_filepath = __get_resource(local_name)
-        with tarfile.open(target_filepath) as file:
-            def is_within_directory(directory, target):
-
-                abs_directory = os.path.abspath(directory)
-                abs_target = os.path.abspath(target)
-
-                prefix = os.path.commonprefix([abs_directory, abs_target])
-
-                return prefix == abs_directory
-
-            def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
-
-                for member in tar.getmembers():
-                    member_path = os.path.join(path, member.name)
-                    if not is_within_directory(path, member_path):
-                        raise Exception("Attempted Path Traversal in Tar File")
-
-                tar.extractall(path, members, numeric_owner=numeric_owner)
-
-            safe_extract(file, __get_embedding_dir(target_filepath))

arekit/contrib/utils/io_utils/opinions.py

@@ -1,37 +0,0 @@
-from os.path import join
-
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.contrib.utils.io_utils.utils import filename_template
-
-
-class OpinionsIO(BaseSamplesIO):
-
-    def __init__(self, target_dir, reader=None, prefix="opinion"):
-        assert(isinstance(reader, BaseReader))
-        self.__target_dir = target_dir
-        self.__prefix = prefix
-        self.__reader = reader
-        self.__target_extension = reader.extension()
-
-    @property
-    def Reader(self):
-        return self.__reader
-
-    def create_target(self, data_type):
-        return self.__get_input_opinions_target(data_type)
-
-    def __get_input_opinions_target(self, data_type):
-        template = filename_template(data_type=data_type)
-        return self.__get_filepath(out_dir=self.__target_dir,
-                                   template=template,
-                                   prefix=self.__prefix,
-                                   extension=self.__target_extension)
-
-    @staticmethod
-    def __get_filepath(out_dir, template, prefix, extension):
-        assert(isinstance(template, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(extension, str))
-        return join(out_dir, "{prefix}-{template}{extension}".format(
-            prefix=prefix, template=template, extension=extension))

arekit/contrib/utils/io_utils/samples.py

@@ -1,79 +0,0 @@
-import logging
-from os.path import join
-
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.contrib.utils.data.writers.base import BaseWriter
-from arekit.contrib.utils.io_utils.utils import filename_template, check_targets_existence
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class SamplesIO(BaseSamplesIO):
-    """ Samples default IO utils for samples.
-            Sample is a text part which include pair of attitude participants.
-            This class allows to provide saver and loader for such entries, bubbed as samples.
-            Samples required for machine learning training/inferring.
-    """
-
-    def __init__(self, target_dir, writer=None, reader=None, prefix="sample"):
-        assert(isinstance(target_dir, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(writer, BaseWriter) or writer is None)
-        assert(isinstance(reader, BaseReader) or reader is None)
-        self.__target_dir = target_dir
-        self.__prefix = prefix
-        self.__writer = writer
-        self.__reader = reader
-
-        self.__target_extension = None
-        if writer is not None:
-            self.__target_extension = writer.extension()
-        elif reader is not None:
-            self.__target_extension = reader.extension()
-
-    # region public methods
-
-    @property
-    def Prefix(self):
-        return self.__prefix
-
-    @property
-    def Reader(self):
-        return self.__reader
-
-    @property
-    def Writer(self):
-        return self.__writer
-
-    def create_target(self, data_type):
-        return self.__get_input_sample_target(data_type)
-
-    def check_targets_existed(self, data_types_iter):
-        for data_type in data_types_iter:
-
-            targets = [
-                self.__get_input_sample_target(data_type=data_type),
-            ]
-
-            if not check_targets_existence(targets=targets):
-                return False
-        return True
-
-    # endregion
-
-    def __get_input_sample_target(self, data_type):
-        template = filename_template(data_type=data_type)
-        return self.__get_filepath(out_dir=self.__target_dir,
-                                   template=template,
-                                   prefix=self.__prefix,
-                                   extension=self.__target_extension)
-
-    @staticmethod
-    def __get_filepath(out_dir, template, prefix, extension):
-        assert(isinstance(template, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(extension, str))
-        return join(out_dir, "{prefix}-{template}{extension}".format(
-            prefix=prefix, template=template, extension=extension))

arekit/contrib/utils/lexicons/lexicon.py

@@ -1,41 +0,0 @@
-from arekit.contrib.utils.data.readers.csv_pd import PandasCsvReader
-
-
-class Lexicon(object):
-
-    @property
-    def ToneKey(self):
-        return 'tone'
-
-    @property
-    def TermKey(self):
-        return 'term'
-
-    def __init__(self, dataframe):
-        self.__lexicon_df = dataframe
-
-    @classmethod
-    def load(cls, filepath, separator=','):
-        reader = PandasCsvReader(compression=None, sep=separator)
-        return cls(reader.read(filepath))
-
-    def get_score(self, lemma):
-        assert(type(lemma) == str)
-        s = self.__lexicon_df[lemma.encode('utf-8') == self.__lexicon_df[self.TermKey]]
-        return s[self.ToneKey].values[0] if len(s) > 0 else 0
-
-    def has_term(self, term):
-        assert(type(term) == str)
-        s = self.__lexicon_df[term.encode('utf-8') == self.__lexicon_df[self.TermKey]]
-        return len(s) > 0
-
-    def __iter__(self):
-        for term in self.__lexicon_df[self.TermKey]:
-            yield term
-
-    def __contains__(self, item):
-        assert(isinstance(item, str))
-        result = self.__lexicon_df[self.__lexicon_df[self.TermKey] == item.encode('utf-8')]
-        return len(result) > 0
-
-

arekit/contrib/utils/lexicons/relation.py

@@ -1,42 +0,0 @@
-from arekit.contrib.utils.data.readers.csv_pd import PandasCsvReader
-
-
-class RelationLexicon(object):
-
-    def __init__(self, dataframe):
-        self.__check(dataframe)
-        self.__lexicon = dataframe
-
-    @classmethod
-    def load(cls, filepath, separator=','):
-        reader = PandasCsvReader(compression=None, sep=separator)
-        return cls(reader.read(filepath))
-
-    @staticmethod
-    def __check(df):
-        for index in df.index:
-            relation = df.loc[index][0]
-            assert(len(relation.split('<->')) == 2)
-
-    @staticmethod
-    def __create_key(l, r):
-        assert(type(l) == str)
-        assert(type(r) == str)
-        return '<->'.join([l, r])
-
-    def get_score(self, left, right):
-        assert(type(left) == str)
-        assert(type(right) == str)
-
-        lr_key = self.__create_key(left, right)
-        rl_key = self.__create_key(right, left)
-
-        lr_score = self.__lexicon[lr_key == self.__lexicon['relation']]
-        rl_score = self.__lexicon[rl_key == self.__lexicon['relation']]
-
-        if len(lr_score) > 0:
-            return lr_score['tone'].values[0]
-        if len(rl_score) > 0:
-            return rl_score['tone'].values[0]
-
-        return None

arekit/contrib/utils/lexicons/rusentilex.py

@@ -1,37 +0,0 @@
-import importlib
-import zipfile
-from os import path
-
-
-from arekit.contrib.source.zip_utils import ZipArchiveUtils
-from arekit.contrib.utils.lexicons.lexicon import Lexicon
-
-
-class RuSentiLexLexicon(Lexicon):
-    """
-    RuSentiLex Lexicon wrapper for csv file stored in /data folder.
-    """
-
-    __INNER_PATH = 'rusentilex.csv'
-
-    @property
-    def ToneKey(self):
-        return 'tone'
-
-    @property
-    def TermKey(self):
-        return 'term'
-
-    @staticmethod
-    def __get_archive_filepath():
-        return path.join(ZipArchiveUtils.get_data_root(), "rusentilex.zip")
-
-    @classmethod
-    def from_zip(cls):
-        """ Using Pandas API to read lexicon.
-        """
-        pd = importlib.import_module("pandas")
-        with zipfile.ZipFile(cls.__get_archive_filepath(), "r") as zip_ref:
-            with zip_ref.open(cls.__INNER_PATH, mode='r') as csv_file:
-                df = pd.read_csv(csv_file, sep=',')
-                return cls(df)