arekit 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/utils/cv/splitters/default.py

@@ -1,53 +0,0 @@
-import random
-
-from arekit.contrib.utils.cv.splitters.base import CrossValidationSplitter
-
-
-class SimpleCrossValidationSplitter(CrossValidationSplitter):
-    """ This splitter assumes to performs folding
-        without extra additional statistics of the related documents.
-    """
-
-    def __init__(self, shuffle=True, seed=1):
-        self.__shuffle = shuffle
-        self.__seed = seed
-
-    # region private methods
-
-    @staticmethod
-    def __chunk_it(sequence, num):
-        avg = len(sequence) / float(num)
-        out = []
-        last = 0.0
-
-        while last < len(sequence):
-            out.append(sequence[int(last):int(last + avg)])
-            last += avg
-
-        return out
-
-    # endregion
-
-    def items_to_cv_pairs(self, doc_ids, cv_count):
-        """
-        Splits array of indices into list of pairs (train_indices_list,
-        test_indices_list)
-        """
-        assert(isinstance(doc_ids, set))
-        assert(isinstance(cv_count, int))
-
-        doc_ids_list = list(doc_ids)
-
-        if self.__shuffle:
-            random.Random(self.__seed).shuffle(doc_ids_list)
-
-        chunks = self.__chunk_it(doc_ids_list, cv_count)
-
-        for test_index, chunk in enumerate(chunks):
-            train_indices = list(range(len(chunks)))
-            train_indices.remove(test_index)
-
-            large = [v for train_index in train_indices for v in chunks[train_index]]
-            small = chunk
-
-            yield large, small

arekit/contrib/utils/cv/splitters/statistical.py

@@ -1,57 +0,0 @@
-import numpy as np
-from arekit.contrib.utils.cv.doc_stat.base import BaseDocumentStatGenerator
-from arekit.contrib.utils.cv.splitters.base import CrossValidationSplitter
-
-
-class StatBasedCrossValidationSplitter(CrossValidationSplitter):
-    """ Sentence-based splitter.
-    """
-
-    def __init__(self, docs_stat, doc_ids):
-        assert(isinstance(docs_stat, BaseDocumentStatGenerator))
-        super(StatBasedCrossValidationSplitter, self).__init__()
-        self.__docs_info = docs_stat.calculate(doc_ids_iter=doc_ids)
-
-    # region private methods
-
-    @staticmethod
-    def __select_group(cv_group_size, item):
-        deltas = []
-        for group_index in range(len(cv_group_size)):
-            delta = StatBasedCrossValidationSplitter.__calc_cv_group_delta(
-                cv_group_size=cv_group_size, item=item, g_index_to_add=group_index)
-            deltas.append(delta)
-
-        return int(np.argmin(deltas))
-
-    @staticmethod
-    def __calc_cv_group_delta(cv_group_size, item, g_index_to_add):
-        sums = []
-        for i in range(len(cv_group_size)):
-            sums.append(sum(cv_group_size[i]))
-
-        sums[g_index_to_add] += item
-        return max(sums) - np.mean(sums)
-
-    # endregion
-
-    def items_to_cv_pairs(self, doc_ids, cv_count):
-        """ Separation with the specific separation, in terms of cv-classes size difference.
-        """
-        assert(isinstance(doc_ids, set))
-        assert(isinstance(cv_count, int))
-
-        sorted_stat = reversed(sorted(self.__docs_info, key=lambda pair: pair[1]))
-        cv_group_docs = [[] for _ in range(cv_count)]
-        cv_group_sizes = [[] for _ in range(cv_count)]
-
-        for doc_id, s_count in sorted_stat:
-            group_index = self.__select_group(cv_group_size=cv_group_sizes, item=s_count)
-            cv_group_docs[group_index].append(doc_id)
-            cv_group_sizes[group_index].append(s_count)
-
-        for g_index in range(len(cv_group_docs)):
-            small = cv_group_docs[g_index]
-            large = [doc_id for doc_id, _ in self.__docs_info if doc_id not in small]
-
-            yield large, small

arekit/contrib/utils/cv/two_class.py

@@ -1,77 +0,0 @@
-from arekit.common.folding.base import BaseDataFolding
-from arekit.contrib.utils.cv.splitters.base import CrossValidationSplitter
-
-
-class TwoClassCVFolding(BaseDataFolding):
-    """ Performs folding onto a pair of data_types,
-        i.e. two-class cv-folding algorithm
-    """
-
-    def __init__(self, supported_data_types, doc_ids_to_fold, cv_count, splitter):
-        assert(isinstance(splitter, CrossValidationSplitter))
-        assert(isinstance(cv_count, int) and cv_count > 0)
-
-        if len(supported_data_types) > 2:
-            raise NotImplementedError("Experiments with such amount of data-types are not supported!")
-
-        super(TwoClassCVFolding, self).__init__(doc_ids_to_fold=doc_ids_to_fold,
-                                                supported_data_types=supported_data_types)
-
-        self.__cv_count = cv_count
-        self.__splitter = splitter
-        self.__state_index = 0
-
-    # region Properties
-
-    @property
-    def StateIndex(self):
-        return self.__state_index
-
-    @property
-    def CVCount(self):
-        return self.__cv_count
-
-    # endregion
-
-    def __assign_index(self, i):
-        self.__state_index = i
-
-    # region BaseFolding
-
-    def iter_states(self):
-        """ Performs iteration over states supported by folding algorithm
-            Default:
-                considering a single state.
-        """
-        for state_index in range(self.__cv_count):
-            self.__assign_index(state_index)
-            yield None
-
-    def fold_doc_ids_set(self):
-
-        # Access to protected fields
-        data_types = self._supported_data_types
-        doc_ids = self._doc_ids_to_fold_set
-
-        if len(data_types) == 1:
-            # By default we provide the same output since
-            # there is no need to perform splitting onto single part
-            return {
-                data_types[0]: list(doc_ids)
-            }
-
-        if self.__splitter is None:
-            raise NotImplementedError("Splitter has not been intialized!")
-
-        it = self.__splitter.items_to_cv_pairs(doc_ids=set(doc_ids),
-                                               cv_count=self.__cv_count)
-
-        for index, pair in enumerate(it):
-            large, small = pair
-            if index == self.__state_index:
-                return {
-                    data_types[0]: large,
-                    data_types[1]: small
-                }
-
-    # endregion

arekit/contrib/utils/data/doc_ops/dict_based.py

@@ -1,13 +0,0 @@
-from arekit.common.experiment.api.ops_doc import DocumentOperations
-
-
-class DictionaryBasedDocumentOperations(DocumentOperations):
-
-    def __init__(self, d):
-        assert(isinstance(d, dict))
-        super(DictionaryBasedDocumentOperations, self).__init__()
-        self.__d = d
-
-    def by_id(self, doc_id):
-        assert(isinstance(doc_id, int))
-        return self.__d[doc_id]

arekit/contrib/utils/data/ext.py

@@ -1,31 +0,0 @@
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.contrib.utils.data.writers.base import BaseWriter
-from arekit.contrib.utils.data.writers.csv_native import NativeCsvWriter
-from arekit.contrib.utils.data.writers.json_opennre import OpenNREJsonWriter
-
-
-PANDAS_CSV_EXTENSION = ".tsv.gz"
-OPENNRE_EXTENSION = ".jsonl"
-
-
-def create_writer_extension(writer):
-    assert(isinstance(writer, BaseWriter))
-
-    if isinstance(writer, OpenNREJsonWriter):
-        return OPENNRE_EXTENSION
-    if isinstance(writer, NativeCsvWriter):
-        return ".csv"
-    else:
-        # consider ".tsv.gz" and assuming it is a Pandas.
-        return PANDAS_CSV_EXTENSION
-
-
-def create_reader_extension(writer):
-    assert(isinstance(writer, BaseReader))
-
-    if isinstance(writer, OpenNREJsonWriter):
-        return OPENNRE_EXTENSION
-    else:
-        # consider ".tsv.gz" and assuming it is a Pandas.
-        # other options are not available in 0.23.1
-        return PANDAS_CSV_EXTENSION

arekit/contrib/utils/data/views/linkages/base.py

@@ -1,58 +0,0 @@
-from arekit.common.data import const
-from arekit.common.data.row_ids.base import BaseIDProvider
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.common.linkage.opinions import OpinionsLinkage
-from arekit.contrib.utils.data.views.linkages import utils
-from arekit.contrib.utils.data.views.opinions import BaseOpinionStorageView
-
-
-class BaseOpinionLinkagesView(object):
-    """ Base view onto source in terms of opinion linkages.
-    """
-
-    def __init__(self, ids_provider, storage):
-        assert(isinstance(ids_provider, BaseIDProvider))
-        assert(isinstance(storage, BaseRowsStorage))
-        self._ids_provider = ids_provider
-        self._storage = storage
-
-    # region private methods
-
-    def __iter_doc_opinion_ids(self, row_ids):
-        for row_id in row_ids:
-            yield self._ids_provider.parse_opinion_in_opinion_id(row_id)
-
-    def __iter_opinions_by_linkages(self, linkages_df, opinions_view):
-        for df_linkage in linkages_df:
-            yield self._iter_by_opinions(linked_df=df_linkage, opinions_view=opinions_view)
-
-    # endregion
-
-    # region protected methods
-
-    def _iter_by_opinions(self, linked_df, opinions_view):
-        raise NotImplementedError()
-
-    # endregion
-
-    # region public methods
-
-    def iter_opinion_linkages(self, doc_id, opinions_view):
-        assert(isinstance(opinions_view, BaseOpinionStorageView))
-        doc_df = self._storage.find_by_value(column_name=const.DOC_ID, value=doc_id)
-        row_ids = [row_id for row_id in doc_df[const.ID]]  # TODO. Adopt storage.
-        doc_opin_ids = self.__iter_doc_opinion_ids(row_ids=row_ids)
-
-        doc_opin_id_patterns = map(
-            lambda opinion_id: self._ids_provider.create_pattern(id_value=opinion_id, p_type=BaseIDProvider.OPINION),
-            doc_opin_ids)
-
-        linkages_df = map(
-            lambda opin_id: utils.filter_by_id(doc_df=doc_df, column=const.ID, value=opin_id),
-            doc_opin_id_patterns)
-
-        opinions_iter = self.__iter_opinions_by_linkages(linkages_df, opinions_view=opinions_view)
-
-        return map(lambda opinions: OpinionsLinkage(opinions), opinions_iter)
-
-    # endregion

arekit/contrib/utils/data/views/linkages/multilabel.py

@@ -1,48 +0,0 @@
-import numpy as np
-
-from arekit.common.data import const
-from arekit.common.data.row_ids.multiple import MultipleIDProvider
-from arekit.common.labels.scaler.base import BaseLabelScaler
-from arekit.contrib.utils.data.views.linkages import utils
-from arekit.contrib.utils.data.views.linkages.base import BaseOpinionLinkagesView
-from arekit.contrib.utils.data.views.opinions import BaseOpinionStorageView
-
-
-class MultilableOpinionLinkagesView(BaseOpinionLinkagesView):
-    """ View onto source, where each row, related to opinion, has multiple labels.
-    """
-
-    def __init__(self, labels_scaler, storage):
-        assert(isinstance(labels_scaler, BaseLabelScaler))
-        super(MultilableOpinionLinkagesView, self).__init__(ids_provider=MultipleIDProvider(),
-                                                            storage=storage)
-        self.__labels_scaler = labels_scaler
-
-    # region private methods
-
-    def __get_column_header(self):
-        return [str(self.__labels_scaler.label_to_uint(label))
-                for label in self.__labels_scaler.ordered_suppoted_labels()]
-
-    def __calculate_label(self, row):
-        """
-        Using a single row (probabilities by each class)
-        """
-        labels_prob = [row[label] for label in self.__get_column_header()]
-        return self.__labels_scaler.uint_to_label(value=int(np.argmax(labels_prob)))
-
-    # endregion
-
-    # region protected methods
-
-    def _iter_by_opinions(self, linked_df, opinions_view):
-        assert(isinstance(opinions_view, BaseOpinionStorageView))
-
-        for _, series in linked_df.iterrows():
-            yield utils.compose_opinion_by_opinion_id(
-                ids_provider=self._ids_provider,
-                sample_id=series[const.ID],
-                opinions_view=opinions_view,
-                calc_label_func=lambda: self.__calculate_label(series))
-
-    # endregion

arekit/contrib/utils/data/views/linkages/utils.py

@@ -1,24 +0,0 @@
-from arekit.common.data import const
-from arekit.common.data.row_ids.base import BaseIDProvider
-from arekit.common.opinions.base import Opinion
-from arekit.contrib.utils.data.views.opinions import BaseOpinionStorageView
-
-
-def compose_opinion_by_opinion_id(ids_provider, sample_id, opinions_view, calc_label_func):
-    assert(isinstance(ids_provider, BaseIDProvider))
-    assert(isinstance(sample_id, str))
-    assert(isinstance(opinions_view, BaseOpinionStorageView))
-    assert(callable(calc_label_func))
-
-    opinion_id = ids_provider.convert_sample_id_to_opinion_id(sample_id=sample_id)
-    row = opinions_view.row_by_id(opinion_id=opinion_id)
-
-    return Opinion(source_value=row[const.SOURCE],
-                   target_value=row[const.TARGET],
-                   sentiment=calc_label_func())
-
-
-# TODO. Adopt storage.
-def filter_by_id(doc_df, column, value):
-    assert(isinstance(column, str))
-    return doc_df[doc_df[column].str.contains(value)]

arekit/contrib/utils/data/views/opinions.py

@@ -1,14 +0,0 @@
-from arekit.common.data import const
-from arekit.common.data.storages.base import BaseRowsStorage
-
-
-class BaseOpinionStorageView(object):
-
-    def __init__(self, storage):
-        assert(isinstance(storage, BaseRowsStorage))
-        self._storage = storage
-
-    def row_by_id(self, opinion_id):
-        assert(isinstance(opinion_id, str))
-        return self._storage.find_first_by_value(column_name=const.ID,
-                                                 value=opinion_id)

arekit/contrib/utils/download.py

@@ -1,78 +0,0 @@
-import os
-import tarfile
-from os.path import join, exists
-
-from arekit.common import utils
-from arekit.contrib.utils.np_utils.embedding import NpzEmbeddingHelper
-from arekit.contrib.utils.np_utils.vocab import VocabRepositoryUtils
-
-NEWS_MYSTEM_SKIPGRAM_1000_20_2015 = "news_mystem_skipgram_1000_20_2015.tar.gz"
-
-
-def __get_resource(local_name, check_existance=False, download_if_missed=False):
-    assert(isinstance(local_name, str))
-    filepath = join(utils.get_default_download_dir(), local_name)
-
-    if check_existance and not exists(filepath):
-        if download_if_missed:
-            download()
-            # We try to ger the resource again but won't attempt to download it again.
-            __get_resource(local_name, check_existance=check_existance, download_if_missed=False)
-        else:
-            raise Exception("Resource could not be found: {}".format(filepath))
-
-    return filepath
-
-
-def __get_embedding_dir(filepath):
-    return filepath.replace(".tar.gz", "")
-
-
-def load_embedding_and_vocab(local_name, check_existance=False, download_if_missed=False):
-    tar_gz_archive = __get_resource(local_name, check_existance=check_existance,
-                                    download_if_missed=download_if_missed)
-    target_dir = __get_embedding_dir(tar_gz_archive)
-    embedding = NpzEmbeddingHelper.load_embedding(os.path.join(target_dir, "embedding.npz"))
-    vocab = VocabRepositoryUtils.load(os.path.join(target_dir, "vocab.txt"))
-    return embedding, vocab
-
-
-def download():
-
-    data = {
-        NEWS_MYSTEM_SKIPGRAM_1000_20_2015: "https://www.dropbox.com/s/0omnlgzgnjhxlmf/{filename}?dl=1".format(
-            filename=NEWS_MYSTEM_SKIPGRAM_1000_20_2015),
-    }
-
-    # Perform downloading ...
-    for local_name, url_link in data.items():
-        utils.download(dest_file_path=__get_resource(local_name),
-                       source_url=url_link)
-
-    # Untar files ...
-    for local_name in data.keys():
-
-        if ".tar.gz" not in local_name:
-            continue
-
-        target_filepath = __get_resource(local_name)
-        with tarfile.open(target_filepath) as file:
-            def is_within_directory(directory, target):
-                
-                abs_directory = os.path.abspath(directory)
-                abs_target = os.path.abspath(target)
-            
-                prefix = os.path.commonprefix([abs_directory, abs_target])
-                
-                return prefix == abs_directory
-            
-            def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
-            
-                for member in tar.getmembers():
-                    member_path = os.path.join(path, member.name)
-                    if not is_within_directory(path, member_path):
-                        raise Exception("Attempted Path Traversal in Tar File")
-            
-                tar.extractall(path, members, numeric_owner=numeric_owner) 
-
-            safe_extract(file, __get_embedding_dir(target_filepath))

arekit/contrib/utils/entities/formatters/str_rus_cased_fmt.py

@@ -1,78 +0,0 @@
-from arekit.common.entities.base import Entity
-from arekit.common.entities.str_fmt import StringEntitiesFormatter
-from arekit.common.entities.types import OpinionEntityType
-from arekit.contrib.utils.processing.languages.ru.cases import RussianCases
-from arekit.contrib.utils.processing.languages.ru.number import RussianNumberType
-from arekit.contrib.utils.processing.pos.russian import RussianPOSTagger
-
-
-class RussianEntitiesCasedFormatter(StringEntitiesFormatter):
-
-    # Объект/Субъект
-    obj_subj_cases_map = {
-        RussianCases.UNKN: ['', ''],      # UNKN
-        RussianCases.NOM: ['', "ы"],      # именительный
-        RussianCases.GEN: ['а', 'ов'],    # родительный
-        RussianCases.DAT: ['y', 'ам'],    # дательный
-        RussianCases.ACC: ['', 'ы'],      # винительный
-        RussianCases.INS: ['ом', 'aми'],  # творительный
-        RussianCases.ABL: ['e', 'ах']     # предложный
-    }
-
-    # Сущност
-    entity_cases_map = {
-        RussianCases.UNKN: ['ь', 'и'],     # UNKN
-        RussianCases.NOM: ['ь', "и"],      # именительный
-        RussianCases.GEN: ['и', 'ей'],     # родительный
-        RussianCases.DAT: ['и', 'ям'],     # дательный
-        RussianCases.ACC: ['ь', 'и'],      # винительный
-        RussianCases.INS: ['ью', 'ьями'],  # творительный
-        RussianCases.ABL: ['и', 'ях']      # предложный
-    }
-
-    def __init__(self, pos_tagger):
-        assert(isinstance(pos_tagger, RussianPOSTagger))
-        self.__pos_tagger = pos_tagger
-
-    def to_string(self, original_value, entity_type):
-        assert(isinstance(original_value, Entity))
-        assert(isinstance(entity_type, OpinionEntityType))
-
-        template = None
-        cases_map = None
-
-        if (entity_type == OpinionEntityType.Object) or (entity_type == OpinionEntityType.SynonymObject):
-            template = "объект"
-            cases_map = self.obj_subj_cases_map
-        elif (entity_type == OpinionEntityType.Subject) or (entity_type == OpinionEntityType.SynonymSubject):
-            template = "субъект"
-            cases_map = self.obj_subj_cases_map
-        elif entity_type == OpinionEntityType.Other:
-            template = "сущност"
-            cases_map = self.entity_cases_map
-
-        return self.__get_correct_declention(value=original_value.Value,
-                                             template=template,
-                                             cases_map=cases_map)
-
-    def __get_correct_declention(self, value, template, cases_map):
-        assert(isinstance(value, str))
-        assert(isinstance(template, str))
-        assert(isinstance(cases_map, dict))
-
-        num = self.__pos_tagger.get_term_number(value)
-        case = self.__pos_tagger.get_term_case(value)
-
-        assert(isinstance(num, RussianNumberType))
-        assert(isinstance(case, RussianCases))
-
-        if num == RussianNumberType.UNKN or num == RussianNumberType.Single:
-            num_int = 0
-        else:
-            num_int = 1
-
-        if case not in cases_map:
-            case = RussianCases.UNKN
-
-        return template + (cases_map[case])[num_int]
-

arekit/contrib/utils/entities/formatters/str_rus_nocased_fmt.py

@@ -1,15 +0,0 @@
-from arekit.common.entities.str_fmt import StringEntitiesFormatter
-from arekit.common.entities.types import OpinionEntityType
-
-
-class RussianEntitiesFormatter(StringEntitiesFormatter):
-
-    def to_string(self, original_value, entity_type):
-        assert(isinstance(entity_type, OpinionEntityType))
-
-        if (entity_type == OpinionEntityType.Object) or (entity_type == OpinionEntityType.SynonymObject):
-            return "объект"
-        elif (entity_type == OpinionEntityType.Subject) or (entity_type == OpinionEntityType.SynonymSubject):
-            return "субъект"
-        if entity_type == OpinionEntityType.Other:
-            return "сущность"

arekit/contrib/utils/entities/formatters/str_simple_fmt.py

@@ -1,24 +0,0 @@
-from arekit.common.entities.str_fmt import StringEntitiesFormatter
-from arekit.common.entities.types import OpinionEntityType
-
-
-class StringEntitiesSimpleFormatter(StringEntitiesFormatter):
-    """
-    Utilized for picking a related word in word embedding.
-    """
-
-    def to_string(self, original_value, entity_type):
-        """
-        Returns: str (unicode)
-            Value that assumes to be utilized in Word2Vec model embedding search.
-        """
-        assert(isinstance(entity_type, OpinionEntityType))
-
-        if entity_type == OpinionEntityType.Other:
-            return "e"
-        elif entity_type == OpinionEntityType.Object or entity_type == OpinionEntityType.SynonymObject:
-            return "object"
-        elif entity_type == OpinionEntityType.Subject or entity_type == OpinionEntityType.SynonymSubject:
-            return "subject"
-
-        return None

arekit/contrib/utils/entities/formatters/str_simple_uppercase_fmt.py

@@ -1,21 +0,0 @@
-from arekit.common.entities.base import Entity
-from arekit.common.entities.str_fmt import StringEntitiesFormatter
-from arekit.common.entities.types import OpinionEntityType
-
-
-class SimpleUppercasedEntityFormatter(StringEntitiesFormatter):
-
-    def to_string(self, original_value, entity_type):
-        assert(isinstance(original_value, Entity) or original_value is None)
-        assert(isinstance(entity_type, OpinionEntityType))
-
-        if entity_type == OpinionEntityType.Other:
-            mask = "ENTITY"
-        elif entity_type == OpinionEntityType.Subject or entity_type == OpinionEntityType.SynonymSubject:
-            mask = "E_SUBJ"
-        elif entity_type == OpinionEntityType.Object or entity_type == OpinionEntityType.SynonymObject:
-            mask = "E_OBJ"
-        else:
-            raise NotImplementedError()
-
-        return mask

arekit/contrib/utils/io_utils/opinions.py

@@ -1,39 +0,0 @@
-from os.path import join
-
-from arekit.contrib.utils.data.ext import create_reader_extension
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.contrib.utils.io_utils.utils import filename_template
-
-
-class OpinionsIO(BaseSamplesIO):
-
-    def __init__(self, target_dir, reader=None, prefix="opinion", target_extension=".tsv.gz"):
-        assert(isinstance(reader, BaseReader))
-        self.__target_dir = target_dir
-        self.__prefix = prefix
-        self.__reader = reader
-        self.__target_extension = create_reader_extension(reader) \
-            if target_extension is None else target_extension
-
-    @property
-    def Reader(self):
-        return self.__reader
-
-    def create_target(self, data_type, data_folding):
-        return self.__get_input_opinions_target(data_type, data_folding=data_folding)
-
-    def __get_input_opinions_target(self, data_type, data_folding):
-        template = filename_template(data_type=data_type, data_folding=data_folding)
-        return self.__get_filepath(out_dir=self.__target_dir,
-                                   template=template,
-                                   prefix=self.__prefix,
-                                   extension=self.__target_extension)
-
-    @staticmethod
-    def __get_filepath(out_dir, template, prefix, extension):
-        assert(isinstance(template, str))
-        assert(isinstance(prefix, str))
-        assert(isinstance(extension, str))
-        return join(out_dir, "{prefix}-{template}{extension}".format(
-            prefix=prefix, template=template, extension=extension))