arekit 0.24.0__py3-none-any.whl → 0.25.1__py3-none-any.whl

arekit/contrib/utils/data/readers/jsonl.py

@@ -1,15 +0,0 @@
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.contrib.utils.data.storages.jsonl_based import JsonlBasedRowsStorage
-
-
-class JsonlReader(BaseReader):
-
-    def extension(self):
-        return ".jsonl"
-
-    def read(self, target):
-        rows = []
-        with open(target, "r") as f:
-            for line in f.readlines():
-                rows.append(line)
-        return JsonlBasedRowsStorage(rows)

arekit/contrib/utils/data/service/balance.py

@@ -1,50 +0,0 @@
-import gc
-import importlib
-from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
-
-
-class PandasBasedStorageBalancing(object):
-
-    @staticmethod
-    def create_balanced_from(storage, column_name, free_origin=True):
-        """ Performs oversampled balancing.
-
-            Note: it is quite important to remove previously created storage
-            in order to avoid memory leaking.
-
-            storage: PandasBasedRowsStorage
-                storage contents to be balanced.
-
-            column_name: str
-                column utilized for balancing.
-
-            free_origin: bool
-                indicates whether there is a need to release the resources
-                utilized for the original storage.
-        """
-        assert(isinstance(storage, PandasBasedRowsStorage))
-
-        original_df = storage.DataFrame
-
-        max_size = original_df[column_name].value_counts().max()
-
-        dframes = []
-        for class_index, group in original_df.groupby(column_name):
-            dframes.append(group.sample(max_size - len(group), replace=True))
-
-        # Clear resources.
-        pd = importlib.import_module("pandas")
-        balanced_df = pd.concat(dframes + [original_df])
-
-        # Removing temporary created dataframe.
-        for df in dframes:
-            del df
-
-        # Marking the original dataframe as released
-        # in terms of the allocated memory for it.
-        if free_origin:
-            storage.free()
-
-        gc.collect()
-
-        return PandasBasedRowsStorage(df=balanced_df)

arekit/contrib/utils/data/writers/csv_native.py

@@ -1,63 +0,0 @@
-import csv
-import os
-from os.path import dirname
-
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-
-class NativeCsvWriter(BaseWriter):
-
-    def __init__(self, delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL, header=True):
-        self.__target_f = None
-        self.__writer = None
-        self.__create_writer_func = lambda f: csv.writer(
-            f, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
-        self.__header = header
-        self.__header_written = None
-
-    def extension(self):
-        return ".csv"
-
-    @staticmethod
-    def __iter_storage_column_names(storage):
-        """ Iter only those columns that existed in storage.
-        """
-        for col_name in storage.iter_column_names():
-            if col_name in storage.RowCache:
-                yield col_name
-
-    def open_target(self, target):
-        os.makedirs(dirname(target), exist_ok=True)
-        self.__target_f = open(target, "w")
-        self.__writer = self.__create_writer_func(self.__target_f)
-        self.__header_written = not self.__header
-
-    def close_target(self):
-        self.__target_f.close()
-
-    def commit_line(self, storage):
-        assert(isinstance(storage, RowCacheStorage))
-        assert(self.__writer is not None)
-
-        if not self.__header_written:
-            self.__writer.writerow(list(self.__iter_storage_column_names(storage)))
-            self.__header_written = True
-
-        line_data = list(map(lambda col_name: storage.RowCache[col_name],
-                             self.__iter_storage_column_names(storage)))
-        self.__writer.writerow(line_data)
-
-    def write_all(self, storage, target):
-        """ Writes all the `storage` rows
-            into the `target` filepath, formatted as CSV.
-        """
-        assert(isinstance(storage, BaseRowsStorage))
-
-        with open(target, "w") as f:
-            writer = self.__create_writer_func(f)
-            for _, row in storage:
-                #content = [row[col_name] for col_name in storage.iter_column_names()]
-                content = [v for v in row]
-                writer.writerow(content)

arekit/contrib/utils/data/writers/csv_pd.py

@@ -1,40 +0,0 @@
-import logging
-
-from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
-from arekit.common.utils import create_dir_if_not_exists
-from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class PandasCsvWriter(BaseWriter):
-
-    def __init__(self, write_header):
-        super(PandasCsvWriter, self).__init__()
-        self.__write_header = write_header
-
-    def extension(self):
-        return ".tsv.gz"
-
-    def write_all(self, storage, target):
-        assert(isinstance(storage, PandasBasedRowsStorage))
-        assert(isinstance(target, str))
-
-        create_dir_if_not_exists(target)
-
-        # Temporary hack, remove it in future.
-        df = storage.DataFrame
-
-        logger.info("Saving... {length}: {filepath}".format(length=len(storage), filepath=target))
-        df.to_csv(target,
-                  sep='\t',
-                  encoding='utf-8',
-                  columns=[c for c in df.columns if c != BaseColumnsProvider.ROW_ID],
-                  index=False,
-                  float_format="%.0f",
-                  compression='gzip',
-                  header=self.__write_header)
-
-        logger.info("Saving completed!")

arekit/contrib/utils/data/writers/json_opennre.py

@@ -1,132 +0,0 @@
-import json
-import logging
-import os
-from os.path import dirname
-
-from arekit.common.data import const
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-logger = logging.getLogger(__name__)
-
-
-class OpenNREJsonWriter(BaseWriter):
-    """ This is a bag-based writer for the samples.
-        Project page: https://github.com/thunlp/OpenNRE
-
-        Every bag presented as follows:
-            {
-              'text' or 'token': ...,
-              'h': {'pos': [start, end], 'id': ... },
-              't': {'pos': [start, end], 'id': ... }
-              'id': "id_of_the_text_opinion"
-            }
-
-        In terms of the linked opinions (i0, i1, etc.) we consider id of the first opinion in linkage.
-        During the dataset reading stage via OpenNRE, these linkages automaticaly groups into bags.
-    """
-
-    def __init__(self, text_columns, encoding="utf-8", na_value="NA", keep_extra_columns=True,
-                 skip_extra_existed=True):
-        """ text_columns: list
-                column names that expected to be joined into a single (token) column.
-        """
-        assert(isinstance(text_columns, list))
-        assert(isinstance(encoding, str))
-        self.__text_columns = text_columns
-        self.__encoding = encoding
-        self.__target_f = None
-        self.__keep_extra_columns = keep_extra_columns
-        self.__na_value = na_value
-        self.__skip_extra_existed = skip_extra_existed
-
-    def extension(self):
-        return ".jsonl"
-
-    @staticmethod
-    def __format_row(row, na_value, text_columns, keep_extra_columns, skip_extra_existed):
-        """ Formatting that is compatible with the OpenNRE.
-        """
-        assert(isinstance(na_value, str))
-
-        sample_id = row[const.ID]
-        s_ind = int(row[const.S_IND])
-        t_ind = int(row[const.T_IND])
-        bag_id = str(row[const.OPINION_ID])
-
-        # Gather tokens.
-        tokens = []
-        for text_col in text_columns:
-            if text_col in row:
-                tokens.extend(row[text_col].split())
-
-        # Filtering JSON row.
-        formatted_data = {
-            "id": bag_id,
-            "id_orig": sample_id,
-            "token": tokens,
-            "h": {"pos": [s_ind, s_ind + 1], "id": str(bag_id + "s")},
-            "t": {"pos": [t_ind, t_ind + 1], "id": str(bag_id + "t")},
-            "relation": str(int(row[const.LABEL_UINT])) if const.LABEL_UINT in row else na_value
-        }
-
-        # Register extra fields (optionally).
-        if keep_extra_columns:
-            for key, value in row.items():
-                if key not in formatted_data and key not in text_columns:
-                    formatted_data[key] = value
-                else:
-                    if not skip_extra_existed:
-                        raise Exception(f"key `{key}` is already exist in formatted data "
-                                        f"or a part of the text columns list: {text_columns}")
-
-        return formatted_data
-
-    def open_target(self, target):
-        os.makedirs(dirname(target), exist_ok=True)
-        self.__target_f = open(target, "w")
-        pass
-
-    def close_target(self):
-        self.__target_f.close()
-
-    def commit_line(self, storage):
-        assert(isinstance(storage, RowCacheStorage))
-
-        # Collect existed columns.
-        row_data = {}
-        for col_name in storage.iter_column_names():
-            if col_name not in storage.RowCache:
-                continue
-            row_data[col_name] = storage.RowCache[col_name]
-
-        bag = self.__format_row(row_data, text_columns=self.__text_columns,
-                                keep_extra_columns=self.__keep_extra_columns,
-                                na_value=self.__na_value,
-                                skip_extra_existed=self.__skip_extra_existed)
-
-        self.__write_bag(bag=bag, json_file=self.__target_f)
-
-    @staticmethod
-    def __write_bag(bag, json_file):
-        assert(isinstance(bag, dict))
-        json.dump(bag, json_file, separators=(",", ":"), ensure_ascii=False)
-        json_file.write("\n")
-
-    def write_all(self, storage, target):
-        assert(isinstance(storage, BaseRowsStorage))
-        assert(isinstance(target, str))
-
-        logger.info("Saving... {rows}: {filepath}".format(rows=(len(storage)), filepath=target))
-
-        os.makedirs(os.path.dirname(target), exist_ok=True)
-        with open(target, "w", encoding=self.__encoding) as json_file:
-            for row_index, row in storage:
-                self.__write_bag(bag=self.__format_row(row, text_columns=self.__text_columns,
-                                                       keep_extra_columns=self.__keep_extra_columns,
-                                                       na_value=self.__na_value,
-                                                       skip_extra_existed=self.__skip_extra_existed),
-                                 json_file=json_file)
-
-        logger.info("Saving completed!")

arekit/contrib/utils/data/writers/sqlite_native.py

@@ -1,110 +0,0 @@
-import os
-import sqlite3
-from os.path import dirname
-
-from arekit.common.data import const
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-
-class SQliteWriter(BaseWriter):
-
-    def __init__(self, table_name="contents", index_column_names=None, skip_existed=False, clear_table=True):
-        """ index_column_names: list or None
-                column names should be considered to build a unique index;
-                if None, the default 'const.ID' will be considered for row indexation.
-        """
-        assert (isinstance(index_column_names, list) or index_column_names is None)
-        self.__index_column_names = index_column_names if index_column_names is not None else [const.ID]
-        self.__table_name = table_name
-        self.__conn = None
-        self.__cur = None
-        self.__need_init_table = True
-        self.__origin_column_names = None
-        self.__skip_existed = skip_existed
-        self.__clear_table = clear_table
-
-    def extension(self):
-        return ".sqlite"
-
-    @staticmethod
-    def __iter_storage_column_names(storage):
-        """ Iter only those columns that existed in storage.
-        """
-        assert (isinstance(storage, RowCacheStorage))
-        for col_name, col_type in zip(storage.iter_column_names(), storage.iter_column_types()):
-            if col_name in storage.RowCache:
-                yield col_name, col_type
-
-    def __init_table(self, column_data):
-        # Compose column name with the related SQLITE type.
-        column_types = ",".join([" ".join([col_name, self.type_to_sqlite(col_type)])
-                                 for col_name, col_type in column_data])
-        # Create table if not exists.
-        self.__cur.execute(f"CREATE TABLE IF NOT EXISTS {self.__table_name}({column_types})")
-        # Table exists, however we may optionally remove the content from it.
-        if self.__clear_table:
-            self.__cur.execute(f"DELETE FROM {self.__table_name};")
-        # Create index.
-        index_name = f"i_{self.__table_name}_id"
-        self.__cur.execute(f"DROP INDEX IF EXISTS {index_name};")
-        self.__cur.execute("CREATE INDEX IF NOT EXISTS {index} ON {table}({columns})".format(
-            index=index_name,
-            table=self.__table_name,
-            columns=", ".join(self.__index_column_names)
-        ))
-        self.__origin_column_names = [col_name for col_name, _ in column_data]
-
-    @staticmethod
-    def type_to_sqlite(col_type):
-        """ This is a simple function that provides conversion from the
-            base numpy types to SQLITE.
-            NOTE: this method represent a quick implementation for supporting
-            types, however it is far away from the generalized implementation.
-        """
-        if isinstance(col_type, str):
-            if 'int' in col_type:
-                return 'INTEGER'
-
-        return "TEXT"
-
-    def open_target(self, target):
-        os.makedirs(dirname(target), exist_ok=True)
-        self.__conn = sqlite3.connect(target)
-        self.__cur = self.__conn.cursor()
-
-    def commit_line(self, storage):
-        assert (isinstance(storage, RowCacheStorage))
-
-        column_data = list(self.__iter_storage_column_names(storage))
-
-        if self.__need_init_table:
-            self.__init_table(column_data)
-            self.__need_init_table = False
-
-        # Check whether the related row is already exist in SQLITE database.
-        row_id = storage.RowCache[const.ID]
-        top_row = self.__cur.execute(f"SELECT EXISTS(SELECT 1 FROM {self.__table_name} WHERE id='{row_id}');")
-        is_exists = top_row.fetchone()[0]
-        if is_exists == 1 and self.__skip_existed:
-            return
-
-        line_data = [storage.RowCache[col_name] for col_name, _ in column_data]
-        parameters = ",".join(["?"] * len(line_data))
-
-        assert (len(self.__origin_column_names) == len(line_data))
-
-        self.__cur.execute(
-            f"INSERT OR REPLACE INTO {self.__table_name} VALUES ({parameters})",
-            tuple(line_data))
-
-        self.__conn.commit()
-
-    def close_target(self):
-        self.__cur = None
-        self.__origin_column_names = None
-        self.__need_init_table = True
-        self.__conn.close()
-
-    def write_all(self, storage, target):
-        pass

arekit/contrib/utils/download.py

@@ -1,77 +0,0 @@
-import os
-import tarfile
-from os.path import join, exists
-
-from arekit.common import utils
-from arekit.contrib.utils.np_utils.embedding import NpzEmbeddingHelper
-from arekit.contrib.utils.np_utils.vocab import VocabRepositoryUtils
-
-NEWS_MYSTEM_SKIPGRAM_1000_20_2015 = "news_mystem_skipgram_1000_20_2015.tar.gz"
-
-
-def __get_resource(local_name, check_existance=False, download_if_missed=False):
-    assert (isinstance(local_name, str))
-    filepath = join(utils.get_default_download_dir(), local_name)
-
-    if check_existance and not exists(filepath):
-        if download_if_missed:
-            download()
-            # We try to ger the resource again but won't attempt to download it again.
-            __get_resource(local_name, check_existance=check_existance, download_if_missed=False)
-        else:
-            raise Exception("Resource could not be found: {}".format(filepath))
-
-    return filepath
-
-
-def __get_embedding_dir(filepath):
-    return filepath.replace(".tar.gz", "")
-
-
-def load_embedding_and_vocab(local_name, check_existance=False, download_if_missed=False):
-    tar_gz_archive = __get_resource(local_name, check_existance=check_existance,
-                                    download_if_missed=download_if_missed)
-    target_dir = __get_embedding_dir(tar_gz_archive)
-    embedding = NpzEmbeddingHelper.load_embedding(os.path.join(target_dir, "embedding.npz"))
-    vocab = VocabRepositoryUtils.load(os.path.join(target_dir, "vocab.txt"))
-    return embedding, vocab
-
-
-def download():
-    data = {
-        NEWS_MYSTEM_SKIPGRAM_1000_20_2015: "https://www.dropbox.com/s/0omnlgzgnjhxlmf/{filename}?dl=1".format(
-            filename=NEWS_MYSTEM_SKIPGRAM_1000_20_2015),
-    }
-
-    # Perform downloading ...
-    for local_name, url_link in data.items():
-        utils.download(dest_file_path=__get_resource(local_name),
-                       source_url=url_link)
-
-    # Untar files ...
-    for local_name in data.keys():
-
-        if ".tar.gz" not in local_name:
-            continue
-
-        target_filepath = __get_resource(local_name)
-        with tarfile.open(target_filepath) as file:
-            def is_within_directory(directory, target):
-
-                abs_directory = os.path.abspath(directory)
-                abs_target = os.path.abspath(target)
-
-                prefix = os.path.commonprefix([abs_directory, abs_target])
-
-                return prefix == abs_directory
-
-            def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
-
-                for member in tar.getmembers():
-                    member_path = os.path.join(path, member.name)
-                    if not is_within_directory(path, member_path):
-                        raise Exception("Attempted Path Traversal in Tar File")
-
-                tar.extractall(path, members, numeric_owner=numeric_owner)
-
-            safe_extract(file, __get_embedding_dir(target_filepath))

arekit/contrib/utils/embeddings/rusvectores.py

@@ -1,58 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.contrib.networks.embedding import Embedding
-
-
-class RusvectoresEmbedding(Embedding):
-    """ Wrapper over models from the following resource.
-        https://rusvectores.org/ru/models/
-
-        NOTE: Usually these are embeddings for texts written in Russian.
-        for the better performance it is expected that we adopt stemmer.
-    """
-
-    def __init__(self, matrix, words, stemmer):
-        assert(isinstance(stemmer, Stemmer) or stemmer is None)
-        super(RusvectoresEmbedding, self).__init__(matrix=matrix, words=words)
-        self.__index_without_pos = self.__create_terms_without_pos()
-        self.__stemmer = stemmer
-        self.__lemmatize_by_default = stemmer is not None
-
-    def try_find_index_by_plain_word(self, word):
-        assert(isinstance(word, str))
-
-        temp = self.__lemmatize_by_default
-        self.__lemmatize_by_default = False
-        index = super(RusvectoresEmbedding, self).try_find_index_by_plain_word(word)
-        self.__lemmatize_by_default = temp
-
-        return index
-
-    def _handler(self, word):
-        return self.__try_find_word_index_pair_lemmatized(word, self.__lemmatize_by_default)
-
-    # region private methods
-
-    def __try_find_word_index_pair_lemmatized(self, term, lemmatize):
-        assert(isinstance(term, str))
-        assert(isinstance(lemmatize, bool))
-
-        if lemmatize:
-            term = self.__stemmer.lemmatize_to_str(term)
-
-        index = self.__index_without_pos[term] \
-            if term in self.__index_without_pos else None
-
-        return term, index
-
-    def __create_terms_without_pos(self):
-        d = {}
-        for word_with_pos, index in self.iter_vocabulary():
-            assert(isinstance(word_with_pos, str))
-            word = word_with_pos.split(u'_')[0]
-            if word in d:
-                continue
-            d[word] = index
-
-        return d
-
-    # endregion

arekit/contrib/utils/embeddings/tokens.py

@@ -1,30 +0,0 @@
-import numpy as np
-
-from arekit.contrib.networks.embedding import Embedding
-from arekit.contrib.utils.processing.text.tokens import Tokens
-
-
-class TokenEmbedding(Embedding):
-    """ Embedding vectors for text punctuation, based on Tokens in parsed text
-    """
-
-    @classmethod
-    def from_supported_tokens(cls, vector_size, random_vector_func):
-        """
-        random_vector_func: func
-            function with parameters (vector_size, seed)
-        """
-        assert(isinstance(vector_size, int))
-        assert(callable(random_vector_func))
-
-        matrix = []
-        tokens_list = list(Tokens.iter_supported_tokens())
-
-        for token_index, _ in enumerate(tokens_list):
-
-            vector = random_vector_func(vector_size, token_index)
-
-            matrix.append(vector)
-
-        return cls(matrix=np.array(matrix),
-                   words=tokens_list)

arekit/contrib/utils/io_utils/embedding.py

@@ -1,72 +0,0 @@
-from os.path import join
-
-from arekit.contrib.networks.embedding_io import BaseEmbeddingIO
-from arekit.contrib.utils.io_utils.utils import check_targets_existence
-from arekit.contrib.utils.np_utils.embedding import NpzEmbeddingHelper
-from arekit.contrib.utils.np_utils.vocab import VocabRepositoryUtils
-
-
-class NpEmbeddingIO(BaseEmbeddingIO):
-    """ Npz-based IO utils for embedding and text-based for vocabulary.
-        This format represents a archived version of the numpy math data, i.e. vectors, numbers, etc.
-
-        Provides additional Input/Output paths generation functions for:
-            - embedding matrix;
-            - embedding vocabulary.
-    """
-
-    def __init__(self, target_dir, prefix_name="sample"):
-        assert(isinstance(target_dir, str))
-
-        self.__target_dir = target_dir
-        self.__term_emb_fn_template = "-".join([prefix_name, "term_embedding"])
-        self.__vocab_fn_template = "-".join([prefix_name, "term_embedding"])
-
-    # region Embedding-related data
-
-    def save_vocab(self, data):
-        target = self.__get_default_vocab_filepath()
-        return VocabRepositoryUtils.save(data=data, target=target)
-
-    def load_vocab(self):
-        source = self.___get_vocab_source()
-        return dict(VocabRepositoryUtils.load(source))
-
-    def save_embedding(self, data):
-        target = self.__get_default_embedding_filepath()
-        NpzEmbeddingHelper.save_embedding(data=data, target=target)
-
-    def load_embedding(self):
-        source = self.__get_term_embedding_source()
-        return NpzEmbeddingHelper.load_embedding(source)
-
-    def check_targets_existed(self):
-        targets = [
-            self.__get_default_vocab_filepath(),
-            self.__get_term_embedding_target()
-        ]
-        return check_targets_existence(targets=targets)
-
-    # endregion
-
-    # region embedding-related data
-
-    def ___get_vocab_source(self):
-        """ It is possible to load a predefined embedding from another experiment
-            using the related filepath provided by model_io.
-        """
-        return self.__get_default_vocab_filepath()
-
-    def __get_term_embedding_target(self):
-        return self.__get_default_embedding_filepath()
-
-    def __get_term_embedding_source(self):
-        return self.__get_default_embedding_filepath()
-
-    def __get_default_vocab_filepath(self):
-        return join(self.__target_dir, self.__vocab_fn_template)
-
-    def __get_default_embedding_filepath(self):
-        return join(self.__target_dir, self.__term_emb_fn_template)
-
-    # endregion