arekit 0.24.0__py3-none-any.whl

arekit/contrib/utils/data/storages/pandas_based.py

@@ -0,0 +1,123 @@
+import importlib
+
+import numpy as np
+
+from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
+from arekit.common.data.storages.base import BaseRowsStorage, logger
+from arekit.common.utils import progress_bar_iter
+
+
+class PandasBasedRowsStorage(BaseRowsStorage):
+    """ Storage Kernel functions implementation,
+        based on the pandas DataFrames.
+    """
+
+    def __init__(self, df=None):
+        self._df = df
+
+    @property
+    def DataFrame(self):
+        # TODO. Temporary hack, however this should be removed in future.
+        return self._df
+
+    @staticmethod
+    def __create_empty(cols_with_types):
+        """ cols_with_types: list of pairs ("name", dtype)
+        """
+        assert(isinstance(cols_with_types, list))
+        data = np.empty(0, dtype=np.dtype(cols_with_types))
+        pd = importlib.import_module("pandas")
+        return pd.DataFrame(data)
+
+    def __filter(self, column_name, value):
+        return self._df[self._df[column_name] == value]
+
+    @staticmethod
+    def __iter_rows_core(df):
+        for row_index, row in df.iterrows():
+            yield row_index, row
+
+    def __fill_with_blank_rows(self, row_id_column_name, rows_count):
+        assert(isinstance(row_id_column_name, str))
+        assert(isinstance(rows_count, int))
+        self._df[row_id_column_name] = list(range(rows_count))
+        self._df.set_index(row_id_column_name, inplace=True)
+
+    # region protected methods
+
+    def iter_column_names(self):
+        return iter(self._df.columns)
+
+    def iter_column_types(self):
+        return iter(self._df.dtypes)
+
+    def _set_row_value(self, row_ind, column, value):
+        self._df.at[row_ind, column] = value
+
+    def _iter_rows(self):
+        for row_index, row in self.__iter_rows_core(self._df):
+            yield row_index, row.to_dict()
+
+    def _get_rows_count(self):
+        return len(self._df)
+
+    # endregion
+
+    # region public methods
+
+    def fill(self, iter_rows_func, columns_provider, row_handler=None, rows_count=None, desc=""):
+        """ NOTE: We provide the rows counting which is required
+            in order to know an expected amount of rows in advace
+            due to the specifics of the pandas memory allocation
+            for the DataFrames.
+            The latter allows us avoid rows appending, which
+            may significantly affects on performance once the size
+            of DataFrame becomes relatively large.
+        """
+        assert(isinstance(columns_provider, BaseColumnsProvider))
+
+        logger.info("Rows calculation process started. [Required by Pandas-Based storage kernel]")
+        logged_rows_it = progress_bar_iter(
+            iterable=iter_rows_func(True),
+            desc="Calculating rows count ({reason})".format(reason=desc),
+            unit="rows")
+        rows_count = sum(1 for _ in logged_rows_it)
+
+        logger.info("Filling with blank rows: {}".format(rows_count))
+        self.__fill_with_blank_rows(row_id_column_name=columns_provider.ROW_ID,
+                                    rows_count=rows_count)
+        logger.info("Completed!")
+
+        super(PandasBasedRowsStorage, self).fill(iter_rows_func=iter_rows_func,
+                                                 row_handler=row_handler,
+                                                 columns_provider=columns_provider,
+                                                 rows_count=rows_count)
+
+    def get_row(self, row_index):
+        return self._df.iloc[row_index]
+
+    def get_cell(self, row_index, column_name):
+        return self._df.iloc[row_index][column_name]
+
+    def iter_column_values(self, column_name, dtype=None):
+        values = self._df[column_name]
+        if dtype is None:
+            return values
+        return values.astype(dtype)
+
+    def find_by_value(self, column_name, value):
+        return self.__filter(column_name=column_name, value=value)
+
+    def init_empty(self, columns_provider):
+        cols_with_types = columns_provider.get_columns_list_with_types()
+        self._df = self.__create_empty(cols_with_types)
+
+    def iter_shuffled(self):
+        shuffled_df = self._df.sample(frac=1)
+        return self.__iter_rows_core(shuffled_df)
+
+    def free(self):
+        del self._df
+        super(PandasBasedRowsStorage, self).free()
+
+    # endregion

arekit/contrib/utils/data/storages/row_cache.py

@@ -0,0 +1,48 @@
+from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
+from arekit.common.data.storages.base import BaseRowsStorage
+
+
+class RowCacheStorage(BaseRowsStorage):
+    """ Row Caching storage kernel, based on python dictionary.
+    """
+
+    def __init__(self, force_collect_columns=None):
+        """ This is a particular/related solution for the following issue:
+            https://github.com/nicolay-r/AREkit/issues/464
+            force_collect_columns: list
+                columns that supposed to be additionally considered in output.
+        """
+        assert(isinstance(force_collect_columns, list) or force_collect_columns is None)
+        self.__f = None
+        self.__row_cache = {}
+        self.__column_names = []
+        self.__column_types = []
+        self.__force_collect_columns = [] if force_collect_columns is None else force_collect_columns
+
+    @property
+    def RowCache(self):
+        return self.__row_cache
+
+    def init_empty(self, columns_provider):
+        assert (isinstance(columns_provider, BaseColumnsProvider))
+
+        self.__column_names.clear()
+        for col_name, col_type in columns_provider.get_columns_list_with_types():
+            self.__column_names.append(col_name)
+            self.__column_types.append(col_type)
+
+        # Expand with columns that are forced to be provided.
+        existed_set = set(self.__column_names)
+        self.__column_names += [c for c in self.__force_collect_columns if c not in existed_set]
+
+    def iter_column_names(self):
+        return iter(self.__column_names)
+
+    def iter_column_types(self):
+        return iter(self.__column_types)
+
+    def _set_row_value(self, row_ind, column, value):
+        self.__row_cache[column] = value
+
+    def _begin_filling_row(self, row_ind):
+        self.__row_cache.clear()

arekit/contrib/utils/data/writers/base.py

@@ -0,0 +1,27 @@
+class BaseWriter(object):
+
+    def extension(self):
+        """ Expected output extension type.
+        """
+        raise NotImplementedError()
+
+    def open_target(self, target):
+        pass
+
+    def commit_line(self, storage):
+        pass
+
+    def close_target(self):
+        pass
+
+    def write_all(self, storage, target):
+        """ Performs the writing process of the whole storage.
+            The implementation and support of the related operation
+            may vary and depends on the nature of the storage, which
+            briefly might keep all the data in memory (available)
+            or cache only temporary information (unavailable)
+
+            storage: BaseRowsStorage
+            target: str
+        """
+        raise NotImplementedError()

arekit/contrib/utils/data/writers/csv_native.py

@@ -0,0 +1,63 @@
+import csv
+import os
+from os.path import dirname
+
+from arekit.common.data.storages.base import BaseRowsStorage
+from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
+from arekit.contrib.utils.data.writers.base import BaseWriter
+
+
+class NativeCsvWriter(BaseWriter):
+
+    def __init__(self, delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL, header=True):
+        self.__target_f = None
+        self.__writer = None
+        self.__create_writer_func = lambda f: csv.writer(
+            f, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
+        self.__header = header
+        self.__header_written = None
+
+    def extension(self):
+        return ".csv"
+
+    @staticmethod
+    def __iter_storage_column_names(storage):
+        """ Iter only those columns that existed in storage.
+        """
+        for col_name in storage.iter_column_names():
+            if col_name in storage.RowCache:
+                yield col_name
+
+    def open_target(self, target):
+        os.makedirs(dirname(target), exist_ok=True)
+        self.__target_f = open(target, "w")
+        self.__writer = self.__create_writer_func(self.__target_f)
+        self.__header_written = not self.__header
+
+    def close_target(self):
+        self.__target_f.close()
+
+    def commit_line(self, storage):
+        assert(isinstance(storage, RowCacheStorage))
+        assert(self.__writer is not None)
+
+        if not self.__header_written:
+            self.__writer.writerow(list(self.__iter_storage_column_names(storage)))
+            self.__header_written = True
+
+        line_data = list(map(lambda col_name: storage.RowCache[col_name],
+                             self.__iter_storage_column_names(storage)))
+        self.__writer.writerow(line_data)
+
+    def write_all(self, storage, target):
+        """ Writes all the `storage` rows
+            into the `target` filepath, formatted as CSV.
+        """
+        assert(isinstance(storage, BaseRowsStorage))
+
+        with open(target, "w") as f:
+            writer = self.__create_writer_func(f)
+            for _, row in storage:
+                #content = [row[col_name] for col_name in storage.iter_column_names()]
+                content = [v for v in row]
+                writer.writerow(content)

arekit/contrib/utils/data/writers/csv_pd.py

@@ -0,0 +1,40 @@
+import logging
+
+from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
+from arekit.common.utils import create_dir_if_not_exists
+from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
+from arekit.contrib.utils.data.writers.base import BaseWriter
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class PandasCsvWriter(BaseWriter):
+
+    def __init__(self, write_header):
+        super(PandasCsvWriter, self).__init__()
+        self.__write_header = write_header
+
+    def extension(self):
+        return ".tsv.gz"
+
+    def write_all(self, storage, target):
+        assert(isinstance(storage, PandasBasedRowsStorage))
+        assert(isinstance(target, str))
+
+        create_dir_if_not_exists(target)
+
+        # Temporary hack, remove it in future.
+        df = storage.DataFrame
+
+        logger.info("Saving... {length}: {filepath}".format(length=len(storage), filepath=target))
+        df.to_csv(target,
+                  sep='\t',
+                  encoding='utf-8',
+                  columns=[c for c in df.columns if c != BaseColumnsProvider.ROW_ID],
+                  index=False,
+                  float_format="%.0f",
+                  compression='gzip',
+                  header=self.__write_header)
+
+        logger.info("Saving completed!")

arekit/contrib/utils/data/writers/json_opennre.py

@@ -0,0 +1,132 @@
+import json
+import logging
+import os
+from os.path import dirname
+
+from arekit.common.data import const
+from arekit.common.data.storages.base import BaseRowsStorage
+from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
+from arekit.contrib.utils.data.writers.base import BaseWriter
+
+logger = logging.getLogger(__name__)
+
+
+class OpenNREJsonWriter(BaseWriter):
+    """ This is a bag-based writer for the samples.
+        Project page: https://github.com/thunlp/OpenNRE
+
+        Every bag presented as follows:
+            {
+              'text' or 'token': ...,
+              'h': {'pos': [start, end], 'id': ... },
+              't': {'pos': [start, end], 'id': ... }
+              'id': "id_of_the_text_opinion"
+            }
+
+        In terms of the linked opinions (i0, i1, etc.) we consider id of the first opinion in linkage.
+        During the dataset reading stage via OpenNRE, these linkages automaticaly groups into bags.
+    """
+
+    def __init__(self, text_columns, encoding="utf-8", na_value="NA", keep_extra_columns=True,
+                 skip_extra_existed=True):
+        """ text_columns: list
+                column names that expected to be joined into a single (token) column.
+        """
+        assert(isinstance(text_columns, list))
+        assert(isinstance(encoding, str))
+        self.__text_columns = text_columns
+        self.__encoding = encoding
+        self.__target_f = None
+        self.__keep_extra_columns = keep_extra_columns
+        self.__na_value = na_value
+        self.__skip_extra_existed = skip_extra_existed
+
+    def extension(self):
+        return ".jsonl"
+
+    @staticmethod
+    def __format_row(row, na_value, text_columns, keep_extra_columns, skip_extra_existed):
+        """ Formatting that is compatible with the OpenNRE.
+        """
+        assert(isinstance(na_value, str))
+
+        sample_id = row[const.ID]
+        s_ind = int(row[const.S_IND])
+        t_ind = int(row[const.T_IND])
+        bag_id = str(row[const.OPINION_ID])
+
+        # Gather tokens.
+        tokens = []
+        for text_col in text_columns:
+            if text_col in row:
+                tokens.extend(row[text_col].split())
+
+        # Filtering JSON row.
+        formatted_data = {
+            "id": bag_id,
+            "id_orig": sample_id,
+            "token": tokens,
+            "h": {"pos": [s_ind, s_ind + 1], "id": str(bag_id + "s")},
+            "t": {"pos": [t_ind, t_ind + 1], "id": str(bag_id + "t")},
+            "relation": str(int(row[const.LABEL_UINT])) if const.LABEL_UINT in row else na_value
+        }
+
+        # Register extra fields (optionally).
+        if keep_extra_columns:
+            for key, value in row.items():
+                if key not in formatted_data and key not in text_columns:
+                    formatted_data[key] = value
+                else:
+                    if not skip_extra_existed:
+                        raise Exception(f"key `{key}` is already exist in formatted data "
+                                        f"or a part of the text columns list: {text_columns}")
+
+        return formatted_data
+
+    def open_target(self, target):
+        os.makedirs(dirname(target), exist_ok=True)
+        self.__target_f = open(target, "w")
+        pass
+
+    def close_target(self):
+        self.__target_f.close()
+
+    def commit_line(self, storage):
+        assert(isinstance(storage, RowCacheStorage))
+
+        # Collect existed columns.
+        row_data = {}
+        for col_name in storage.iter_column_names():
+            if col_name not in storage.RowCache:
+                continue
+            row_data[col_name] = storage.RowCache[col_name]
+
+        bag = self.__format_row(row_data, text_columns=self.__text_columns,
+                                keep_extra_columns=self.__keep_extra_columns,
+                                na_value=self.__na_value,
+                                skip_extra_existed=self.__skip_extra_existed)
+
+        self.__write_bag(bag=bag, json_file=self.__target_f)
+
+    @staticmethod
+    def __write_bag(bag, json_file):
+        assert(isinstance(bag, dict))
+        json.dump(bag, json_file, separators=(",", ":"), ensure_ascii=False)
+        json_file.write("\n")
+
+    def write_all(self, storage, target):
+        assert(isinstance(storage, BaseRowsStorage))
+        assert(isinstance(target, str))
+
+        logger.info("Saving... {rows}: {filepath}".format(rows=(len(storage)), filepath=target))
+
+        os.makedirs(os.path.dirname(target), exist_ok=True)
+        with open(target, "w", encoding=self.__encoding) as json_file:
+            for row_index, row in storage:
+                self.__write_bag(bag=self.__format_row(row, text_columns=self.__text_columns,
+                                                       keep_extra_columns=self.__keep_extra_columns,
+                                                       na_value=self.__na_value,
+                                                       skip_extra_existed=self.__skip_extra_existed),
+                                 json_file=json_file)
+
+        logger.info("Saving completed!")

arekit/contrib/utils/data/writers/sqlite_native.py

@@ -0,0 +1,110 @@
+import os
+import sqlite3
+from os.path import dirname
+
+from arekit.common.data import const
+from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
+from arekit.contrib.utils.data.writers.base import BaseWriter
+
+
+class SQliteWriter(BaseWriter):
+
+    def __init__(self, table_name="contents", index_column_names=None, skip_existed=False, clear_table=True):
+        """ index_column_names: list or None
+                column names should be considered to build a unique index;
+                if None, the default 'const.ID' will be considered for row indexation.
+        """
+        assert (isinstance(index_column_names, list) or index_column_names is None)
+        self.__index_column_names = index_column_names if index_column_names is not None else [const.ID]
+        self.__table_name = table_name
+        self.__conn = None
+        self.__cur = None
+        self.__need_init_table = True
+        self.__origin_column_names = None
+        self.__skip_existed = skip_existed
+        self.__clear_table = clear_table
+
+    def extension(self):
+        return ".sqlite"
+
+    @staticmethod
+    def __iter_storage_column_names(storage):
+        """ Iter only those columns that existed in storage.
+        """
+        assert (isinstance(storage, RowCacheStorage))
+        for col_name, col_type in zip(storage.iter_column_names(), storage.iter_column_types()):
+            if col_name in storage.RowCache:
+                yield col_name, col_type
+
+    def __init_table(self, column_data):
+        # Compose column name with the related SQLITE type.
+        column_types = ",".join([" ".join([col_name, self.type_to_sqlite(col_type)])
+                                 for col_name, col_type in column_data])
+        # Create table if not exists.
+        self.__cur.execute(f"CREATE TABLE IF NOT EXISTS {self.__table_name}({column_types})")
+        # Table exists, however we may optionally remove the content from it.
+        if self.__clear_table:
+            self.__cur.execute(f"DELETE FROM {self.__table_name};")
+        # Create index.
+        index_name = f"i_{self.__table_name}_id"
+        self.__cur.execute(f"DROP INDEX IF EXISTS {index_name};")
+        self.__cur.execute("CREATE INDEX IF NOT EXISTS {index} ON {table}({columns})".format(
+            index=index_name,
+            table=self.__table_name,
+            columns=", ".join(self.__index_column_names)
+        ))
+        self.__origin_column_names = [col_name for col_name, _ in column_data]
+
+    @staticmethod
+    def type_to_sqlite(col_type):
+        """ This is a simple function that provides conversion from the
+            base numpy types to SQLITE.
+            NOTE: this method represent a quick implementation for supporting
+            types, however it is far away from the generalized implementation.
+        """
+        if isinstance(col_type, str):
+            if 'int' in col_type:
+                return 'INTEGER'
+
+        return "TEXT"
+
+    def open_target(self, target):
+        os.makedirs(dirname(target), exist_ok=True)
+        self.__conn = sqlite3.connect(target)
+        self.__cur = self.__conn.cursor()
+
+    def commit_line(self, storage):
+        assert (isinstance(storage, RowCacheStorage))
+
+        column_data = list(self.__iter_storage_column_names(storage))
+
+        if self.__need_init_table:
+            self.__init_table(column_data)
+            self.__need_init_table = False
+
+        # Check whether the related row is already exist in SQLITE database.
+        row_id = storage.RowCache[const.ID]
+        top_row = self.__cur.execute(f"SELECT EXISTS(SELECT 1 FROM {self.__table_name} WHERE id='{row_id}');")
+        is_exists = top_row.fetchone()[0]
+        if is_exists == 1 and self.__skip_existed:
+            return
+
+        line_data = [storage.RowCache[col_name] for col_name, _ in column_data]
+        parameters = ",".join(["?"] * len(line_data))
+
+        assert (len(self.__origin_column_names) == len(line_data))
+
+        self.__cur.execute(
+            f"INSERT OR REPLACE INTO {self.__table_name} VALUES ({parameters})",
+            tuple(line_data))
+
+        self.__conn.commit()
+
+    def close_target(self):
+        self.__cur = None
+        self.__origin_column_names = None
+        self.__need_init_table = True
+        self.__conn.close()
+
+    def write_all(self, storage, target):
+        pass

arekit/contrib/utils/download.py

@@ -0,0 +1,77 @@
+import os
+import tarfile
+from os.path import join, exists
+
+from arekit.common import utils
+from arekit.contrib.utils.np_utils.embedding import NpzEmbeddingHelper
+from arekit.contrib.utils.np_utils.vocab import VocabRepositoryUtils
+
+NEWS_MYSTEM_SKIPGRAM_1000_20_2015 = "news_mystem_skipgram_1000_20_2015.tar.gz"
+
+
+def __get_resource(local_name, check_existance=False, download_if_missed=False):
+    assert (isinstance(local_name, str))
+    filepath = join(utils.get_default_download_dir(), local_name)
+
+    if check_existance and not exists(filepath):
+        if download_if_missed:
+            download()
+            # We try to ger the resource again but won't attempt to download it again.
+            __get_resource(local_name, check_existance=check_existance, download_if_missed=False)
+        else:
+            raise Exception("Resource could not be found: {}".format(filepath))
+
+    return filepath
+
+
+def __get_embedding_dir(filepath):
+    return filepath.replace(".tar.gz", "")
+
+
+def load_embedding_and_vocab(local_name, check_existance=False, download_if_missed=False):
+    tar_gz_archive = __get_resource(local_name, check_existance=check_existance,
+                                    download_if_missed=download_if_missed)
+    target_dir = __get_embedding_dir(tar_gz_archive)
+    embedding = NpzEmbeddingHelper.load_embedding(os.path.join(target_dir, "embedding.npz"))
+    vocab = VocabRepositoryUtils.load(os.path.join(target_dir, "vocab.txt"))
+    return embedding, vocab
+
+
+def download():
+    data = {
+        NEWS_MYSTEM_SKIPGRAM_1000_20_2015: "https://www.dropbox.com/s/0omnlgzgnjhxlmf/{filename}?dl=1".format(
+            filename=NEWS_MYSTEM_SKIPGRAM_1000_20_2015),
+    }
+
+    # Perform downloading ...
+    for local_name, url_link in data.items():
+        utils.download(dest_file_path=__get_resource(local_name),
+                       source_url=url_link)
+
+    # Untar files ...
+    for local_name in data.keys():
+
+        if ".tar.gz" not in local_name:
+            continue
+
+        target_filepath = __get_resource(local_name)
+        with tarfile.open(target_filepath) as file:
+            def is_within_directory(directory, target):
+
+                abs_directory = os.path.abspath(directory)
+                abs_target = os.path.abspath(target)
+
+                prefix = os.path.commonprefix([abs_directory, abs_target])
+
+                return prefix == abs_directory
+
+            def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
+
+                for member in tar.getmembers():
+                    member_path = os.path.join(path, member.name)
+                    if not is_within_directory(path, member_path):
+                        raise Exception("Attempted Path Traversal in Tar File")
+
+                tar.extractall(path, members, numeric_owner=numeric_owner)
+
+            safe_extract(file, __get_embedding_dir(target_filepath))

arekit/contrib/utils/embeddings/rusvectores.py

@@ -0,0 +1,58 @@
+from arekit.common.text.stemmer import Stemmer
+from arekit.contrib.networks.embedding import Embedding
+
+
+class RusvectoresEmbedding(Embedding):
+    """ Wrapper over models from the following resource.
+        https://rusvectores.org/ru/models/
+
+        NOTE: Usually these are embeddings for texts written in Russian.
+        for the better performance it is expected that we adopt stemmer.
+    """
+
+    def __init__(self, matrix, words, stemmer):
+        assert(isinstance(stemmer, Stemmer) or stemmer is None)
+        super(RusvectoresEmbedding, self).__init__(matrix=matrix, words=words)
+        self.__index_without_pos = self.__create_terms_without_pos()
+        self.__stemmer = stemmer
+        self.__lemmatize_by_default = stemmer is not None
+
+    def try_find_index_by_plain_word(self, word):
+        assert(isinstance(word, str))
+
+        temp = self.__lemmatize_by_default
+        self.__lemmatize_by_default = False
+        index = super(RusvectoresEmbedding, self).try_find_index_by_plain_word(word)
+        self.__lemmatize_by_default = temp
+
+        return index
+
+    def _handler(self, word):
+        return self.__try_find_word_index_pair_lemmatized(word, self.__lemmatize_by_default)
+
+    # region private methods
+
+    def __try_find_word_index_pair_lemmatized(self, term, lemmatize):
+        assert(isinstance(term, str))
+        assert(isinstance(lemmatize, bool))
+
+        if lemmatize:
+            term = self.__stemmer.lemmatize_to_str(term)
+
+        index = self.__index_without_pos[term] \
+            if term in self.__index_without_pos else None
+
+        return term, index
+
+    def __create_terms_without_pos(self):
+        d = {}
+        for word_with_pos, index in self.iter_vocabulary():
+            assert(isinstance(word_with_pos, str))
+            word = word_with_pos.split(u'_')[0]
+            if word in d:
+                continue
+            d[word] = index
+
+        return d
+
+    # endregion