arekit 0.25.0__py3-none-any.whl → 0.25.2__py3-none-any.whl

arekit/contrib/networks/input/rows_parser.py

@@ -1,47 +0,0 @@
-import arekit.contrib.networks.input.const as const
-from arekit.common.data.rows_fmt import process_indices_list
-
-
-def create_nn_column_formatters(no_value_func=lambda: None, args_sep=","):
-    assert(callable(no_value_func))
-
-    empty_list = []
-
-    def str_to_list(value):
-        return process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-
-    def list_to_str(inds_iter):
-        return args_sep.join([str(i) for i in inds_iter])
-
-    return {
-        const.FrameVariantIndices: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-                if isinstance(value, str) else empty_list
-        },
-        const.FrameConnotations: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-                if isinstance(value, str) else empty_list
-        },
-        const.SynonymObject: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-        },
-        const.SynonymSubject: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-        },
-        const.PosTags: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: str_to_list(value)
-        }
-    }
-
-
-def create_nn_val_writer_fmt(fmt_type, args_sep=","):
-    assert(isinstance(fmt_type, str))
-    d = create_nn_column_formatters(args_sep=args_sep)
-    for k, v in d.items():
-        d[k] = v[fmt_type]
-    return d

arekit/contrib/networks/input/term_types.py

@@ -1,13 +0,0 @@
-class TermTypes(object):
-    """ Types of input terms that may occur within the
-        input sequence of the neural network moodels.
-    """
-
-    WORD = "word"
-    ENTITY = "entity"
-    FRAME = "frame"
-    TOKEN = "token"
-
-    @staticmethod
-    def iter_types():
-        return [TermTypes.WORD, TermTypes.ENTITY, TermTypes.FRAME, TermTypes.TOKEN]

arekit/contrib/networks/input/terms_mapping.py

@@ -1,60 +0,0 @@
-from arekit.common.data.input.terms_mapper import OpinionContainingTextTermsMapper
-from arekit.common.entities.base import Entity
-from arekit.common.frames.text_variant import TextFrameVariant
-from arekit.contrib.networks.input.term_types import TermTypes
-
-
-class VectorizedNetworkTermMapping(OpinionContainingTextTermsMapper):
-    """ For every element returns: (word, embedded vector)
-    """
-
-    def __init__(self, string_entities_formatter, vectorizers):
-        """string_emb_entity_formatter:
-            Utilized in order to obtain embedding value from predefined_embeding for entities
-           vectorizers:
-            dict
-        """
-        assert(isinstance(vectorizers, dict))
-
-        for term_type in TermTypes.iter_types():
-            assert(term_type in vectorizers)
-
-        super(VectorizedNetworkTermMapping, self).__init__(
-            entity_formatter=string_entities_formatter)
-
-        self.__vectorizers = vectorizers
-
-    def map_term(self, term_type, term):
-        """Universal term mapping method.
-
-        Args:
-            term_type (TermTypes): The type of term to map.
-            term (str): The term to map.
-
-        Returns:
-            The mapped term.
-        """
-        return self.__vectorizers[term_type].create_term_embedding(term=term)
-
-    def map_word(self, w_ind, word):
-        return self.map_term(TermTypes.WORD, word)
-
-    def map_text_frame_variant(self, fv_ind, text_frame_variant):
-        assert(isinstance(text_frame_variant, TextFrameVariant))
-        return self.map_term(TermTypes.FRAME, text_frame_variant.Variant.get_value())
-
-    def map_token(self, t_ind, token):
-        """ It assumes to be composed for all the supported types.
-        """
-        return self.map_term(TermTypes.TOKEN, token.get_token_value())
-
-    def map_entity(self, e_ind, entity):
-        assert(isinstance(entity, Entity))
-
-        # Value extraction
-        str_formatted_entity = super(VectorizedNetworkTermMapping, self).map_entity(
-            e_ind=e_ind,
-            entity=entity)
-
-        # Vector extraction
-        return self.map_term(TermTypes.ENTITY, str_formatted_entity)

arekit/contrib/networks/vectorizer.py

@@ -1,6 +0,0 @@
-class BaseVectorizer(object):
-    """ Custom API for vectorization
-    """
-
-    def create_term_embedding(self, term):
-        raise NotImplementedError()

arekit/contrib/utils/data/readers/base.py

@@ -1,7 +0,0 @@
-class BaseReader(object):
-
-    def extension(self):
-        raise NotImplementedError()
-
-    def read(self, target):
-        raise NotImplementedError()

arekit/contrib/utils/data/readers/csv_pd.py

@@ -1,38 +0,0 @@
-import importlib
-
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
-
-
-class PandasCsvReader(BaseReader):
-    """ Represents a CSV-based reader, implmented via pandas API.
-    """
-
-    def __init__(self, sep='\t', header='infer', compression='infer', encoding='utf-8', col_types=None,
-                 custom_extension=None):
-        self.__sep = sep
-        self.__compression = compression
-        self.__encoding = encoding
-        self.__header = header
-        self.__custom_extension = custom_extension
-
-        # Special assignation of types for certain columns.
-        self.__col_types = col_types
-        if self.__col_types is None:
-            self.__col_types = dict()
-
-    def extension(self):
-        return ".tsv.gz" if self.__custom_extension is None else self.__custom_extension
-
-    def __from_csv(self, filepath):
-        pd = importlib.import_module("pandas")
-        return pd.read_csv(filepath,
-                           sep=self.__sep,
-                           encoding=self.__encoding,
-                           compression=self.__compression,
-                           dtype=self.__col_types,
-                           header=self.__header)
-
-    def read(self, target):
-        df = self.__from_csv(filepath=target)
-        return PandasBasedRowsStorage(df)

arekit/contrib/utils/data/readers/jsonl.py

@@ -1,15 +0,0 @@
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.contrib.utils.data.storages.jsonl_based import JsonlBasedRowsStorage
-
-
-class JsonlReader(BaseReader):
-
-    def extension(self):
-        return ".jsonl"
-
-    def read(self, target):
-        rows = []
-        with open(target, "r") as f:
-            for line in f.readlines():
-                rows.append(line)
-        return JsonlBasedRowsStorage(rows)

arekit/contrib/utils/data/readers/sqlite.py

@@ -1,14 +0,0 @@
-from arekit.contrib.utils.data.readers.base import BaseReader
-from arekit.contrib.utils.data.storages.sqlite_based import SQliteBasedRowsStorage
-
-
-class SQliteReader(BaseReader):
-
-    def __init__(self, table_name):
-        self.__table_name = table_name
-
-    def extension(self):
-        return ".sqlite"
-
-    def read(self, target):
-        return SQliteBasedRowsStorage(path=target, table_name=self.__table_name)

arekit/contrib/utils/data/service/balance.py

@@ -1,50 +0,0 @@
-import gc
-import importlib
-from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
-
-
-class PandasBasedStorageBalancing(object):
-
-    @staticmethod
-    def create_balanced_from(storage, column_name, free_origin=True):
-        """ Performs oversampled balancing.
-
-            Note: it is quite important to remove previously created storage
-            in order to avoid memory leaking.
-
-            storage: PandasBasedRowsStorage
-                storage contents to be balanced.
-
-            column_name: str
-                column utilized for balancing.
-
-            free_origin: bool
-                indicates whether there is a need to release the resources
-                utilized for the original storage.
-        """
-        assert(isinstance(storage, PandasBasedRowsStorage))
-
-        original_df = storage.DataFrame
-
-        max_size = original_df[column_name].value_counts().max()
-
-        dframes = []
-        for class_index, group in original_df.groupby(column_name):
-            dframes.append(group.sample(max_size - len(group), replace=True))
-
-        # Clear resources.
-        pd = importlib.import_module("pandas")
-        balanced_df = pd.concat(dframes + [original_df])
-
-        # Removing temporary created dataframe.
-        for df in dframes:
-            del df
-
-        # Marking the original dataframe as released
-        # in terms of the allocated memory for it.
-        if free_origin:
-            storage.free()
-
-        gc.collect()
-
-        return PandasBasedRowsStorage(df=balanced_df)

arekit/contrib/utils/data/storages/pandas_based.py

@@ -1,123 +0,0 @@
-import importlib
-
-import numpy as np
-
-from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
-from arekit.common.data.storages.base import BaseRowsStorage, logger
-from arekit.common.utils import progress_bar_iter
-
-
-class PandasBasedRowsStorage(BaseRowsStorage):
-    """ Storage Kernel functions implementation,
-        based on the pandas DataFrames.
-    """
-
-    def __init__(self, df=None):
-        self._df = df
-
-    @property
-    def DataFrame(self):
-        # TODO. Temporary hack, however this should be removed in future.
-        return self._df
-
-    @staticmethod
-    def __create_empty(cols_with_types):
-        """ cols_with_types: list of pairs ("name", dtype)
-        """
-        assert(isinstance(cols_with_types, list))
-        data = np.empty(0, dtype=np.dtype(cols_with_types))
-        pd = importlib.import_module("pandas")
-        return pd.DataFrame(data)
-
-    def __filter(self, column_name, value):
-        return self._df[self._df[column_name] == value]
-
-    @staticmethod
-    def __iter_rows_core(df):
-        for row_index, row in df.iterrows():
-            yield row_index, row
-
-    def __fill_with_blank_rows(self, row_id_column_name, rows_count):
-        assert(isinstance(row_id_column_name, str))
-        assert(isinstance(rows_count, int))
-        self._df[row_id_column_name] = list(range(rows_count))
-        self._df.set_index(row_id_column_name, inplace=True)
-
-    # region protected methods
-
-    def iter_column_names(self):
-        return iter(self._df.columns)
-
-    def iter_column_types(self):
-        return iter(self._df.dtypes)
-
-    def _set_row_value(self, row_ind, column, value):
-        self._df.at[row_ind, column] = value
-
-    def _iter_rows(self):
-        for row_index, row in self.__iter_rows_core(self._df):
-            yield row_index, row.to_dict()
-
-    def _get_rows_count(self):
-        return len(self._df)
-
-    # endregion
-
-    # region public methods
-
-    def fill(self, iter_rows_func, columns_provider, row_handler=None, rows_count=None, desc=""):
-        """ NOTE: We provide the rows counting which is required
-            in order to know an expected amount of rows in advace
-            due to the specifics of the pandas memory allocation
-            for the DataFrames.
-            The latter allows us avoid rows appending, which
-            may significantly affects on performance once the size
-            of DataFrame becomes relatively large.
-        """
-        assert(isinstance(columns_provider, BaseColumnsProvider))
-
-        logger.info("Rows calculation process started. [Required by Pandas-Based storage kernel]")
-        logged_rows_it = progress_bar_iter(
-            iterable=iter_rows_func(True),
-            desc="Calculating rows count ({reason})".format(reason=desc),
-            unit="rows")
-        rows_count = sum(1 for _ in logged_rows_it)
-
-        logger.info("Filling with blank rows: {}".format(rows_count))
-        self.__fill_with_blank_rows(row_id_column_name=columns_provider.ROW_ID,
-                                    rows_count=rows_count)
-        logger.info("Completed!")
-
-        super(PandasBasedRowsStorage, self).fill(iter_rows_func=iter_rows_func,
-                                                 row_handler=row_handler,
-                                                 columns_provider=columns_provider,
-                                                 rows_count=rows_count)
-
-    def get_row(self, row_index):
-        return self._df.iloc[row_index]
-
-    def get_cell(self, row_index, column_name):
-        return self._df.iloc[row_index][column_name]
-
-    def iter_column_values(self, column_name, dtype=None):
-        values = self._df[column_name]
-        if dtype is None:
-            return values
-        return values.astype(dtype)
-
-    def find_by_value(self, column_name, value):
-        return self.__filter(column_name=column_name, value=value)
-
-    def init_empty(self, columns_provider):
-        cols_with_types = columns_provider.get_columns_list_with_types()
-        self._df = self.__create_empty(cols_with_types)
-
-    def iter_shuffled(self):
-        shuffled_df = self._df.sample(frac=1)
-        return self.__iter_rows_core(shuffled_df)
-
-    def free(self):
-        del self._df
-        super(PandasBasedRowsStorage, self).free()
-
-    # endregion

arekit/contrib/utils/data/writers/csv_native.py

@@ -1,63 +0,0 @@
-import csv
-import os
-from os.path import dirname
-
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-
-class NativeCsvWriter(BaseWriter):
-
-    def __init__(self, delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL, header=True):
-        self.__target_f = None
-        self.__writer = None
-        self.__create_writer_func = lambda f: csv.writer(
-            f, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
-        self.__header = header
-        self.__header_written = None
-
-    def extension(self):
-        return ".csv"
-
-    @staticmethod
-    def __iter_storage_column_names(storage):
-        """ Iter only those columns that existed in storage.
-        """
-        for col_name in storage.iter_column_names():
-            if col_name in storage.RowCache:
-                yield col_name
-
-    def open_target(self, target):
-        os.makedirs(dirname(target), exist_ok=True)
-        self.__target_f = open(target, "w")
-        self.__writer = self.__create_writer_func(self.__target_f)
-        self.__header_written = not self.__header
-
-    def close_target(self):
-        self.__target_f.close()
-
-    def commit_line(self, storage):
-        assert(isinstance(storage, RowCacheStorage))
-        assert(self.__writer is not None)
-
-        if not self.__header_written:
-            self.__writer.writerow(list(self.__iter_storage_column_names(storage)))
-            self.__header_written = True
-
-        line_data = list(map(lambda col_name: storage.RowCache[col_name],
-                             self.__iter_storage_column_names(storage)))
-        self.__writer.writerow(line_data)
-
-    def write_all(self, storage, target):
-        """ Writes all the `storage` rows
-            into the `target` filepath, formatted as CSV.
-        """
-        assert(isinstance(storage, BaseRowsStorage))
-
-        with open(target, "w") as f:
-            writer = self.__create_writer_func(f)
-            for _, row in storage:
-                #content = [row[col_name] for col_name in storage.iter_column_names()]
-                content = [v for v in row]
-                writer.writerow(content)

arekit/contrib/utils/data/writers/csv_pd.py

@@ -1,40 +0,0 @@
-import logging
-
-from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
-from arekit.common.utils import create_dir_if_not_exists
-from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class PandasCsvWriter(BaseWriter):
-
-    def __init__(self, write_header):
-        super(PandasCsvWriter, self).__init__()
-        self.__write_header = write_header
-
-    def extension(self):
-        return ".tsv.gz"
-
-    def write_all(self, storage, target):
-        assert(isinstance(storage, PandasBasedRowsStorage))
-        assert(isinstance(target, str))
-
-        create_dir_if_not_exists(target)
-
-        # Temporary hack, remove it in future.
-        df = storage.DataFrame
-
-        logger.info("Saving... {length}: {filepath}".format(length=len(storage), filepath=target))
-        df.to_csv(target,
-                  sep='\t',
-                  encoding='utf-8',
-                  columns=[c for c in df.columns if c != BaseColumnsProvider.ROW_ID],
-                  index=False,
-                  float_format="%.0f",
-                  compression='gzip',
-                  header=self.__write_header)
-
-        logger.info("Saving completed!")

arekit/contrib/utils/data/writers/json_opennre.py

@@ -1,132 +0,0 @@
-import json
-import logging
-import os
-from os.path import dirname
-
-from arekit.common.data import const
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-logger = logging.getLogger(__name__)
-
-
-class OpenNREJsonWriter(BaseWriter):
-    """ This is a bag-based writer for the samples.
-        Project page: https://github.com/thunlp/OpenNRE
-
-        Every bag presented as follows:
-            {
-              'text' or 'token': ...,
-              'h': {'pos': [start, end], 'id': ... },
-              't': {'pos': [start, end], 'id': ... }
-              'id': "id_of_the_text_opinion"
-            }
-
-        In terms of the linked opinions (i0, i1, etc.) we consider id of the first opinion in linkage.
-        During the dataset reading stage via OpenNRE, these linkages automaticaly groups into bags.
-    """
-
-    def __init__(self, text_columns, encoding="utf-8", na_value="NA", keep_extra_columns=True,
-                 skip_extra_existed=True):
-        """ text_columns: list
-                column names that expected to be joined into a single (token) column.
-        """
-        assert(isinstance(text_columns, list))
-        assert(isinstance(encoding, str))
-        self.__text_columns = text_columns
-        self.__encoding = encoding
-        self.__target_f = None
-        self.__keep_extra_columns = keep_extra_columns
-        self.__na_value = na_value
-        self.__skip_extra_existed = skip_extra_existed
-
-    def extension(self):
-        return ".jsonl"
-
-    @staticmethod
-    def __format_row(row, na_value, text_columns, keep_extra_columns, skip_extra_existed):
-        """ Formatting that is compatible with the OpenNRE.
-        """
-        assert(isinstance(na_value, str))
-
-        sample_id = row[const.ID]
-        s_ind = int(row[const.S_IND])
-        t_ind = int(row[const.T_IND])
-        bag_id = str(row[const.OPINION_ID])
-
-        # Gather tokens.
-        tokens = []
-        for text_col in text_columns:
-            if text_col in row:
-                tokens.extend(row[text_col].split())
-
-        # Filtering JSON row.
-        formatted_data = {
-            "id": bag_id,
-            "id_orig": sample_id,
-            "token": tokens,
-            "h": {"pos": [s_ind, s_ind + 1], "id": str(bag_id + "s")},
-            "t": {"pos": [t_ind, t_ind + 1], "id": str(bag_id + "t")},
-            "relation": str(int(row[const.LABEL_UINT])) if const.LABEL_UINT in row else na_value
-        }
-
-        # Register extra fields (optionally).
-        if keep_extra_columns:
-            for key, value in row.items():
-                if key not in formatted_data and key not in text_columns:
-                    formatted_data[key] = value
-                else:
-                    if not skip_extra_existed:
-                        raise Exception(f"key `{key}` is already exist in formatted data "
-                                        f"or a part of the text columns list: {text_columns}")
-
-        return formatted_data
-
-    def open_target(self, target):
-        os.makedirs(dirname(target), exist_ok=True)
-        self.__target_f = open(target, "w")
-        pass
-
-    def close_target(self):
-        self.__target_f.close()
-
-    def commit_line(self, storage):
-        assert(isinstance(storage, RowCacheStorage))
-
-        # Collect existed columns.
-        row_data = {}
-        for col_name in storage.iter_column_names():
-            if col_name not in storage.RowCache:
-                continue
-            row_data[col_name] = storage.RowCache[col_name]
-
-        bag = self.__format_row(row_data, text_columns=self.__text_columns,
-                                keep_extra_columns=self.__keep_extra_columns,
-                                na_value=self.__na_value,
-                                skip_extra_existed=self.__skip_extra_existed)
-
-        self.__write_bag(bag=bag, json_file=self.__target_f)
-
-    @staticmethod
-    def __write_bag(bag, json_file):
-        assert(isinstance(bag, dict))
-        json.dump(bag, json_file, separators=(",", ":"), ensure_ascii=False)
-        json_file.write("\n")
-
-    def write_all(self, storage, target):
-        assert(isinstance(storage, BaseRowsStorage))
-        assert(isinstance(target, str))
-
-        logger.info("Saving... {rows}: {filepath}".format(rows=(len(storage)), filepath=target))
-
-        os.makedirs(os.path.dirname(target), exist_ok=True)
-        with open(target, "w", encoding=self.__encoding) as json_file:
-            for row_index, row in storage:
-                self.__write_bag(bag=self.__format_row(row, text_columns=self.__text_columns,
-                                                       keep_extra_columns=self.__keep_extra_columns,
-                                                       na_value=self.__na_value,
-                                                       skip_extra_existed=self.__skip_extra_existed),
-                                 json_file=json_file)
-
-        logger.info("Saving completed!")