arekit 0.25.0__py3-none-any.whl → 0.25.2__py3-none-any.whl

arekit/contrib/utils/data/writers/sqlite_native.py

@@ -1,114 +0,0 @@
-import os
-import sqlite3
-from os.path import dirname
-
-from arekit.common.data import const
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-
-class SQliteWriter(BaseWriter):
-    """ TODO. This implementation is dedicated for the writing concepts of the data
-        serialization pipeline. However we add the SQLite3 service, it would be
-        right to refactor and utlize some core functionality from the core/service/sqlite.py
-    """
-
-    def __init__(self, table_name="contents", index_column_names=None, skip_existed=False, clear_table=True):
-        """ index_column_names: list or None
-                column names should be considered to build a unique index;
-                if None, the default 'const.ID' will be considered for row indexation.
-        """
-        assert (isinstance(index_column_names, list) or index_column_names is None)
-        self.__index_column_names = index_column_names if index_column_names is not None else [const.ID]
-        self.__table_name = table_name
-        self.__conn = None
-        self.__cur = None
-        self.__need_init_table = True
-        self.__origin_column_names = None
-        self.__skip_existed = skip_existed
-        self.__clear_table = clear_table
-
-    def extension(self):
-        return ".sqlite"
-
-    @staticmethod
-    def __iter_storage_column_names(storage):
-        """ Iter only those columns that existed in storage.
-        """
-        assert (isinstance(storage, RowCacheStorage))
-        for col_name, col_type in zip(storage.iter_column_names(), storage.iter_column_types()):
-            if col_name in storage.RowCache:
-                yield col_name, col_type
-
-    def __init_table(self, column_data):
-        # Compose column name with the related SQLITE type.
-        column_types = ",".join([" ".join([col_name, self.type_to_sqlite(col_type)])
-                                 for col_name, col_type in column_data])
-        # Create table if not exists.
-        self.__cur.execute(f"CREATE TABLE IF NOT EXISTS {self.__table_name}({column_types})")
-        # Table exists, however we may optionally remove the content from it.
-        if self.__clear_table:
-            self.__cur.execute(f"DELETE FROM {self.__table_name};")
-        # Create index.
-        index_name = f"i_{self.__table_name}_id"
-        self.__cur.execute(f"DROP INDEX IF EXISTS {index_name};")
-        self.__cur.execute("CREATE INDEX IF NOT EXISTS {index} ON {table}({columns})".format(
-            index=index_name,
-            table=self.__table_name,
-            columns=", ".join(self.__index_column_names)
-        ))
-        self.__origin_column_names = [col_name for col_name, _ in column_data]
-
-    @staticmethod
-    def type_to_sqlite(col_type):
-        """ This is a simple function that provides conversion from the
-            base numpy types to SQLITE.
-            NOTE: this method represent a quick implementation for supporting
-            types, however it is far away from the generalized implementation.
-        """
-        if isinstance(col_type, str):
-            if 'int' in col_type:
-                return 'INTEGER'
-
-        return "TEXT"
-
-    def open_target(self, target):
-        os.makedirs(dirname(target), exist_ok=True)
-        self.__conn = sqlite3.connect(target)
-        self.__cur = self.__conn.cursor()
-
-    def commit_line(self, storage):
-        assert (isinstance(storage, RowCacheStorage))
-
-        column_data = list(self.__iter_storage_column_names(storage))
-
-        if self.__need_init_table:
-            self.__init_table(column_data)
-            self.__need_init_table = False
-
-        # Check whether the related row is already exist in SQLITE database.
-        row_id = storage.RowCache[const.ID]
-        top_row = self.__cur.execute(f"SELECT EXISTS(SELECT 1 FROM {self.__table_name} WHERE id='{row_id}');")
-        is_exists = top_row.fetchone()[0]
-        if is_exists == 1 and self.__skip_existed:
-            return
-
-        line_data = [storage.RowCache[col_name] for col_name, _ in column_data]
-        parameters = ",".join(["?"] * len(line_data))
-
-        assert (len(self.__origin_column_names) == len(line_data))
-
-        self.__cur.execute(
-            f"INSERT OR REPLACE INTO {self.__table_name} VALUES ({parameters})",
-            tuple(line_data))
-
-        self.__conn.commit()
-
-    def close_target(self):
-        self.__cur = None
-        self.__origin_column_names = None
-        self.__need_init_table = True
-        self.__conn.close()
-
-    def write_all(self, storage, target):
-        pass

arekit/contrib/utils/embeddings/rusvectores.py

@@ -1,58 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.contrib.networks.embedding import Embedding
-
-
-class RusvectoresEmbedding(Embedding):
-    """ Wrapper over models from the following resource.
-        https://rusvectores.org/ru/models/
-
-        NOTE: Usually these are embeddings for texts written in Russian.
-        for the better performance it is expected that we adopt stemmer.
-    """
-
-    def __init__(self, matrix, words, stemmer):
-        assert(isinstance(stemmer, Stemmer) or stemmer is None)
-        super(RusvectoresEmbedding, self).__init__(matrix=matrix, words=words)
-        self.__index_without_pos = self.__create_terms_without_pos()
-        self.__stemmer = stemmer
-        self.__lemmatize_by_default = stemmer is not None
-
-    def try_find_index_by_plain_word(self, word):
-        assert(isinstance(word, str))
-
-        temp = self.__lemmatize_by_default
-        self.__lemmatize_by_default = False
-        index = super(RusvectoresEmbedding, self).try_find_index_by_plain_word(word)
-        self.__lemmatize_by_default = temp
-
-        return index
-
-    def _handler(self, word):
-        return self.__try_find_word_index_pair_lemmatized(word, self.__lemmatize_by_default)
-
-    # region private methods
-
-    def __try_find_word_index_pair_lemmatized(self, term, lemmatize):
-        assert(isinstance(term, str))
-        assert(isinstance(lemmatize, bool))
-
-        if lemmatize:
-            term = self.__stemmer.lemmatize_to_str(term)
-
-        index = self.__index_without_pos[term] \
-            if term in self.__index_without_pos else None
-
-        return term, index
-
-    def __create_terms_without_pos(self):
-        d = {}
-        for word_with_pos, index in self.iter_vocabulary():
-            assert(isinstance(word_with_pos, str))
-            word = word_with_pos.split(u'_')[0]
-            if word in d:
-                continue
-            d[word] = index
-
-        return d
-
-    # endregion

arekit/contrib/utils/embeddings/tokens.py

@@ -1,30 +0,0 @@
-import numpy as np
-
-from arekit.contrib.networks.embedding import Embedding
-from arekit.contrib.utils.processing.text.tokens import Tokens
-
-
-class TokenEmbedding(Embedding):
-    """ Embedding vectors for text punctuation, based on Tokens in parsed text
-    """
-
-    @classmethod
-    def from_supported_tokens(cls, vector_size, random_vector_func):
-        """
-        random_vector_func: func
-            function with parameters (vector_size, seed)
-        """
-        assert(isinstance(vector_size, int))
-        assert(callable(random_vector_func))
-
-        matrix = []
-        tokens_list = list(Tokens.iter_supported_tokens())
-
-        for token_index, _ in enumerate(tokens_list):
-
-            vector = random_vector_func(vector_size, token_index)
-
-            matrix.append(vector)
-
-        return cls(matrix=np.array(matrix),
-                   words=tokens_list)

arekit/contrib/utils/entities/formatters/str_display.py

@@ -1,11 +0,0 @@
-from arekit.common.entities.base import Entity
-from arekit.common.entities.str_fmt import StringEntitiesFormatter
-
-
-class StringEntitiesDisplayValueFormatter(StringEntitiesFormatter):
-    """ Provides the contents of the DisplayValue property.
-    """
-
-    def to_string(self, original_value, entity_type):
-        assert(isinstance(original_value, Entity))
-        return original_value.DisplayValue

arekit/contrib/utils/io_utils/embedding.py

@@ -1,72 +0,0 @@
-from os.path import join
-
-from arekit.contrib.networks.embedding_io import BaseEmbeddingIO
-from arekit.contrib.utils.io_utils.utils import check_targets_existence
-from arekit.contrib.utils.np_utils.embedding import NpzEmbeddingHelper
-from arekit.contrib.utils.np_utils.vocab import VocabRepositoryUtils
-
-
-class NpEmbeddingIO(BaseEmbeddingIO):
-    """ Npz-based IO utils for embedding and text-based for vocabulary.
-        This format represents a archived version of the numpy math data, i.e. vectors, numbers, etc.
-
-        Provides additional Input/Output paths generation functions for:
-            - embedding matrix;
-            - embedding vocabulary.
-    """
-
-    def __init__(self, target_dir, prefix_name="sample"):
-        assert(isinstance(target_dir, str))
-
-        self.__target_dir = target_dir
-        self.__term_emb_fn_template = "-".join([prefix_name, "term_embedding"])
-        self.__vocab_fn_template = "-".join([prefix_name, "term_embedding"])
-
-    # region Embedding-related data
-
-    def save_vocab(self, data):
-        target = self.__get_default_vocab_filepath()
-        return VocabRepositoryUtils.save(data=data, target=target)
-
-    def load_vocab(self):
-        source = self.___get_vocab_source()
-        return dict(VocabRepositoryUtils.load(source))
-
-    def save_embedding(self, data):
-        target = self.__get_default_embedding_filepath()
-        NpzEmbeddingHelper.save_embedding(data=data, target=target)
-
-    def load_embedding(self):
-        source = self.__get_term_embedding_source()
-        return NpzEmbeddingHelper.load_embedding(source)
-
-    def check_targets_existed(self):
-        targets = [
-            self.__get_default_vocab_filepath(),
-            self.__get_term_embedding_target()
-        ]
-        return check_targets_existence(targets=targets)
-
-    # endregion
-
-    # region embedding-related data
-
-    def ___get_vocab_source(self):
-        """ It is possible to load a predefined embedding from another experiment
-            using the related filepath provided by model_io.
-        """
-        return self.__get_default_vocab_filepath()
-
-    def __get_term_embedding_target(self):
-        return self.__get_default_embedding_filepath()
-
-    def __get_term_embedding_source(self):
-        return self.__get_default_embedding_filepath()
-
-    def __get_default_vocab_filepath(self):
-        return join(self.__target_dir, self.__vocab_fn_template)
-
-    def __get_default_embedding_filepath(self):
-        return join(self.__target_dir, self.__term_emb_fn_template)
-
-    # endregion

arekit/contrib/utils/np_utils/embedding.py

@@ -1,22 +0,0 @@
-import logging
-
-from arekit.contrib.utils.np_utils.npz_utils import NpzRepositoryUtils
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class NpzEmbeddingHelper:
-
-    @staticmethod
-    def save_embedding(data, target):
-        NpzRepositoryUtils.save(data=data, target=target)
-        logger.info("Saving embedding [size={shape}]: {filepath}".format(shape=data.shape,
-                                                                         filepath=target))
-
-    @staticmethod
-    def load_embedding(source):
-        embedding = NpzRepositoryUtils.load(source)
-        logger.info("Embedding read [size={size}]: {filepath}".format(size=embedding.shape,
-                                                                      filepath=source))
-        return embedding

arekit/contrib/utils/np_utils/npz_utils.py

@@ -1,13 +0,0 @@
-import numpy as np
-
-
-class NpzRepositoryUtils(object):
-
-    @staticmethod
-    def save(data, target):
-        np.savez(target, data)
-
-    @staticmethod
-    def load(source):
-        data = np.load(source)
-        return data['arr_0']

arekit/contrib/utils/np_utils/vocab.py

@@ -1,20 +0,0 @@
-import logging
-
-import numpy as np
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class VocabRepositoryUtils(object):
-
-    @staticmethod
-    def save(data, target):
-        logger.info("Saving vocabulary [size={size}]: {filepath}".format(size=len(data), filepath=target))
-        np.savetxt(target, data, fmt='%s')
-
-    @staticmethod
-    def load(source):
-        vocab = np.loadtxt(source, dtype=str, comments=None)
-        logger.info("Loading vocabulary [size={size}]: {filepath}".format(size=len(vocab), filepath=source))
-        return vocab

arekit/contrib/utils/pipelines/items/sampling/base.py

@@ -1,94 +0,0 @@
-from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
-from arekit.common.experiment.data_type import DataType
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.contrib.utils.serializer import InputDataSerializationHelper
-
-
-class BaseSerializerPipelineItem(BasePipelineItem):
-
-    def __init__(self, rows_provider, samples_io, save_labels_func, storage, **kwargs):
-        """ sample_rows_formatter:
-                how we format input texts for a BERT model, for example:
-                    - single text
-                    - two sequences, separated by [SEP] token
-
-            save_labels_func: function
-                data_type -> bool
-        """
-        assert(isinstance(rows_provider, BaseSampleRowProvider))
-        assert(isinstance(samples_io, BaseSamplesIO))
-        assert(callable(save_labels_func))
-        assert(isinstance(storage, BaseRowsStorage))
-        super(BaseSerializerPipelineItem, self).__init__(**kwargs)
-
-        self._rows_provider = rows_provider
-        self._samples_io = samples_io
-        self._save_labels_func = save_labels_func
-        self._storage = storage
-
-    def _serialize_iteration(self, data_type, pipeline, data_folding, doc_ids):
-        assert(isinstance(data_type, DataType))
-        assert(isinstance(pipeline, list))
-        assert(isinstance(data_folding, dict) or data_folding is None)
-        assert(isinstance(doc_ids, list) or doc_ids is None)
-        assert(doc_ids is not None or data_folding is not None)
-
-        repos = {
-            "sample": InputDataSerializationHelper.create_samples_repo(
-                keep_labels=self._save_labels_func(data_type),
-                rows_provider=self._rows_provider,
-                storage=self._storage),
-        }
-
-        writer_and_targets = {
-            "sample": (self._samples_io.Writer,
-                       self._samples_io.create_target(data_type=data_type)),
-        }
-
-        for description, repo in repos.items():
-
-            if data_folding is None:
-                # Consider only the predefined doc_ids.
-                doc_ids_iter = doc_ids
-            else:
-                # Take particular data_type.
-                doc_ids_iter = data_folding[data_type]
-                # Consider only predefined doc_ids.
-                if doc_ids is not None:
-                    doc_ids_iter = set(doc_ids_iter).intersection(doc_ids)
-
-            InputDataSerializationHelper.fill_and_write(
-                repo=repo,
-                pipeline=pipeline,
-                doc_ids_iter=doc_ids_iter,
-                desc="{desc} [{data_type}]".format(desc=description, data_type=data_type),
-                writer=writer_and_targets[description][0],
-                target=writer_and_targets[description][1])
-
-    def _handle_iteration(self, data_type_pipelines, data_folding, doc_ids):
-        """ Performing data serialization for a particular iteration
-        """
-        assert(isinstance(data_type_pipelines, dict))
-        for data_type, pipeline in data_type_pipelines.items():
-            self._serialize_iteration(data_type=data_type, pipeline=pipeline, data_folding=data_folding,
-                                      doc_ids=doc_ids)
-
-    def apply_core(self, input_data, pipeline_ctx):
-        """
-            data_type_pipelines: dict of, for example:
-                {
-                    DataType.Train: BasePipeline,
-                    DataType.Test: BasePipeline
-                }
-
-                data_type_pipelines: doc_id -> parsed_doc -> annot -> opinion linkages
-                    for example, function: sentiment_attitude_extraction_default_pipeline
-                doc_ids: optional
-                    this parameter allows to limit amount of documents considered for sampling
-        """
-        assert("data_type_pipelines" in pipeline_ctx)
-        self._handle_iteration(data_type_pipelines=pipeline_ctx.provide("data_type_pipelines"),
-                               doc_ids=pipeline_ctx.provide_or_none("doc_ids"),
-                               data_folding=pipeline_ctx.provide_or_none("data_folding"))

arekit/contrib/utils/pipelines/items/sampling/networks.py

@@ -1,55 +0,0 @@
-from arekit.contrib.networks.input.embedding.matrix import create_term_embedding_matrix
-from arekit.contrib.networks.input.embedding.offsets import TermsEmbeddingOffsets
-from arekit.contrib.networks.embedding import Embedding
-from arekit.contrib.networks.input.providers.sample import NetworkSampleRowProvider
-from arekit.contrib.utils.io_utils.embedding import NpEmbeddingIO
-from arekit.contrib.utils.pipelines.items.sampling.base import BaseSerializerPipelineItem
-
-
-class NetworksInputSerializerPipelineItem(BaseSerializerPipelineItem):
-
-    def __init__(self, save_labels_func, rows_provider, samples_io, emb_io, storage, save_embedding=True, **kwargs):
-        """ This pipeline item allows to perform a data preparation for neural network models.
-
-            considering a list of the whole data_types with the related pipelines,
-            which are supported and required in a handler. It is necessary to know
-            data_types in advance as it allows to create a complete vocabulary of input terms,
-            with the related embeddings.
-        """
-        assert(isinstance(emb_io, NpEmbeddingIO))
-        assert(isinstance(rows_provider, NetworkSampleRowProvider))
-        assert(isinstance(save_embedding, bool))
-        super(NetworksInputSerializerPipelineItem, self).__init__(
-            rows_provider=rows_provider,
-            samples_io=samples_io,
-            save_labels_func=save_labels_func,
-            storage=storage,
-            **kwargs)
-
-        self.__emb_io = emb_io
-        self.__save_embedding = save_embedding
-
-    def _handle_iteration(self, data_type_pipelines, data_folding, doc_ids):
-        """ Performing data serialization for a particular iteration
-        """
-        assert(isinstance(data_type_pipelines, dict))
-
-        # Prepare for the present iteration.
-        self._rows_provider.clear_embedding_pairs()
-
-        super(NetworksInputSerializerPipelineItem, self)._handle_iteration(
-            data_type_pipelines=data_type_pipelines, data_folding=data_folding, doc_ids=doc_ids)
-
-        if not (self.__save_embedding and self._rows_provider.HasEmbeddingPairs):
-            return
-
-        # Save embedding information additionally.
-        term_embedding = Embedding.from_word_embedding_pairs_iter(self._rows_provider.iter_term_embedding_pairs())
-        embedding_matrix = create_term_embedding_matrix(term_embedding=term_embedding)
-        vocab = list(TermsEmbeddingOffsets.extract_vocab(words_embedding=term_embedding))
-
-        # Save embedding matrix
-        self.__emb_io.save_embedding(data=embedding_matrix)
-        self.__emb_io.save_vocab(data=vocab)
-
-        del embedding_matrix

arekit/contrib/utils/pipelines/items/text/entities_default.py

@@ -1,23 +0,0 @@
-from arekit.common.entities.base import Entity
-from arekit.common.pipeline.items.base import BasePipelineItem
-
-
-class TextEntitiesParser(BasePipelineItem):
-
-    def __init__(self, **kwargs):
-        super(TextEntitiesParser, self).__init__(**kwargs)
-
-    @staticmethod
-    def __process_word(word):
-        assert(isinstance(word, str))
-
-        # If this is a special word which is related to the [entity] mention.
-        if word[0] == "[" and word[-1] == "]":
-            entity = Entity(value=word[1:-1], e_type=None)
-            return entity
-
-        return word
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(input_data, list))
-        return [self.__process_word(w) for w in input_data]

arekit/contrib/utils/pipelines/items/text/frames_lemmatized.py

@@ -1,36 +0,0 @@
-from arekit.common.text.stemmer import Stemmer
-from arekit.contrib.utils.pipelines.items.text.frames import FrameVariantsParser
-from arekit.contrib.utils.processing.languages.ru.mods import RussianLanguageMods
-
-
-class LemmasBasedFrameVariantsParser(FrameVariantsParser):
-
-    def __init__(self, frame_variants, stemmer, locale_mods=RussianLanguageMods, save_lemmas=False, **kwargs):
-        assert(isinstance(stemmer, Stemmer))
-        assert(isinstance(save_lemmas, bool))
-        super(LemmasBasedFrameVariantsParser, self).__init__(frame_variants=frame_variants, **kwargs)
-
-        self.__frame_variants = frame_variants
-        self.__stemmer = stemmer
-        self.__save_lemmas = save_lemmas
-        self.__max_variant_len = max([len(variant) for _, variant in frame_variants.iter_variants()])
-        self.__locale_mods = locale_mods
-
-    def __lemmatize_term(self, term):
-        # we first split onto words for lemmatization and then join all of them.
-        lemma = "".join(self.__stemmer.lemmatize_to_list(term))
-        # then we replace certain chars according to the locale restrictions.
-        return self.__locale_mods.replace_specific_word_chars(lemma)
-
-    def __provide_lemmatized_terms(self, terms):
-        """
-        Compose a list of lemmatized versions of parsed_doc
-        PS: Might be significantly slow, depending on stemmer were used.
-        """
-        assert(isinstance(terms, list))
-        return [self.__lemmatize_term(term) if isinstance(term, str) else term for term in terms]
-
-    def apply_core(self, input_data, pipeline_ctx):
-        lemmas = self.__provide_lemmatized_terms(input_data)
-        processed_it = self._iter_processed(terms=lemmas, origin=lemmas if self.__save_lemmas else input_data)
-        return list(processed_it)

arekit/contrib/utils/pipelines/items/text/frames_negation.py

@@ -1,33 +0,0 @@
-from arekit.common.frames.text_variant import TextFrameVariant
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.contrib.utils.processing.languages.mods import BaseLanguageMods
-from arekit.contrib.utils.processing.languages.ru.mods import RussianLanguageMods
-
-
-class FrameVariantsSentimentNegation(BasePipelineItem):
-
-    def __init__(self, locale_mods=RussianLanguageMods, **kwargs):
-        assert(issubclass(locale_mods, BaseLanguageMods))
-        super(FrameVariantsSentimentNegation, self).__init__(**kwargs)
-        self._locale_mods = locale_mods
-
-    @staticmethod
-    def __get_preposition(terms, index):
-        return terms[index-1] if index > 0 else None
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(input_data, list))
-        assert(isinstance(pipeline_ctx, PipelineContext))
-
-        for curr_ind, term in enumerate(input_data):
-
-            if not isinstance(term, TextFrameVariant):
-                continue
-
-            prep_term = self.__get_preposition(terms=input_data, index=curr_ind)
-            is_negated = self._locale_mods.is_negation_word(prep_term) if prep_term is not None else False
-
-            term.set_is_negated(is_negated)
-
-        return input_data