arekit 0.25.0__py3-none-any.whl → 0.25.1__py3-none-any.whl

arekit/common/service/sqlite.py

@@ -1,36 +0,0 @@
-import sqlite3
-
-
-class SQLiteProvider(object):
-
-    @staticmethod
-    def write(data_it, target, data2col_func, table_name, columns, sqlite3_column_types,
-              id_column_name="id", id_column_type="TEXT"):
-        assert(callable(data2col_func))
-        assert(isinstance(columns, list))
-        assert(isinstance(sqlite3_column_types, list))
-        assert(len(columns) == len(sqlite3_column_types))
-
-        with sqlite3.connect(target) as con:
-            cur = con.cursor()
-
-            # Provide the ID column.
-            columns = [id_column_name] + columns
-            sqlite3_column_types = [id_column_type] + sqlite3_column_types
-
-            # Compose the whole columns list.
-            content = ", ".join([" ".join(item) for item in zip(columns, sqlite3_column_types)])
-            cur.execute(f"CREATE TABLE IF NOT EXISTS {table_name}({content})")
-            cur.execute(f"CREATE INDEX IF NOT EXISTS i_id ON {table_name}({id_column_name})")
-
-            for uid, data in data_it:
-                r = cur.execute(f"SELECT EXISTS(SELECT 1 FROM {table_name} WHERE {id_column_name}='{uid}');")
-                ans = r.fetchone()[0]
-                if ans == 1:
-                    continue
-
-                params = ", ".join(tuple(['?'] * (len(columns))))
-                cur.execute(f"INSERT INTO {table_name} VALUES ({params})", [str(uid)] + data2col_func(data))
-                con.commit()
-
-            cur.close()

arekit/contrib/networks/embedding.py

@@ -1,149 +0,0 @@
-from collections.abc import Iterable
-import numpy as np
-
-
-class Embedding(object):
-    """
-    Represents default wrapper over W2V API.
-    """
-
-    def __init__(self, matrix, words):
-        assert(isinstance(matrix, np.ndarray) and len(matrix.shape) == 2)
-        assert(isinstance(words, np.ndarray))
-        assert(len(words) == matrix.shape[0])
-        self._matrix = matrix
-        self.__words = words
-        self.__index_by_word = self.__create_index(words)
-
-    # region properties
-
-    @property
-    def VectorSize(self):
-        return self._matrix.shape[1]
-
-    @property
-    def VocabularySize(self):
-        return self._matrix.shape[0]
-
-    # endregion
-
-    # region classmethods
-
-    @classmethod
-    def from_word_embedding_pairs_iter(cls, word_embedding_pairs):
-        assert(isinstance(word_embedding_pairs, Iterable))
-
-        matrix = []
-        words = []
-        used = set()
-        for word, vector in word_embedding_pairs:
-
-            if word in used:
-                continue
-
-            used.add(word)
-
-            matrix.append(vector)
-            words.append(word)
-
-        return cls(matrix=np.array(matrix) if len(matrix) > 0 else np.empty(shape=(0, 0)),
-                   words=np.array(words))
-
-    @classmethod
-    def from_list_with_embedding_func(cls, words_iter, embedding_func):
-        assert(isinstance(words_iter, Iterable))
-        assert(callable(embedding_func))
-
-        matrix = []
-        words = []
-        used = set()
-        for word in words_iter:
-
-            if word in used:
-                continue
-            used.add(word)
-
-            vector = embedding_func(word)
-            matrix.append(vector)
-            words.append(word)
-
-        return cls(matrix=np.array(matrix),
-                   words=words)
-
-    # endregion
-
-    # region private methods
-
-    @staticmethod
-    def __create_index(words):
-        index = {}
-        for i, word in enumerate(words):
-            index[word] = i
-        return index
-
-    def __try_find_word_index_pair(self, word):
-        """
-        Assumes to pefrom term transformation (optional)
-        in order to find a term in an inner vocabulary
-
-        returns: pair
-            (processed_term, index)
-        """
-        assert(isinstance(word, str))
-
-        has_index = self.__index_by_word[word] if word in self.__index_by_word else None
-        word = word if has_index else None
-        return word, has_index
-
-    def __hadler_core(self, word):
-        """
-        Core word handler.
-        Assumes to perform word stripping.
-        """
-        stripped_word = word.strip()
-        return self._handler(stripped_word)
-
-    # endregion
-
-    def iter_vocabulary(self):
-        for word in self.__words:
-            yield word, self.__index_by_word[word]
-
-    def get_vector_by_index(self, index):
-        assert(isinstance(index, int))
-        return self._matrix[index]
-
-    def get_word_by_index(self, index):
-        assert(isinstance(index, int))
-        return self.__words[index]
-
-    def try_find_index_by_word(self, word):
-        assert(isinstance(word, str))
-        _, index = self.__hadler_core(word)
-        return index
-
-    def try_find_index_by_plain_word(self, word):
-        assert(isinstance(word, str))
-        _, index = self.__hadler_core(word)
-        return index
-
-    def try_get_related_word(self, word):
-        word, _ = self.__hadler_core(word)
-        return word
-
-    def _handler(self, word):
-        return self.__try_find_word_index_pair(word)
-
-    # region overriden methods
-
-    def __contains__(self, word):
-        assert(isinstance(word, str))
-        _, index = self.__hadler_core(word)
-        return index is not None
-
-    def __getitem__(self, word):
-        assert(isinstance(word, str))
-        _, index = self.__hadler_core(word)
-        return self._matrix[index]
-
-    # endregion

arekit/contrib/networks/embedding_io.py

@@ -1,18 +0,0 @@
-class BaseEmbeddingIO(object):
-    """ API for loading and saving embedding and vocabulary related data.
-    """
-
-    def save_vocab(self, data):
-        raise NotImplementedError()
-
-    def load_vocab(self,):
-        raise NotImplementedError()
-
-    def save_embedding(self, data):
-        raise NotImplementedError()
-
-    def load_embedding(self):
-        raise NotImplementedError()
-
-    def check_targets_existed(self):
-        raise NotImplementedError()

arekit/contrib/networks/input/const.py

@@ -1,6 +0,0 @@
-# Additional input columns
-FrameVariantIndices = "frames"
-FrameConnotations = "frame_connots_uint"
-SynonymObject = "syn_objs"
-SynonymSubject = "syn_subjs"
-PosTags = "pos_tags"

arekit/contrib/networks/input/ctx_serialization.py

@@ -1,28 +0,0 @@
-from arekit.common.data.input.providers.label.multiple import MultipleLabelProvider
-from arekit.contrib.utils.processing.pos.base import POSTagger
-
-
-class NetworkSerializationContext(object):
-
-    def __init__(self, labels_scaler, frame_roles_label_scaler, frames_connotation_provider, pos_tagger=None):
-        assert(isinstance(pos_tagger, POSTagger) or pos_tagger is None)
-        self.__label_provider = MultipleLabelProvider(labels_scaler)
-        self.__frame_roles_label_scaler = frame_roles_label_scaler
-        self.__frames_connotation_provider = frames_connotation_provider
-        self.__pos_tagger = pos_tagger
-
-    @property
-    def LabelProvider(self):
-        return self.__label_provider
-
-    @property
-    def FrameRolesLabelScaler(self):
-        return self.__frame_roles_label_scaler
-
-    @property
-    def FramesConnotationProvider(self):
-        return self.__frames_connotation_provider
-
-    @property
-    def PosTagger(self):
-        return self.__pos_tagger

arekit/contrib/networks/input/embedding/matrix.py

@@ -1,29 +0,0 @@
-import logging
-import numpy as np
-
-from arekit.contrib.networks.input.embedding.offsets import TermsEmbeddingOffsets
-from arekit.contrib.networks.embedding import Embedding
-
-logger = logging.getLogger(__name__)
-
-
-def create_term_embedding_matrix(term_embedding):
-    """
-    Compose complete embedding matrix, which includes:
-        - word embeddings
-        - entity embeddings
-        - token embeddings
-
-    returns: np.ndarray(words_count, embedding_size)
-        embedding matrix which includes embedding both for words and
-        entities
-    """
-    assert(isinstance(term_embedding, Embedding))
-
-    embedding_offsets = TermsEmbeddingOffsets(words_count=term_embedding.VocabularySize)
-    matrix = np.zeros((embedding_offsets.TotalCount, term_embedding.VectorSize))
-
-    for word, index in term_embedding.iter_vocabulary():
-        matrix[embedding_offsets.get_word_index(index)] = term_embedding.get_vector_by_index(index)
-
-    return matrix

arekit/contrib/networks/input/embedding/offsets.py

@@ -1,55 +0,0 @@
-import logging
-
-from arekit.contrib.networks.embedding import Embedding
-
-logger = logging.getLogger(__name__)
-
-
-class TermsEmbeddingOffsets(object):
-    """
-    Describes indices distribution within a further TermsEmbedding.
-
-    All parameters shifted by 1 because of a empty placeholder.
-    """
-
-    def __init__(self, words_count):
-        assert(isinstance(words_count, int))
-        self.__words_count = words_count
-
-    # region properties
-
-    @property
-    def TotalCount(self):
-        return 1 + self.__words_count
-
-    # endregion
-
-    # region 'get' methods
-
-    @staticmethod
-    def get_word_index(index):
-        return 1 + index
-
-    # endregion
-
-    @staticmethod
-    def extract_vocab(words_embedding):
-        """
-        returns:
-            enumeration of pairs (word, key)
-        """
-
-        assert(isinstance(words_embedding, Embedding))
-
-        offsets = TermsEmbeddingOffsets(words_count=words_embedding.VocabularySize)
-
-        all_words = [(0, 'PADDING')]
-
-        for word, index in words_embedding.iter_vocabulary():
-            assert(isinstance(word, str))
-            all_words.append((offsets.get_word_index(index), word))
-
-        assert(len(all_words) == offsets.TotalCount)
-
-        for key, word in sorted(all_words, key=lambda item: item[0]):
-            yield word, key

arekit/contrib/networks/input/formatters/pos_mapper.py

@@ -1,22 +0,0 @@
-from arekit.common.context.terms_mapper import TextTermsMapper
-from arekit.contrib.utils.processing.languages.pos import PartOfSpeechType
-from arekit.contrib.utils.processing.pos.base import POSTagger
-
-
-class PosTermsMapper(TextTermsMapper):
-
-    def __init__(self, pos_tagger):
-        assert(isinstance(pos_tagger, POSTagger))
-        self.__pos_tagger = pos_tagger
-
-    def map_word(self, w_ind, word):
-        return self.__pos_tagger.get_term_pos(word)
-
-    def map_token(self, t_ind, token):
-        return PartOfSpeechType.Unknown
-
-    def map_text_frame_variant(self, fv_ind, text_frame_variant):
-        return self.__pos_tagger.get_term_pos(text_frame_variant.Variant.get_value())
-
-    def map_entity(self, e_ind, entity):
-        return PartOfSpeechType.Unknown

arekit/contrib/networks/input/providers/sample.py

@@ -1,129 +0,0 @@
-import collections
-
-from arekit.common.data.input.providers.label.base import LabelProvider
-from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
-from arekit.common.entities.base import Entity
-from arekit.common.frames.text_variant import TextFrameVariant
-from arekit.common.labels.scaler.sentiment import SentimentLabelScaler
-from arekit.common.docs.parsed.base import ParsedDocument
-from arekit.contrib.networks.input.formatters.pos_mapper import PosTermsMapper
-from arekit.contrib.networks.input import const
-from arekit.contrib.networks.input.providers.term_connotation import extract_uint_frame_variant_connotation
-from arekit.contrib.networks.input.rows_parser import create_nn_val_writer_fmt
-
-
-class NetworkSampleRowProvider(BaseSampleRowProvider):
-
-    def __init__(self,
-                 label_provider,
-                 text_provider,
-                 frames_connotation_provider,
-                 frame_role_label_scaler,
-                 term_embedding_pairs=None,
-                 pos_terms_mapper=None):
-        """ term_embedding_pairs: dict or None
-                additional structure, utilized to collect all the embedding pairs during the
-                rows providing stage.
-        """
-        assert(isinstance(label_provider, LabelProvider))
-        assert(isinstance(frame_role_label_scaler, SentimentLabelScaler))
-        assert(isinstance(pos_terms_mapper, PosTermsMapper) or pos_terms_mapper is None)
-        assert(isinstance(term_embedding_pairs, collections.OrderedDict) or term_embedding_pairs is None)
-
-        super(NetworkSampleRowProvider, self).__init__(label_provider=label_provider,
-                                                       text_provider=text_provider)
-
-        self.__frames_connotation_provider = frames_connotation_provider
-        self.__frame_role_label_scaler = frame_role_label_scaler
-        self.__pos_terms_mapper = pos_terms_mapper
-        self.__term_embedding_pairs = term_embedding_pairs
-        self.__nn_val_fmt = create_nn_val_writer_fmt(fmt_type="writer")
-
-    @property
-    def HasEmbeddingPairs(self):
-        return self.__term_embedding_pairs is not None
-
-    def iter_term_embedding_pairs(self):
-        """ Provide the contents of the embedded pairs.
-        """
-        return iter(self.__term_embedding_pairs.items())
-
-    def clear_embedding_pairs(self):
-        """ Release the contents of the collected embedding pairs.
-        """
-
-        # Check whether we actually collect embedding pairs.
-        if self.__term_embedding_pairs is None:
-            return
-
-        self.__term_embedding_pairs.clear()
-
-    def _fill_row_core(self, row, text_opinion_linkage, index_in_linked, etalon_label,
-                       parsed_doc, sentence_ind, s_ind, t_ind):
-        assert(isinstance(parsed_doc, ParsedDocument))
-
-        super(NetworkSampleRowProvider, self)._fill_row_core(
-            row=row,
-            text_opinion_linkage=text_opinion_linkage,
-            index_in_linked=index_in_linked,
-            etalon_label=etalon_label,
-            parsed_doc=parsed_doc,
-            sentence_ind=sentence_ind,
-            s_ind=s_ind, t_ind=t_ind)
-
-        # Extracting list of terms, utilized in further.
-        terms_iter, actual_s_ind, actual_t_ind = self._provide_sentence_terms(
-            parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
-        terms = list(terms_iter)
-
-        # Compose frame indices.
-        uint_frame_inds = list(self.__iter_indices(terms=terms, filter=lambda t: isinstance(t, TextFrameVariant)))
-
-        # Compose frame connotations.
-        uint_frame_connotations = list(
-            map(lambda variant: extract_uint_frame_variant_connotation(
-                    text_frame_variant=variant,
-                    frames_connotation_provider=self.__frames_connotation_provider,
-                    three_label_scaler=self.__frame_role_label_scaler),
-                [terms[frame_ind] for frame_ind in uint_frame_inds]))
-
-        vm = {
-            const.FrameVariantIndices: uint_frame_inds,
-            const.FrameConnotations: uint_frame_connotations,
-            const.SynonymSubject: self.__create_synonyms_set(terms=terms, term_ind=actual_s_ind),
-            const.SynonymObject: self.__create_synonyms_set(terms=terms, term_ind=actual_t_ind),
-            const.PosTags: None if self.__pos_terms_mapper is None else [int(pos_tag) for pos_tag in self.__pos_terms_mapper.iter_mapped(terms)]
-        }
-
-        self._apply_row_data(row=row, vm=vm, val_fmt=self.__nn_val_fmt)
-
-    # region private methods
-
-    def __create_synonyms_set(self, terms, term_ind):
-        entity = terms[term_ind]
-        assert(isinstance(entity, Entity))
-
-        # Searching for other synonyms among all the terms.
-        group_ind = entity.GroupIndex
-        it = self.__iter_indices(terms=terms, filter=lambda t: self.__syn_check(term=t, group_ind=group_ind))
-        inds_set = set(it)
-
-        # Guarantee the presence of the term_ind
-        inds_set.add(term_ind)
-
-        return sorted(inds_set)
-
-    @staticmethod
-    def __iter_indices(terms, filter):
-        for t_ind, term in enumerate(terms):
-            if filter(term):
-                yield t_ind
-
-    def __syn_check(self, term, group_ind):
-        if not isinstance(term, Entity):
-            return False
-        if group_ind is None:
-            return False
-        return term.GroupIndex == group_ind
-
-    # endregion

arekit/contrib/networks/input/providers/term_connotation.py

@@ -1,23 +0,0 @@
-from arekit.common.frames.connotations.descriptor import FrameConnotationDescriptor
-from arekit.common.frames.connotations.provider import FrameConnotationProvider
-from arekit.common.frames.text_variant import TextFrameVariant
-from arekit.common.labels.scaler.sentiment import SentimentLabelScaler
-
-
-def extract_uint_frame_variant_connotation(text_frame_variant, frames_connotation_provider, three_label_scaler):
-    assert (isinstance(text_frame_variant, TextFrameVariant))
-    assert (isinstance(frames_connotation_provider, FrameConnotationProvider))
-    assert (isinstance(three_label_scaler, SentimentLabelScaler))
-
-    frame_id = text_frame_variant.Variant.FrameID
-    connot_descriptor = frames_connotation_provider.try_provide(frame_id)
-
-    if connot_descriptor is None:
-        return three_label_scaler.label_to_uint(label=three_label_scaler.get_no_label_instance())
-
-    assert (isinstance(connot_descriptor, FrameConnotationDescriptor))
-
-    target_label = three_label_scaler.invert_label(connot_descriptor.Label) \
-        if text_frame_variant.IsNegated else connot_descriptor.Label
-
-    return three_label_scaler.label_to_uint(target_label)

arekit/contrib/networks/input/providers/text.py

@@ -1,24 +0,0 @@
-from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
-from arekit.contrib.networks.input.terms_mapping import VectorizedNetworkTermMapping
-
-
-class NetworkSingleTextProvider(BaseSingleTextProvider):
-    """
-    Performs iteration process over (string, embedding) term pairs.
-    """
-
-    def __init__(self, text_terms_mapper, pair_handling_func):
-        assert(isinstance(text_terms_mapper, VectorizedNetworkTermMapping))
-        assert(callable(pair_handling_func))
-        super(NetworkSingleTextProvider, self).__init__(text_terms_mapper=text_terms_mapper)
-        self.__write_embedding_pair_func = pair_handling_func
-
-    def _mapped_data_to_str(self, m_data):
-        # In this case, m_term consist of
-        # 1. Term;
-        # 2. Embedding.
-        term, _ = m_data
-        return term
-
-    def _handle_mapped_data(self, m_data):
-        self.__write_embedding_pair_func(m_data)

arekit/contrib/networks/input/rows_parser.py

@@ -1,47 +0,0 @@
-import arekit.contrib.networks.input.const as const
-from arekit.common.data.rows_fmt import process_indices_list
-
-
-def create_nn_column_formatters(no_value_func=lambda: None, args_sep=","):
-    assert(callable(no_value_func))
-
-    empty_list = []
-
-    def str_to_list(value):
-        return process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-
-    def list_to_str(inds_iter):
-        return args_sep.join([str(i) for i in inds_iter])
-
-    return {
-        const.FrameVariantIndices: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-                if isinstance(value, str) else empty_list
-        },
-        const.FrameConnotations: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-                if isinstance(value, str) else empty_list
-        },
-        const.SynonymObject: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-        },
-        const.SynonymSubject: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
-        },
-        const.PosTags: {
-            "writer": lambda value: list_to_str(value),
-            "parser": lambda value: str_to_list(value)
-        }
-    }
-
-
-def create_nn_val_writer_fmt(fmt_type, args_sep=","):
-    assert(isinstance(fmt_type, str))
-    d = create_nn_column_formatters(args_sep=args_sep)
-    for k, v in d.items():
-        d[k] = v[fmt_type]
-    return d

arekit/contrib/networks/input/term_types.py

@@ -1,13 +0,0 @@
-class TermTypes(object):
-    """ Types of input terms that may occur within the
-        input sequence of the neural network moodels.
-    """
-
-    WORD = "word"
-    ENTITY = "entity"
-    FRAME = "frame"
-    TOKEN = "token"
-
-    @staticmethod
-    def iter_types():
-        return [TermTypes.WORD, TermTypes.ENTITY, TermTypes.FRAME, TermTypes.TOKEN]

arekit/contrib/networks/input/terms_mapping.py

@@ -1,60 +0,0 @@
-from arekit.common.data.input.terms_mapper import OpinionContainingTextTermsMapper
-from arekit.common.entities.base import Entity
-from arekit.common.frames.text_variant import TextFrameVariant
-from arekit.contrib.networks.input.term_types import TermTypes
-
-
-class VectorizedNetworkTermMapping(OpinionContainingTextTermsMapper):
-    """ For every element returns: (word, embedded vector)
-    """
-
-    def __init__(self, string_entities_formatter, vectorizers):
-        """string_emb_entity_formatter:
-            Utilized in order to obtain embedding value from predefined_embeding for entities
-           vectorizers:
-            dict
-        """
-        assert(isinstance(vectorizers, dict))
-
-        for term_type in TermTypes.iter_types():
-            assert(term_type in vectorizers)
-
-        super(VectorizedNetworkTermMapping, self).__init__(
-            entity_formatter=string_entities_formatter)
-
-        self.__vectorizers = vectorizers
-
-    def map_term(self, term_type, term):
-        """Universal term mapping method.
-
-        Args:
-            term_type (TermTypes): The type of term to map.
-            term (str): The term to map.
-
-        Returns:
-            The mapped term.
-        """
-        return self.__vectorizers[term_type].create_term_embedding(term=term)
-
-    def map_word(self, w_ind, word):
-        return self.map_term(TermTypes.WORD, word)
-
-    def map_text_frame_variant(self, fv_ind, text_frame_variant):
-        assert(isinstance(text_frame_variant, TextFrameVariant))
-        return self.map_term(TermTypes.FRAME, text_frame_variant.Variant.get_value())
-
-    def map_token(self, t_ind, token):
-        """ It assumes to be composed for all the supported types.
-        """
-        return self.map_term(TermTypes.TOKEN, token.get_token_value())
-
-    def map_entity(self, e_ind, entity):
-        assert(isinstance(entity, Entity))
-
-        # Value extraction
-        str_formatted_entity = super(VectorizedNetworkTermMapping, self).map_entity(
-            e_ind=e_ind,
-            entity=entity)
-
-        # Vector extraction
-        return self.map_term(TermTypes.ENTITY, str_formatted_entity)

arekit/contrib/networks/vectorizer.py

@@ -1,6 +0,0 @@
-class BaseVectorizer(object):
-    """ Custom API for vectorization
-    """
-
-    def create_term_embedding(self, term):
-        raise NotImplementedError()

arekit/contrib/utils/data/readers/base.py

@@ -1,7 +0,0 @@
-class BaseReader(object):
-
-    def extension(self):
-        raise NotImplementedError()
-
-    def read(self, target):
-        raise NotImplementedError()