arekit 0.24.0__py3-none-any.whl

arekit/contrib/utils/embeddings/tokens.py

@@ -0,0 +1,30 @@
+import numpy as np
+
+from arekit.contrib.networks.embedding import Embedding
+from arekit.contrib.utils.processing.text.tokens import Tokens
+
+
+class TokenEmbedding(Embedding):
+    """ Embedding vectors for text punctuation, based on Tokens in parsed text
+    """
+
+    @classmethod
+    def from_supported_tokens(cls, vector_size, random_vector_func):
+        """
+        random_vector_func: func
+            function with parameters (vector_size, seed)
+        """
+        assert(isinstance(vector_size, int))
+        assert(callable(random_vector_func))
+
+        matrix = []
+        tokens_list = list(Tokens.iter_supported_tokens())
+
+        for token_index, _ in enumerate(tokens_list):
+
+            vector = random_vector_func(vector_size, token_index)
+
+            matrix.append(vector)
+
+        return cls(matrix=np.array(matrix),
+                   words=tokens_list)

arekit/contrib/utils/entities/filter.py

@@ -0,0 +1,7 @@
+class EntityFilter(object):
+
+    def __init__(self):
+        pass
+
+    def is_ignored(self, entity, e_type):
+        raise NotImplementedError()

arekit/contrib/utils/entities/formatters/str_display.py

@@ -0,0 +1,11 @@
+from arekit.common.entities.base import Entity
+from arekit.common.entities.str_fmt import StringEntitiesFormatter
+
+
+class StringEntitiesDisplayValueFormatter(StringEntitiesFormatter):
+    """ Provides the contents of the DisplayValue property.
+    """
+
+    def to_string(self, original_value, entity_type):
+        assert(isinstance(original_value, Entity))
+        return original_value.DisplayValue

arekit/contrib/utils/entities/formatters/str_simple_sharp_prefixed_fmt.py

@@ -0,0 +1,15 @@
+from arekit.common.entities.str_fmt import StringEntitiesFormatter
+from arekit.common.entities.types import OpinionEntityType
+
+
+class SharpPrefixedEntitiesSimpleFormatter(StringEntitiesFormatter):
+
+    def to_string(self, original_value, entity_type):
+        assert(isinstance(entity_type, OpinionEntityType))
+
+        if (entity_type == OpinionEntityType.Object) or (entity_type == OpinionEntityType.SynonymObject):
+            return "#O"
+        elif (entity_type == OpinionEntityType.Subject) or (entity_type == OpinionEntityType.SynonymSubject):
+            return "#S"
+        elif entity_type == OpinionEntityType.Other:
+            return "#E"

arekit/contrib/utils/io_utils/embedding.py

@@ -0,0 +1,72 @@
+from os.path import join
+
+from arekit.contrib.networks.embedding_io import BaseEmbeddingIO
+from arekit.contrib.utils.io_utils.utils import check_targets_existence
+from arekit.contrib.utils.np_utils.embedding import NpzEmbeddingHelper
+from arekit.contrib.utils.np_utils.vocab import VocabRepositoryUtils
+
+
+class NpEmbeddingIO(BaseEmbeddingIO):
+    """ Npz-based IO utils for embedding and text-based for vocabulary.
+        This format represents a archived version of the numpy math data, i.e. vectors, numbers, etc.
+
+        Provides additional Input/Output paths generation functions for:
+            - embedding matrix;
+            - embedding vocabulary.
+    """
+
+    def __init__(self, target_dir, prefix_name="sample"):
+        assert(isinstance(target_dir, str))
+
+        self.__target_dir = target_dir
+        self.__term_emb_fn_template = "-".join([prefix_name, "term_embedding"])
+        self.__vocab_fn_template = "-".join([prefix_name, "term_embedding"])
+
+    # region Embedding-related data
+
+    def save_vocab(self, data):
+        target = self.__get_default_vocab_filepath()
+        return VocabRepositoryUtils.save(data=data, target=target)
+
+    def load_vocab(self):
+        source = self.___get_vocab_source()
+        return dict(VocabRepositoryUtils.load(source))
+
+    def save_embedding(self, data):
+        target = self.__get_default_embedding_filepath()
+        NpzEmbeddingHelper.save_embedding(data=data, target=target)
+
+    def load_embedding(self):
+        source = self.__get_term_embedding_source()
+        return NpzEmbeddingHelper.load_embedding(source)
+
+    def check_targets_existed(self):
+        targets = [
+            self.__get_default_vocab_filepath(),
+            self.__get_term_embedding_target()
+        ]
+        return check_targets_existence(targets=targets)
+
+    # endregion
+
+    # region embedding-related data
+
+    def ___get_vocab_source(self):
+        """ It is possible to load a predefined embedding from another experiment
+            using the related filepath provided by model_io.
+        """
+        return self.__get_default_vocab_filepath()
+
+    def __get_term_embedding_target(self):
+        return self.__get_default_embedding_filepath()
+
+    def __get_term_embedding_source(self):
+        return self.__get_default_embedding_filepath()
+
+    def __get_default_vocab_filepath(self):
+        return join(self.__target_dir, self.__vocab_fn_template)
+
+    def __get_default_embedding_filepath(self):
+        return join(self.__target_dir, self.__term_emb_fn_template)
+
+    # endregion

arekit/contrib/utils/io_utils/opinions.py

@@ -0,0 +1,37 @@
+from os.path import join
+
+from arekit.contrib.utils.data.readers.base import BaseReader
+from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
+from arekit.contrib.utils.io_utils.utils import filename_template
+
+
+class OpinionsIO(BaseSamplesIO):
+
+    def __init__(self, target_dir, reader=None, prefix="opinion"):
+        assert(isinstance(reader, BaseReader))
+        self.__target_dir = target_dir
+        self.__prefix = prefix
+        self.__reader = reader
+        self.__target_extension = reader.extension()
+
+    @property
+    def Reader(self):
+        return self.__reader
+
+    def create_target(self, data_type):
+        return self.__get_input_opinions_target(data_type)
+
+    def __get_input_opinions_target(self, data_type):
+        template = filename_template(data_type=data_type)
+        return self.__get_filepath(out_dir=self.__target_dir,
+                                   template=template,
+                                   prefix=self.__prefix,
+                                   extension=self.__target_extension)
+
+    @staticmethod
+    def __get_filepath(out_dir, template, prefix, extension):
+        assert(isinstance(template, str))
+        assert(isinstance(prefix, str))
+        assert(isinstance(extension, str))
+        return join(out_dir, "{prefix}-{template}{extension}".format(
+            prefix=prefix, template=template, extension=extension))

arekit/contrib/utils/io_utils/samples.py

@@ -0,0 +1,79 @@
+import logging
+from os.path import join
+
+from arekit.contrib.utils.data.readers.base import BaseReader
+from arekit.common.experiment.api.base_samples_io import BaseSamplesIO
+from arekit.contrib.utils.data.writers.base import BaseWriter
+from arekit.contrib.utils.io_utils.utils import filename_template, check_targets_existence
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class SamplesIO(BaseSamplesIO):
+    """ Samples default IO utils for samples.
+            Sample is a text part which include pair of attitude participants.
+            This class allows to provide saver and loader for such entries, bubbed as samples.
+            Samples required for machine learning training/inferring.
+    """
+
+    def __init__(self, target_dir, writer=None, reader=None, prefix="sample"):
+        assert(isinstance(target_dir, str))
+        assert(isinstance(prefix, str))
+        assert(isinstance(writer, BaseWriter) or writer is None)
+        assert(isinstance(reader, BaseReader) or reader is None)
+        self.__target_dir = target_dir
+        self.__prefix = prefix
+        self.__writer = writer
+        self.__reader = reader
+
+        self.__target_extension = None
+        if writer is not None:
+            self.__target_extension = writer.extension()
+        elif reader is not None:
+            self.__target_extension = reader.extension()
+
+    # region public methods
+
+    @property
+    def Prefix(self):
+        return self.__prefix
+
+    @property
+    def Reader(self):
+        return self.__reader
+
+    @property
+    def Writer(self):
+        return self.__writer
+
+    def create_target(self, data_type):
+        return self.__get_input_sample_target(data_type)
+
+    def check_targets_existed(self, data_types_iter):
+        for data_type in data_types_iter:
+
+            targets = [
+                self.__get_input_sample_target(data_type=data_type),
+            ]
+
+            if not check_targets_existence(targets=targets):
+                return False
+        return True
+
+    # endregion
+
+    def __get_input_sample_target(self, data_type):
+        template = filename_template(data_type=data_type)
+        return self.__get_filepath(out_dir=self.__target_dir,
+                                   template=template,
+                                   prefix=self.__prefix,
+                                   extension=self.__target_extension)
+
+    @staticmethod
+    def __get_filepath(out_dir, template, prefix, extension):
+        assert(isinstance(template, str))
+        assert(isinstance(prefix, str))
+        assert(isinstance(extension, str))
+        return join(out_dir, "{prefix}-{template}{extension}".format(
+            prefix=prefix, template=template, extension=extension))

arekit/contrib/utils/io_utils/utils.py

@@ -0,0 +1,39 @@
+from collections.abc import Iterable
+import logging
+from os.path import join, exists
+
+from arekit.common.experiment.data_type import DataType
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+def join_dir_with_subfolder_name(subfolder_name, dir):
+    """ Returns subfolder in in directory
+    """
+    assert(isinstance(subfolder_name, str))
+    assert(isinstance(dir, str))
+
+    target_dir = join(dir, "{}/".format(subfolder_name))
+    return target_dir
+
+
+def filename_template(data_type):
+    assert(isinstance(data_type, DataType))
+    return "{data_type}-0".format(data_type=data_type.name.lower())
+
+
+def check_targets_existence(targets):
+    assert (isinstance(targets, Iterable))
+
+    result = True
+    for filepath in targets:
+        assert(isinstance(filepath, str))
+
+        existed = exists(filepath)
+        logger.info("Check existence [{is_existed}]: {fp}".format(is_existed=existed, fp=filepath))
+        if not existed:
+            result = False
+
+    return result

arekit/contrib/utils/lexicons/lexicon.py

@@ -0,0 +1,41 @@
+from arekit.contrib.utils.data.readers.csv_pd import PandasCsvReader
+
+
+class Lexicon(object):
+
+    @property
+    def ToneKey(self):
+        return 'tone'
+
+    @property
+    def TermKey(self):
+        return 'term'
+
+    def __init__(self, dataframe):
+        self.__lexicon_df = dataframe
+
+    @classmethod
+    def load(cls, filepath, separator=','):
+        reader = PandasCsvReader(compression=None, sep=separator)
+        return cls(reader.read(filepath))
+
+    def get_score(self, lemma):
+        assert(type(lemma) == str)
+        s = self.__lexicon_df[lemma.encode('utf-8') == self.__lexicon_df[self.TermKey]]
+        return s[self.ToneKey].values[0] if len(s) > 0 else 0
+
+    def has_term(self, term):
+        assert(type(term) == str)
+        s = self.__lexicon_df[term.encode('utf-8') == self.__lexicon_df[self.TermKey]]
+        return len(s) > 0
+
+    def __iter__(self):
+        for term in self.__lexicon_df[self.TermKey]:
+            yield term
+
+    def __contains__(self, item):
+        assert(isinstance(item, str))
+        result = self.__lexicon_df[self.__lexicon_df[self.TermKey] == item.encode('utf-8')]
+        return len(result) > 0
+
+

arekit/contrib/utils/lexicons/relation.py

@@ -0,0 +1,42 @@
+from arekit.contrib.utils.data.readers.csv_pd import PandasCsvReader
+
+
+class RelationLexicon(object):
+
+    def __init__(self, dataframe):
+        self.__check(dataframe)
+        self.__lexicon = dataframe
+
+    @classmethod
+    def load(cls, filepath, separator=','):
+        reader = PandasCsvReader(compression=None, sep=separator)
+        return cls(reader.read(filepath))
+
+    @staticmethod
+    def __check(df):
+        for index in df.index:
+            relation = df.loc[index][0]
+            assert(len(relation.split('<->')) == 2)
+
+    @staticmethod
+    def __create_key(l, r):
+        assert(type(l) == str)
+        assert(type(r) == str)
+        return '<->'.join([l, r])
+
+    def get_score(self, left, right):
+        assert(type(left) == str)
+        assert(type(right) == str)
+
+        lr_key = self.__create_key(left, right)
+        rl_key = self.__create_key(right, left)
+
+        lr_score = self.__lexicon[lr_key == self.__lexicon['relation']]
+        rl_score = self.__lexicon[rl_key == self.__lexicon['relation']]
+
+        if len(lr_score) > 0:
+            return lr_score['tone'].values[0]
+        if len(rl_score) > 0:
+            return rl_score['tone'].values[0]
+
+        return None

arekit/contrib/utils/lexicons/rusentilex.py

@@ -0,0 +1,37 @@
+import importlib
+import zipfile
+from os import path
+
+
+from arekit.contrib.source.zip_utils import ZipArchiveUtils
+from arekit.contrib.utils.lexicons.lexicon import Lexicon
+
+
+class RuSentiLexLexicon(Lexicon):
+    """
+    RuSentiLex Lexicon wrapper for csv file stored in /data folder.
+    """
+
+    __INNER_PATH = 'rusentilex.csv'
+
+    @property
+    def ToneKey(self):
+        return 'tone'
+
+    @property
+    def TermKey(self):
+        return 'term'
+
+    @staticmethod
+    def __get_archive_filepath():
+        return path.join(ZipArchiveUtils.get_data_root(), "rusentilex.zip")
+
+    @classmethod
+    def from_zip(cls):
+        """ Using Pandas API to read lexicon.
+        """
+        pd = importlib.import_module("pandas")
+        with zipfile.ZipFile(cls.__get_archive_filepath(), "r") as zip_ref:
+            with zip_ref.open(cls.__INNER_PATH, mode='r') as csv_file:
+                df = pd.read_csv(csv_file, sep=',')
+                return cls(df)

arekit/contrib/utils/nn/rows.py

@@ -0,0 +1,83 @@
+import collections
+
+from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
+from arekit.common.data.input.terms_mapper import OpinionContainingTextTermsMapper
+from arekit.common.entities.str_fmt import StringEntitiesFormatter
+from arekit.contrib.networks.input.ctx_serialization import NetworkSerializationContext
+from arekit.contrib.networks.input.formatters.pos_mapper import PosTermsMapper
+from arekit.contrib.networks.input.providers.sample import NetworkSampleRowProvider
+from arekit.contrib.networks.input.providers.text import NetworkSingleTextProvider
+from arekit.contrib.networks.input.term_types import TermTypes
+from arekit.contrib.networks.input.terms_mapping import VectorizedNetworkTermMapping
+from arekit.contrib.utils.processing.lemmatization.mystem import MystemWrapper
+from arekit.contrib.utils.resources import load_embedding_news_mystem_skipgram_1000_20_2015
+from arekit.contrib.utils.vectorizers.bpe import BPEVectorizer
+from arekit.contrib.utils.vectorizers.random_norm import RandomNormalVectorizer
+
+
+def __add_term_embedding(dict_data, term, emb_vector):
+    if term in dict_data:
+        return
+    dict_data[term] = emb_vector
+
+
+def create_rows_provider(str_entity_fmt, ctx, vectorizers="default"):
+    """ This method is corresponds to the default initialization of
+        the rows provider for data sampling pipeline.
+
+        vectorizers:
+            NONE: no need to vectorize, just provide text (using SingleTextProvider).
+            DEFAULT: we consider an application of stemmer for Russian Language.
+            DICT: in which for every type there is an assigned Vectorizer
+                vectorization of term types.
+                {
+                    TermType.Word: Vectorizer,
+                    TermType.Entity: Vectorizer,
+                    ...
+                }
+    """
+    assert(isinstance(str_entity_fmt, StringEntitiesFormatter))
+    assert(isinstance(ctx, NetworkSerializationContext))
+    assert(isinstance(vectorizers, dict) or vectorizers == "default" or vectorizers is None)
+
+    term_embedding_pairs = None
+
+    if vectorizers is not None:
+
+        if vectorizers == "default":
+            # initialize default vectorizer for Russian language.
+            embedding = load_embedding_news_mystem_skipgram_1000_20_2015(stemmer=MystemWrapper(), auto_download=True)
+            bpe_vectorizer = BPEVectorizer(embedding=embedding, max_part_size=3)
+            norm_vectorizer = RandomNormalVectorizer(vector_size=embedding.VectorSize,
+                                                     token_offset=12345)
+            vectorizers = {
+                TermTypes.WORD: bpe_vectorizer,
+                TermTypes.ENTITY: bpe_vectorizer,
+                TermTypes.FRAME: bpe_vectorizer,
+                TermTypes.TOKEN: norm_vectorizer
+            }
+
+        # Setup term-embedding pairs collection instance.
+        term_embedding_pairs = collections.OrderedDict()
+
+        # Use text provider with vectorizers.
+        text_provider = NetworkSingleTextProvider(
+            text_terms_mapper=VectorizedNetworkTermMapping(
+                vectorizers=vectorizers,
+                string_entities_formatter=str_entity_fmt),
+            pair_handling_func=lambda pair: __add_term_embedding(
+                dict_data=term_embedding_pairs,
+                term=pair[0],
+                emb_vector=pair[1]))
+    else:
+        # Create text provider which without vectorizers.
+        text_provider = BaseSingleTextProvider(
+            text_terms_mapper=OpinionContainingTextTermsMapper(str_entity_fmt))
+
+    return NetworkSampleRowProvider(
+        label_provider=ctx.LabelProvider,
+        text_provider=text_provider,
+        frames_connotation_provider=ctx.FramesConnotationProvider,
+        frame_role_label_scaler=ctx.FrameRolesLabelScaler,
+        pos_terms_mapper=PosTermsMapper(ctx.PosTagger) if ctx.PosTagger is not None else None,
+        term_embedding_pairs=term_embedding_pairs)

arekit/contrib/utils/np_utils/embedding.py

@@ -0,0 +1,22 @@
+import logging
+
+from arekit.contrib.utils.np_utils.npz_utils import NpzRepositoryUtils
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class NpzEmbeddingHelper:
+
+    @staticmethod
+    def save_embedding(data, target):
+        NpzRepositoryUtils.save(data=data, target=target)
+        logger.info("Saving embedding [size={shape}]: {filepath}".format(shape=data.shape,
+                                                                         filepath=target))
+
+    @staticmethod
+    def load_embedding(source):
+        embedding = NpzRepositoryUtils.load(source)
+        logger.info("Embedding read [size={size}]: {filepath}".format(size=embedding.shape,
+                                                                      filepath=source))
+        return embedding

arekit/contrib/utils/np_utils/npz_utils.py

@@ -0,0 +1,13 @@
+import numpy as np
+
+
+class NpzRepositoryUtils(object):
+
+    @staticmethod
+    def save(data, target):
+        np.savez(target, data)
+
+    @staticmethod
+    def load(source):
+        data = np.load(source)
+        return data['arr_0']

arekit/contrib/utils/np_utils/vocab.py

@@ -0,0 +1,20 @@
+import logging
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class VocabRepositoryUtils(object):
+
+    @staticmethod
+    def save(data, target):
+        logger.info("Saving vocabulary [size={size}]: {filepath}".format(size=len(data), filepath=target))
+        np.savetxt(target, data, fmt='%s')
+
+    @staticmethod
+    def load(source):
+        vocab = np.loadtxt(source, dtype=str, comments=None)
+        logger.info("Loading vocabulary [size={size}]: {filepath}".format(size=len(vocab), filepath=source))
+        return vocab