arekit 0.24.0__py3-none-any.whl

arekit/common/log_utils.py

@@ -0,0 +1,29 @@
+import logging
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+def log_synonym_existed(value):
+    logger.info("Collection already has a value '{}'. Skipped".format(value.encode('utf-8')))
+
+
+def log_synonym_for_entity_does_not_exist(entity_value, end_type, raise_exception):
+    message = "'{s}' for end {e} does not exist in read-only SynonymsCollection".format(
+        s=entity_value,
+        e=end_type)
+
+    if raise_exception:
+        raise Exception(message)
+    else:
+        logger.info(message)
+
+
+def log_opinion_already_exist(opinion, raise_exception, display_log):
+    message = "'{s}->{t}' already exists in collection".format(s=opinion.SourceValue,
+                                                                t=opinion.TargetValue).encode('utf-8')
+
+    if raise_exception:
+        raise Exception(message)
+    elif display_log:
+        logger.info(message + ' [REJECTED]')

arekit/common/model/labeling/base.py

@@ -0,0 +1,24 @@
+from arekit.common.labels.scaler.base import BaseLabelScaler
+
+
+class LabelsHelper(object):
+
+    def __init__(self, label_scaler):
+        assert(isinstance(label_scaler, BaseLabelScaler))
+        self._label_scaler = label_scaler
+
+    def label_from_uint(self, value):
+        return self._label_scaler.uint_to_label(value=value)
+
+    def label_to_uint(self, label):
+        return self._label_scaler.label_to_uint(label=label)
+
+    def get_classes_count(self):
+        return len(self._label_scaler.ordered_suppoted_labels())
+
+    def aggregate_labels(self, labels_list, label_calc_mode):
+        raise NotImplementedError()
+
+    @staticmethod
+    def compose_opinion(text_opinion, label):
+        raise NotImplementedError()

arekit/common/model/labeling/modes.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class LabelCalculationMode(Enum):
+
+    FIRST_APPEARED = 'take_first_appeared'
+
+    AVERAGE = 'average'

arekit/common/model/labeling/single.py

@@ -0,0 +1,24 @@
+import numpy as np
+
+from arekit.common.model.labeling.base import LabelsHelper
+from arekit.common.model.labeling.modes import LabelCalculationMode
+
+
+class SingleLabelsHelper(LabelsHelper):
+
+    def aggregate_labels(self, labels_list, label_calc_mode):
+        assert(isinstance(labels_list, list))
+        assert(isinstance(label_calc_mode, LabelCalculationMode))
+
+        label = None
+
+        if label_calc_mode == LabelCalculationMode.FIRST_APPEARED:
+            label = labels_list[0]
+
+        if label_calc_mode == LabelCalculationMode.AVERAGE:
+            int_labels = [self._label_scaler.label_to_int(label)
+                          for label in labels_list]
+            label = self._label_scaler.int_to_label(int(np.sign(sum(int_labels))))
+
+        return label
+

arekit/common/opinions/annot/algo/base.py

@@ -0,0 +1,4 @@
+class BaseOpinionAnnotationAlgorithm(object):
+
+    def iter_opinions(self, parsed_doc, existed_opinions=None):
+        pass

arekit/common/opinions/annot/algo/pair_based.py

@@ -0,0 +1,99 @@
+from arekit.common.entities.types import OpinionEntityType
+from arekit.common.labels.provider.base import BasePairLabelProvider
+from arekit.common.docs.entity import DocumentEntity
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider, DistanceType
+from arekit.common.docs.parsed.providers.opinion_pairs import OpinionPairsProvider
+from arekit.common.opinions.annot.algo.base import BaseOpinionAnnotationAlgorithm
+from arekit.common.opinions.base import Opinion
+
+
+class PairBasedOpinionAnnotationAlgorithm(BaseOpinionAnnotationAlgorithm):
+    """ Is a pair-based annotation algorithm which assumes to compose source-target entity pairs
+        This is a default annotator which found its application in Sentiment Attitude Extraction task [1].
+
+        References:
+            [1] Extracting Sentiment Attitudes from Analytical Texts https://arxiv.org/pdf/1808.08932.pdf
+    """
+
+    def __init__(self, dist_in_terms_bound, label_provider, entity_index_func, dist_in_sents=0,
+                 is_entity_ignored_func=None):
+        """
+        dist_in_terms_bound: int
+            max allowed distance in term (less than passed value)
+        is_entity_ignored_func: func
+            entity, type -> bool
+        """
+        assert(isinstance(dist_in_terms_bound, int) or dist_in_terms_bound is None)
+        assert(isinstance(label_provider, BasePairLabelProvider))
+        assert(callable(entity_index_func))
+        assert(isinstance(dist_in_sents, int))
+        assert(callable(is_entity_ignored_func) or is_entity_ignored_func is None)
+
+        self.__label_provider = label_provider
+        self.__dist_in_terms_bound = dist_in_terms_bound
+        self.__dist_in_sents = dist_in_sents
+        self.__is_entity_ignored_func = is_entity_ignored_func
+        self.__entity_index_func = entity_index_func
+
+    # region private methods
+
+    @staticmethod
+    def __create_key_by_entity_pair(e1, e2):
+        assert(isinstance(e1, DocumentEntity))
+        assert(isinstance(e2, DocumentEntity))
+        return "{}_{}".format(e1.IdInDocument, e2.IdInDocument)
+
+    def __try_create_pair_key(self, entity_service, e1, e2, existed_opinions):
+        assert(isinstance(entity_service, EntityServiceProvider))
+        assert(isinstance(e1, DocumentEntity))
+        assert(isinstance(e2, DocumentEntity))
+
+        if e1.IdInDocument == e2.IdInDocument:
+            return
+
+        if self.__is_entity_ignored_func is not None:
+            if self.__is_entity_ignored_func(e1, OpinionEntityType.Subject):
+                return
+            if self.__is_entity_ignored_func(e2, OpinionEntityType.Object):
+                return
+
+        s_dist = entity_service.calc_dist_between_entities(e1=e1, e2=e2, distance_type=DistanceType.InSentences)
+
+        if s_dist > self.__dist_in_sents:
+            return
+
+        t_dist = entity_service.calc_dist_between_entities(e1=e1, e2=e2, distance_type=DistanceType.InTerms)
+
+        if self.__dist_in_terms_bound is not None and t_dist > self.__dist_in_terms_bound:
+            return
+
+        if existed_opinions is not None:
+            o = Opinion(source_value=e1.Value,
+                        target_value=e2.Value,
+                        label=self.__label_provider.provide(source=e1, target=e2))
+            if existed_opinions.has_synonymous_opinion(opinion=o):
+                return
+
+        return self.__create_key_by_entity_pair(e1=e1, e2=e2)
+
+    # endregion
+
+    def iter_opinions(self, parsed_doc, existed_opinions=None):
+        assert(isinstance(parsed_doc, ParsedDocument))
+
+        def __filter_pair_func(e1, e2):
+            key = self.__try_create_pair_key(entity_service=entity_service_provider,
+                                             e1=e1, e2=e2,
+                                             existed_opinions=existed_opinions)
+
+            return key is not None
+
+        # Initialize providers.
+        opinions_provider = OpinionPairsProvider(entity_index_func=self.__entity_index_func)
+        entity_service_provider = EntityServiceProvider(entity_index_func=self.__entity_index_func)
+        opinions_provider.init_parsed_doc(parsed_doc)
+        entity_service_provider.init_parsed_doc(parsed_doc)
+
+        return opinions_provider.iter_from_all(label_provider=self.__label_provider,
+                                               filter_func=__filter_pair_func)

arekit/common/opinions/annot/algo/predefined.py

@@ -0,0 +1,16 @@
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.opinions.annot.algo.base import BaseOpinionAnnotationAlgorithm
+
+
+class PredefinedOpinionAnnotationAlgorithm(BaseOpinionAnnotationAlgorithm):
+    """ A placeholder of the algorithm which is consider to return
+        a predefined list of opinions, provided by a given document_id.
+    """
+
+    def __init__(self, get_opinions_by_doc_id_func):
+        assert(callable(get_opinions_by_doc_id_func))
+        self.__get_opinions_by_doc_id_func = get_opinions_by_doc_id_func
+
+    def iter_opinions(self, parsed_doc, existed_opinions=None):
+        assert(isinstance(parsed_doc, ParsedDocument))
+        return self.__get_opinions_by_doc_id_func(parsed_doc.RelatedDocID)

arekit/common/opinions/annot/algo_based.py

@@ -0,0 +1,55 @@
+import logging
+
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.opinions.annot.algo.base import BaseOpinionAnnotationAlgorithm
+from arekit.common.opinions.annot.base import BaseOpinionAnnotator
+from arekit.common.opinions.collection import OpinionCollection
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class AlgorithmBasedOpinionAnnotator(BaseOpinionAnnotator):
+    """ Algorithm-based annotator
+    """
+
+    def __init__(self, annot_algo, create_empty_collection_func, get_doc_existed_opinions_func=None):
+        """ create_empty_collection_func: func
+                function that creates an empty OpinionCollection
+            get_doc_existed_opinions_func: func or None
+                function that provides existed opinions for a document;
+                if None, then we consider an absence of the existed document-level opinions.
+        """
+        assert(isinstance(annot_algo, BaseOpinionAnnotationAlgorithm))
+        assert(callable(get_doc_existed_opinions_func) or get_doc_existed_opinions_func is None)
+        super(AlgorithmBasedOpinionAnnotator, self).__init__()
+
+        self.__annot_algo = annot_algo
+        self.__create_empty_collection_func = create_empty_collection_func
+        self.__get_existed_opinions_func = (lambda _: None) \
+            if get_doc_existed_opinions_func is None else get_doc_existed_opinions_func
+
+    # region private methods
+
+    def _annot_collection_core(self, parsed_doc):
+        assert(isinstance(parsed_doc, ParsedDocument))
+
+        opinions = self.__get_existed_opinions_func(parsed_doc.RelatedDocID)
+        assert(isinstance(opinions, OpinionCollection) or opinions is None)
+        
+        annotated_opinions_it = self.__annot_algo.iter_opinions(
+            parsed_doc=parsed_doc, existed_opinions=opinions)
+
+        collection = self.__create_empty_collection_func()
+        assert(isinstance(collection, OpinionCollection))
+
+        # Filling. Keep all the opinions without duplications.
+        for opinion in annotated_opinions_it:
+            if collection.has_synonymous_opinion(opinion):
+                continue
+            collection.add_opinion(opinion)
+
+        return collection
+
+    # endregion
+

arekit/common/opinions/annot/base.py

@@ -0,0 +1,15 @@
+class BaseOpinionAnnotator(object):
+    """
+    Performs annotation for a particular data_type
+    using OpinOps and DocOps API.
+    """
+
+    def _annot_collection_core(self, parsed_doc):
+        raise NotImplementedError
+
+    # region public methods
+
+    def annotate_collection(self, parsed_doc):
+        return self._annot_collection_core(parsed_doc=parsed_doc)
+
+    # endregion

arekit/common/opinions/base.py

@@ -0,0 +1,74 @@
+from arekit.common.labels.base import Label
+from arekit.common.opinions.enums import OpinionEndTypes
+from arekit.common.synonyms.base import SynonymsCollection
+
+
+class Opinion(object):
+    """ Source opinion description
+    """
+
+    def __init__(self, source_value, target_value, label):
+        assert(isinstance(source_value, str))
+        assert(isinstance(target_value, str))
+        assert(isinstance(label, Label))
+        self.__source_value = source_value
+        self.__target_value = target_value
+        self.__label = label
+        self.__tag = None
+
+    # region properties
+
+    @property
+    def SourceValue(self):
+        return self.__source_value
+
+    @property
+    def TargetValue(self):
+        return self.__target_value
+
+    @property
+    def Label(self):
+        return self.__label
+
+    @property
+    def Tag(self):
+        return self.__tag
+
+    # endregion
+
+    def __get_end_synonym_inds(self, synonyms):
+        s_ind = synonyms.get_synonym_group_index(self.__source_value)
+        t_ind = synonyms.get_synonym_group_index(self.__target_value)
+        return s_ind, t_ind
+
+    # region public methods
+
+    def get_value(self, end_type):
+        assert(isinstance(end_type, OpinionEndTypes))
+
+        if end_type == OpinionEndTypes.Source:
+            return self.SourceValue
+
+        if end_type == OpinionEndTypes.Target:
+            return self.TargetValue
+
+        raise Exception("Unknown end_type='{e_type}'".format(e_type=end_type))
+
+    def set_tag(self, value):
+        self.__tag = value
+
+    def is_loop(self, synonyms):
+        s_ind, t_ind = self.__get_end_synonym_inds(synonyms)
+        return s_ind == t_ind
+
+    def create_synonym_id(self, synonyms):
+        assert(isinstance(synonyms, SynonymsCollection))
+        s_ind, t_ind = self.__get_end_synonym_inds(synonyms)
+        return "{}_{}".format(s_ind, t_ind)
+
+    def has_synonym_for_end(self, synonyms, end_type):
+        assert(isinstance(synonyms, SynonymsCollection))
+        assert(isinstance(end_type, OpinionEndTypes))
+        return synonyms.contains_synonym_value(self.get_value(end_type))
+
+    # endregion

arekit/common/opinions/collection.py

@@ -0,0 +1,150 @@
+from collections.abc import Iterable
+
+from arekit.common import log_utils
+from arekit.common.labels.base import Label
+from arekit.common.opinions.base import Opinion
+from arekit.common.opinions.enums import OpinionEndTypes
+from arekit.common.synonyms.base import SynonymsCollection
+
+
+class OpinionCollection(object):
+    """
+    Document-level Collection of labeled opinions
+    """
+
+    def __init__(self, synonyms,
+                 opinions=None,
+                 error_on_duplicates=True,
+                 error_on_synonym_end_missed=True):
+        """
+        opinions: list
+            list of opinions
+        synonyms: SynonymsCollection
+        raise_exception_on_duplicates: bool
+            denotes whether there is a need to fire exception for duplicates in opinions list.
+        """
+        assert(isinstance(opinions, Iterable) or isinstance(opinions, type(None)))
+        assert(isinstance(synonyms, SynonymsCollection))
+        assert(isinstance(error_on_duplicates, bool))
+        assert(isinstance(error_on_synonym_end_missed, bool))
+
+        self.__by_synonyms = {}
+        self.__ordered_opinion_keys = []
+        self.__synonyms = synonyms
+        self.__error_on_duplicates = error_on_duplicates
+        self.__error_on_synonym_end_missed = error_on_synonym_end_missed
+
+        if opinions is None:
+            return
+
+        for opinion in opinions:
+            self.__register_opinion(
+                opinion=opinion,
+                error_on_existence=error_on_duplicates,
+                error_on_synonym_end_missed=error_on_synonym_end_missed)
+
+    # region public methods
+
+    def try_get_synonyms_opinion(self, opinion, label=None):
+        return self.__try_get_synonyms_opinion(opinion=opinion, label=label)
+
+    def has_synonymous_opinion(self, opinion, label=None):
+        return self.__try_get_synonyms_opinion(opinion=opinion, label=label) is not None
+
+    def add_opinion(self, opinion):
+        assert(isinstance(opinion, Opinion))
+        self.__register_opinion(opinion=opinion,
+                                error_on_existence=True,
+                                error_on_synonym_end_missed=True)
+
+    # endregion
+
+    # region private methods
+
+    def __try_get_synonyms_opinion(self, opinion, label=None):
+        assert(isinstance(opinion, Opinion))
+        assert(label is None or isinstance(label, Label))
+
+        for end_type in OpinionEndTypes:
+            if not opinion.has_synonym_for_end(synonyms=self.__synonyms, end_type=end_type):
+                return None
+
+        s_id = opinion.create_synonym_id(self.__synonyms)
+        if s_id not in self.__by_synonyms:
+            return None
+
+        f_o = self.__by_synonyms[s_id]
+        if label is None:
+            return f_o
+        elif f_o.Label == label:
+            return f_o
+        else:
+            return None
+
+    def __add_synonym(self, value):
+        self.__synonyms.add_synonym_value(value)
+
+    def __register_opinion(self, opinion,
+                           error_on_existence,
+                           error_on_synonym_end_missed,
+                           show_duplications=False):
+        assert(isinstance(error_on_existence, bool))
+        assert(isinstance(error_on_synonym_end_missed, bool))
+
+        for end_type in OpinionEndTypes:
+            value = opinion.get_value(end_type)
+            if opinion.has_synonym_for_end(synonyms=self.__synonyms, end_type=end_type):
+                # OK.
+                continue
+            if not self.__synonyms.IsReadOnly:
+                # OK. Registering new synonyms as it is possible.
+                self.__add_synonym(value)
+                continue
+
+            log_utils.log_synonym_for_entity_does_not_exist(
+                entity_value=value,
+                end_type=end_type,
+                raise_exception=error_on_synonym_end_missed)
+
+            # Rejecting.
+            return False
+
+        if opinion.is_loop(self.__synonyms):
+            # Ignoring loops.
+            return False
+
+        key = opinion.create_synonym_id(self.__synonyms)
+
+        assert(isinstance(key, str))
+        if key in self.__by_synonyms:
+
+            log_utils.log_opinion_already_exist(opinion=opinion,
+                                                raise_exception=error_on_existence,
+                                                display_log=show_duplications)
+
+            # Rejecting.
+            return False
+
+        # Perform registration.
+        self.__by_synonyms[key] = opinion
+        self.__ordered_opinion_keys.append(key)
+
+        return True
+
+    # endregion
+
+    # region base methods
+
+    def __len__(self):
+        return len(self.__by_synonyms)
+
+    def __iter__(self):
+        for key in self.__ordered_opinion_keys:
+            yield self.__by_synonyms[key]
+
+    def __getitem__(self, item):
+        assert(isinstance(item, int))
+        key = self.__ordered_opinion_keys[item]
+        return self.__by_synonyms[key]
+
+    # endregion

arekit/common/opinions/enums.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class OpinionEndTypes(Enum):
+    Source = 1
+    Target = 2

arekit/common/opinions/provider.py

@@ -0,0 +1,4 @@
+class OpinionCollectionsProvider(object):
+
+    def iter_opinions(self, source, encoding, labels_formatter, error_on_non_supported):
+        raise NotImplementedError()

arekit/common/opinions/writer.py

@@ -0,0 +1,4 @@
+class OpinionCollectionWriter(object):
+
+    def serialize(self, collection, target, encoding, labels_formatter, error_on_non_supported):
+        raise NotImplementedError()

arekit/common/pipeline/base.py

@@ -0,0 +1,25 @@
+from arekit.common.pipeline.context import PipelineContext
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class BasePipeline(object):
+
+    def __init__(self, pipeline):
+        assert(isinstance(pipeline, list))
+        self.__pipeline = pipeline
+
+    def run(self, input_data, params_dict=None, parent_ctx=None):
+        assert(isinstance(params_dict, dict) or params_dict is None)
+
+        pipeline_ctx = PipelineContext(d=params_dict if params_dict is not None else dict(),
+                                       parent_ctx=parent_ctx)
+
+        for item in filter(lambda itm: itm is not None, self.__pipeline):
+            assert(isinstance(item, BasePipelineItem))
+            input_data = item.apply(input_data=input_data, pipeline_ctx=pipeline_ctx)
+
+        return input_data
+
+    def append(self, item):
+        assert(isinstance(item, BasePipelineItem))
+        self.__pipeline.append(item)

arekit/common/pipeline/context.py

@@ -0,0 +1,36 @@
+from arekit.common.pipeline.conts import PARENT_CTX
+
+
+class PipelineContext(object):
+    """ Context of parameters utilized in pipeline
+    """
+
+    def __init__(self, d, parent_ctx=None):
+        assert(isinstance(d, dict))
+        assert(isinstance(parent_ctx, PipelineContext) or parent_ctx is None)
+        assert(PARENT_CTX not in d)
+        self._d = d
+        self._d[PARENT_CTX] = parent_ctx
+
+    def __provide(self, param):
+        return self._d[param]
+
+    # region public
+
+    def provide(self, param):
+        return self.__provide(param)
+
+    def provide_or_none(self, param):
+        return self.__provide(param) if param in self._d else None
+
+    def update(self, param, value):
+        self._d[param] = value
+
+    # endregion
+
+    # region base methods
+
+    def __contains__(self, item):
+        return item in self._d
+
+    # endregion

arekit/common/pipeline/conts.py

@@ -0,0 +1,2 @@
+# Reference to the parent pipeline context.
+PARENT_CTX = "parent_ctx"

arekit/common/pipeline/items/base.py

@@ -0,0 +1,12 @@
+class BasePipelineItem(object):
+    """ Single pipeline item that might be instatiated and embedded into pipeline.
+    """
+
+    def apply_core(self, input_data, pipeline_ctx):
+        raise NotImplementedError()
+
+    def apply(self, input_data, pipeline_ctx=None):
+        """ Performs input processing an update it for a further pipeline items.
+        """
+        output_data = self.apply_core(input_data=input_data, pipeline_ctx=pipeline_ctx)
+        return output_data if output_data is not None else input_data

arekit/common/pipeline/items/flatten.py

@@ -0,0 +1,14 @@
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class FlattenIterPipelineItem(BasePipelineItem):
+    """ Considered to flat iterations of items that represent iterations.
+    """
+
+    def __flat_iter(self, iter_data):
+        for iter_item in iter_data:
+            for item in iter_item:
+                yield item
+
+    def apply_core(self, input_data, pipeline_ctx):
+        return self.__flat_iter(input_data)

arekit/common/pipeline/items/handle.py

@@ -0,0 +1,17 @@
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class HandleIterPipelineItem(BasePipelineItem):
+
+    def __init__(self, handle_func=None):
+        assert(callable(handle_func))
+        self.__handle_func = handle_func
+
+    def __updated_data(self, items_iter):
+        for item in items_iter:
+            # Perform item handling
+            self.__handle_func(item)
+            yield item
+
+    def apply_core(self, input_data, pipeline_ctx):
+        return self.__updated_data(input_data)

arekit/common/pipeline/items/iter.py

@@ -0,0 +1,11 @@
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class FilterPipelineItem(BasePipelineItem):
+
+    def __init__(self, filter_func=None):
+        assert(callable(filter_func))
+        self.__filter_func = filter_func
+
+    def apply_core(self, input_data, pipeline_ctx):
+        return filter(self.__filter_func, input_data)