arekit 0.24.0__py3-none-any.whl

arekit/common/docs/objects_parser.py

@@ -0,0 +1,37 @@
+from arekit.common.pipeline.items.base import BasePipelineItem
+from arekit.common.text.partitioning.base import BasePartitioning
+from arekit.common.pipeline.context import PipelineContext
+
+
+class SentenceObjectsParserPipelineItem(BasePipelineItem):
+
+    def __init__(self, partitioning):
+        assert(isinstance(partitioning, BasePartitioning))
+        self.__partitioning = partitioning
+
+    # region protected
+
+    def _get_text(self, pipeline_ctx):
+        return None
+
+    def _get_parts_provider_func(self, input_data, pipeline_ctx):
+        raise NotImplementedError()
+
+    # endregion
+
+    def apply_core(self, input_data, pipeline_ctx):
+        assert(isinstance(pipeline_ctx, PipelineContext))
+        external_input = self._get_text(pipeline_ctx)
+        actual_input = input_data if external_input is None else external_input
+        parts_it = self._get_parts_provider_func(input_data=actual_input, pipeline_ctx=pipeline_ctx)
+        return self.__partitioning.provide(text=actual_input, parts_it=parts_it)
+
+    # region base
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    # endregion

arekit/common/docs/parsed/base.py

@@ -0,0 +1,101 @@
+from collections.abc import Iterable
+
+from arekit.common.entities.base import Entity
+from arekit.common.text.enums import TermFormat
+from arekit.common.text.parsed import BaseParsedText
+
+
+class ParsedDocument(object):
+    """
+    This class represents an information of the processed doc in following directions:
+        - doc words
+        - tokens
+        - entities (positions).
+        - frames (FrameVariants)
+    It allows:
+        - Expand parsed sentences with other objects:
+            modify_parsed_sentences(func)
+
+    Limitations:
+        IN MEMORY implementation (`add` method)
+    """
+
+    def __init__(self, doc_id, parsed_sentences):
+        """
+        parsed_sentences: iterable of ParsedSentence type
+            NOTE: Considered sentences with labeled Entities in it!
+        """
+        assert(isinstance(parsed_sentences, Iterable))
+
+        self.__doc_id = doc_id
+        self.__parsed_sentences = list(parsed_sentences)
+
+    # region properties
+
+    @property
+    def RelatedDocID(self):
+        return self.__doc_id
+
+    # endregion
+
+    # region private methods
+
+    def __iter_all_raw_terms(self, filter_func=None, term_only=False):
+        assert(callable(filter_func) or filter_func is None)
+        assert(isinstance(term_only, bool))
+
+        for s_ind, sentence in enumerate(self.__parsed_sentences):
+            for ind_in_sent, term in self.__iter_sentence_raw_terms(sentence, filter_func=filter_func):
+
+                if term_only:
+                    yield term
+                else:
+                    yield s_ind, ind_in_sent, term
+
+    @staticmethod
+    def __iter_sentence_raw_terms(sentence, filter_func):
+        assert(isinstance(sentence, BaseParsedText))
+        assert(callable(filter_func) or filter_func is None)
+
+        for ind_in_sent, term in enumerate(sentence.iter_terms(TermFormat.Raw)):
+
+            if filter_func is not None:
+                if not filter_func(term):
+                    continue
+
+            yield ind_in_sent, term
+
+    # endregion
+
+    # region public 'iter' methods
+
+    def get_sentence(self, s_ind):
+        assert(isinstance(s_ind, int))
+        return self.__parsed_sentences[s_ind]
+
+    def iter_entities(self):
+        for entity in self.__iter_all_raw_terms(term_only=True, filter_func=lambda t: isinstance(t, Entity)):
+            yield entity
+
+    def iter_terms(self, filter_func=None, term_only=True):
+        for term in self.__iter_all_raw_terms(term_only=term_only, filter_func=filter_func):
+            yield term
+
+    def iter_sentence_terms(self, sentence_index, return_id, filter_func=None):
+        assert(isinstance(sentence_index, int))
+        assert(isinstance(return_id, bool))
+        assert(callable(filter_func) or filter_func is None)
+
+        it = self.__iter_sentence_raw_terms(sentence=self.__parsed_sentences[sentence_index],
+                                            filter_func=filter_func)
+
+        for ind_in_sent, term in it:
+            if return_id:
+                yield ind_in_sent, term
+            else:
+                yield term
+    # endregion
+
+    def __iter__(self):
+        for sentence in self.__parsed_sentences:
+            yield sentence

arekit/common/docs/parsed/providers/base.py

@@ -0,0 +1,68 @@
+from arekit.common.entities.base import Entity
+from arekit.common.docs.entity import DocumentEntity
+from arekit.common.docs.parsed.base import ParsedDocument
+
+
+class BaseParsedDocumentServiceProvider(object):
+
+    def __init__(self, entity_index_func=None):
+        """ Outside enity indexing function
+            entity_index_func: provides id for a given entity, i.e.
+                func(entity) -> int (id)
+        """
+        assert(callable(entity_index_func) or entity_index_func is None)
+        self._doc_entities = None
+        self.__entity_map = {}
+        self.__entity_index_func = entity_index_func
+
+    @property
+    def Name(self):
+        raise NotImplementedError()
+
+    def init_parsed_doc(self, parsed_doc):
+        assert(isinstance(parsed_doc, ParsedDocument))
+
+        def __iter_childs_and_root_node(entity):
+            """ Note: Entity has childs and we would like to iterate over childs
+                to conider them as well as keep the root Node.
+            """
+            # We first add childs.
+            for child_entity in entity.iter_childs():
+                yield child_entity, True
+
+            # Return Root node.
+            yield entity, False
+
+        self._doc_entities = []
+        self.__entity_map.clear()
+
+        current_id = 0
+        for _, entity in enumerate(parsed_doc.iter_entities()):
+
+            child_doc_entities = []
+            for tree_entity, is_child in __iter_childs_and_root_node(entity):
+
+                doc_entity = DocumentEntity(id_in_doc=current_id,
+                                            value=tree_entity.Value,
+                                            e_type=tree_entity.Type,
+                                            display_value=tree_entity.DisplayValue,
+                                            childs=None if is_child else child_doc_entities,
+                                            group_index=tree_entity.GroupIndex)
+                current_id += 1
+
+                if is_child:
+                    child_doc_entities.append(doc_entity)
+
+                self._doc_entities.append(doc_entity)
+
+                if self.__entity_index_func is not None:
+                    self.__entity_map[self.__entity_index_func(tree_entity)] = doc_entity
+
+    def get_document_entity(self, entity):
+        """ Maps entity to the related one with DocumentEntity type
+        """
+        assert(isinstance(entity, Entity))
+        return self.__entity_map[self.__entity_index_func(entity)]
+
+    def contains_entity(self, entity):
+        return self.__entity_index_func(entity) in self.__entity_map

arekit/common/docs/parsed/providers/base_pairs.py

@@ -0,0 +1,51 @@
+from arekit.common.labels.provider.base import BasePairLabelProvider
+from arekit.common.docs.parsed.providers.base import BaseParsedDocumentServiceProvider
+
+
+class BasePairProvider(BaseParsedDocumentServiceProvider):
+
+    @property
+    def Name(self):
+        raise NotImplementedError()
+
+    def _create_pair(self, source_entity, target_entity, label):
+        raise NotImplementedError()
+
+    # region private methods
+
+    def _iter_from_entities(self, src_entity_doc_ids, tgt_entity_doc_ids, label_provider, filter_func=None):
+        assert(isinstance(src_entity_doc_ids, list))
+        assert(isinstance(tgt_entity_doc_ids, list))
+        assert(isinstance(label_provider, BasePairLabelProvider))
+        assert(callable(filter_func) or filter_func is None)
+
+        for src_e_doc_id in src_entity_doc_ids:
+            for tgt_e_doc_id in tgt_entity_doc_ids:
+                assert(isinstance(src_e_doc_id, int))
+                assert(isinstance(tgt_e_doc_id, int))
+
+                # Extract entities by doc_id.
+                source_entity = self._doc_entities[src_e_doc_id]
+                target_entity = self._doc_entities[tgt_e_doc_id]
+
+                if filter_func is not None and not filter_func(source_entity, target_entity):
+                    continue
+
+                if source_entity == target_entity:
+                    continue
+
+                label = label_provider.provide(source=source_entity,
+                                               target=target_entity)
+
+                yield self._create_pair(source_entity=source_entity,
+                                        target_entity=target_entity,
+                                        label=label)
+
+    # endregion
+
+    def iter_from_all(self, label_provider, filter_func):
+        assert(isinstance(label_provider, BasePairLabelProvider))
+        return self._iter_from_entities(src_entity_doc_ids=list(map(lambda e: e.IdInDocument, self._doc_entities)),
+                                        tgt_entity_doc_ids=list(map(lambda e: e.IdInDocument, self._doc_entities)),
+                                        label_provider=label_provider,
+                                        filter_func=filter_func)

arekit/common/docs/parsed/providers/entity_service.py

@@ -0,0 +1,175 @@
+from enum import Enum
+
+from arekit.common.entities.base import Entity
+from arekit.common.docs.entity import DocumentEntity
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.docs.parsed.providers.base import BaseParsedDocumentServiceProvider
+from arekit.common.docs.parsed.term_position import TermPositionTypes, TermPosition
+from arekit.common.text_opinions.base import TextOpinion
+
+
+class EntityEndType(Enum):
+    """ Pair end type
+    """
+    Source = 1
+    Target = 2
+
+
+class DistanceType(Enum):
+    InTerms = 1
+    InSentences = 2
+
+    @staticmethod
+    def to_position_type(dist_type):
+        assert(isinstance(dist_type, DistanceType))
+
+        if dist_type == DistanceType.InTerms:
+            return TermPositionTypes.IndexInDocument
+
+        if dist_type == DistanceType.InSentences:
+            return TermPositionTypes.SentenceIndex
+
+
+class EntityServiceProvider(BaseParsedDocumentServiceProvider):
+    """ This class provides a helper functions for TextOpinions, which become a part of TextOpinionCollection.
+        The latter is important because of the dependency from Owner.
+        We utilize 'extract' prefix in methods to emphasize that these are methods of helper.
+
+        Wrapper over:
+            parsed doc, positions, text_opinions
+    """
+
+    NAME = "entity-service-provider"
+
+    def __init__(self, entity_index_func):
+        assert(callable(entity_index_func))
+        super(EntityServiceProvider, self).__init__(entity_index_func=entity_index_func)
+        # Initialize API.
+        self.__iter_raw_terms_func = None
+        # Initialize entity positions.
+        self.__entity_positions = None
+
+    @property
+    def Name(self):
+        return self.NAME
+
+    def init_parsed_doc(self, parsed_doc):
+        super(EntityServiceProvider, self).init_parsed_doc(parsed_doc)
+        assert(isinstance(parsed_doc, ParsedDocument))
+        self.__iter_raw_terms_func = lambda: parsed_doc.iter_terms(filter_func=None, term_only=False)
+        self.__entity_positions = self.__calculate_entity_positions()
+
+    # region public 'extract' methods
+
+    def extract_entity_value(self, text_opinion, end_type):
+        return self.__extract_entity_value(text_opinion=text_opinion, end_type=end_type)
+
+    def extract_entity_position(self, text_opinion, end_type, position_type=None):
+        return self.__get_entity_position(text_opinion=text_opinion,
+                                          end_type=end_type,
+                                          position_type=position_type)
+
+    # endregion
+
+    # region public 'calculate' methods
+
+    @staticmethod
+    def calc_dist_between_text_opinion_end_indices(pos1_ind, pos2_ind):
+        return EntityServiceProvider.__calc_distance_by_inds(pos1_ind=pos1_ind, pos2_ind=pos2_ind)
+
+    def calc_dist_between_text_opinion_ends(self, text_opinion, distance_type):
+        assert(isinstance(text_opinion, TextOpinion))
+        assert(isinstance(distance_type, DistanceType))
+
+        e1_id = self.__get_end_id(text_opinion=text_opinion, end_type=EntityEndType.Source)
+        e2_id = self.__get_end_id(text_opinion=text_opinion, end_type=EntityEndType.Target)
+
+        return self.__calc_distance(
+            pos1=self.get_entity_position(id_in_document=e1_id),
+            pos2=self.get_entity_position(id_in_document=e2_id),
+            position_type=DistanceType.to_position_type(distance_type))
+
+    def calc_dist_between_entities(self, e1, e2, distance_type):
+        assert(isinstance(e1, DocumentEntity))
+        assert(isinstance(e2, DocumentEntity))
+        assert(isinstance(distance_type, DistanceType))
+
+        return self.__calc_distance(
+            pos1=self.get_entity_position(e1.IdInDocument),
+            pos2=self.get_entity_position(e2.IdInDocument),
+            position_type=DistanceType.to_position_type(distance_type))
+
+    def get_entity_position(self, id_in_document, position_type=None):
+        """ returns: TermPosition or int
+        """
+        assert(isinstance(position_type, TermPositionTypes) or position_type is None)
+
+        e_pos = self.__entity_positions[id_in_document]
+        assert(isinstance(e_pos, TermPosition))
+
+        if position_type is None:
+            return e_pos
+
+        return e_pos.get_index(position_type)
+
+    def get_entity_value(self, id_in_document):
+        entity = self._doc_entities[id_in_document]
+        assert(isinstance(entity, Entity))
+        return entity.Value
+
+    # endregion
+
+    # region private methods
+
+    def __extract_entity_value(self, text_opinion, end_type):
+        assert(isinstance(text_opinion, TextOpinion))
+        end_id = self.__get_end_id(text_opinion=text_opinion, end_type=end_type)
+        return self.get_entity_value(end_id)
+
+    def __get_entity_position(self, text_opinion, end_type, position_type=None):
+        assert(isinstance(text_opinion, TextOpinion))
+        end_id = self.__get_end_id(text_opinion=text_opinion, end_type=end_type)
+        return self.get_entity_position(end_id, position_type)
+
+    def __calc_distance(self, pos1, pos2, position_type=TermPositionTypes.IndexInDocument):
+        assert(isinstance(pos1, TermPosition))
+        assert(isinstance(pos2, TermPosition))
+        return self.__calc_distance_by_inds(pos1_ind=pos1.get_index(position_type),
+                                            pos2_ind=pos2.get_index(position_type))
+
+    @staticmethod
+    def __calc_distance_by_inds(pos1_ind, pos2_ind):
+        return abs(pos1_ind - pos2_ind)
+
+    @staticmethod
+    def __get_end_id(text_opinion, end_type):
+        assert(isinstance(text_opinion, TextOpinion))
+        assert(end_type == EntityEndType.Source or end_type == EntityEndType.Target)
+        return text_opinion.SourceId if end_type == EntityEndType.Source else text_opinion.TargetId
+
+    def __calculate_entity_positions(self):
+        """ Note: here we consider the same order as in self._entities.
+        """
+        t_ind_in_doc = -1
+
+        positions = {}
+        for s_ind, t_ind_in_sent, term in self.__iter_raw_terms_func():
+
+            t_ind_in_doc += 1
+
+            if not isinstance(term, Entity):
+                continue
+
+            # We consider that entities within a single tree has the same positions.
+            for tree_entity in list(term.iter_childs()) + [term]:
+
+                key = self.get_document_entity(tree_entity).IdInDocument
+                assert(key not in positions)
+
+                positions[key] = TermPosition(term_ind_in_doc=t_ind_in_doc,
+                                              term_ind_in_sent=t_ind_in_sent,
+                                              s_ind=s_ind)
+
+        return positions
+
+    # endregion

arekit/common/docs/parsed/providers/opinion_pairs.py

@@ -0,0 +1,20 @@
+from arekit.common.entities.base import Entity
+from arekit.common.docs.parsed.providers.base_pairs import BasePairProvider
+from arekit.common.opinions.base import Opinion
+
+
+class OpinionPairsProvider(BasePairProvider):
+
+    NAME = "opinion-pairs-provider"
+
+    @property
+    def Name(self):
+        return self.NAME
+
+    def _create_pair(self, source_entity, target_entity, label):
+        assert(isinstance(source_entity, Entity))
+        assert(isinstance(target_entity, Entity))
+
+        return Opinion(source_value=source_entity.Value,
+                       target_value=target_entity.Value,
+                       label=label)

arekit/common/docs/parsed/providers/text_opinion_pairs.py

@@ -0,0 +1,78 @@
+import logging
+
+from arekit.common.entities.collection import EntityCollection
+from arekit.common.docs.entity import DocumentEntity
+from arekit.common.docs.parsed.providers.base_pairs import BasePairProvider
+from arekit.common.opinions.base import Opinion
+from arekit.common.text_opinions.base import TextOpinion
+from arekit.common.labels.provider.constant import ConstantLabelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class TextOpinionPairsProvider(BasePairProvider):
+    """ Document Related text opinion provider.
+    """
+
+    NAME = "text-opinion-pairs-provider"
+
+    def __init__(self, value_to_group_id_func):
+        super(TextOpinionPairsProvider, self).__init__()
+        self.__value_to_group_id_func = value_to_group_id_func
+        self.__doc_id = None
+        self.__entities_collection = None
+
+    @property
+    def Name(self):
+        return self.NAME
+
+    def _create_pair(self, source_entity, target_entity, label):
+        assert(isinstance(source_entity, DocumentEntity))
+        assert(isinstance(target_entity, DocumentEntity))
+
+        return TextOpinion(doc_id=self.__doc_id,
+                           source_id=source_entity.IdInDocument,
+                           target_id=target_entity.IdInDocument,
+                           label=label,
+                           text_opinion_id=None)
+
+    def init_parsed_doc(self, parsed_doc):
+        super(TextOpinionPairsProvider, self).init_parsed_doc(parsed_doc)
+        self.__doc_id = parsed_doc.RelatedDocID
+        self.__entities_collection = EntityCollection(
+            entities=list(self._doc_entities),
+            value_to_group_id_func=self.__value_to_group_id_func)
+
+    def iter_from_opinion(self, opinion, debug=False):
+        """ Provides text-level opinion extraction by document-level opinions
+            (Opinion class instances), for a particular document (doc_id),
+            with the related entity collection.
+        """
+        assert(isinstance(opinion, Opinion))
+
+        key = EntityCollection.KeyType.BY_SYNONYMS
+        source_entities = self.__entities_collection.try_get_entities(opinion.SourceValue, group_key=key)
+        target_entities = self.__entities_collection.try_get_entities(opinion.TargetValue, group_key=key)
+
+        if source_entities is None:
+            if debug:
+                logger.info("Appropriate entity for '{}'->'...' has not been found".format(
+                    opinion.SourceValue))
+            return
+            yield
+
+        if target_entities is None:
+            if debug:
+                logger.info("Appropriate entity for '...'->'{}' has not been found".format(
+                    opinion.TargetValue))
+            return
+            yield
+
+        label_provider = ConstantLabelProvider(label_instance=opinion.Label)
+
+        pairs_it = self._iter_from_entities(src_entity_doc_ids=list(map(lambda e: e.IdInDocument, source_entities)),
+                                            tgt_entity_doc_ids=list(map(lambda e: e.IdInDocument, target_entities)),
+                                            label_provider=label_provider)
+
+        for pair in pairs_it:
+            yield pair

arekit/common/docs/parsed/service.py

@@ -0,0 +1,31 @@
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.docs.parsed.providers.base import BaseParsedDocumentServiceProvider
+
+
+class ParsedDocumentService(object):
+    """ Represents a collection of providers, combined with the parsed doc.
+    """
+
+    def __init__(self, parsed_doc, providers):
+        assert(isinstance(parsed_doc, ParsedDocument))
+        assert(isinstance(providers, list))
+        self.__parsed_doc = parsed_doc
+        self.__providers = {}
+
+        for provider in providers:
+            assert(isinstance(provider, BaseParsedDocumentServiceProvider))
+            assert(provider.Name not in self.__providers)
+
+            # Link provider with the related name.
+            self.__providers[provider.Name] = provider
+
+            # Post initialize with the related parsed doc.
+            provider.init_parsed_doc(self.__parsed_doc)
+
+
+    @property
+    def ParsedDocument(self):
+        return self.__parsed_doc
+
+    def get_provider(self, name):
+        return self.__providers[name]

arekit/common/docs/parsed/term_position.py

@@ -0,0 +1,42 @@
+from enum import Enum
+
+
+class TermPosition:
+
+    def __init__(self, term_ind_in_doc, term_ind_in_sent, s_ind):
+        self.__t_ind_in_doc = term_ind_in_doc
+        self.__t_ind_in_sent = term_ind_in_sent
+        self.__s_ind = s_ind
+
+    def get_index(self, position_type):
+        assert(isinstance(position_type, TermPositionTypes))
+
+        if position_type == TermPositionTypes.IndexInDocument:
+            return self.__t_ind_in_doc
+        if position_type == TermPositionTypes.IndexInSentence:
+            return self.__t_ind_in_sent
+        if position_type == TermPositionTypes.SentenceIndex:
+            return self.__s_ind
+
+
+class TermPositionTypes(Enum):
+
+    """
+    Corresponds to an index of a related term in a whole document
+    (document considered as a sequence of terms)
+    """
+    IndexInDocument = 1
+
+    """
+    Corresponds to an index of a related term in a certain sentence.
+    """
+    IndexInSentence = 2
+
+    """
+    Corresponds to an index of a sentence in a whole document.
+    """
+    SentenceIndex = 3
+
+
+
+

arekit/common/docs/parser.py

@@ -0,0 +1,34 @@
+from arekit.common.docs.base import Document
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.pipeline.context import PipelineContext
+from arekit.common.text.parser import BaseTextParser
+
+
+class DocumentParser(object):
+
+    @staticmethod
+    def __get_sent(doc, sent_ind):
+        return doc.get_sentence(sent_ind)
+
+    @staticmethod
+    def parse(doc, text_parser, parent_ppl_ctx=None):
+        assert(isinstance(doc, Document))
+        assert(isinstance(text_parser, BaseTextParser))
+        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
+
+        parsed_sentences = [text_parser.run(input_data=DocumentParser.__get_sent(doc, sent_ind).Text,
+                                            params_dict=DocumentParser.__create_ppl_params(doc=doc, sent_ind=sent_ind),
+                                            parent_ctx=parent_ppl_ctx)
+                            for sent_ind in range(doc.SentencesCount)]
+
+        return ParsedDocument(doc_id=doc.ID,
+                              parsed_sentences=parsed_sentences)
+
+    @staticmethod
+    def __create_ppl_params(doc, sent_ind):
+        assert(isinstance(doc, Document))
+        return {
+            "s_ind": sent_ind,                                     # sentence index. (as Metadata)
+            "doc_id": doc.ID,                                      # document index. (as Metadata)
+            "sentence": DocumentParser.__get_sent(doc, sent_ind),  # Required for special sources.
+        }

arekit/common/docs/sentence.py

@@ -0,0 +1,14 @@
+
+class BaseDocumentSentence(object):
+
+    def __init__(self, text):
+        self.__text = text
+
+    @property
+    def Text(self):
+        """
+        Any type, i.e.
+            - str: original text as string
+            - list of words: separated by words/tokens
+        """
+        return self.__text