arekit 0.24.0__py3-none-any.whl

arekit/common/bound.py

@@ -0,0 +1,48 @@
+class Bound:
+
+    def __init__(self, pos, length):
+        assert(isinstance(pos, int))
+        assert(isinstance(length, int))
+        self.__pos = pos
+        self.__length = length
+
+    # region properties
+
+    @property
+    def Position(self):
+        return self.__pos
+
+    @property
+    def Length(self):
+        return self.__length
+
+    # endregion
+
+    def itersects_with(self, other):
+        begin = self.__pos
+        end = self.__pos + self.__length
+        other_begin = other.Position
+        other_end_included = other.Position + other.Length - 1
+        if end > other_begin >= begin:
+            return True
+        if end > other_end_included >= begin:
+            return True
+        if other_begin < begin and end <= other_end_included:
+            return True
+        return False
+
+    def intersect(self, other):
+        begin = self.__pos
+        end = self.__pos + self.__length
+        other_begin = other.Position
+        other_end = other.Position + other.Length
+        actual_begin = min(begin, other_begin)
+        actual_length = max(end, other_end) - actual_begin
+        return Bound(pos=actual_begin, length=actual_length)
+
+    def contains(self, other):
+        begin = self.__pos
+        end = self.__pos + self.__length
+        other_begin = other.Position
+        other_end = other.Position + other.Length
+        return begin <= other_begin and end >= other_end

arekit/common/context/terms_mapper.py

@@ -0,0 +1,51 @@
+from collections.abc import Iterable
+
+from arekit.common.context.token import Token
+from arekit.common.entities.base import Entity
+from arekit.common.frames.text_variant import TextFrameVariant
+
+
+class TextTermsMapper(object):
+
+    def iter_mapped(self, terms):
+        """ Performs mapping operation of each terms in a sequence
+        """
+        assert(isinstance(terms, Iterable))
+
+        self._before_mapping()
+
+        for i, term in enumerate(terms):
+
+            if isinstance(term, str):
+                m_term = self.map_word(i, term)
+            elif isinstance(term, Token):
+                m_term = self.map_token(i, term)
+            elif isinstance(term, TextFrameVariant):
+                m_term = self.map_text_frame_variant(i, term)
+            elif isinstance(term, Entity):
+                m_term = self.map_entity(i, term)
+            else:
+                raise Exception("Unsupported type {}".format(term))
+
+            if m_term is not None:
+                yield m_term
+
+        self._after_mapping()
+
+    def _before_mapping(self):
+        pass
+
+    def _after_mapping(self):
+        pass
+
+    def map_word(self, w_ind, word):
+        raise NotImplementedError()
+
+    def map_token(self, t_ind, token):
+        raise NotImplementedError()
+
+    def map_text_frame_variant(self, fv_ind, text_frame_variant):
+        raise NotImplementedError()
+
+    def map_entity(self, e_ind, entity):
+        raise NotImplementedError()

arekit/common/context/token.py

@@ -0,0 +1,16 @@
+class Token:
+    """
+    Token that stores original and resulted token values
+    i.e.: term=',', token_value='<[COMMA]>'
+    """
+    def __init__(self, term, token_value):
+        assert(isinstance(term, str))
+        assert(isinstance(token_value, str))
+        self.__meta_value = term
+        self.__token_value = token_value
+
+    def get_meta_value(self):
+        return self.__meta_value
+
+    def get_token_value(self):
+        return self.__token_value

arekit/common/data/const.py

@@ -0,0 +1,21 @@
+ID = 'id'
+DOC_ID = 'doc_id'
+TEXT = 'text_a'
+LABEL_UINT = 'label_uint'
+LABEL_STR = 'label_str'
+
+# Global identifier of the opinion in the sampled data.
+OPINION_ID = "opinion_id"
+OPINION_LINKAGE_ID = "linkage_id"
+
+# Corresponds to fields with attitude ends. (indices, INT)
+S_IND = 's_ind'
+T_IND = 't_ind'
+
+# Provide sentence index.
+SENT_IND = 'sent_ind'
+
+# Entity parameters
+ENTITY_VALUES = 'entity_values'
+ENTITY_TYPES = 'entity_types'
+ENTITIES = 'entities'

arekit/common/data/doc_provider.py

@@ -0,0 +1,6 @@
+class DocumentProvider(object):
+    """ Provides operations with documents
+    """
+
+    def by_id(self, doc_id):
+        raise NotImplementedError()

arekit/common/data/input/providers/columns/base.py

@@ -0,0 +1,9 @@
+
+class BaseColumnsProvider(object):
+
+    ROW_ID = 'row_id'
+
+    def get_columns_list_with_types(self):
+        dtypes_list = list()
+        dtypes_list.append((BaseColumnsProvider.ROW_ID, 'int32'))
+        return dtypes_list

arekit/common/data/input/providers/columns/sample.py

@@ -0,0 +1,59 @@
+from arekit.common.data import const
+from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
+
+
+class SampleColumnsProvider(BaseColumnsProvider):
+    """
+    [id, label, text_a] -- for train
+    [id, text_a] -- for test
+    """
+
+    def __init__(self, store_labels):
+        super(SampleColumnsProvider, self).__init__()
+        self.__store_labels = store_labels
+        self.__text_column_names = None
+
+    # region properties
+
+    @property
+    def StoreLabels(self):
+        return self.__store_labels
+
+    @property
+    def TextColumnNames(self):
+        return self.__text_column_names
+
+    # endregion
+
+    def get_columns_list_with_types(self):
+        """
+        Composing df with the following columns:
+            [id, label, type, text_a]
+        """
+        dtypes_list = super(SampleColumnsProvider, self).get_columns_list_with_types()
+
+        dtypes_list.append((const.ID, str))
+        dtypes_list.append((const.DOC_ID, str))
+
+        # insert labels
+        if self.__store_labels:
+            dtypes_list.append((const.LABEL_UINT, 'int32'))
+            dtypes_list.append((const.LABEL_STR, str))
+
+        # insert text columns
+        for col_name in self.__text_column_names:
+            dtypes_list.append((col_name, str))
+
+        # insert indices
+        dtypes_list.append((const.S_IND, 'int32'))
+        dtypes_list.append((const.T_IND, 'int32'))
+
+        # opinion-extraction task related fields
+        dtypes_list.append((const.OPINION_ID, 'int32'))
+        dtypes_list.append((const.OPINION_LINKAGE_ID, 'int32'))
+
+        return dtypes_list
+
+    def set_text_column_names(self, text_column_names):
+        assert(isinstance(text_column_names, list))
+        self.__text_column_names = text_column_names

arekit/common/data/input/providers/const.py

@@ -0,0 +1,3 @@
+# Idle mode related to iteration format, which is considered only for assessing the expected
+# amount of sampled data, while its contents could be avoided at all.
+IDLE_MODE = "idle_mode"

arekit/common/data/input/providers/contents.py

@@ -0,0 +1,9 @@
+
+class ContentsProvider(object):
+    """ This is a main provider of the contents of further sampled output.
+    """
+
+    def from_doc_ids(self, doc_ids, idle_mode=False):
+        """ This is a main method is expected to be implemented.
+        """
+        pass

arekit/common/data/input/providers/instances/base.py

@@ -0,0 +1,14 @@
+from arekit.common.linkage.base import LinkedDataWrapper
+
+
+class BaseLinkedDataInstancesProvider(object):
+
+    def iter_instances(self, linked_data):
+        raise NotImplementedError()
+
+    @staticmethod
+    def provide_label(linked_data):
+        """ Implementation based on the first element of the linkage.
+        """
+        assert(isinstance(linked_data, LinkedDataWrapper))
+        return linked_data.First.Label

arekit/common/data/input/providers/instances/multiple.py

@@ -0,0 +1,27 @@
+from arekit.common.data.input.providers.instances.base import BaseLinkedDataInstancesProvider
+from arekit.common.linkage.text_opinions import TextOpinionsLinkage
+from arekit.common.text_opinions.base import TextOpinion
+
+
+class MultipleInstancesLinkedTextOpinionsProvider(BaseLinkedDataInstancesProvider):
+
+    def __init__(self, supported_labels):
+        assert(isinstance(supported_labels, list))
+        self.__supported_labels = supported_labels
+
+    def iter_instances(self, linked_data):
+        """ Enumerate all opinions as if it would be with the different label types.
+        """
+        for label in self.__supported_labels:
+            yield self.__modify_first_and_copy_linked_wrap(linked_data, label)
+
+    @staticmethod
+    def __modify_first_and_copy_linked_wrap(text_opinions_linkage, label):
+        assert (isinstance(text_opinions_linkage, TextOpinionsLinkage))
+
+        linkage = list(text_opinions_linkage)
+        text_opinion_copy = TextOpinion.create_copy(other=linkage[0])
+        text_opinion_copy.set_label(label=label)
+        linkage[0] = text_opinion_copy
+
+        return TextOpinionsLinkage(linked_data=linkage)

arekit/common/data/input/providers/instances/single.py

@@ -0,0 +1,8 @@
+from arekit.common.data.input.providers.instances.base import BaseLinkedDataInstancesProvider
+
+
+class SingleInstanceLinkedDataProvider(BaseLinkedDataInstancesProvider):
+
+    def iter_instances(self, linked_data):
+        yield linked_data
+        return

arekit/common/data/input/providers/label/base.py

@@ -0,0 +1,24 @@
+from arekit.common.labels.scaler.base import BaseLabelScaler
+
+
+class LabelProvider(object):
+
+    def __init__(self, label_scaler):
+        assert(isinstance(label_scaler, BaseLabelScaler))
+        self.__label_scaler = label_scaler
+
+    @property
+    def LabelScaler(self):
+        return self.__label_scaler
+
+    @property
+    def SupportedLabels(self):
+        return self.__label_scaler.ordered_suppoted_labels()
+
+    @property
+    def OutputLabelsUint(self):
+        raise NotImplementedError()
+
+    def calculate_output_uint_label(self, expected_uint_label, etalon_uint_label):
+        raise NotImplementedError()
+

arekit/common/data/input/providers/label/binary.py

@@ -0,0 +1,11 @@
+from arekit.common.data.input.providers.label.base import LabelProvider
+
+
+class BinaryLabelProvider(LabelProvider):
+
+    def calculate_output_uint_label(self, expected_uint_label, etalon_uint_label):
+        return 1 if expected_uint_label == etalon_uint_label else 0
+
+    @property
+    def OutputLabelsUint(self):
+        return [0, 1]

arekit/common/data/input/providers/label/multiple.py

@@ -0,0 +1,15 @@
+from arekit.common.data.input.providers.label.base import LabelProvider
+
+
+class MultipleLabelProvider(LabelProvider):
+
+    def __init__(self, label_scaler):
+        super(MultipleLabelProvider, self).__init__(label_scaler=label_scaler)
+
+    def calculate_output_uint_label(self, expected_uint_label, etalon_uint_label):
+        return expected_uint_label
+
+    @property
+    def OutputLabelsUint(self):
+        return [self.LabelScaler.label_to_uint(label) for label in self.SupportedLabels]
+

arekit/common/data/input/providers/rows/base.py

@@ -0,0 +1,64 @@
+from collections import Counter
+from collections.abc import Iterable
+import logging
+
+from arekit.common.data.input.providers.contents import ContentsProvider
+from arekit.common.linkage.base import LinkedDataWrapper
+from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider
+from arekit.common.linkage.meta import MetaEmptyLinkedDataWrapper
+
+logger = logging.getLogger(__name__)
+
+
+class BaseRowProvider(object):
+    """ Base provider for rows that suppose to be filled into BaseRowsStorage.
+    """
+
+    def __init__(self):
+        self.__rows_counter = None
+
+    # region protected methods
+
+    # TODO. This might be also generalized.
+    # TODO. Idle-mode is also a implementation and task specific parameter, i.e. might be removed from here.
+    def _provide_rows(self, parsed_doc, entity_service, text_opinion_linkage, idle_mode):
+        raise NotImplementedError()
+
+    def _count_row(self):
+        index = self.__rows_counter["rows_iterated"]
+        self.__rows_counter["rows_iterated"] += 1
+        return index
+
+    # endregion
+
+    def __iter_rows(self, linked_data, idle_mode):
+        parsed_doc_service = linked_data.Tag
+        return self._provide_rows(parsed_doc=parsed_doc_service.ParsedDocument,
+                                  entity_service=parsed_doc_service.get_provider(EntityServiceProvider.NAME),
+                                  text_opinion_linkage=linked_data,
+                                  idle_mode=idle_mode)
+
+    def iter_by_rows(self, contents_provider, doc_ids_iter, idle_mode):
+        assert(isinstance(contents_provider, ContentsProvider))
+        assert(isinstance(doc_ids_iter, Iterable))
+
+        self.__rows_counter = Counter()
+
+        for linked_data in contents_provider.from_doc_ids(doc_ids=doc_ids_iter, idle_mode=idle_mode):
+            assert(isinstance(linked_data, LinkedDataWrapper))
+
+            if isinstance(linked_data, MetaEmptyLinkedDataWrapper):
+                if idle_mode:
+                    # In the case of the IDLE mode we do not consider the meta-data.
+                    data_it = []
+                else:
+                    # Consider the actual linked data instance.
+                    data_it = [linked_data]
+            else:
+                # Consider the actual rows of the related linked data.
+                data_it = self.__iter_rows(linked_data=linked_data, idle_mode=idle_mode)
+
+            for data in data_it:
+                yield linked_data.RelatedDocID, data
+
+        self.__rows_counter = None

arekit/common/data/input/providers/rows/samples.py

@@ -0,0 +1,227 @@
+from collections import OrderedDict
+
+from arekit.common.data import const
+from arekit.common.data.input.providers.instances.multiple import MultipleInstancesLinkedTextOpinionsProvider
+from arekit.common.data.input.providers.instances.single import SingleInstanceLinkedDataProvider
+from arekit.common.data.input.providers.label.base import LabelProvider
+from arekit.common.data.input.providers.label.binary import BinaryLabelProvider
+from arekit.common.data.input.providers.label.multiple import MultipleLabelProvider
+from arekit.common.data.input.providers.rows.base import BaseRowProvider
+from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
+from arekit.common.data.rows_fmt import create_base_column_fmt
+from arekit.common.entities.base import Entity
+from arekit.common.labels.base import Label
+
+from arekit.common.linkage.text_opinions import TextOpinionsLinkage
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.docs.parsed.providers.entity_service import EntityEndType, EntityServiceProvider
+from arekit.common.docs.parsed.term_position import TermPositionTypes
+from arekit.common.text_opinions.base import TextOpinion
+
+
+# TODO. This is actually a text-opinion related sampler.
+# TODO. Here we may expose all the text-opinion related params.
+# TODO. With more generalized API in base class.
+class BaseSampleRowProvider(BaseRowProvider):
+    """ Rows provider for samples storage.
+    """
+
+    def __init__(self, label_provider, text_provider):
+        assert(isinstance(label_provider, LabelProvider))
+        assert(isinstance(text_provider, BaseSingleTextProvider))
+        super(BaseSampleRowProvider, self).__init__()
+
+        self._label_provider = label_provider
+        self.__text_provider = text_provider
+        self.__instances_provider = self.__create_instances_provider(label_provider)
+        self.__store_labels = None
+        self._val_fmt = create_base_column_fmt(fmt_type="writer")
+
+    # region properties
+
+    @property
+    def LabelProvider(self):
+        return self._label_provider
+
+    @property
+    def TextProvider(self):
+        return self.__text_provider
+
+    # endregion
+
+    # region protected methods
+
+    def _provide_sentence_terms(self, parsed_doc, sentence_ind, s_ind, t_ind):
+        terms_iter = parsed_doc.iter_sentence_terms(sentence_index=sentence_ind, return_id=False)
+        return list(terms_iter), s_ind, t_ind
+
+    # TODO. This is a very task-specific description, too many data provided.
+    # TODO. Switch this API to dict of params
+    def _fill_row_core(self, row, text_opinion_linkage, index_in_linked, etalon_label,
+                       parsed_doc, sentence_ind, s_ind, t_ind):
+        assert(isinstance(self.__store_labels, bool))
+
+        sentence_terms, actual_s_ind, actual_t_ind = self._provide_sentence_terms(
+            parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
+
+        # Entity indices from the related context.
+        entities = list(filter(lambda term: isinstance(term, Entity), sentence_terms))
+
+        # Values mapping.
+        vm = {
+            const.ID: self._count_row(),
+            const.OPINION_ID: text_opinion_linkage.First.TextOpinionID,
+            const.OPINION_LINKAGE_ID: index_in_linked,
+            const.DOC_ID: text_opinion_linkage.First.DocID,
+            const.SENT_IND: sentence_ind,
+            const.ENTITY_VALUES: entities,
+            const.ENTITY_TYPES: entities,
+            const.ENTITIES: [str(i) for i, t in enumerate(sentence_terms) if isinstance(t, Entity)],
+            const.S_IND: actual_s_ind,
+            const.T_IND: actual_t_ind,
+            const.LABEL_UINT: None,
+            const.LABEL_STR: None
+        }
+
+        # Compose text value.
+        def __assign_value(column, value):
+            vm[column] = value
+
+        expected_label = text_opinion_linkage.get_linked_label()
+
+        self.__text_provider.add_text_in_row(
+            set_text_func=__assign_value, sentence_terms=sentence_terms,
+            s_ind=actual_s_ind, t_ind=actual_t_ind,
+            expected_label=expected_label)
+
+        if self.__store_labels:
+            l2i = self._label_provider.LabelScaler.label_to_uint
+            ui2l = self._label_provider.LabelScaler.uint_to_label
+            uint_label = self._label_provider.calculate_output_uint_label(
+                expected_uint_label=l2i(expected_label), etalon_uint_label=l2i(etalon_label))
+            vm[const.LABEL_UINT] = uint_label
+            vm[const.LABEL_STR] = type(ui2l(uint_label)).__name__
+
+        self._apply_row_data(row=row, vm=vm, val_fmt=self._val_fmt)
+
+    @staticmethod
+    def _apply_row_data(row, vm, val_fmt):
+        for k, v in vm.items():
+            if v is None:
+                continue
+            row[k] = v if k not in val_fmt else val_fmt[k](v)
+
+    def _provide_rows(self, parsed_doc, entity_service, text_opinion_linkage, idle_mode):
+        assert(isinstance(idle_mode, bool))
+
+        row_dict = OrderedDict()
+
+        for index_in_linked in range(len(text_opinion_linkage)):
+
+            rows_it = self.__provide_rows(
+                parsed_doc=parsed_doc,
+                entity_service=entity_service,
+                row_dict=row_dict,
+                text_opinion_linkage=text_opinion_linkage,
+                index_in_linked=index_in_linked,
+                idle_mode=idle_mode)
+
+            for row in rows_it:
+                yield row
+
+    # endregion
+
+    # region private methods
+
+    @staticmethod
+    def __create_instances_provider(label_provider):
+        # TODO. #473 related: these label providers are based on text opinion extraction task!
+        if isinstance(label_provider, BinaryLabelProvider):
+            return MultipleInstancesLinkedTextOpinionsProvider(label_provider.SupportedLabels)
+        if isinstance(label_provider, MultipleLabelProvider):
+            return SingleInstanceLinkedDataProvider()
+
+    def __provide_rows(self, row_dict, parsed_doc, entity_service,
+                       text_opinion_linkage, index_in_linked, idle_mode):
+        """
+        Providing Rows depending on row_id_formatter type
+        """
+        assert(isinstance(parsed_doc, ParsedDocument))
+        assert(isinstance(row_dict, OrderedDict))
+        assert(isinstance(text_opinion_linkage, TextOpinionsLinkage))
+
+        etalon_label = self.__instances_provider.provide_label(text_opinion_linkage)
+        for instance in self.__instances_provider.iter_instances(text_opinion_linkage):
+            yield self.__create_row(row=row_dict,
+                                    row_id=0,
+                                    parsed_doc=parsed_doc,
+                                    entity_service=entity_service,
+                                    text_opinions_linkage=instance,
+                                    index_in_linked=index_in_linked,
+                                    # TODO. provide uint_label
+                                    etalon_label=etalon_label,
+                                    idle_mode=idle_mode)
+
+    def __create_row(self, row, row_id, parsed_doc, entity_service, text_opinions_linkage,
+                     index_in_linked, etalon_label, idle_mode):
+        """
+        Composing row in following format:
+            [id, label, type, text_a]
+
+        returns: OrderedDict
+            row with key values
+        """
+        assert(isinstance(row, OrderedDict))
+        assert(isinstance(text_opinions_linkage, TextOpinionsLinkage))
+        assert(isinstance(index_in_linked, int))
+        assert(isinstance(etalon_label, Label))
+        assert(isinstance(idle_mode, bool))
+
+        if idle_mode:
+            return None
+
+        text_opinion = text_opinions_linkage[index_in_linked]
+
+        s_ind, t_ind = self.__get_opinion_end_indices(entity_service, text_opinion)
+
+        row.clear()
+
+        source_s_ind = entity_service.extract_entity_position(
+            text_opinion=text_opinion, end_type=EntityEndType.Source,
+            position_type=TermPositionTypes.SentenceIndex)
+
+        target_s_ind = entity_service.extract_entity_position(
+            text_opinion=text_opinion, end_type=EntityEndType.Target,
+            position_type=TermPositionTypes.SentenceIndex)
+
+        if target_s_ind != source_s_ind:
+            raise Exception("Limitation: Multi-Sentence text_opinions are not supported.")
+
+        self._fill_row_core(row=row,
+                            parsed_doc=parsed_doc,
+                            sentence_ind=source_s_ind,
+                            text_opinion_linkage=text_opinions_linkage,
+                            index_in_linked=index_in_linked,
+                            etalon_label=etalon_label,
+                            s_ind=s_ind,
+                            t_ind=t_ind)
+        return row
+
+    @staticmethod
+    def __get_opinion_end_indices(service, text_opinion):
+        assert(isinstance(service, EntityServiceProvider))
+        assert(isinstance(text_opinion, TextOpinion))
+
+        s_ind = service.get_entity_position(text_opinion.SourceId).get_index(
+            position_type=TermPositionTypes.IndexInSentence)
+
+        t_ind = service.get_entity_position(text_opinion.TargetId).get_index(
+            position_type=TermPositionTypes.IndexInSentence)
+
+        return s_ind, t_ind
+
+    # endregion
+
+    def set_store_labels(self, store_labels):
+        assert(isinstance(store_labels, bool))
+        self.__store_labels = store_labels