arekit 0.24.0__py3-none-any.whl

arekit/common/data/input/providers/sample/cropped.py

@@ -0,0 +1,43 @@
+from arekit.common.data.input.providers.label.multiple import MultipleLabelProvider
+from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
+
+
+class CroppedSampleRowProvider(BaseSampleRowProvider):
+    """ Sample provided which has `crop_window` that allows to slice
+        the potentially large samples and guarantee the presence of
+        attitude inside.
+    """
+
+    def __init__(self, crop_window_size, label_scaler, text_provider):
+        assert(isinstance(crop_window_size, int) and crop_window_size > 0)
+        super(CroppedSampleRowProvider, self).__init__(label_provider=MultipleLabelProvider(label_scaler),
+                                                       text_provider=text_provider)
+        self.__crop_window_size = crop_window_size
+
+    @staticmethod
+    def __calc_window_bounds(window_size, s_ind, t_ind, input_length):
+        """ returns: [_from, _to)
+        """
+        assert(isinstance(s_ind, int))
+        assert(isinstance(t_ind, int))
+        assert(isinstance(input_length, int))
+        assert(input_length >= s_ind and input_length >= t_ind)
+
+        def __in():
+            return _from <= s_ind < _to and _from <= t_ind < _to
+
+        _from = 0
+        _to = window_size
+        while not __in():
+            _from += 1
+            _to += 1
+
+        return _from, _to
+
+    def _provide_sentence_terms(self, parsed_doc, sentence_ind, s_ind, t_ind):
+        terms_iter, src_ind, tgt_ind = super(CroppedSampleRowProvider, self)._provide_sentence_terms(
+            parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
+        terms = list(terms_iter)
+        _from, _to = self.__calc_window_bounds(window_size=self.__crop_window_size,
+                                               s_ind=s_ind, t_ind=t_ind, input_length=len(terms))
+        return terms[_from:_to], src_ind - _from, tgt_ind - _from

arekit/common/data/input/providers/text/single.py

@@ -0,0 +1,49 @@
+from arekit.common.data import const
+from arekit.common.data.input.terms_mapper import OpinionContainingTextTermsMapper
+from arekit.common.labels.base import Label
+
+
+class BaseSingleTextProvider(object):
+
+    TEXT_A = const.TEXT
+    TERMS_SEPARATOR = " "
+
+    def __init__(self, text_terms_mapper):
+        assert(isinstance(text_terms_mapper, OpinionContainingTextTermsMapper))
+        self._mapper = text_terms_mapper
+
+    def iter_columns(self):
+        yield BaseSingleTextProvider.TEXT_A
+
+    @staticmethod
+    def _process_text(text):
+        assert(isinstance(text, str))
+        return text.strip()
+
+    def _mapped_data_to_str(self, m_data):
+        return m_data
+
+    def _handle_mapped_data(self, m_data):
+        # Optionally handle mapped data.
+        pass
+
+    def _handle_terms_and_compose_text(self, sentence_terms):
+        assert(isinstance(sentence_terms, list))
+
+        str_terms = []
+
+        for m_data in self._mapper.iter_mapped(sentence_terms):
+            str_terms.append(self._mapped_data_to_str(m_data=m_data))
+            self._handle_mapped_data(m_data=m_data)
+
+        return self.TERMS_SEPARATOR.join(str_terms)
+
+    def add_text_in_row(self, set_text_func, sentence_terms, s_ind, t_ind, expected_label):
+        assert(callable(set_text_func))
+        assert(isinstance(sentence_terms, list))
+        assert(isinstance(expected_label, Label))
+
+        self._mapper.set_s_ind(s_ind)
+        self._mapper.set_t_ind(t_ind)
+        set_text_func(column=self.TEXT_A,
+                      value=self._process_text(text=self._handle_terms_and_compose_text(sentence_terms)))

arekit/common/data/input/repositories/base.py

@@ -0,0 +1,68 @@
+from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
+from arekit.common.data.input.providers.contents import ContentsProvider
+from arekit.common.data.input.providers.rows.base import BaseRowProvider
+from arekit.common.data.storages.base import BaseRowsStorage
+from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
+from arekit.contrib.utils.data.writers.base import BaseWriter
+
+
+class BaseInputRepository(object):
+
+    def __init__(self, columns_provider, rows_provider, storage):
+        assert(isinstance(columns_provider, BaseColumnsProvider))
+        assert(isinstance(rows_provider, BaseRowProvider))
+        assert(isinstance(storage, BaseRowsStorage))
+
+        self._columns_provider = columns_provider
+        self._rows_provider = rows_provider
+        self._storage = storage
+
+        # Do setup operations.
+        self._setup_columns_provider()
+        self._setup_rows_provider()
+
+    # region protected methods
+
+    def _setup_columns_provider(self):
+        pass
+
+    def _setup_rows_provider(self):
+        pass
+
+    # endregion
+
+    def populate(self, contents_provider, doc_ids, desc="", writer=None, target=None):
+        assert(isinstance(contents_provider, ContentsProvider))
+        assert(isinstance(self._storage, BaseRowsStorage))
+        assert(isinstance(doc_ids, list))
+        assert(isinstance(writer, BaseWriter) or writer is None)
+        assert(isinstance(target, str) or target is None)
+
+        def iter_rows(idle_mode):
+            return self._rows_provider.iter_by_rows(
+                contents_provider=contents_provider,
+                doc_ids_iter=doc_ids,
+                idle_mode=idle_mode)
+
+        self._storage.init_empty(columns_provider=self._columns_provider)
+
+        is_async_write_mode_on = writer is not None and target is not None
+
+        if is_async_write_mode_on:
+            writer.open_target(target)
+
+        self._storage.fill(lambda idle_mode: iter_rows(idle_mode),
+                           columns_provider=self._columns_provider,
+                           row_handler=lambda: writer.commit_line(self._storage) if is_async_write_mode_on else None,
+                           desc=desc)
+
+        if is_async_write_mode_on:
+            writer.close_target()
+
+    def push(self, writer, target, free_storage=True):
+        if not isinstance(self._storage, RowCacheStorage):
+            writer.write_all(self._storage, target)
+
+        # After writing we free the contents of the storage.
+        if free_storage:
+            self._storage.free()

arekit/common/data/input/repositories/sample.py

@@ -0,0 +1,22 @@
+import logging
+
+from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
+from arekit.common.data.input.repositories.base import BaseInputRepository
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class BaseInputSamplesRepository(BaseInputRepository):
+
+    def _setup_rows_provider(self):
+        """ Setup store labels.
+        """
+        assert(isinstance(self._rows_provider, BaseSampleRowProvider))
+        self._rows_provider.set_store_labels(self._columns_provider.StoreLabels)
+
+    def _setup_columns_provider(self):
+        """ Setup text column names.
+        """
+        text_column_names = list(self._rows_provider.TextProvider.iter_columns())
+        self._columns_provider.set_text_column_names(text_column_names)

arekit/common/data/input/sample.py

@@ -0,0 +1,66 @@
+from collections import OrderedDict
+
+from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider, DistanceType
+from arekit.common.text_opinions.base import TextOpinion
+
+
+class InputSampleBase(object):
+    """
+    Description of a single sample (context) of a model
+    """
+
+    def __init__(self, shift_index_dbg, input_sample_id, values):
+        assert(isinstance(shift_index_dbg, int))
+        assert(isinstance(input_sample_id, str))
+        assert(isinstance(values, list))
+        self._shift_index_dbg = shift_index_dbg
+        self.__input_sample_id = input_sample_id
+        self.__values = OrderedDict(values)
+
+    # region properties
+
+    @property
+    def ID(self):
+        return self.__input_sample_id
+
+    # endregion
+
+    @staticmethod
+    def check_ability_to_create_sample(entity_service, window_size, text_opinion):
+        """
+        Main text_opinion filtering rules
+        """
+        assert(isinstance(entity_service, EntityServiceProvider))
+        assert(isinstance(text_opinion, TextOpinion))
+        assert(isinstance(window_size, int) and window_size > 0)
+
+        is_not_same_ends = False
+        is_in_window = False
+        is_same_sentence = False
+
+        if text_opinion.SourceId != text_opinion.TargetId:
+            is_not_same_ends = True
+
+        dist_between_entities = entity_service.calc_dist_between_text_opinion_ends(
+            text_opinion=text_opinion,
+            distance_type=DistanceType.InTerms)
+
+        if InputSampleBase._check_ends_could_be_fitted_in_window(dist_between_entities, window_size):
+            is_in_window = True
+
+        dist_in_sents = entity_service.calc_dist_between_text_opinion_ends(
+            text_opinion=text_opinion,
+            distance_type=DistanceType.InSentences)
+
+        if dist_in_sents == 0:
+            is_same_sentence = True
+
+        return is_not_same_ends and is_in_window and is_same_sentence
+
+    @staticmethod
+    def _check_ends_could_be_fitted_in_window(actual_dist, window):
+        return actual_dist < window
+
+    def __iter__(self):
+        for key, value in self.__values.items():
+            yield key, value

arekit/common/data/input/terms_mapper.py

@@ -0,0 +1,88 @@
+from arekit.common.context.terms_mapper import TextTermsMapper
+from arekit.common.context.token import Token
+from arekit.common.entities.base import Entity
+from arekit.common.entities.str_fmt import StringEntitiesFormatter
+from arekit.common.entities.types import OpinionEntityType
+from arekit.common.frames.text_variant import TextFrameVariant
+
+
+class OpinionContainingTextTermsMapper(TextTermsMapper):
+    """
+    Provides an ability to setup s_obj, t_obj
+    The latter might be utilized with synonyms collection
+    """
+
+    def __init__(self, entity_formatter):
+        assert(isinstance(entity_formatter, StringEntitiesFormatter))
+        self.__entities_formatter = entity_formatter
+        self.__s_ind = None
+        self.__t_ind = None
+        self.__s_group = None
+        self.__t_group = None
+
+    @property
+    def StringEntitiesFormatter(self):
+        return self.__entities_formatter
+
+    def __syn_group(self, entity):
+        """ Note: here we guarantee that entity has GroupIndex.
+        """
+        assert(isinstance(entity, Entity))
+        return entity.GroupIndex if entity is not None else None
+
+    def set_s_ind(self, s_ind):
+        assert(isinstance(s_ind, int))
+        self.__s_ind = s_ind
+
+    def set_t_ind(self, t_ind):
+        assert(isinstance(t_ind, int))
+        self.__t_ind = t_ind
+
+    def _after_mapping(self):
+        """ In order to prevent bugs.
+            Every index should be declared before mapping.
+        """
+        self.__s_ind = None
+        self.__t_ind = None
+
+    def iter_mapped(self, terms):
+        terms_list = list(terms)
+        self.__s_group = self.__syn_group(terms_list[self.__s_ind] if self.__s_ind is not None else None)
+        self.__t_group = self.__syn_group(terms_list[self.__t_ind] if self.__t_ind is not None else None)
+        return super(OpinionContainingTextTermsMapper, self).iter_mapped(terms)
+
+    def map_entity(self, e_ind, entity):
+
+        entity_type = OpinionEntityType.Other
+        if e_ind == self.__s_ind:
+            entity_type = OpinionEntityType.Subject
+        elif e_ind == self.__t_ind:
+            entity_type = OpinionEntityType.Object
+        elif self.__is_in_same_group(self.__syn_group(entity), self.__s_group):
+            entity_type = OpinionEntityType.SynonymSubject
+        elif self.__is_in_same_group(self.__syn_group(entity), self.__t_group):
+            entity_type = OpinionEntityType.SynonymObject
+
+        return self.__entities_formatter.to_string(original_value=entity,
+                                                   entity_type=entity_type)
+
+    @staticmethod
+    def __is_in_same_group(g1, g2):
+
+        if g1 is None or g2 is None:
+            # In such scenario we cannot guarantee
+            # that g1 and g2 belong to the same group.
+            return False
+
+        return g1 == g2
+
+    def map_word(self, w_ind, word):
+        return word.strip()
+
+    def map_text_frame_variant(self, fv_ind, text_frame_variant):
+        assert(isinstance(text_frame_variant, TextFrameVariant))
+        return text_frame_variant.Variant.get_value().strip()
+
+    def map_token(self, t_ind, token):
+        assert(isinstance(token, Token))
+        return token.get_meta_value()

arekit/common/data/rows_fmt.py

@@ -0,0 +1,82 @@
+from arekit.common.data import const
+from arekit.common.utils import filter_whitespaces, split_by_whitespaces
+
+
+def process_values_list(value, args_sep):
+    return value.split(args_sep)
+
+
+def process_indices_list(value, no_value_func, args_sep):
+    return no_value_func() if not value else [int(v) for v in str(value).split(args_sep)]
+
+
+def process_text(value):
+    """ The core method of the input text processing.
+    """
+    assert(isinstance(value, str) or isinstance(value, list))
+    return filter_whitespaces([term for term in split_by_whitespaces(value)]
+                              if isinstance(value, str) else value)
+
+
+def create_base_column_value_fmt(no_value_func=lambda: None, args_sep=","):
+
+    self_func = lambda value: value
+
+    return {
+        const.ID: {
+            "writer": self_func,
+            "parser": self_func
+        },
+        const.DOC_ID: {
+            "writer": self_func,
+            "parser": self_func,
+        },
+        const.S_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.T_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.SENT_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.OPINION_ID: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.OPINION_LINKAGE_ID: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.ENTITY_VALUES: {
+            "writer": lambda entities: args_sep.join([e.DisplayValue.replace(args_sep, '') for e in entities]),
+            "parser": lambda value: process_values_list(value, args_sep=args_sep),
+        },
+        const.ENTITY_TYPES: {
+            "writer": lambda entities: args_sep.join([e.Type.replace(args_sep, '') for e in entities]),
+            "parser": lambda value: process_values_list(value, args_sep=args_sep)
+        },
+        const.ENTITIES: {
+            "writer": lambda entity_inds: args_sep.join(entity_inds),
+            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
+        },
+        const.TEXT: {
+            "writer": self_func,
+            "parser": lambda value: process_text(value)
+        },
+        const.LABEL_UINT: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        }
+    }
+
+
+def create_base_column_fmt(fmt_type, args_sep=","):
+    assert(isinstance(fmt_type, str))
+    d = create_base_column_value_fmt(args_sep=args_sep)
+    for k, v in d.items():
+        d[k] = v[fmt_type]
+    return d

arekit/common/data/rows_parser.py

@@ -0,0 +1,43 @@
+class ParsedSampleRow(object):
+    """ Provides a parsed information for a sample row.
+    """
+
+    def __init__(self, row, columns_fmts, no_value_func):
+        """ row: dict
+                dict of the pairs ("field_name", value)
+            columns_fmt: list
+                list of the formatters, where every formatter represent a dictionary.
+            no_value_func: func
+                the default value the conveys the absence of the parameter-value.
+        """
+        assert(isinstance(row, dict))
+        assert(isinstance(columns_fmts, list))
+        assert(callable(no_value_func))
+
+        self.__uint_label = None
+        self.__params = {}
+        self.__no_value = no_value_func
+
+        for key, value in row.items():
+
+            for columns_fmt in columns_fmts:
+                assert(isinstance(columns_fmt, dict))
+
+                if key not in columns_fmt:
+                    continue
+
+                self.__params[key] = columns_fmt[key](value)
+                break
+
+    def __value_or_none(self, key):
+        return self.__params[key] if key in self.__params else self.__no_value()
+
+    def __getitem__(self, item):
+        assert (isinstance(item, str) or item is None)
+        if item not in self.__params:
+            return self.__no_value()
+        return self.__params[item] if item is not None else self.__no_value()
+
+    @classmethod
+    def parse(cls, row, columns_fmts, no_value_func):
+        return cls(row=row, columns_fmts=columns_fmts, no_value_func=no_value_func)

arekit/common/data/storages/base.py

@@ -0,0 +1,109 @@
+import gc
+import logging
+
+from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
+from arekit.common.linkage.meta import MetaEmptyLinkedDataWrapper
+from arekit.common.utils import progress_bar_conditional
+
+logger = logging.getLogger(__name__)
+
+
+class BaseRowsStorage(object):
+
+    # region protected methods
+
+    def _begin_filling_row(self, row_ind):
+        pass
+
+    # endregion
+
+    # region abstract methods
+
+    def _set_row_value(self, row_ind, column, value):
+        raise NotImplemented()
+
+    def _iter_rows(self):
+        """ returns: tuple(int, list)
+                provides the index (int) and the related content of the row (dict)
+        """
+        raise NotImplemented()
+
+    def _get_rows_count(self):
+        raise NotImplemented()
+
+    def find_by_value(self, column_name, value):
+        raise NotImplemented()
+
+    def find_first_by_value(self, column_name, value):
+        raise NotImplemented()
+
+    def iter_column_values(self, column_name, dtype=None):
+        raise NotImplemented()
+
+    def get_row(self, row_index):
+        raise NotImplemented()
+
+    def get_cell(self, row_index, column_name):
+        raise NotImplemented()
+
+    def init_empty(self, columns_provider):
+        raise NotImplemented()
+
+    def iter_shuffled(self):
+        raise NotImplemented()
+
+    def iter_column_names(self):
+        raise NotImplemented()
+
+    def iter_column_types(self):
+        raise NotImplemented()
+
+    # endregion
+
+    def fill(self, iter_rows_func, columns_provider, row_handler=None, rows_count=None, desc=""):
+        assert(callable(iter_rows_func))
+        assert(isinstance(columns_provider, BaseColumnsProvider))
+        assert(callable(row_handler) or row_handler is None)
+
+        doc_ids_seen = set()
+
+        def postfix_func(item):
+            doc_id, _ = item
+            doc_ids_seen.add(doc_id)
+            return {
+                "docs_seen": len(doc_ids_seen),
+                "doc_now": str(doc_id)
+            }
+
+        pbar_it = progress_bar_conditional(
+            iterable=iter_rows_func(False),
+            # We skip meta information data.
+            condition_func=lambda item: not isinstance(item[1], MetaEmptyLinkedDataWrapper),
+            postfix_func=postfix_func,
+            desc="{fmt}".format(fmt=desc),
+            total=rows_count)
+
+        for row_index, item in enumerate(pbar_it):
+            _, row_values = item
+            self._begin_filling_row(row_index)
+            for column, value in row_values.items():
+                self._set_row_value(row_ind=row_index,
+                                    column=column,
+                                    value=value)
+            if row_handler is not None:
+                row_handler()
+
+    def free(self):
+        gc.collect()
+
+    # endregion
+
+    # region base methods
+
+    def __iter__(self):
+        return self._iter_rows()
+
+    def __len__(self):
+        return self._get_rows_count()
+
+    # endregion

arekit/common/data/views/samples.py

@@ -0,0 +1,26 @@
+from arekit.common.data import const
+from arekit.common.data.storages.base import BaseRowsStorage
+
+
+# TODO. This is a particular type of view, and expected to be off the core.
+class LinkedSamplesStorageView(object):
+
+    def iter_from_storage(self, storage):
+        assert(isinstance(storage, BaseRowsStorage))
+        undefined = -1
+
+        linked = []
+        current_opinion_id = undefined
+        for row_index, opinion_id in enumerate(storage.iter_column_values(const.OPINION_ID)):
+            if current_opinion_id != undefined:
+                if opinion_id != current_opinion_id:
+                    yield linked
+                    linked = []
+                    current_opinion_id = opinion_id
+            else:
+                current_opinion_id = opinion_id
+
+            linked.append(storage.get_row(row_index))
+
+        if len(linked) > 0:
+            yield linked

arekit/common/docs/base.py

@@ -0,0 +1,30 @@
+class Document(object):
+
+    def __init__(self, doc_id, sentences):
+        assert(isinstance(sentences, list))
+        self.__id = doc_id
+        self._sentences = sentences
+
+    # region properties
+
+    @property
+    def ID(self):
+        return self.__id
+
+    @property
+    def SentencesCount(self):
+        """ Provides total amount of sentences within a doc
+            At present is useful for:
+            -   CV-splitters, which may rely on sentences count.
+            -   Text parsing.
+        """
+        return len(self._sentences)
+
+    # endregion
+
+    def iter_sentences(self):
+        for sentence in self._sentences:
+            yield sentence
+
+    def get_sentence(self, s_ind):
+        return self._sentences[s_ind]

arekit/common/docs/entities_grouping.py

@@ -0,0 +1,16 @@
+from arekit.common.entities.base import Entity
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class EntitiesGroupingPipelineItem(BasePipelineItem):
+
+    def __init__(self, value_to_group_id_func):
+        assert(callable(value_to_group_id_func))
+        self.__value_to_group_id_func = value_to_group_id_func
+
+    def apply_core(self, input_data, pipeline_ctx):
+        assert(isinstance(input_data, list))
+
+        for entity in filter(lambda term: isinstance(term, Entity), input_data):
+            group_index = self.__value_to_group_id_func(entity.Value)
+            entity.set_group_index(group_index)

arekit/common/docs/entity.py

@@ -0,0 +1,18 @@
+from arekit.common.entities.base import Entity
+
+
+class DocumentEntity(Entity):
+
+    def __init__(self, value, display_value, e_type, childs, id_in_doc, group_index):
+        """ id_in_doc: Id, utilized witin the internal services
+        """
+        super(DocumentEntity, self).__init__(value=value,
+                                             e_type=e_type,
+                                             display_value=display_value,
+                                             childs=childs,
+                                             group_index=group_index)
+        self.__id = id_in_doc
+
+    @property
+    def IdInDocument(self):
+        return self.__id