arekit 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/common/context/terms_mapper.py

@@ -1,4 +1,4 @@
-import collections
+from collections.abc import Iterable
 
 from arekit.common.context.token import Token
 from arekit.common.entities.base import Entity
@@ -10,7 +10,7 @@ class TextTermsMapper(object):
     def iter_mapped(self, terms):
         """ Performs mapping operation of each terms in a sequence
         """
-        assert(isinstance(terms, collections.Iterable))
+        assert(isinstance(terms, Iterable))
 
         self._before_mapping()
 

arekit/common/data/const.py

@@ -1,11 +1,12 @@
 ID = 'id'
 DOC_ID = 'doc_id'
 TEXT = 'text_a'
-LABEL = 'label'
+LABEL_UINT = 'label_uint'
+LABEL_STR = 'label_str'
 
-# Corresponds to fields with attitude ends. (values, STRING)
-SOURCE = 'source'
-TARGET = 'target'
+# Global identifier of the opinion in the sampled data.
+OPINION_ID = "opinion_id"
+OPINION_LINKAGE_ID = "linkage_id"
 
 # Corresponds to fields with attitude ends. (indices, INT)
 S_IND = 's_ind'

arekit/common/{experiment/api/ops_doc.py → data/doc_provider.py}

@@ -1,4 +1,4 @@
-class DocumentOperations(object):
+class DocumentProvider(object):
     """ Provides operations with documents
     """
 

arekit/common/data/input/providers/columns/sample.py

@@ -37,7 +37,8 @@ class SampleColumnsProvider(BaseColumnsProvider):
 
         # insert labels
         if self.__store_labels:
-            dtypes_list.append((const.LABEL, 'int32'))
+            dtypes_list.append((const.LABEL_UINT, 'int32'))
+            dtypes_list.append((const.LABEL_STR, str))
 
         # insert text columns
         for col_name in self.__text_column_names:
@@ -47,6 +48,10 @@ class SampleColumnsProvider(BaseColumnsProvider):
         dtypes_list.append((const.S_IND, 'int32'))
         dtypes_list.append((const.T_IND, 'int32'))
 
+        # opinion-extraction task related fields
+        dtypes_list.append((const.OPINION_ID, 'int32'))
+        dtypes_list.append((const.OPINION_LINKAGE_ID, 'int32'))
+
         return dtypes_list
 
     def set_text_column_names(self, text_column_names):

arekit/common/data/input/providers/instances/base.py

@@ -11,4 +11,4 @@ class BaseLinkedDataInstancesProvider(object):
         """ Implementation based on the first element of the linkage.
         """
         assert(isinstance(linked_data, LinkedDataWrapper))
-        return linked_data.First.Sentiment
+        return linked_data.First.Label

arekit/common/data/input/providers/rows/base.py

@@ -1,10 +1,11 @@
-import collections
+from collections import Counter
+from collections.abc import Iterable
 import logging
 
 from arekit.common.data.input.providers.contents import ContentsProvider
 from arekit.common.linkage.base import LinkedDataWrapper
-from arekit.common.news.parsed.providers.entity_service import EntityServiceProvider
-from arekit.common.news.parsed.service import ParsedNewsService
+from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider
+from arekit.common.linkage.meta import MetaEmptyLinkedDataWrapper
 
 logger = logging.getLogger(__name__)
 
@@ -13,29 +14,51 @@ class BaseRowProvider(object):
     """ Base provider for rows that suppose to be filled into BaseRowsStorage.
     """
 
+    def __init__(self):
+        self.__rows_counter = None
+
     # region protected methods
 
     # TODO. This might be also generalized.
     # TODO. Idle-mode is also a implementation and task specific parameter, i.e. might be removed from here.
-    def _provide_rows(self, parsed_news, entity_service, text_opinion_linkage, idle_mode):
+    def _provide_rows(self, parsed_doc, entity_service, text_opinion_linkage, idle_mode):
         raise NotImplementedError()
 
+    def _count_row(self):
+        index = self.__rows_counter["rows_iterated"]
+        self.__rows_counter["rows_iterated"] += 1
+        return index
+
     # endregion
 
+    def __iter_rows(self, linked_data, idle_mode):
+        parsed_doc_service = linked_data.Tag
+        return self._provide_rows(parsed_doc=parsed_doc_service.ParsedDocument,
+                                  entity_service=parsed_doc_service.get_provider(EntityServiceProvider.NAME),
+                                  text_opinion_linkage=linked_data,
+                                  idle_mode=idle_mode)
+
     def iter_by_rows(self, contents_provider, doc_ids_iter, idle_mode):
         assert(isinstance(contents_provider, ContentsProvider))
-        assert(isinstance(doc_ids_iter, collections.Iterable))
+        assert(isinstance(doc_ids_iter, Iterable))
+
+        self.__rows_counter = Counter()
 
         for linked_data in contents_provider.from_doc_ids(doc_ids=doc_ids_iter, idle_mode=idle_mode):
             assert(isinstance(linked_data, LinkedDataWrapper))
-            assert(isinstance(linked_data.Tag, ParsedNewsService))
 
-            parsed_news_service = linked_data.Tag
+            if isinstance(linked_data, MetaEmptyLinkedDataWrapper):
+                if idle_mode:
+                    # In the case of the IDLE mode we do not consider the meta-data.
+                    data_it = []
+                else:
+                    # Consider the actual linked data instance.
+                    data_it = [linked_data]
+            else:
+                # Consider the actual rows of the related linked data.
+                data_it = self.__iter_rows(linked_data=linked_data, idle_mode=idle_mode)
 
-            rows_it = self._provide_rows(parsed_news=parsed_news_service.ParsedNews,
-                                         entity_service=parsed_news_service.get_provider(EntityServiceProvider.NAME),
-                                         text_opinion_linkage=linked_data,
-                                         idle_mode=idle_mode)
+            for data in data_it:
+                yield linked_data.RelatedDocID, data
 
-            for row in rows_it:
-                yield linked_data.RelatedDocID, row
+        self.__rows_counter = None

arekit/common/data/input/providers/rows/samples.py

@@ -8,15 +8,14 @@ from arekit.common.data.input.providers.label.binary import BinaryLabelProvider
 from arekit.common.data.input.providers.label.multiple import MultipleLabelProvider
 from arekit.common.data.input.providers.rows.base import BaseRowProvider
 from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
-from arekit.common.data.row_ids.binary import BinaryIDProvider
-from arekit.common.data.row_ids.multiple import MultipleIDProvider
+from arekit.common.data.rows_fmt import create_base_column_fmt
 from arekit.common.entities.base import Entity
 from arekit.common.labels.base import Label
 
 from arekit.common.linkage.text_opinions import TextOpinionsLinkage
-from arekit.common.news.parsed.base import ParsedNews
-from arekit.common.news.parsed.providers.entity_service import EntityEndType, EntityServiceProvider
-from arekit.common.news.parsed.term_position import TermPositionTypes
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.docs.parsed.providers.entity_service import EntityEndType, EntityServiceProvider
+from arekit.common.docs.parsed.term_position import TermPositionTypes
 from arekit.common.text_opinions.base import TextOpinion
 
 
@@ -34,9 +33,9 @@ class BaseSampleRowProvider(BaseRowProvider):
 
         self._label_provider = label_provider
         self.__text_provider = text_provider
-        self.__row_ids_provider = self.__create_row_ids_provider(label_provider)
         self.__instances_provider = self.__create_instances_provider(label_provider)
         self.__store_labels = None
+        self._val_fmt = create_base_column_fmt(fmt_type="writer")
 
     # region properties
 
@@ -52,56 +51,67 @@ class BaseSampleRowProvider(BaseRowProvider):
 
     # region protected methods
 
-    def _provide_sentence_terms(self, parsed_news, sentence_ind, s_ind, t_ind):
-        terms_iter = parsed_news.iter_sentence_terms(sentence_index=sentence_ind, return_id=False)
+    def _provide_sentence_terms(self, parsed_doc, sentence_ind, s_ind, t_ind):
+        terms_iter = parsed_doc.iter_sentence_terms(sentence_index=sentence_ind, return_id=False)
         return list(terms_iter), s_ind, t_ind
 
     # TODO. This is a very task-specific description, too many data provided.
     # TODO. Switch this API to dict of params
     def _fill_row_core(self, row, text_opinion_linkage, index_in_linked, etalon_label,
-                       parsed_news, sentence_ind, s_ind, t_ind):
+                       parsed_doc, sentence_ind, s_ind, t_ind):
         assert(isinstance(self.__store_labels, bool))
 
-        def __assign_value(column, value):
-            row[column] = value
-
-        row[const.ID] = self.__row_ids_provider.create_sample_id(
-            linked_opinions=text_opinion_linkage,
-            index_in_linked=index_in_linked,
-            label_scaler=self._label_provider.LabelScaler)
+        sentence_terms, actual_s_ind, actual_t_ind = self._provide_sentence_terms(
+            parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
 
-        row[const.DOC_ID] = text_opinion_linkage.First.DocID
+        # Entity indices from the related context.
+        entities = list(filter(lambda term: isinstance(term, Entity), sentence_terms))
 
-        row[const.SENT_IND] = sentence_ind
+        # Values mapping.
+        vm = {
+            const.ID: self._count_row(),
+            const.OPINION_ID: text_opinion_linkage.First.TextOpinionID,
+            const.OPINION_LINKAGE_ID: index_in_linked,
+            const.DOC_ID: text_opinion_linkage.First.DocID,
+            const.SENT_IND: sentence_ind,
+            const.ENTITY_VALUES: entities,
+            const.ENTITY_TYPES: entities,
+            const.ENTITIES: [str(i) for i, t in enumerate(sentence_terms) if isinstance(t, Entity)],
+            const.S_IND: actual_s_ind,
+            const.T_IND: actual_t_ind,
+            const.LABEL_UINT: None,
+            const.LABEL_STR: None
+        }
+
+        # Compose text value.
+        def __assign_value(column, value):
+            vm[column] = value
 
         expected_label = text_opinion_linkage.get_linked_label()
 
-        if self.__store_labels:
-            row[const.LABEL] = self._label_provider.calculate_output_uint_label(
-                expected_uint_label=self._label_provider.LabelScaler.label_to_uint(expected_label),
-                etalon_uint_label=self._label_provider.LabelScaler.label_to_uint(etalon_label))
-
-        sentence_terms, actual_s_ind, actual_t_ind = self._provide_sentence_terms(
-            parsed_news=parsed_news, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
-
         self.__text_provider.add_text_in_row(
-            set_text_func=lambda column, value: __assign_value(column, value),
-            sentence_terms=sentence_terms,
-            s_ind=actual_s_ind,
-            t_ind=actual_t_ind,
+            set_text_func=__assign_value, sentence_terms=sentence_terms,
+            s_ind=actual_s_ind, t_ind=actual_t_ind,
             expected_label=expected_label)
 
-        # Entity indicies from the related context.
-        entities = list(filter(lambda term: isinstance(term, Entity), sentence_terms))
-        entity_inds = [str(i) for i, t in enumerate(sentence_terms) if isinstance(t, Entity)]
-        row[const.ENTITY_VALUES] = ",".join([e.DisplayValue.replace(',', '') for e in entities])
-        row[const.ENTITY_TYPES] = ",".join([e.Type.replace(',', '') for e in entities])
-        row[const.ENTITIES] = ",".join(entity_inds)
+        if self.__store_labels:
+            l2i = self._label_provider.LabelScaler.label_to_uint
+            ui2l = self._label_provider.LabelScaler.uint_to_label
+            uint_label = self._label_provider.calculate_output_uint_label(
+                expected_uint_label=l2i(expected_label), etalon_uint_label=l2i(etalon_label))
+            vm[const.LABEL_UINT] = uint_label
+            vm[const.LABEL_STR] = type(ui2l(uint_label)).__name__
 
-        row[const.S_IND] = actual_s_ind
-        row[const.T_IND] = actual_t_ind
+        self._apply_row_data(row=row, vm=vm, val_fmt=self._val_fmt)
+
+    @staticmethod
+    def _apply_row_data(row, vm, val_fmt):
+        for k, v in vm.items():
+            if v is None:
+                continue
+            row[k] = v if k not in val_fmt else val_fmt[k](v)
 
-    def _provide_rows(self, parsed_news, entity_service, text_opinion_linkage, idle_mode):
+    def _provide_rows(self, parsed_doc, entity_service, text_opinion_linkage, idle_mode):
         assert(isinstance(idle_mode, bool))
 
         row_dict = OrderedDict()
@@ -109,7 +119,7 @@ class BaseSampleRowProvider(BaseRowProvider):
         for index_in_linked in range(len(text_opinion_linkage)):
 
             rows_it = self.__provide_rows(
-                parsed_news=parsed_news,
+                parsed_doc=parsed_doc,
                 entity_service=entity_service,
                 row_dict=row_dict,
                 text_opinion_linkage=text_opinion_linkage,
@@ -123,36 +133,28 @@ class BaseSampleRowProvider(BaseRowProvider):
 
     # region private methods
 
-    @staticmethod
-    def __create_row_ids_provider(label_provider):
-        # TODO. #376 related. This should be removed after refactoring, because
-        # TODO. we consider an ordinary IDs, that not based on the other data.
-        if isinstance(label_provider, BinaryLabelProvider):
-            return BinaryIDProvider()
-        if isinstance(label_provider, MultipleLabelProvider):
-            return MultipleIDProvider()
-
     @staticmethod
     def __create_instances_provider(label_provider):
-        # TODO. #473 related: thiese label providers are based on text opinion extraction task!
+        # TODO. #473 related: these label providers are based on text opinion extraction task!
         if isinstance(label_provider, BinaryLabelProvider):
             return MultipleInstancesLinkedTextOpinionsProvider(label_provider.SupportedLabels)
         if isinstance(label_provider, MultipleLabelProvider):
             return SingleInstanceLinkedDataProvider()
 
-    def __provide_rows(self, row_dict, parsed_news, entity_service,
+    def __provide_rows(self, row_dict, parsed_doc, entity_service,
                        text_opinion_linkage, index_in_linked, idle_mode):
         """
         Providing Rows depending on row_id_formatter type
         """
-        assert(isinstance(parsed_news, ParsedNews))
+        assert(isinstance(parsed_doc, ParsedDocument))
         assert(isinstance(row_dict, OrderedDict))
         assert(isinstance(text_opinion_linkage, TextOpinionsLinkage))
 
         etalon_label = self.__instances_provider.provide_label(text_opinion_linkage)
         for instance in self.__instances_provider.iter_instances(text_opinion_linkage):
             yield self.__create_row(row=row_dict,
-                                    parsed_news=parsed_news,
+                                    row_id=0,
+                                    parsed_doc=parsed_doc,
                                     entity_service=entity_service,
                                     text_opinions_linkage=instance,
                                     index_in_linked=index_in_linked,
@@ -160,7 +162,7 @@ class BaseSampleRowProvider(BaseRowProvider):
                                     etalon_label=etalon_label,
                                     idle_mode=idle_mode)
 
-    def __create_row(self, row, parsed_news, entity_service, text_opinions_linkage,
+    def __create_row(self, row, row_id, parsed_doc, entity_service, text_opinions_linkage,
                      index_in_linked, etalon_label, idle_mode):
         """
         Composing row in following format:
@@ -196,7 +198,7 @@ class BaseSampleRowProvider(BaseRowProvider):
             raise Exception("Limitation: Multi-Sentence text_opinions are not supported.")
 
         self._fill_row_core(row=row,
-                            parsed_news=parsed_news,
+                            parsed_doc=parsed_doc,
                             sentence_ind=source_s_ind,
                             text_opinion_linkage=text_opinions_linkage,
                             index_in_linked=index_in_linked,

arekit/common/data/input/providers/sample/cropped.py

@@ -34,9 +34,9 @@ class CroppedSampleRowProvider(BaseSampleRowProvider):
 
         return _from, _to
 
-    def _provide_sentence_terms(self, parsed_news, sentence_ind, s_ind, t_ind):
+    def _provide_sentence_terms(self, parsed_doc, sentence_ind, s_ind, t_ind):
         terms_iter, src_ind, tgt_ind = super(CroppedSampleRowProvider, self)._provide_sentence_terms(
-            parsed_news=parsed_news, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
+            parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
         terms = list(terms_iter)
         _from, _to = self.__calc_window_bounds(window_size=self.__crop_window_size,
                                                s_ind=s_ind, t_ind=t_ind, input_length=len(terms))

arekit/common/data/input/sample.py

@@ -1,6 +1,6 @@
 from collections import OrderedDict
 
-from arekit.common.news.parsed.providers.entity_service import EntityServiceProvider, DistanceType
+from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider, DistanceType
 from arekit.common.text_opinions.base import TextOpinion
 
 

arekit/common/data/rows_fmt.py

@@ -0,0 +1,82 @@
+from arekit.common.data import const
+from arekit.common.utils import filter_whitespaces, split_by_whitespaces
+
+
+def process_values_list(value, args_sep):
+    return value.split(args_sep)
+
+
+def process_indices_list(value, no_value_func, args_sep):
+    return no_value_func() if not value else [int(v) for v in str(value).split(args_sep)]
+
+
+def process_text(value):
+    """ The core method of the input text processing.
+    """
+    assert(isinstance(value, str) or isinstance(value, list))
+    return filter_whitespaces([term for term in split_by_whitespaces(value)]
+                              if isinstance(value, str) else value)
+
+
+def create_base_column_value_fmt(no_value_func=lambda: None, args_sep=","):
+
+    self_func = lambda value: value
+
+    return {
+        const.ID: {
+            "writer": self_func,
+            "parser": self_func
+        },
+        const.DOC_ID: {
+            "writer": self_func,
+            "parser": self_func,
+        },
+        const.S_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.T_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.SENT_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.OPINION_ID: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.OPINION_LINKAGE_ID: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.ENTITY_VALUES: {
+            "writer": lambda entities: args_sep.join([e.DisplayValue.replace(args_sep, '') for e in entities]),
+            "parser": lambda value: process_values_list(value, args_sep=args_sep),
+        },
+        const.ENTITY_TYPES: {
+            "writer": lambda entities: args_sep.join([e.Type.replace(args_sep, '') for e in entities]),
+            "parser": lambda value: process_values_list(value, args_sep=args_sep)
+        },
+        const.ENTITIES: {
+            "writer": lambda entity_inds: args_sep.join(entity_inds),
+            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
+        },
+        const.TEXT: {
+            "writer": self_func,
+            "parser": lambda value: process_text(value)
+        },
+        const.LABEL_UINT: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        }
+    }
+
+
+def create_base_column_fmt(fmt_type, args_sep=","):
+    assert(isinstance(fmt_type, str))
+    d = create_base_column_value_fmt(args_sep=args_sep)
+    for k, v in d.items():
+        d[k] = v[fmt_type]
+    return d

arekit/common/data/rows_parser.py

@@ -0,0 +1,43 @@
+class ParsedSampleRow(object):
+    """ Provides a parsed information for a sample row.
+    """
+
+    def __init__(self, row, columns_fmts, no_value_func):
+        """ row: dict
+                dict of the pairs ("field_name", value)
+            columns_fmt: list
+                list of the formatters, where every formatter represent a dictionary.
+            no_value_func: func
+                the default value the conveys the absence of the parameter-value.
+        """
+        assert(isinstance(row, dict))
+        assert(isinstance(columns_fmts, list))
+        assert(callable(no_value_func))
+
+        self.__uint_label = None
+        self.__params = {}
+        self.__no_value = no_value_func
+
+        for key, value in row.items():
+
+            for columns_fmt in columns_fmts:
+                assert(isinstance(columns_fmt, dict))
+
+                if key not in columns_fmt:
+                    continue
+
+                self.__params[key] = columns_fmt[key](value)
+                break
+
+    def __value_or_none(self, key):
+        return self.__params[key] if key in self.__params else self.__no_value()
+
+    def __getitem__(self, item):
+        assert (isinstance(item, str) or item is None)
+        if item not in self.__params:
+            return self.__no_value()
+        return self.__params[item] if item is not None else self.__no_value()
+
+    @classmethod
+    def parse(cls, row, columns_fmts, no_value_func):
+        return cls(row=row, columns_fmts=columns_fmts, no_value_func=no_value_func)

arekit/common/data/storages/base.py

@@ -2,7 +2,8 @@ import gc
 import logging
 
 from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
-from arekit.common.utils import progress_bar
+from arekit.common.linkage.meta import MetaEmptyLinkedDataWrapper
+from arekit.common.utils import progress_bar_conditional
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +55,9 @@ class BaseRowsStorage(object):
     def iter_column_names(self):
         raise NotImplemented()
 
+    def iter_column_types(self):
+        raise NotImplemented()
+
     # endregion
 
     def fill(self, iter_rows_func, columns_provider, row_handler=None, rows_count=None, desc=""):
@@ -61,30 +65,31 @@ class BaseRowsStorage(object):
         assert(isinstance(columns_provider, BaseColumnsProvider))
         assert(callable(row_handler) or row_handler is None)
 
-        pbar_it = progress_bar(iterable=iter_rows_func(False),
-                               desc="{fmt}".format(fmt=desc),
-                               total=rows_count)
-
         doc_ids_seen = set()
 
-        for row_index, row in enumerate(pbar_it):
-
-            doc_id, row_values = row
-
+        def postfix_func(item):
+            doc_id, _ = item
+            doc_ids_seen.add(doc_id)
+            return {
+                "docs_seen": len(doc_ids_seen),
+                "doc_now": str(doc_id)
+            }
+
+        pbar_it = progress_bar_conditional(
+            iterable=iter_rows_func(False),
+            # We skip meta information data.
+            condition_func=lambda item: not isinstance(item[1], MetaEmptyLinkedDataWrapper),
+            postfix_func=postfix_func,
+            desc="{fmt}".format(fmt=desc),
+            total=rows_count)
+
+        for row_index, item in enumerate(pbar_it):
+            _, row_values = item
             self._begin_filling_row(row_index)
-
             for column, value in row_values.items():
                 self._set_row_value(row_ind=row_index,
                                     column=column,
                                     value=value)
-
-            # Provide information about amount of processed documents.
-            doc_ids_seen.add(doc_id)
-            pbar_it.set_postfix({
-                "docs_seen": len(doc_ids_seen),
-                "doc_now": doc_id,
-            })
-
             if row_handler is not None:
                 row_handler()
 

arekit/common/data/views/samples.py

@@ -1,23 +1,17 @@
 from arekit.common.data import const
-from arekit.common.data.row_ids.base import BaseIDProvider
 from arekit.common.data.storages.base import BaseRowsStorage
 
 
+# TODO. This is a particular type of view, and expected to be off the core.
 class LinkedSamplesStorageView(object):
 
-    def __init__(self, row_ids_provider):
-        assert(isinstance(row_ids_provider, BaseIDProvider))
-        self.__row_ids_provider = row_ids_provider
-
     def iter_from_storage(self, storage):
         assert(isinstance(storage, BaseRowsStorage))
         undefined = -1
 
         linked = []
         current_opinion_id = undefined
-        for row_index, sample_id in enumerate(storage.iter_column_values(const.ID)):
-            sample_id = str(sample_id)
-            opinion_id = self.__row_ids_provider.parse_opinion_in_sample_id(sample_id)
+        for row_index, opinion_id in enumerate(storage.iter_column_values(const.OPINION_ID)):
             if current_opinion_id != undefined:
                 if opinion_id != current_opinion_id:
                     yield linked

arekit/common/{news → docs}/base.py

@@ -1,4 +1,4 @@
-class News(object):
+class Document(object):
 
     def __init__(self, doc_id, sentences):
         assert(isinstance(sentences, list))
@@ -13,7 +13,7 @@ class News(object):
 
     @property
     def SentencesCount(self):
-        """ Provides total amount of sentences within a news
+        """ Provides total amount of sentences within a doc
             At present is useful for:
             -   CV-splitters, which may rely on sentences count.
             -   Text parsing.

arekit/common/{news → docs}/entities_grouping.py

@@ -4,8 +4,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class EntitiesGroupingPipelineItem(BasePipelineItem):
 
-    def __init__(self, value_to_group_id_func):
+    def __init__(self, value_to_group_id_func, **kwargs):
         assert(callable(value_to_group_id_func))
+        super(EntitiesGroupingPipelineItem, self).__init__(**kwargs)
         self.__value_to_group_id_func = value_to_group_id_func
 
     def apply_core(self, input_data, pipeline_ctx):

arekit/common/{news → docs}/entity.py

@@ -3,12 +3,13 @@ from arekit.common.entities.base import Entity
 
 class DocumentEntity(Entity):
 
-    def __init__(self, value, display_value, e_type, id_in_doc, group_index):
+    def __init__(self, value, display_value, e_type, childs, id_in_doc, group_index):
         """ id_in_doc: Id, utilized witin the internal services
         """
         super(DocumentEntity, self).__init__(value=value,
                                              e_type=e_type,
                                              display_value=display_value,
+                                             childs=childs,
                                              group_index=group_index)
         self.__id = id_in_doc
 

arekit/common/{news → docs}/parsed/base.py

@@ -1,14 +1,14 @@
-import collections
+from collections.abc import Iterable
 
 from arekit.common.entities.base import Entity
 from arekit.common.text.enums import TermFormat
 from arekit.common.text.parsed import BaseParsedText
 
 
-class ParsedNews(object):
+class ParsedDocument(object):
     """
-    This class represents an information of the processed news in following directions:
-        - news words
+    This class represents an information of the processed doc in following directions:
+        - doc words
         - tokens
         - entities (positions).
         - frames (FrameVariants)
@@ -25,7 +25,7 @@ class ParsedNews(object):
         parsed_sentences: iterable of ParsedSentence type
             NOTE: Considered sentences with labeled Entities in it!
         """
-        assert(isinstance(parsed_sentences, collections.Iterable))
+        assert(isinstance(parsed_sentences, Iterable))
 
         self.__doc_id = doc_id
         self.__parsed_sentences = list(parsed_sentences)