arekit 0.23.1__tar.gz → 0.25.0__tar.gz

arekit-0.25.0/PKG-INFO

@@ -0,0 +1,82 @@
+Metadata-Version: 2.1
+Name: arekit
+Version: 0.25.0
+Summary: Document level Attitude and Relation Extraction toolkit (AREkit) for sampling and prompting mass-media news into datasets for ML-model training
+Home-page: https://github.com/nicolay-r/AREkit
+Author: Nicolay Rusnachenko
+Author-email: rusnicolay@gmail.com
+License: MIT License
+Keywords: natural language processing,relation extraction,sentiment analysis
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: tqdm
+Requires-Dist: enum34==1.1.10
+Requires-Dist: numpy>=1.14.5
+Requires-Dist: pymystem3==0.2.0
+
+# AREkit 0.25.0
+
+![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+
+<p align="center">
+    <img src="logo.png"/>
+</p>
+
+**AREkit** (Attitude and Relation Extraction Toolkit) --
+is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+
+## Description
+
+
+This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+
+<p align="center">
+    <img src="docs/arekit-pipeline-concept.png"/>
+</p>
+
+> Figure: AREkit pipelines design. More on 
+> **[ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction](https://link.springer.com/chapter/10.1007/978-3-031-56069-9_23)** paper
+
+In particular, this framework serves the following features: 
+* ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+* 🔗 EL (entity-linking) API support for objects, 
+* ➰ avoidance of cyclic connections,
+* :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+* 📑 relations annotations and filtering rules,
+* *️⃣ entities formatting or masking, and more.
+
+The core functionality includes: 
+* API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+for sentence level relations preparation (dubbed as contexts);
+* API for contexts extraction;
+* Relations transferring from sentence-level onto document-level, and more.
+
+## Installation 
+
+```bash
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+```
+
+## Usage
+
+Please follow the **[tutorial section on project Wiki](https://github.com/nicolay-r/AREkit/wiki/Tutorials)** for mode details.
+
+## How to cite
+A great research is also accompanied by the faithful reference. 
+if you use or extend our work, please cite as follows:
+
+```bibtex
+@inproceedings{rusnachenko2024arelight,
+  title={ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction},
+  author={Rusnachenko, Nicolay and Liang, Huizhi and Kolomeets, Maxim and Shi, Lei},
+  booktitle={European Conference on Information Retrieval},
+  year={2024},
+  organization={Springer}
+}
+```

arekit-0.25.0/README.md

@@ -0,0 +1,60 @@
+# AREkit 0.25.0
+
+![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+
+<p align="center">
+    <img src="logo.png"/>
+</p>
+
+**AREkit** (Attitude and Relation Extraction Toolkit) --
+is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+
+## Description
+
+
+This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+
+<p align="center">
+    <img src="docs/arekit-pipeline-concept.png"/>
+</p>
+
+> Figure: AREkit pipelines design. More on 
+> **[ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction](https://link.springer.com/chapter/10.1007/978-3-031-56069-9_23)** paper
+
+In particular, this framework serves the following features: 
+* ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+* 🔗 EL (entity-linking) API support for objects, 
+* ➰ avoidance of cyclic connections,
+* :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+* 📑 relations annotations and filtering rules,
+* *️⃣ entities formatting or masking, and more.
+
+The core functionality includes: 
+* API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+for sentence level relations preparation (dubbed as contexts);
+* API for contexts extraction;
+* Relations transferring from sentence-level onto document-level, and more.
+
+## Installation 
+
+```bash
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+```
+
+## Usage
+
+Please follow the **[tutorial section on project Wiki](https://github.com/nicolay-r/AREkit/wiki/Tutorials)** for mode details.
+
+## How to cite
+A great research is also accompanied by the faithful reference. 
+if you use or extend our work, please cite as follows:
+
+```bibtex
+@inproceedings{rusnachenko2024arelight,
+  title={ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction},
+  author={Rusnachenko, Nicolay and Liang, Huizhi and Kolomeets, Maxim and Shi, Lei},
+  booktitle={European Conference on Information Retrieval},
+  year={2024},
+  organization={Springer}
+}
+```

{arekit-0.23.1 → arekit-0.25.0}/arekit/common/context/terms_mapper.py

@@ -1,4 +1,4 @@
-import collections
+from collections.abc import Iterable
 
 from arekit.common.context.token import Token
 from arekit.common.entities.base import Entity
@@ -10,7 +10,7 @@ class TextTermsMapper(object):
     def iter_mapped(self, terms):
         """ Performs mapping operation of each terms in a sequence
         """
-        assert(isinstance(terms, collections.Iterable))
+        assert(isinstance(terms, Iterable))
 
         self._before_mapping()
 

{arekit-0.23.1 → arekit-0.25.0}/arekit/common/data/const.py

@@ -1,11 +1,12 @@
 ID = 'id'
 DOC_ID = 'doc_id'
 TEXT = 'text_a'
-LABEL = 'label'
+LABEL_UINT = 'label_uint'
+LABEL_STR = 'label_str'
 
-# Corresponds to fields with attitude ends. (values, STRING)
-SOURCE = 'source'
-TARGET = 'target'
+# Global identifier of the opinion in the sampled data.
+OPINION_ID = "opinion_id"
+OPINION_LINKAGE_ID = "linkage_id"
 
 # Corresponds to fields with attitude ends. (indices, INT)
 S_IND = 's_ind'

arekit-0.23.1/arekit/common/experiment/api/ops_doc.py → arekit-0.25.0/arekit/common/data/doc_provider.py

@@ -1,4 +1,4 @@
-class DocumentOperations(object):
+class DocumentProvider(object):
     """ Provides operations with documents
     """
 

{arekit-0.23.1 → arekit-0.25.0}/arekit/common/data/input/providers/columns/sample.py

@@ -37,7 +37,8 @@ class SampleColumnsProvider(BaseColumnsProvider):
 
         # insert labels
         if self.__store_labels:
-            dtypes_list.append((const.LABEL, 'int32'))
+            dtypes_list.append((const.LABEL_UINT, 'int32'))
+            dtypes_list.append((const.LABEL_STR, str))
 
         # insert text columns
         for col_name in self.__text_column_names:
@@ -47,6 +48,10 @@ class SampleColumnsProvider(BaseColumnsProvider):
         dtypes_list.append((const.S_IND, 'int32'))
         dtypes_list.append((const.T_IND, 'int32'))
 
+        # opinion-extraction task related fields
+        dtypes_list.append((const.OPINION_ID, 'int32'))
+        dtypes_list.append((const.OPINION_LINKAGE_ID, 'int32'))
+
         return dtypes_list
 
     def set_text_column_names(self, text_column_names):

{arekit-0.23.1 → arekit-0.25.0}/arekit/common/data/input/providers/instances/base.py

@@ -11,4 +11,4 @@ class BaseLinkedDataInstancesProvider(object):
         """ Implementation based on the first element of the linkage.
         """
         assert(isinstance(linked_data, LinkedDataWrapper))
-        return linked_data.First.Sentiment
+        return linked_data.First.Label

arekit-0.25.0/arekit/common/data/input/providers/rows/base.py

@@ -0,0 +1,64 @@
+from collections import Counter
+from collections.abc import Iterable
+import logging
+
+from arekit.common.data.input.providers.contents import ContentsProvider
+from arekit.common.linkage.base import LinkedDataWrapper
+from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider
+from arekit.common.linkage.meta import MetaEmptyLinkedDataWrapper
+
+logger = logging.getLogger(__name__)
+
+
+class BaseRowProvider(object):
+    """ Base provider for rows that suppose to be filled into BaseRowsStorage.
+    """
+
+    def __init__(self):
+        self.__rows_counter = None
+
+    # region protected methods
+
+    # TODO. This might be also generalized.
+    # TODO. Idle-mode is also a implementation and task specific parameter, i.e. might be removed from here.
+    def _provide_rows(self, parsed_doc, entity_service, text_opinion_linkage, idle_mode):
+        raise NotImplementedError()
+
+    def _count_row(self):
+        index = self.__rows_counter["rows_iterated"]
+        self.__rows_counter["rows_iterated"] += 1
+        return index
+
+    # endregion
+
+    def __iter_rows(self, linked_data, idle_mode):
+        parsed_doc_service = linked_data.Tag
+        return self._provide_rows(parsed_doc=parsed_doc_service.ParsedDocument,
+                                  entity_service=parsed_doc_service.get_provider(EntityServiceProvider.NAME),
+                                  text_opinion_linkage=linked_data,
+                                  idle_mode=idle_mode)
+
+    def iter_by_rows(self, contents_provider, doc_ids_iter, idle_mode):
+        assert(isinstance(contents_provider, ContentsProvider))
+        assert(isinstance(doc_ids_iter, Iterable))
+
+        self.__rows_counter = Counter()
+
+        for linked_data in contents_provider.from_doc_ids(doc_ids=doc_ids_iter, idle_mode=idle_mode):
+            assert(isinstance(linked_data, LinkedDataWrapper))
+
+            if isinstance(linked_data, MetaEmptyLinkedDataWrapper):
+                if idle_mode:
+                    # In the case of the IDLE mode we do not consider the meta-data.
+                    data_it = []
+                else:
+                    # Consider the actual linked data instance.
+                    data_it = [linked_data]
+            else:
+                # Consider the actual rows of the related linked data.
+                data_it = self.__iter_rows(linked_data=linked_data, idle_mode=idle_mode)
+
+            for data in data_it:
+                yield linked_data.RelatedDocID, data
+
+        self.__rows_counter = None

{arekit-0.23.1 → arekit-0.25.0}/arekit/common/data/input/providers/rows/samples.py

@@ -8,15 +8,14 @@ from arekit.common.data.input.providers.label.binary import BinaryLabelProvider
 from arekit.common.data.input.providers.label.multiple import MultipleLabelProvider
 from arekit.common.data.input.providers.rows.base import BaseRowProvider
 from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
-from arekit.common.data.row_ids.binary import BinaryIDProvider
-from arekit.common.data.row_ids.multiple import MultipleIDProvider
+from arekit.common.data.rows_fmt import create_base_column_fmt
 from arekit.common.entities.base import Entity
 from arekit.common.labels.base import Label
 
 from arekit.common.linkage.text_opinions import TextOpinionsLinkage
-from arekit.common.news.parsed.base import ParsedNews
-from arekit.common.news.parsed.providers.entity_service import EntityEndType, EntityServiceProvider
-from arekit.common.news.parsed.term_position import TermPositionTypes
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.docs.parsed.providers.entity_service import EntityEndType, EntityServiceProvider
+from arekit.common.docs.parsed.term_position import TermPositionTypes
 from arekit.common.text_opinions.base import TextOpinion
 
 
@@ -34,9 +33,9 @@ class BaseSampleRowProvider(BaseRowProvider):
 
         self._label_provider = label_provider
         self.__text_provider = text_provider
-        self.__row_ids_provider = self.__create_row_ids_provider(label_provider)
         self.__instances_provider = self.__create_instances_provider(label_provider)
         self.__store_labels = None
+        self._val_fmt = create_base_column_fmt(fmt_type="writer")
 
     # region properties
 
@@ -52,56 +51,67 @@ class BaseSampleRowProvider(BaseRowProvider):
 
     # region protected methods
 
-    def _provide_sentence_terms(self, parsed_news, sentence_ind, s_ind, t_ind):
-        terms_iter = parsed_news.iter_sentence_terms(sentence_index=sentence_ind, return_id=False)
+    def _provide_sentence_terms(self, parsed_doc, sentence_ind, s_ind, t_ind):
+        terms_iter = parsed_doc.iter_sentence_terms(sentence_index=sentence_ind, return_id=False)
         return list(terms_iter), s_ind, t_ind
 
     # TODO. This is a very task-specific description, too many data provided.
     # TODO. Switch this API to dict of params
     def _fill_row_core(self, row, text_opinion_linkage, index_in_linked, etalon_label,
-                       parsed_news, sentence_ind, s_ind, t_ind):
+                       parsed_doc, sentence_ind, s_ind, t_ind):
         assert(isinstance(self.__store_labels, bool))
 
-        def __assign_value(column, value):
-            row[column] = value
-
-        row[const.ID] = self.__row_ids_provider.create_sample_id(
-            linked_opinions=text_opinion_linkage,
-            index_in_linked=index_in_linked,
-            label_scaler=self._label_provider.LabelScaler)
+        sentence_terms, actual_s_ind, actual_t_ind = self._provide_sentence_terms(
+            parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
 
-        row[const.DOC_ID] = text_opinion_linkage.First.DocID
+        # Entity indices from the related context.
+        entities = list(filter(lambda term: isinstance(term, Entity), sentence_terms))
 
-        row[const.SENT_IND] = sentence_ind
+        # Values mapping.
+        vm = {
+            const.ID: self._count_row(),
+            const.OPINION_ID: text_opinion_linkage.First.TextOpinionID,
+            const.OPINION_LINKAGE_ID: index_in_linked,
+            const.DOC_ID: text_opinion_linkage.First.DocID,
+            const.SENT_IND: sentence_ind,
+            const.ENTITY_VALUES: entities,
+            const.ENTITY_TYPES: entities,
+            const.ENTITIES: [str(i) for i, t in enumerate(sentence_terms) if isinstance(t, Entity)],
+            const.S_IND: actual_s_ind,
+            const.T_IND: actual_t_ind,
+            const.LABEL_UINT: None,
+            const.LABEL_STR: None
+        }
+
+        # Compose text value.
+        def __assign_value(column, value):
+            vm[column] = value
 
         expected_label = text_opinion_linkage.get_linked_label()
 
-        if self.__store_labels:
-            row[const.LABEL] = self._label_provider.calculate_output_uint_label(
-                expected_uint_label=self._label_provider.LabelScaler.label_to_uint(expected_label),
-                etalon_uint_label=self._label_provider.LabelScaler.label_to_uint(etalon_label))
-
-        sentence_terms, actual_s_ind, actual_t_ind = self._provide_sentence_terms(
-            parsed_news=parsed_news, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
-
         self.__text_provider.add_text_in_row(
-            set_text_func=lambda column, value: __assign_value(column, value),
-            sentence_terms=sentence_terms,
-            s_ind=actual_s_ind,
-            t_ind=actual_t_ind,
+            set_text_func=__assign_value, sentence_terms=sentence_terms,
+            s_ind=actual_s_ind, t_ind=actual_t_ind,
             expected_label=expected_label)
 
-        # Entity indicies from the related context.
-        entities = list(filter(lambda term: isinstance(term, Entity), sentence_terms))
-        entity_inds = [str(i) for i, t in enumerate(sentence_terms) if isinstance(t, Entity)]
-        row[const.ENTITY_VALUES] = ",".join([e.DisplayValue.replace(',', '') for e in entities])
-        row[const.ENTITY_TYPES] = ",".join([e.Type.replace(',', '') for e in entities])
-        row[const.ENTITIES] = ",".join(entity_inds)
+        if self.__store_labels:
+            l2i = self._label_provider.LabelScaler.label_to_uint
+            ui2l = self._label_provider.LabelScaler.uint_to_label
+            uint_label = self._label_provider.calculate_output_uint_label(
+                expected_uint_label=l2i(expected_label), etalon_uint_label=l2i(etalon_label))
+            vm[const.LABEL_UINT] = uint_label
+            vm[const.LABEL_STR] = type(ui2l(uint_label)).__name__
 
-        row[const.S_IND] = actual_s_ind
-        row[const.T_IND] = actual_t_ind
+        self._apply_row_data(row=row, vm=vm, val_fmt=self._val_fmt)
+
+    @staticmethod
+    def _apply_row_data(row, vm, val_fmt):
+        for k, v in vm.items():
+            if v is None:
+                continue
+            row[k] = v if k not in val_fmt else val_fmt[k](v)
 
-    def _provide_rows(self, parsed_news, entity_service, text_opinion_linkage, idle_mode):
+    def _provide_rows(self, parsed_doc, entity_service, text_opinion_linkage, idle_mode):
         assert(isinstance(idle_mode, bool))
 
         row_dict = OrderedDict()
@@ -109,7 +119,7 @@ class BaseSampleRowProvider(BaseRowProvider):
         for index_in_linked in range(len(text_opinion_linkage)):
 
             rows_it = self.__provide_rows(
-                parsed_news=parsed_news,
+                parsed_doc=parsed_doc,
                 entity_service=entity_service,
                 row_dict=row_dict,
                 text_opinion_linkage=text_opinion_linkage,
@@ -123,36 +133,28 @@ class BaseSampleRowProvider(BaseRowProvider):
 
     # region private methods
 
-    @staticmethod
-    def __create_row_ids_provider(label_provider):
-        # TODO. #376 related. This should be removed after refactoring, because
-        # TODO. we consider an ordinary IDs, that not based on the other data.
-        if isinstance(label_provider, BinaryLabelProvider):
-            return BinaryIDProvider()
-        if isinstance(label_provider, MultipleLabelProvider):
-            return MultipleIDProvider()
-
     @staticmethod
     def __create_instances_provider(label_provider):
-        # TODO. #473 related: thiese label providers are based on text opinion extraction task!
+        # TODO. #473 related: these label providers are based on text opinion extraction task!
         if isinstance(label_provider, BinaryLabelProvider):
             return MultipleInstancesLinkedTextOpinionsProvider(label_provider.SupportedLabels)
         if isinstance(label_provider, MultipleLabelProvider):
             return SingleInstanceLinkedDataProvider()
 
-    def __provide_rows(self, row_dict, parsed_news, entity_service,
+    def __provide_rows(self, row_dict, parsed_doc, entity_service,
                        text_opinion_linkage, index_in_linked, idle_mode):
         """
         Providing Rows depending on row_id_formatter type
         """
-        assert(isinstance(parsed_news, ParsedNews))
+        assert(isinstance(parsed_doc, ParsedDocument))
         assert(isinstance(row_dict, OrderedDict))
         assert(isinstance(text_opinion_linkage, TextOpinionsLinkage))
 
         etalon_label = self.__instances_provider.provide_label(text_opinion_linkage)
         for instance in self.__instances_provider.iter_instances(text_opinion_linkage):
             yield self.__create_row(row=row_dict,
-                                    parsed_news=parsed_news,
+                                    row_id=0,
+                                    parsed_doc=parsed_doc,
                                     entity_service=entity_service,
                                     text_opinions_linkage=instance,
                                     index_in_linked=index_in_linked,
@@ -160,7 +162,7 @@ class BaseSampleRowProvider(BaseRowProvider):
                                     etalon_label=etalon_label,
                                     idle_mode=idle_mode)
 
-    def __create_row(self, row, parsed_news, entity_service, text_opinions_linkage,
+    def __create_row(self, row, row_id, parsed_doc, entity_service, text_opinions_linkage,
                      index_in_linked, etalon_label, idle_mode):
         """
         Composing row in following format:
@@ -196,7 +198,7 @@ class BaseSampleRowProvider(BaseRowProvider):
             raise Exception("Limitation: Multi-Sentence text_opinions are not supported.")
 
         self._fill_row_core(row=row,
-                            parsed_news=parsed_news,
+                            parsed_doc=parsed_doc,
                             sentence_ind=source_s_ind,
                             text_opinion_linkage=text_opinions_linkage,
                             index_in_linked=index_in_linked,

{arekit-0.23.1 → arekit-0.25.0}/arekit/common/data/input/providers/sample/cropped.py

@@ -34,9 +34,9 @@ class CroppedSampleRowProvider(BaseSampleRowProvider):
 
         return _from, _to
 
-    def _provide_sentence_terms(self, parsed_news, sentence_ind, s_ind, t_ind):
+    def _provide_sentence_terms(self, parsed_doc, sentence_ind, s_ind, t_ind):
         terms_iter, src_ind, tgt_ind = super(CroppedSampleRowProvider, self)._provide_sentence_terms(
-            parsed_news=parsed_news, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
+            parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
         terms = list(terms_iter)
         _from, _to = self.__calc_window_bounds(window_size=self.__crop_window_size,
                                                s_ind=s_ind, t_ind=t_ind, input_length=len(terms))

{arekit-0.23.1 → arekit-0.25.0}/arekit/common/data/input/sample.py

@@ -1,6 +1,6 @@
 from collections import OrderedDict
 
-from arekit.common.news.parsed.providers.entity_service import EntityServiceProvider, DistanceType
+from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider, DistanceType
 from arekit.common.text_opinions.base import TextOpinion
 
 

arekit-0.25.0/arekit/common/data/rows_fmt.py

@@ -0,0 +1,82 @@
+from arekit.common.data import const
+from arekit.common.utils import filter_whitespaces, split_by_whitespaces
+
+
+def process_values_list(value, args_sep):
+    return value.split(args_sep)
+
+
+def process_indices_list(value, no_value_func, args_sep):
+    return no_value_func() if not value else [int(v) for v in str(value).split(args_sep)]
+
+
+def process_text(value):
+    """ The core method of the input text processing.
+    """
+    assert(isinstance(value, str) or isinstance(value, list))
+    return filter_whitespaces([term for term in split_by_whitespaces(value)]
+                              if isinstance(value, str) else value)
+
+
+def create_base_column_value_fmt(no_value_func=lambda: None, args_sep=","):
+
+    self_func = lambda value: value
+
+    return {
+        const.ID: {
+            "writer": self_func,
+            "parser": self_func
+        },
+        const.DOC_ID: {
+            "writer": self_func,
+            "parser": self_func,
+        },
+        const.S_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.T_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.SENT_IND: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.OPINION_ID: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.OPINION_LINKAGE_ID: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        },
+        const.ENTITY_VALUES: {
+            "writer": lambda entities: args_sep.join([e.DisplayValue.replace(args_sep, '') for e in entities]),
+            "parser": lambda value: process_values_list(value, args_sep=args_sep),
+        },
+        const.ENTITY_TYPES: {
+            "writer": lambda entities: args_sep.join([e.Type.replace(args_sep, '') for e in entities]),
+            "parser": lambda value: process_values_list(value, args_sep=args_sep)
+        },
+        const.ENTITIES: {
+            "writer": lambda entity_inds: args_sep.join(entity_inds),
+            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
+        },
+        const.TEXT: {
+            "writer": self_func,
+            "parser": lambda value: process_text(value)
+        },
+        const.LABEL_UINT: {
+            "writer": self_func,
+            "parser": lambda value: int(value)
+        }
+    }
+
+
+def create_base_column_fmt(fmt_type, args_sep=","):
+    assert(isinstance(fmt_type, str))
+    d = create_base_column_value_fmt(args_sep=args_sep)
+    for k, v in d.items():
+        d[k] = v[fmt_type]
+    return d

arekit-0.25.0/arekit/common/data/rows_parser.py

@@ -0,0 +1,43 @@
+class ParsedSampleRow(object):
+    """ Provides a parsed information for a sample row.
+    """
+
+    def __init__(self, row, columns_fmts, no_value_func):
+        """ row: dict
+                dict of the pairs ("field_name", value)
+            columns_fmt: list
+                list of the formatters, where every formatter represent a dictionary.
+            no_value_func: func
+                the default value the conveys the absence of the parameter-value.
+        """
+        assert(isinstance(row, dict))
+        assert(isinstance(columns_fmts, list))
+        assert(callable(no_value_func))
+
+        self.__uint_label = None
+        self.__params = {}
+        self.__no_value = no_value_func
+
+        for key, value in row.items():
+
+            for columns_fmt in columns_fmts:
+                assert(isinstance(columns_fmt, dict))
+
+                if key not in columns_fmt:
+                    continue
+
+                self.__params[key] = columns_fmt[key](value)
+                break
+
+    def __value_or_none(self, key):
+        return self.__params[key] if key in self.__params else self.__no_value()
+
+    def __getitem__(self, item):
+        assert (isinstance(item, str) or item is None)
+        if item not in self.__params:
+            return self.__no_value()
+        return self.__params[item] if item is not None else self.__no_value()
+
+    @classmethod
+    def parse(cls, row, columns_fmts, no_value_func):
+        return cls(row=row, columns_fmts=columns_fmts, no_value_func=no_value_func)