arekit 0.24.0__tar.gz → 0.25.1__tar.gz

arekit-0.25.1/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.1
+Name: arekit
+Version: 0.25.1
+Summary: Document level Attitude and Relation Extraction toolkit (AREkit) for sampling and prompting mass-media news into datasets for ML-model training
+Home-page: https://github.com/nicolay-r/AREkit
+Author: Nicolay Rusnachenko
+Author-email: rusnicolay@gmail.com
+License: MIT License
+Keywords: natural language processing,relation extraction,sentiment analysis
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: tqdm
+Requires-Dist: enum34==1.1.10
+Requires-Dist: numpy>=1.14.5
+
+# AREkit 0.25.1
+
+![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+
+<p align="center">
+    <img src="logo.png"/>
+</p>
+
+**AREkit** (Attitude and Relation Extraction Toolkit) --
+is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+
+## Description
+
+
+This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
+
+<p align="center">
+    <img src="docs/arekit-pipeline-concept.png"/>
+</p>
+
+> Figure: AREkit pipelines design. More on 
+> **[ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction](https://link.springer.com/chapter/10.1007/978-3-031-56069-9_23)** paper
+
+In particular, this framework serves the following features: 
+* ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+* 🔗 EL (entity-linking) API support for objects, 
+* ➰ avoidance of cyclic connections,
+* :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+* 📑 relations annotations and filtering rules,
+* *️⃣ entities formatting or masking, and more.
+
+The core functionality includes: 
+* API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+for sentence level relations preparation (dubbed as contexts);
+* API for contexts extraction;
+* Relations transferring from sentence-level onto document-level, and more.
+
+## Installation 
+
+```bash
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.1-rc
+```
+
+## Usage
+
+Please follow the **[tutorial section on project Wiki](https://github.com/nicolay-r/AREkit/wiki/Tutorials)** for mode details.
+
+## How to cite
+A great research is also accompanied by the faithful reference. 
+if you use or extend our work, please cite as follows:
+
+```bibtex
+@inproceedings{rusnachenko2024arelight,
+  title={ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction},
+  author={Rusnachenko, Nicolay and Liang, Huizhi and Kolomeets, Maxim and Shi, Lei},
+  booktitle={European Conference on Information Retrieval},
+  year={2024},
+  organization={Springer}
+}
+```

arekit-0.25.1/README.md

@@ -0,0 +1,60 @@
+# AREkit 0.25.1
+
+![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+
+<p align="center">
+    <img src="logo.png"/>
+</p>
+
+**AREkit** (Attitude and Relation Extraction Toolkit) --
+is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+
+## Description
+
+
+This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
+
+<p align="center">
+    <img src="docs/arekit-pipeline-concept.png"/>
+</p>
+
+> Figure: AREkit pipelines design. More on 
+> **[ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction](https://link.springer.com/chapter/10.1007/978-3-031-56069-9_23)** paper
+
+In particular, this framework serves the following features: 
+* ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+* 🔗 EL (entity-linking) API support for objects, 
+* ➰ avoidance of cyclic connections,
+* :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+* 📑 relations annotations and filtering rules,
+* *️⃣ entities formatting or masking, and more.
+
+The core functionality includes: 
+* API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+for sentence level relations preparation (dubbed as contexts);
+* API for contexts extraction;
+* Relations transferring from sentence-level onto document-level, and more.
+
+## Installation 
+
+```bash
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.1-rc
+```
+
+## Usage
+
+Please follow the **[tutorial section on project Wiki](https://github.com/nicolay-r/AREkit/wiki/Tutorials)** for mode details.
+
+## How to cite
+A great research is also accompanied by the faithful reference. 
+if you use or extend our work, please cite as follows:
+
+```bibtex
+@inproceedings{rusnachenko2024arelight,
+  title={ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction},
+  author={Rusnachenko, Nicolay and Liang, Huizhi and Kolomeets, Maxim and Shi, Lei},
+  booktitle={European Conference on Information Retrieval},
+  year={2024},
+  organization={Springer}
+}
+```

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/data/storages/base.py

@@ -10,6 +10,9 @@ logger = logging.getLogger(__name__)
 
 class BaseRowsStorage(object):
 
+    def __init__(self, log_out=None):
+        self.__log_out = log_out
+
     # region protected methods
 
     def _begin_filling_row(self, row_ind):
@@ -31,27 +34,12 @@ class BaseRowsStorage(object):
     def _get_rows_count(self):
         raise NotImplemented()
 
-    def find_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def find_first_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def iter_column_values(self, column_name, dtype=None):
-        raise NotImplemented()
-
     def get_row(self, row_index):
         raise NotImplemented()
 
-    def get_cell(self, row_index, column_name):
-        raise NotImplemented()
-
     def init_empty(self, columns_provider):
         raise NotImplemented()
 
-    def iter_shuffled(self):
-        raise NotImplemented()
-
     def iter_column_names(self):
         raise NotImplemented()
 
@@ -81,6 +69,7 @@ class BaseRowsStorage(object):
             condition_func=lambda item: not isinstance(item[1], MetaEmptyLinkedDataWrapper),
             postfix_func=postfix_func,
             desc="{fmt}".format(fmt=desc),
+            file=self.__log_out,
             total=rows_count)
 
         for row_index, item in enumerate(pbar_it):

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/docs/entities_grouping.py

@@ -4,8 +4,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class EntitiesGroupingPipelineItem(BasePipelineItem):
 
-    def __init__(self, value_to_group_id_func):
+    def __init__(self, value_to_group_id_func, **kwargs):
         assert(callable(value_to_group_id_func))
+        super(EntitiesGroupingPipelineItem, self).__init__(**kwargs)
         self.__value_to_group_id_func = value_to_group_id_func
 
     def apply_core(self, input_data, pipeline_ctx):

arekit-0.25.1/arekit/common/docs/parser.py

@@ -0,0 +1,39 @@
+from arekit.common.docs.base import Document
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.pipeline.batching import BatchingPipelineLauncher
+from arekit.common.pipeline.context import PipelineContext
+from arekit.common.pipeline.utils import BatchIterator
+from arekit.common.text.parsed import BaseParsedText
+from arekit.common.utils import progress_bar_defined
+
+
+class DocumentParsers(object):
+
+    @staticmethod
+    def parse_batch(doc, pipeline_items, batch_size, parent_ppl_ctx=None, src_key="input", show_progress=False):
+        """ This document parser is based on batch of sentences.
+        """
+        assert(isinstance(batch_size, int) and batch_size > 0)
+        assert(isinstance(doc, Document))
+        assert(isinstance(pipeline_items, list))
+        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
+
+        parsed_sentences = []
+
+        data_it = BatchIterator(data_iter=iter(range(doc.SentencesCount)), batch_size=batch_size)
+        progress_it = progress_bar_defined(data_it, total=round(doc.SentencesCount / batch_size),
+                                           disable=not show_progress)
+
+        for batch in progress_it:
+
+            # Composing the context from a single sentence.
+            ctx = PipelineContext({src_key: [doc.get_sentence(s_ind) for s_ind in batch]},
+                                  parent_ctx=parent_ppl_ctx)
+
+            # Apply all the operations.
+            BatchingPipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key=src_key)
+
+            # Collecting the result.
+            parsed_sentences += [BaseParsedText(terms=result) for result in ctx.provide("result")]
+
+        return ParsedDocument(doc_id=doc.ID, parsed_sentences=parsed_sentences)

arekit-0.25.1/arekit/common/pipeline/base.py

@@ -0,0 +1,21 @@
+from arekit.common.pipeline.context import PipelineContext
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class BasePipelineLauncher:
+
+    @staticmethod
+    def run(pipeline, pipeline_ctx, src_key=None, has_input=True):
+        assert(isinstance(pipeline, list))
+        assert(isinstance(pipeline_ctx, PipelineContext))
+        assert(isinstance(src_key, str) or src_key is None)
+
+        for ind, item in enumerate(filter(lambda itm: itm is not None, pipeline)):
+            assert(isinstance(item, BasePipelineItem))
+            do_force_key = src_key is not None and ind == 0
+            input_data = item.get_source(pipeline_ctx, force_key=src_key if do_force_key else None) \
+                if has_input or ind > 0 else None
+            item_result = item.apply(input_data=input_data, pipeline_ctx=pipeline_ctx)
+            pipeline_ctx.update(param=item.ResultKey, value=item_result, is_new_key=False)
+
+        return pipeline_ctx

arekit-0.25.1/arekit/common/pipeline/batching.py

@@ -0,0 +1,28 @@
+from arekit.common.pipeline.context import PipelineContext
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class BatchingPipelineLauncher:
+
+    @staticmethod
+    def run(pipeline, pipeline_ctx, src_key=None):
+        assert(isinstance(pipeline, list))
+        assert(isinstance(pipeline_ctx, PipelineContext))
+        assert(isinstance(src_key, str) or src_key is None)
+
+        for ind, item in enumerate(filter(lambda itm: itm is not None, pipeline)):
+            assert (isinstance(item, BasePipelineItem))
+
+            # Handle the content of the batch or batch itself.
+            content = item.get_source(pipeline_ctx, call_func=False, force_key=src_key if ind == 0 else None)
+            handled_batch = [item._src_func(i) if item._src_func is not None else i for i in content]
+
+            if item.SupportBatching:
+                batch_result = list(item.apply(input_data=handled_batch, pipeline_ctx=pipeline_ctx))
+            else:
+                batch_result = [item.apply(input_data=input_data, pipeline_ctx=pipeline_ctx)
+                                for input_data in handled_batch]
+
+            pipeline_ctx.update(param=item.ResultKey, value=batch_result, is_new_key=False)
+
+        return pipeline_ctx

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/pipeline/context.py

@@ -13,6 +13,8 @@ class PipelineContext(object):
         self._d[PARENT_CTX] = parent_ctx
 
     def __provide(self, param):
+        if param not in self._d:
+            raise Exception(f"Key `{param}` is not in dictionary.\n{self._d}")
         return self._d[param]
 
     # region public
@@ -23,7 +25,9 @@ class PipelineContext(object):
     def provide_or_none(self, param):
         return self.__provide(param) if param in self._d else None
 
-    def update(self, param, value):
+    def update(self, param, value, is_new_key=False):
+        if is_new_key and param in self._d:
+            raise Exception(f"Key `{param}` is already presented in pipeline context dictionary.")
         self._d[param] = value
 
     # endregion

arekit-0.25.1/arekit/common/pipeline/items/base.py

@@ -0,0 +1,49 @@
+from arekit.common.pipeline.context import PipelineContext
+
+
+class BasePipelineItem(object):
+    """ Single pipeline item that might be instantiated and embedded into pipeline.
+    """
+
+    def __init__(self, src_key="result", result_key="result", src_func=None):
+        assert(isinstance(src_key, str) or src_key is None)
+        assert(callable(src_func) or src_func is None)
+        self.__src_key = src_key
+        self._src_func = src_func
+        self.__result_key = result_key
+
+    @property
+    def ResultKey(self):
+        return self.__result_key
+
+    @property
+    def SupportBatching(self):
+        """ By default pipeline item is not designed for batching.
+        """
+        return False
+
+    def get_source(self, src_ctx, call_func=True, force_key=None):
+        """ Extract input element for processing.
+        """
+        assert(isinstance(src_ctx, PipelineContext))
+
+        # If there is no information about key, then we consider absence of the source.
+        if self.__src_key is None:
+            return None
+
+        # Extracting actual source.
+        src_data = src_ctx.provide(self.__src_key if force_key is None else force_key)
+        if self._src_func is not None and call_func:
+            src_data = self._src_func(src_data)
+
+        return src_data
+
+    def apply_core(self, input_data, pipeline_ctx):
+        """By default we do nothing."""
+        pass
+
+    def apply(self, input_data, pipeline_ctx=None):
+        """ Performs input processing an update it for a further pipeline items.
+        """
+        output_data = self.apply_core(input_data=input_data, pipeline_ctx=pipeline_ctx)
+        return output_data if output_data is not None else input_data

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/pipeline/items/flatten.py

@@ -5,10 +5,14 @@ class FlattenIterPipelineItem(BasePipelineItem):
     """ Considered to flat iterations of items that represent iterations.
     """
 
+    def __init__(self, **kwargs):
+        super(FlattenIterPipelineItem, self).__init__(**kwargs)
+        pass
+
     def __flat_iter(self, iter_data):
         for iter_item in iter_data:
             for item in iter_item:
                 yield item
 
     def apply_core(self, input_data, pipeline_ctx):
-        return self.__flat_iter(input_data)
+        return self.__flat_iter(input_data)

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/pipeline/items/handle.py

@@ -3,8 +3,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class HandleIterPipelineItem(BasePipelineItem):
 
-    def __init__(self, handle_func=None):
+    def __init__(self, handle_func=None, **kwargs):
         assert(callable(handle_func))
+        super(HandleIterPipelineItem, self).__init__(**kwargs)
         self.__handle_func = handle_func
 
     def __updated_data(self, items_iter):

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/pipeline/items/iter.py

@@ -3,8 +3,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class FilterPipelineItem(BasePipelineItem):
 
-    def __init__(self, filter_func=None):
+    def __init__(self, filter_func=None, **kwargs):
         assert(callable(filter_func))
+        super(FilterPipelineItem, self).__init__(**kwargs)
         self.__filter_func = filter_func
 
     def apply_core(self, input_data, pipeline_ctx):

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/pipeline/items/map.py

@@ -3,8 +3,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class MapPipelineItem(BasePipelineItem):
 
-    def __init__(self, map_func=None):
+    def __init__(self, map_func=None, **kwargs):
         assert(callable(map_func))
+        super(MapPipelineItem, self).__init__(**kwargs)
         self._map_func = map_func
 
     def apply_core(self, input_data, pipeline_ctx):

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/pipeline/items/map_nested.py

@@ -9,5 +9,9 @@ class MapNestedPipelineItem(MapPipelineItem):
         suppose to be mapped with the passed pipeline context.
     """
 
+    def __init__(self, **kwargs):
+        super(MapNestedPipelineItem, self).__init__(**kwargs)
+        pass
+
     def apply_core(self, input_data, pipeline_ctx):
         return map(lambda item: self._map_func(item, pipeline_ctx), input_data)

arekit-0.25.1/arekit/common/pipeline/utils.py

@@ -0,0 +1,32 @@
+class BatchIterator:
+
+    def __init__(self, data_iter, batch_size, end_value=None):
+        assert(isinstance(batch_size, int) and batch_size > 0)
+        assert(callable(end_value) or end_value is None)
+        self.__data_iter = data_iter
+        self.__index = 0
+        self.__batch_size = batch_size
+        self.__end_value = end_value
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        buffer = []
+        while True:
+            try:
+                data = next(self.__data_iter)
+            except StopIteration:
+                break
+            buffer.append(data)
+            if len(buffer) == self.__batch_size:
+                break
+
+        if len(buffer) > 0:
+            self.__index += 1
+            return buffer
+
+        if self.__end_value is None:
+            raise StopIteration
+        else:
+            return self.__end_value()

arekit-0.24.0/arekit/common/text/partitioning/str.py → arekit-0.25.1/arekit/common/text/partitioning.py

@@ -1,28 +1,34 @@
 from collections.abc import Iterable
 
 from arekit.common.bound import Bound
-from arekit.common.text.partitioning.base import BasePartitioning
 
 
-class StringPartitioning(BasePartitioning):
+class Partitioning(object):
     """ NOTE: considering that provided parts
         has no intersections between each other
     """
 
+    list_reg_types = {
+        "str": lambda p, item: p.append(item),
+        "list": lambda p, item: p.extend(item)
+    }
+
+    def __init__(self, text_fmt):
+        assert(isinstance(text_fmt, str) and text_fmt in self.list_reg_types)
+        self.__reg_part = self.list_reg_types[text_fmt]
+
     def provide(self, text, parts_it):
-        assert(isinstance(text, str))
         assert(isinstance(parts_it, Iterable))
 
-        start = 0
         parts = []
+        start = 0
+
         for value, bound in parts_it:
             assert(isinstance(bound, Bound))
             assert(bound.Position >= start)
 
             # Release everything till the current value position.
-            part = text[start:bound.Position]
-
-            parts.append(part)
+            self.__reg_part(p=parts, item=text[start:bound.Position])
 
             # Release the entity value.
             parts.extend([value])
@@ -30,7 +36,6 @@ class StringPartitioning(BasePartitioning):
             start = bound.Position + bound.Length
 
         # Release everything after the last entity.
-        last_part = text[start:len(text)]
-        parts.extend([last_part])
+        self.__reg_part(p=parts, item=text[start:len(text)])
 
         return parts

{arekit-0.24.0 → arekit-0.25.1}/arekit/common/utils.py

@@ -1,6 +1,4 @@
-import sys
 import os
-import requests
 from tqdm import tqdm
 
 
@@ -28,14 +26,14 @@ def split_by_whitespaces(text):
     return text.split()
 
 
-def progress_bar(iterable, total, desc="", unit="it"):
+def progress_bar(iterable, total, desc="", unit="it", file=None, disable=False):
     if total is not None:
-        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit)
+        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit, file=file, disable=disable)
     else:
-        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit)
+        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit, file=file, disable=disable)
 
 
-def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it"):
+def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it", file=None):
     """ This progress-bar updates only on the
         specific conditions during the iteration process.
     """
@@ -48,7 +46,7 @@ def progress_bar_conditional(iterable, condition_func, total, postfix_func=None,
             yield 0
 
     pbar_it = progress_bar(iterable=__iter_infinite_placeholder(),
-                           desc=desc, unit=unit, total=total)
+                           desc=desc, unit=unit, total=total, file=file)
     element = iter(pbar_it)
 
     # Initialize with 0.
@@ -66,7 +64,7 @@ def progress_bar_conditional(iterable, condition_func, total, postfix_func=None,
             pbar_it.set_postfix(postfix_func(item))
 
 
-def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
+def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it", file=None, disable=False):
     return tqdm(iterable=iterable,
                 total=total,
                 desc=desc,
@@ -74,56 +72,17 @@ def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
                 position=0,
                 leave=True,
                 unit=unit,
+                file=file,
+                disable=disable,
                 miniters=total / miniters if total is not None else total)
 
 
-def progress_bar_iter(iterable, desc="", unit='it'):
+def progress_bar_iter(iterable, desc="", unit='it', file=None, disable=False):
     return tqdm(iterable=iterable,
                 desc=desc,
                 position=0,
                 leave=True,
                 ncols=120,
+                file=file,
+                disable=disable,
                 unit=unit)
-
-
-def get_default_download_dir():
-    """ Refered to NLTK toolkit approach
-        https://github.com/nltk/nltk/blob/8e771679cee1b4a9540633cc3ea17f4421ffd6c0/nltk/downloader.py#L1051
-    """
-
-    # On Windows, use %APPDATA%
-    if sys.platform == "win32" and "APPDATA" in os.environ:
-        homedir = os.environ["APPDATA"]
-
-    # Otherwise, install in the user's home directory.
-    else:
-        homedir = os.path.expanduser("~/")
-        if homedir == "~/":
-            raise ValueError("Could not find a default download directory")
-
-    return os.path.join(homedir, ".arekit")
-
-
-def download(dest_file_path, source_url):
-    """ Refered to https://github.com/nicolay-r/ner-bilstm-crf-tensorflow/blob/master/ner/utils.py
-        Simple http file downloader
-    """
-    print(('Downloading from {src} to {dest}'.format(src=source_url, dest=dest_file_path)))
-
-    sys.stdout.flush()
-    datapath = os.path.dirname(dest_file_path)
-
-    if not os.path.exists(datapath):
-        os.makedirs(datapath, mode=0o755)
-
-    dest_file_path = os.path.abspath(dest_file_path)
-
-    r = requests.get(source_url, stream=True)
-    total_length = int(r.headers.get('content-length', 0))
-
-    with open(dest_file_path, 'wb') as f:
-        pbar = tqdm(total=total_length, unit='B', unit_scale=True)
-        for chunk in r.iter_content(chunk_size=32 * 1024):
-            if chunk:  # filter out keep-alive new chunks
-                pbar.update(len(chunk))
-                f.write(chunk)

{arekit-0.24.0 → arekit-0.25.1}/arekit/contrib/utils/data/contents/opinions.py

@@ -2,7 +2,8 @@ from arekit.common.data.input.providers.const import IDLE_MODE
 from arekit.common.data.input.providers.contents import ContentsProvider
 from arekit.common.linkage.base import LinkedDataWrapper
 from arekit.common.linkage.text_opinions import TextOpinionsLinkage
-from arekit.common.pipeline.base import BasePipeline
+from arekit.common.pipeline.base import BasePipelineLauncher
+from arekit.common.pipeline.context import PipelineContext
 from arekit.common.text_opinions.base import TextOpinion
 
 
@@ -13,7 +14,7 @@ class InputTextOpinionProvider(ContentsProvider):
             results in a TextOpinionLinkage instances.
             pipeline: id -> ... -> TextOpinionLinkage[]
         """
-        assert(isinstance(pipeline, BasePipeline))
+        assert(isinstance(pipeline, list))
         self.__pipeline = pipeline
         self.__current_id = None
 
@@ -30,7 +31,16 @@ class InputTextOpinionProvider(ContentsProvider):
 
     def from_doc_ids(self, doc_ids, idle_mode=False):
         self.__current_id = 0
-        for linkage in self.__pipeline.run(doc_ids, params_dict={IDLE_MODE: idle_mode}):
+
+        ctx = PipelineContext(d={
+            "result": doc_ids,
+            IDLE_MODE: idle_mode
+        })
+
+        # Launching pipeline with the passed context
+        BasePipelineLauncher.run(pipeline=self.__pipeline, pipeline_ctx=ctx)
+
+        for linkage in ctx.provide("result"):
             assert(isinstance(linkage, LinkedDataWrapper))
             if isinstance(linkage, TextOpinionsLinkage):
                 self.__assign_ids(linkage)

{arekit-0.24.0 → arekit-0.25.1}/arekit/contrib/utils/data/storages/jsonl_based.py

@@ -5,8 +5,9 @@ from arekit.common.data.storages.base import BaseRowsStorage
 
 class JsonlBasedRowsStorage(BaseRowsStorage):
 
-    def __init__(self, rows):
+    def __init__(self, rows, **kwargs):
         assert(isinstance(rows, list))
+        super(JsonlBasedRowsStorage, self).__init__(**kwargs)
         self.__rows = rows
 
     def _iter_rows(self):