arekit 0.25.0__py3-none-any.whl → 0.25.1__py3-none-any.whl

arekit/common/data/storages/base.py

@@ -10,6 +10,9 @@ logger = logging.getLogger(__name__)
 
 class BaseRowsStorage(object):
 
+    def __init__(self, log_out=None):
+        self.__log_out = log_out
+
     # region protected methods
 
     def _begin_filling_row(self, row_ind):
@@ -31,27 +34,12 @@ class BaseRowsStorage(object):
     def _get_rows_count(self):
         raise NotImplemented()
 
-    def find_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def find_first_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def iter_column_values(self, column_name, dtype=None):
-        raise NotImplemented()
-
     def get_row(self, row_index):
         raise NotImplemented()
 
-    def get_cell(self, row_index, column_name):
-        raise NotImplemented()
-
     def init_empty(self, columns_provider):
         raise NotImplemented()
 
-    def iter_shuffled(self):
-        raise NotImplemented()
-
     def iter_column_names(self):
         raise NotImplemented()
 
@@ -81,6 +69,7 @@ class BaseRowsStorage(object):
             condition_func=lambda item: not isinstance(item[1], MetaEmptyLinkedDataWrapper),
             postfix_func=postfix_func,
             desc="{fmt}".format(fmt=desc),
+            file=self.__log_out,
             total=rows_count)
 
         for row_index, item in enumerate(pbar_it):

arekit/common/docs/parser.py

@@ -1,42 +1,14 @@
-from tqdm import tqdm
 from arekit.common.docs.base import Document
 from arekit.common.docs.parsed.base import ParsedDocument
-from arekit.common.pipeline.base import BasePipelineLauncher
 from arekit.common.pipeline.batching import BatchingPipelineLauncher
 from arekit.common.pipeline.context import PipelineContext
 from arekit.common.pipeline.utils import BatchIterator
 from arekit.common.text.parsed import BaseParsedText
+from arekit.common.utils import progress_bar_defined
 
 
 class DocumentParsers(object):
 
-    @staticmethod
-    def parse(doc, pipeline_items, parent_ppl_ctx=None, src_key="input", show_progress=False):
-        """ This document parser is based on single text parts (sentences)
-            that passes sequentially through the pipeline of transformations.
-        """
-        assert(isinstance(doc, Document))
-        assert(isinstance(pipeline_items, list))
-        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
-
-        parsed_sentences = []
-
-        data_it = range(doc.SentencesCount)
-        progress_it = tqdm(data_it, disable=not show_progress)
-
-        for sent_ind in progress_it:
-
-            # Composing the context from a single sentence.
-            ctx = PipelineContext({src_key: doc.get_sentence(sent_ind)}, parent_ctx=parent_ppl_ctx)
-
-            # Apply all the operations.
-            BasePipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key=src_key)
-
-            # Collecting the result.
-            parsed_sentences.append(BaseParsedText(terms=ctx.provide("result")))
-
-        return ParsedDocument(doc_id=doc.ID, parsed_sentences=parsed_sentences)
-
     @staticmethod
     def parse_batch(doc, pipeline_items, batch_size, parent_ppl_ctx=None, src_key="input", show_progress=False):
         """ This document parser is based on batch of sentences.
@@ -49,7 +21,8 @@ class DocumentParsers(object):
         parsed_sentences = []
 
         data_it = BatchIterator(data_iter=iter(range(doc.SentencesCount)), batch_size=batch_size)
-        progress_it = tqdm(data_it, total=round(doc.SentencesCount / batch_size), disable=not show_progress)
+        progress_it = progress_bar_defined(data_it, total=round(doc.SentencesCount / batch_size),
+                                           disable=not show_progress)
 
         for batch in progress_it:
 

arekit/common/pipeline/items/base.py

@@ -2,7 +2,7 @@ from arekit.common.pipeline.context import PipelineContext
 
 
 class BasePipelineItem(object):
-    """ Single pipeline item that might be instatiated and embedded into pipeline.
+    """ Single pipeline item that might be instantiated and embedded into pipeline.
     """
 
     def __init__(self, src_key="result", result_key="result", src_func=None):

arekit/common/utils.py

@@ -1,4 +1,3 @@
-import sys
 import os
 from tqdm import tqdm
 
@@ -27,14 +26,14 @@ def split_by_whitespaces(text):
     return text.split()
 
 
-def progress_bar(iterable, total, desc="", unit="it"):
+def progress_bar(iterable, total, desc="", unit="it", file=None, disable=False):
     if total is not None:
-        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit)
+        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit, file=file, disable=disable)
     else:
-        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit)
+        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit, file=file, disable=disable)
 
 
-def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it"):
+def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it", file=None):
     """ This progress-bar updates only on the
         specific conditions during the iteration process.
     """
@@ -47,7 +46,7 @@ def progress_bar_conditional(iterable, condition_func, total, postfix_func=None,
             yield 0
 
     pbar_it = progress_bar(iterable=__iter_infinite_placeholder(),
-                           desc=desc, unit=unit, total=total)
+                           desc=desc, unit=unit, total=total, file=file)
     element = iter(pbar_it)
 
     # Initialize with 0.
@@ -65,7 +64,7 @@ def progress_bar_conditional(iterable, condition_func, total, postfix_func=None,
             pbar_it.set_postfix(postfix_func(item))
 
 
-def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
+def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it", file=None, disable=False):
     return tqdm(iterable=iterable,
                 total=total,
                 desc=desc,
@@ -73,13 +72,17 @@ def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
                 position=0,
                 leave=True,
                 unit=unit,
+                file=file,
+                disable=disable,
                 miniters=total / miniters if total is not None else total)
 
 
-def progress_bar_iter(iterable, desc="", unit='it'):
+def progress_bar_iter(iterable, desc="", unit='it', file=None, disable=False):
     return tqdm(iterable=iterable,
                 desc=desc,
                 position=0,
                 leave=True,
                 ncols=120,
+                file=file,
+                disable=disable,
                 unit=unit)

arekit/contrib/utils/data/storages/jsonl_based.py

@@ -5,8 +5,9 @@ from arekit.common.data.storages.base import BaseRowsStorage
 
 class JsonlBasedRowsStorage(BaseRowsStorage):
 
-    def __init__(self, rows):
+    def __init__(self, rows, **kwargs):
         assert(isinstance(rows, list))
+        super(JsonlBasedRowsStorage, self).__init__(**kwargs)
         self.__rows = rows
 
     def _iter_rows(self):

arekit/contrib/utils/data/storages/pandas_based.py

@@ -12,7 +12,8 @@ class PandasBasedRowsStorage(BaseRowsStorage):
         based on the pandas DataFrames.
     """
 
-    def __init__(self, df=None):
+    def __init__(self, df=None, **kwargs):
+        super(PandasBasedRowsStorage, self).__init__(**kwargs)
         self._df = df
 
     @property
@@ -96,26 +97,10 @@ class PandasBasedRowsStorage(BaseRowsStorage):
     def get_row(self, row_index):
         return self._df.iloc[row_index]
 
-    def get_cell(self, row_index, column_name):
-        return self._df.iloc[row_index][column_name]
-
-    def iter_column_values(self, column_name, dtype=None):
-        values = self._df[column_name]
-        if dtype is None:
-            return values
-        return values.astype(dtype)
-
-    def find_by_value(self, column_name, value):
-        return self.__filter(column_name=column_name, value=value)
-
     def init_empty(self, columns_provider):
         cols_with_types = columns_provider.get_columns_list_with_types()
         self._df = self.__create_empty(cols_with_types)
 
-    def iter_shuffled(self):
-        shuffled_df = self._df.sample(frac=1)
-        return self.__iter_rows_core(shuffled_df)
-
     def free(self):
         del self._df
         super(PandasBasedRowsStorage, self).free()

arekit/contrib/utils/data/storages/row_cache.py

@@ -6,13 +6,14 @@ class RowCacheStorage(BaseRowsStorage):
     """ Row Caching storage kernel, based on python dictionary.
     """
 
-    def __init__(self, force_collect_columns=None):
+    def __init__(self, force_collect_columns=None, **kwargs):
         """ This is a particular/related solution for the following issue:
             https://github.com/nicolay-r/AREkit/issues/464
             force_collect_columns: list
                 columns that supposed to be additionally considered in output.
         """
         assert(isinstance(force_collect_columns, list) or force_collect_columns is None)
+        super(RowCacheStorage, self).__init__(**kwargs)
         self.__f = None
         self.__row_cache = {}
         self.__column_names = []

arekit/contrib/utils/data/storages/sqlite_based.py

@@ -4,7 +4,8 @@ from arekit.common.data.storages.base import BaseRowsStorage
 
 class SQliteBasedRowsStorage(BaseRowsStorage):
 
-    def __init__(self, path, table_name):
+    def __init__(self, path, table_name, **kwargs):
+        super(SQliteBasedRowsStorage, self).__init__(**kwargs)
         self.__path = path
         self.__table_name = table_name
         self.__conn = None

arekit/contrib/utils/pipelines/text_opinion/extraction.py

@@ -15,7 +15,7 @@ from arekit.contrib.utils.pipelines.text_opinion.filters.limitation import Frame
 def __iter_text_opinion_linkages(parsed_doc, annotators, entity_index_func,
                                  text_opinion_filters, use_meta):
     """ use_meta: bool
-            this is mainly for tqdm and other console parameters to stay up-to-date
+            this is mainly for the progress-bar and other console parameters to stay up-to-date
             with the state in the case we do not have that much output results
             across multiple amount of documents.
     """
@@ -62,12 +62,13 @@ def __iter_text_opinion_linkages(parsed_doc, annotators, entity_index_func,
         yield MetaEmptyLinkedDataWrapper(doc_id=parsed_doc.RelatedDocID)
 
 
-def text_opinion_extraction_pipeline(pipeline_items, get_doc_by_id_func, annotators, entity_index_func,
+def text_opinion_extraction_pipeline(pipeline_items, get_doc_by_id_func, annotators, entity_index_func, batch_size,
                                      text_opinion_filters=None, use_meta_between_docs=True):
     assert(callable(get_doc_by_id_func))
     assert(isinstance(annotators, list))
     assert(isinstance(text_opinion_filters, list) or text_opinion_filters is None)
     assert(isinstance(use_meta_between_docs, bool))
+    assert(isinstance(batch_size, int) and batch_size > 0)
 
     extra_filters = [] if text_opinion_filters is None else text_opinion_filters
     actual_text_opinion_filters = [FrameworkLimitationsTextOpinionFilter()] + extra_filters
@@ -77,8 +78,8 @@ def text_opinion_extraction_pipeline(pipeline_items, get_doc_by_id_func, annotat
         MapPipelineItem(map_func=lambda doc_id: get_doc_by_id_func(doc_id)),
 
         # (doc, ppl_ctx) -> (parsed_doc)
-        MapNestedPipelineItem(map_func=lambda doc, ppl_ctx: DocumentParsers.parse(
-            doc=doc, pipeline_items=pipeline_items, parent_ppl_ctx=ppl_ctx)),
+        MapNestedPipelineItem(map_func=lambda doc, ppl_ctx: DocumentParsers.parse_batch(
+            doc=doc, pipeline_items=pipeline_items, parent_ppl_ctx=ppl_ctx, batch_size=batch_size)),
 
         # (parsed_doc) -> (text_opinions)
         MapPipelineItem(map_func=lambda parsed_doc: __iter_text_opinion_linkages(

{arekit-0.25.0.dist-info → arekit-0.25.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arekit
-Version: 0.25.0
+Version: 0.25.1
 Summary: Document level Attitude and Relation Extraction toolkit (AREkit) for sampling and prompting mass-media news into datasets for ML-model training
 Home-page: https://github.com/nicolay-r/AREkit
 Author: Nicolay Rusnachenko
@@ -18,9 +18,8 @@ License-File: LICENSE
 Requires-Dist: tqdm
 Requires-Dist: enum34==1.1.10
 Requires-Dist: numpy>=1.14.5
-Requires-Dist: pymystem3==0.2.0
 
-# AREkit 0.25.0
+# AREkit 0.25.1
 
 ![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
 
@@ -34,7 +33,7 @@ is a python toolkit, devoted to document level Attitude and Relation Extraction
 ## Description
 
 
-This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
 
 <p align="center">
     <img src="docs/arekit-pipeline-concept.png"/>
@@ -60,7 +59,7 @@ for sentence level relations preparation (dubbed as contexts);
 ## Installation 
 
 ```bash
-pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.1-rc
 ```
 
 ## Usage

{arekit-0.25.0.dist-info → arekit-0.25.1.dist-info}/RECORD

@@ -2,7 +2,7 @@ arekit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/common/bound.py,sha256=lPpHY6ct_CU9e4qXeYjhJfWbTj6Sb_NVtZ1CJheQPNE,1402
 arekit/common/log_utils.py,sha256=OfEQxbExkuRAl9dxlgFEqcFhI4HHoMYT7WE8ud0IPOM,924
-arekit/common/utils.py,sha256=eVRGhRy882ow-63Glncc3pJ-_43KSI0ukBePjC8ogAY,2394
+arekit/common/utils.py,sha256=N061ENJJgvsB338Q9cixc6RWyuikSPQq4Tc8mmgwy9s,2659
 arekit/common/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/common/context/terms_mapper.py,sha256=QA02Cv7D2JKTlXkez_0w0J8HuvNziNF2vrqLgy4Bwc8,1447
 arekit/common/context/token.py,sha256=CpWAlvprUnJfCtYvO8lwdfU_ofSKAOGOudXTwppyzSk,459
@@ -35,18 +35,13 @@ arekit/common/data/input/providers/sample/__init__.py,sha256=47DEQpj8HBSa-_TImW-
 arekit/common/data/input/providers/sample/cropped.py,sha256=jJSos4Si-qy-wb-QmomXxxgURR1UhJnvY0tZoowlfVc,1885
 arekit/common/data/input/providers/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/common/data/input/providers/text/single.py,sha256=vm3sShIYZcmses-hmZX9cOfveWXCYGwvKLgQ0qs3VXQ,1604
-arekit/common/data/input/repositories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/common/data/input/repositories/base.py,sha256=4DmLVORc85gu6bxtXVZgxi176NxnIaqHz2tVebMyGZ8,2557
-arekit/common/data/input/repositories/sample.py,sha256=LAdpaA1N_nq1iInLwkWQVvL6HGH64JYWSJ9tywU0llY,784
 arekit/common/data/storages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/common/data/storages/base.py,sha256=L9OLpVOZwlAXZION0YP1T6ZN1t_dfQpnAPAU4ztSs48,2956
-arekit/common/data/views/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/common/data/views/samples.py,sha256=LDqUDqArGt90ujRB4kDFgDHLmR2_AQoUnzhxpXYWYaM,882
+arekit/common/data/storages/base.py,sha256=psxo5uIc3hUDi5Cgf4j3Cm-935Fy1VQBYzcBzCcCFZE,2661
 arekit/common/docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/common/docs/base.py,sha256=uXUOtpR9BEsDBfDHg4eLqOjfSVOV_o9VPii3nSxLZuY,734
 arekit/common/docs/entities_grouping.py,sha256=_r254fNr0j6BjHuLZBLjj21yWm4_k__5aOcBXcAaQUQ,704
 arekit/common/docs/entity.py,sha256=TxrZMdIEgjk-PgCyskCkVis2KAw_M7vTBp3ppP6G05M,662
-arekit/common/docs/parser.py,sha256=514lQNrZiwU_mxgyuWBkDhqjS5SVAvcIHx9GQUTuVG8,2883
+arekit/common/docs/parser.py,sha256=dzWjpbbYt-C9UU9sSy_Holnm0kQxJqtz1_6va6kS_L4,1780
 arekit/common/docs/sentence.py,sha256=nZCCFj2yk71POoXCBfEMN3pteM2qQdj60eEzxMVY_3k,302
 arekit/common/docs/parsed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/common/docs/parsed/base.py,sha256=WPstqOpBuLKjtz6UO_bI0DpOPF3Sm0wYEVwjtldbPXE,3175
@@ -115,14 +110,12 @@ arekit/common/pipeline/context.py,sha256=Fw25lBVakHNAXjtkdEqopR-Jh59cDKGWD2jCJxB
 arekit/common/pipeline/conts.py,sha256=NAQNsHt1kK3HnxWv3M6yXi0c7C6Mx6ZZ6KZc0yE0eas,70
 arekit/common/pipeline/utils.py,sha256=5VqH1LtRa4tYUbyiRvWdBmP4biFhTKq9vhr8QiRFFkY,882
 arekit/common/pipeline/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/common/pipeline/items/base.py,sha256=dWIZVGJjYuURLCiZj8YQHWtsS725SOi9SPZaCPV7NvI,1694
+arekit/common/pipeline/items/base.py,sha256=15-z8ERQ0QxaRszs7sHQduU0KIBJIm8B0V2nwCva6d0,1695
 arekit/common/pipeline/items/flatten.py,sha256=9T4jWqPGv4UDxajlM0Nm0-gvwUgqqYB8XH0efTum9a0,542
 arekit/common/pipeline/items/handle.py,sha256=QS5Byj7-o5jmFi0ag58NE3zm2-JzVIunIgc3Pn1ij6g,578
 arekit/common/pipeline/items/iter.py,sha256=Tk9WdUMPOq20s7jEWEpU4PmillnVtQ8nIa2ct7iw-3s,406
 arekit/common/pipeline/items/map.py,sha256=G5wBdjaaxePD0pijrxsfpJACeP7kzj7HerjCkNIhmII,381
 arekit/common/pipeline/items/map_nested.py,sha256=vs0GdJNr3qSF9p2yd1nWji5E1HGzECbvOfN2MqoHc2A,630
-arekit/common/service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/common/service/sqlite.py,sha256=1jLIszkcJGeT0hUos8Y0Chp3o9XRUfljG2P9q0T2_Ds,1440
 arekit/common/synonyms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/common/synonyms/base.py,sha256=YxD-CKCjlEtar1zTdumnfC3vKgbP2wLODR9mMEwbbnA,4237
 arekit/common/synonyms/grouping.py,sha256=fi7QQbBvsTvvP2CPTesSPEsPNmGfc6euqj-HPhVvtlg,698
@@ -141,29 +134,9 @@ arekit/contrib/bert/input/providers/cropped_sample.py,sha256=46uHHhAe8cGxV2JlfO3
 arekit/contrib/bert/input/providers/text_pair.py,sha256=_1d-he0n42y3ksj8RjJlNHgHnaQUEq0aQhUdTPRMKgg,2817
 arekit/contrib/bert/terms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/bert/terms/mapper.py,sha256=oHX-lsaZYjBFLjngzSKT5z_JPJCHbclUsEe4i4fup_8,992
-arekit/contrib/networks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/networks/embedding.py,sha256=lrLdB6CdmnmzwavAL6MZuLHceNM3PsZZiWLZ4BjGeXc,3845
-arekit/contrib/networks/embedding_io.py,sha256=hV1MBr9wu9-10gQgnAzLuC-l897aB-8KNcw4h69B5VM,460
-arekit/contrib/networks/vectorizer.py,sha256=KKV_f0GZD10ZpeYgqZfvMapJtsKa3NBddR6W_GdYqrM,155
-arekit/contrib/networks/input/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/networks/input/const.py,sha256=nPeuO-G6MILNlIkGc5HzSDj_RmTwLflReF7n5htFAUI,176
-arekit/contrib/networks/input/ctx_serialization.py,sha256=eCOw4xjp8A7Z2WFanshooS3MqSy7dbZ8ywf_DA2LZO8,982
-arekit/contrib/networks/input/rows_parser.py,sha256=6_43LbAelveY9yEWMU5BdvQlpWwm4RDOjUEmqHuPYdE,1807
-arekit/contrib/networks/input/term_types.py,sha256=P8E5LKegZE5ZEh4vNtC55Lu8USbQt8_Eo14op_anmvU,348
-arekit/contrib/networks/input/terms_mapping.py,sha256=NAnuTAbj7tBTe1Ga4js2IfnUdAWlTV9fcgSQEgYqQUQ,2129
-arekit/contrib/networks/input/embedding/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/networks/input/embedding/matrix.py,sha256=BFn7eXhiqXY7c4tUfy1fzemIqRnZYx_GiEv873QnIEs,952
-arekit/contrib/networks/input/embedding/offsets.py,sha256=HrBfbFD03o_Y0ZvEGTd-FRxmPx55_5vqItTranMFy88,1313
-arekit/contrib/networks/input/formatters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/networks/input/formatters/pos_mapper.py,sha256=yftPKYU7noVb_q0KAflHf7bqjuUXt5siIgbnwMEoWrw,773
-arekit/contrib/networks/input/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/networks/input/providers/sample.py,sha256=MHtXhhBD-kM0yzTACTbY14KMPIuhiLgUKEYXfhvumfo,5445
-arekit/contrib/networks/input/providers/term_connotation.py,sha256=Q90pVN4hQgYAk3oBSCPYc6_1xQUQE1b6ksiU_k8frcM,1157
-arekit/contrib/networks/input/providers/text.py,sha256=kucezKm6Ilmy5wuM2jUP5xk9zh1K1Pf8KcMd1prrp8k,917
 arekit/contrib/prompt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/prompt/sample.py,sha256=MxpbDR0ww7WmdtuPu74B8R6QKVXeuzO0CKGOJIYwbRk,3164
 arekit/contrib/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/serializer.py,sha256=D9LJ2ZXeVx3YntV-HqEnt32xW-s4GauwD97XRVlqr0g,1626
 arekit/contrib/utils/bert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/bert/samplers.py,sha256=ZVe3rbUAH0Jw1xR_yHE1DoUJf3CI0pDgbBQQzlLWevc,989
 arekit/contrib/utils/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -172,54 +145,28 @@ arekit/contrib/utils/data/contents/opinions.py,sha256=MSV7NytEe15adKhhHCq5KiCj6Z
 arekit/contrib/utils/data/doc_provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/data/doc_provider/dict_based.py,sha256=zUOiiIbj5zby4xqMb0m9N-a6enavJJ7wFmPaGErykWU,371
 arekit/contrib/utils/data/doc_provider/dir_based.py,sha256=FTw3kLV_CYtPoUoHl39IrP6RjLvTecCno9May95jVXw,1916
-arekit/contrib/utils/data/readers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/data/readers/base.py,sha256=zAsZLX5ng0_gb_ysL6wQchptmBHlNgqgQilw295Y5Aw,153
-arekit/contrib/utils/data/readers/csv_pd.py,sha256=Ym49j04Z-_WQN-7xJMiiN1y2TIMnMDtPxy5h0mT3WBQ,1383
-arekit/contrib/utils/data/readers/jsonl.py,sha256=c2bHwnTfNEwb1c8B9fRwaQyeze5x3nOd2UXXAp4MbxQ,426
-arekit/contrib/utils/data/readers/sqlite.py,sha256=U1138XNCIwqycNivxwzwIUnowj3jDkP4M6J_Kvyedbc,416
-arekit/contrib/utils/data/service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/data/service/balance.py,sha256=PgA5B6qSPmt8ITPLsQuCkniE8-u2NO_eQ2m-U9Akh98,1547
 arekit/contrib/utils/data/storages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/data/storages/jsonl_based.py,sha256=Oj5u8aW_UtVDSDxMpIQsgMlZlV-KBD0qVHPVVT3m8nA,450
-arekit/contrib/utils/data/storages/pandas_based.py,sha256=m8z34tO_7NupYd_zQ4L1miTXJQkmMMB90zPFqEeYCNs,4301
-arekit/contrib/utils/data/storages/row_cache.py,sha256=V1InYIqRf5WMWV_JndHNH9JzAjFS3ZL38f4_pDPLo_8,1985
-arekit/contrib/utils/data/storages/sqlite_based.py,sha256=ARwVisVbPKBap_mVdpvTpp28iXgJbCJ3dAj41UYu03Q,609
+arekit/contrib/utils/data/storages/jsonl_based.py,sha256=dz8uizu9t1C215o0HEL8y4LiDKR4aC_-OwDu_xF0xIM,522
+arekit/contrib/utils/data/storages/pandas_based.py,sha256=gMkWUFHZE9Oe1Uy04vEBcUfTIAdh46r5zpjlPAwwG2g,3842
+arekit/contrib/utils/data/storages/row_cache.py,sha256=MRK0uJFvw6O99k2aFb3JLZhLUBo2JUO-WYQ4EeRRu6M,2051
+arekit/contrib/utils/data/storages/sqlite_based.py,sha256=cIYAHyiB4CMftKgrgLqw-L4F1WnhbspjwWLSPqH5NHk,682
 arekit/contrib/utils/data/writers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/data/writers/base.py,sha256=JLwf5WVl_U319sdMev8YOn4OoCcrgNIUZtrOuG1JLjI,766
-arekit/contrib/utils/data/writers/csv_native.py,sha256=7fPxYeu9YDK8Cvjp1n-sbKT63ZuhDIEv3VwghHuKk5k,2252
-arekit/contrib/utils/data/writers/csv_pd.py,sha256=WhBjDJCHUBy_TabngMF42Qicx0ye8xIus0m6c7qotto,1330
-arekit/contrib/utils/data/writers/json_opennre.py,sha256=EkhXmONgtMe7A9VKrs9ElFHc8RoMumjFbkKfwuOVOoU,5067
-arekit/contrib/utils/data/writers/sqlite_native.py,sha256=MnbLU8iPvYvpYgEbOXhBKH_G8DJs0W9iSuhr_TPKBAQ,4601
-arekit/contrib/utils/embeddings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/embeddings/rusvectores.py,sha256=WA0HejE2U5kgeBvh4_vty2QzoAkFXiMk94BK8FHxoxw,1931
-arekit/contrib/utils/embeddings/tokens.py,sha256=z3lJ30JTX9zvZtPgzRl3yANECmuA1qboMDTcJsr_4E4,872
 arekit/contrib/utils/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/entities/filter.py,sha256=aHTExIMFaMdy4QL8iYE23eiby3qLImAakXR6gNqG6fs,145
 arekit/contrib/utils/entities/formatters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/entities/formatters/str_display.py,sha256=N8igv7EVaTFayvLXkyBGtm67KwHaeP-M-L8d7oqBG9Q,401
 arekit/contrib/utils/entities/formatters/str_simple_sharp_prefixed_fmt.py,sha256=rEUIma9O3kOBWIguGtJ69JH-00Dhm0vUBOd5yNcKweY,653
 arekit/contrib/utils/io_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/io_utils/embedding.py,sha256=cBDRv_1LROJ262QaL3QVfGt2W9EvBfbh83oL41PJn60,2543
 arekit/contrib/utils/io_utils/utils.py,sha256=310SIJTsNLn2OZrGPer9W4ZP52PHkjBK3zsyqxVs3h0,537
-arekit/contrib/utils/np_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/np_utils/embedding.py,sha256=G7Ls_ClzbskLLy-opRcVzQlfUfhdwbqoXgk0zoGrmHM,798
-arekit/contrib/utils/np_utils/npz_utils.py,sha256=XoUHNmOlcr2X674R1xKGUJitEpFCIBJ8DOpNEPhtJFk,234
-arekit/contrib/utils/np_utils/vocab.py,sha256=FsS18chMLU4WfMeGwBbvmfB5Qmoj5tZTOo-4zqWPm3Q,580
 arekit/contrib/utils/pipelines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/pipelines/opinion_collections.py,sha256=y9-klVJGCN9mPd7t1ECllAiCnAb3MKVXC1PnYddp5sQ,3195
 arekit/contrib/utils/pipelines/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/pipelines/items/sampling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/pipelines/items/sampling/base.py,sha256=-H-r5GIi9ee7CxxpJs8KnHC91l7Y1dYaWPR_OK17E8g,4245
-arekit/contrib/utils/pipelines/items/sampling/networks.py,sha256=E0EjQ4KRd3oYLFVbie05XJa00JqR26eLRoMrDnuQySQ,2653
 arekit/contrib/utils/pipelines/items/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/pipelines/items/text/entities_default.py,sha256=vNx5ir2mf7a1gg_OeqUsf_p1Fu2k7QIFxVpe-CuwZ84,727
 arekit/contrib/utils/pipelines/items/text/frames.py,sha256=pZQybYfgEQB1DM3PtmsgrtB2Xl0HejmP4rhT0nR_YKE,2586
-arekit/contrib/utils/pipelines/items/text/frames_lemmatized.py,sha256=4rIAAB-_GeWNbu5KyaDm5qttH4o2Bzpdvy-D9YR5bRk,1776
-arekit/contrib/utils/pipelines/items/text/frames_negation.py,sha256=AdoY7lqSAT0RApp0DbqeI7xxyRVF6NPJLAfR59lsIec,1303
-arekit/contrib/utils/pipelines/items/text/tokenizer.py,sha256=FmV5flziDLCNttxrUzRr-FGCcKK6venZEcZ-KwcqwNE,3147
-arekit/contrib/utils/pipelines/items/text/translator.py,sha256=TkXVyZYRbS8P4S2Pnn2GzQMRa-9ba-nS4_zXvsf16vU,5365
 arekit/contrib/utils/pipelines/text_opinion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/pipelines/text_opinion/extraction.py,sha256=QoK0-dfMl27uOOfUhvnbvzYX23jCpZbm97Qs27Na7VA,4133
+arekit/contrib/utils/pipelines/text_opinion/extraction.py,sha256=MT1WMlvVI25JRL0g7W83bV8BGUr7_MNOQBj7ZAHgrnU,4245
 arekit/contrib/utils/pipelines/text_opinion/annot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/pipelines/text_opinion/annot/algo_based.py,sha256=bwS-UR2x3rgp_xqnf6z-73T-eIZE_kltRSGYxgd_WpU,1751
 arekit/contrib/utils/pipelines/text_opinion/filters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -228,32 +175,12 @@ arekit/contrib/utils/pipelines/text_opinion/filters/distance_based.py,sha256=3Pj
 arekit/contrib/utils/pipelines/text_opinion/filters/entity_based.py,sha256=pdWFJaKh4kKIsUuBNp3WNy5Rj80CjWEy2wp-0axFnrI,1254
 arekit/contrib/utils/pipelines/text_opinion/filters/limitation.py,sha256=4AFS5zhocJuYphGO2ZMWmYTtIhGItKDTkB0--AmjgnA,1151
 arekit/contrib/utils/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/processing/languages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/processing/languages/mods.py,sha256=OERKcglI4pJEIQxlWMYuYg_uHnNWVpP-mqhnFsQbY7A,263
-arekit/contrib/utils/processing/languages/pos.py,sha256=etC3ueLGgZorgKEc3TWpeIuv46vs392xPi1lM31Cg0s,278
-arekit/contrib/utils/processing/languages/ru/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/processing/languages/ru/cases.py,sha256=27sIQsU5_0aT4EVuPtKCK-tfi1Q0TH11phV1x5hIzLs,1492
-arekit/contrib/utils/processing/languages/ru/constants.py,sha256=f4z7ivILKqYju9rkagi9_FIvPm1FnWHbXgxigyb3zm4,147
-arekit/contrib/utils/processing/languages/ru/mods.py,sha256=j4xKgRbCC834i9n-RyU607v9Qph9sP_B31WLrKFByRk,343
-arekit/contrib/utils/processing/languages/ru/number.py,sha256=kHyP0Lp_iHVDwkbN7tkZUJpGFQ40QRm-j_1g0dFU-sM,401
-arekit/contrib/utils/processing/languages/ru/pos_service.py,sha256=BWHLPybjmTVNXjJM2QmrZlEDcl7nZY7keLmXZcG_PFM,1125
-arekit/contrib/utils/processing/lemmatization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/processing/lemmatization/mystem.py,sha256=_FRqEGWUlgAbhSJ-dsyoFg_qbbUxePDSAOWWuveRqCo,1340
-arekit/contrib/utils/processing/pos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/processing/pos/base.py,sha256=CrMr3u6lRs2NoV7uch5HZgV71A-0M-pwJfwXjfudHBY,259
-arekit/contrib/utils/processing/pos/mystem_wrap.py,sha256=C9AnRIAZL4e8DMNte9LDuvxS-cbEQpo2AYdQtP9uIJ4,4336
-arekit/contrib/utils/processing/pos/russian.py,sha256=POCo6xKmK7vAEq-kWlODg611kLOtOj37OVc3L_GWL-8,229
-arekit/contrib/utils/processing/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/processing/text/tokens.py,sha256=_3u5Oy1MG_QfHH8wi0x0nA588qSaCp3Wmnp2SzMWjXY,3573
 arekit/contrib/utils/synonyms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arekit/contrib/utils/synonyms/simple.py,sha256=ST9EwuWP88FzbyV8Gi0-biTPgGOsZ7OWyaBWHL_U_eo,557
 arekit/contrib/utils/synonyms/stemmer_based.py,sha256=q19P_XOCWN2_JrBtybAt7ToMIr1ambw4ahr0fSEEHmQ,1400
-arekit/contrib/utils/vectorizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arekit/contrib/utils/vectorizers/bpe.py,sha256=bFS5MZytvU1L21YS5aAeb3FZl7RMjyog4lWwysvKD-8,3047
-arekit/contrib/utils/vectorizers/random_norm.py,sha256=TL86Kz6p59lJqoLg8RwQRTvfhr0e-tiULGHhO4vhBbo,1339
-arekit-0.25.0.data/data/logo.png,sha256=S8OZ4MGGD72Pf5co7ngYbXKkJH1EUhbErUXv1ZjUWiU,45718
-arekit-0.25.0.dist-info/LICENSE,sha256=JO9tIbxAvhwDv73cX-gUStr9yA-TY7wusUeLHRx7JuY,1076
-arekit-0.25.0.dist-info/METADATA,sha256=4DSUy6aTidHG9jFR7jMwQe3uJGER-e8E9vU0q2G20Uo,3145
-arekit-0.25.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-arekit-0.25.0.dist-info/top_level.txt,sha256=4pXuFE8IE0lBsqi6ZsR7figx0H939VIX4_-76YIbkOQ,7
-arekit-0.25.0.dist-info/RECORD,,
+arekit-0.25.1.data/data/logo.png,sha256=S8OZ4MGGD72Pf5co7ngYbXKkJH1EUhbErUXv1ZjUWiU,45718
+arekit-0.25.1.dist-info/LICENSE,sha256=JO9tIbxAvhwDv73cX-gUStr9yA-TY7wusUeLHRx7JuY,1076
+arekit-0.25.1.dist-info/METADATA,sha256=ryWGTL4fYqR36z2qh1UuYBg6UIU6n7_U9Y09KPRS6xk,3177
+arekit-0.25.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+arekit-0.25.1.dist-info/top_level.txt,sha256=4pXuFE8IE0lBsqi6ZsR7figx0H939VIX4_-76YIbkOQ,7
+arekit-0.25.1.dist-info/RECORD,,

arekit/common/data/input/repositories/base.py

@@ -1,68 +0,0 @@
-from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
-from arekit.common.data.input.providers.contents import ContentsProvider
-from arekit.common.data.input.providers.rows.base import BaseRowProvider
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-
-class BaseInputRepository(object):
-
-    def __init__(self, columns_provider, rows_provider, storage):
-        assert(isinstance(columns_provider, BaseColumnsProvider))
-        assert(isinstance(rows_provider, BaseRowProvider))
-        assert(isinstance(storage, BaseRowsStorage))
-
-        self._columns_provider = columns_provider
-        self._rows_provider = rows_provider
-        self._storage = storage
-
-        # Do setup operations.
-        self._setup_columns_provider()
-        self._setup_rows_provider()
-
-    # region protected methods
-
-    def _setup_columns_provider(self):
-        pass
-
-    def _setup_rows_provider(self):
-        pass
-
-    # endregion
-
-    def populate(self, contents_provider, doc_ids, desc="", writer=None, target=None):
-        assert(isinstance(contents_provider, ContentsProvider))
-        assert(isinstance(self._storage, BaseRowsStorage))
-        assert(isinstance(doc_ids, list))
-        assert(isinstance(writer, BaseWriter) or writer is None)
-        assert(isinstance(target, str) or target is None)
-
-        def iter_rows(idle_mode):
-            return self._rows_provider.iter_by_rows(
-                contents_provider=contents_provider,
-                doc_ids_iter=doc_ids,
-                idle_mode=idle_mode)
-
-        self._storage.init_empty(columns_provider=self._columns_provider)
-
-        is_async_write_mode_on = writer is not None and target is not None
-
-        if is_async_write_mode_on:
-            writer.open_target(target)
-
-        self._storage.fill(lambda idle_mode: iter_rows(idle_mode),
-                           columns_provider=self._columns_provider,
-                           row_handler=lambda: writer.commit_line(self._storage) if is_async_write_mode_on else None,
-                           desc=desc)
-
-        if is_async_write_mode_on:
-            writer.close_target()
-
-    def push(self, writer, target, free_storage=True):
-        if not isinstance(self._storage, RowCacheStorage):
-            writer.write_all(self._storage, target)
-
-        # After writing we free the contents of the storage.
-        if free_storage:
-            self._storage.free()

arekit/common/data/input/repositories/sample.py

@@ -1,22 +0,0 @@
-import logging
-
-from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
-from arekit.common.data.input.repositories.base import BaseInputRepository
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class BaseInputSamplesRepository(BaseInputRepository):
-
-    def _setup_rows_provider(self):
-        """ Setup store labels.
-        """
-        assert(isinstance(self._rows_provider, BaseSampleRowProvider))
-        self._rows_provider.set_store_labels(self._columns_provider.StoreLabels)
-
-    def _setup_columns_provider(self):
-        """ Setup text column names.
-        """
-        text_column_names = list(self._rows_provider.TextProvider.iter_columns())
-        self._columns_provider.set_text_column_names(text_column_names)

arekit/common/data/views/samples.py

@@ -1,26 +0,0 @@
-from arekit.common.data import const
-from arekit.common.data.storages.base import BaseRowsStorage
-
-
-# TODO. This is a particular type of view, and expected to be off the core.
-class LinkedSamplesStorageView(object):
-
-    def iter_from_storage(self, storage):
-        assert(isinstance(storage, BaseRowsStorage))
-        undefined = -1
-
-        linked = []
-        current_opinion_id = undefined
-        for row_index, opinion_id in enumerate(storage.iter_column_values(const.OPINION_ID)):
-            if current_opinion_id != undefined:
-                if opinion_id != current_opinion_id:
-                    yield linked
-                    linked = []
-                    current_opinion_id = opinion_id
-            else:
-                current_opinion_id = opinion_id
-
-            linked.append(storage.get_row(row_index))
-
-        if len(linked) > 0:
-            yield linked