arekit 0.25.0__tar.gz → 0.25.1__tar.gz

{arekit-0.25.0 → arekit-0.25.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arekit
-Version: 0.25.0
+Version: 0.25.1
 Summary: Document level Attitude and Relation Extraction toolkit (AREkit) for sampling and prompting mass-media news into datasets for ML-model training
 Home-page: https://github.com/nicolay-r/AREkit
 Author: Nicolay Rusnachenko
@@ -18,9 +18,8 @@ License-File: LICENSE
 Requires-Dist: tqdm
 Requires-Dist: enum34==1.1.10
 Requires-Dist: numpy>=1.14.5
-Requires-Dist: pymystem3==0.2.0
 
-# AREkit 0.25.0
+# AREkit 0.25.1
 
 ![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
 
@@ -34,7 +33,7 @@ is a python toolkit, devoted to document level Attitude and Relation Extraction
 ## Description
 
 
-This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
 
 <p align="center">
     <img src="docs/arekit-pipeline-concept.png"/>
@@ -60,7 +59,7 @@ for sentence level relations preparation (dubbed as contexts);
 ## Installation 
 
 ```bash
-pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.1-rc
 ```
 
 ## Usage

{arekit-0.25.0 → arekit-0.25.1}/README.md

@@ -1,4 +1,4 @@
-# AREkit 0.25.0
+# AREkit 0.25.1
 
 ![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
 
@@ -12,7 +12,7 @@ is a python toolkit, devoted to document level Attitude and Relation Extraction
 ## Description
 
 
-This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
 
 <p align="center">
     <img src="docs/arekit-pipeline-concept.png"/>
@@ -38,7 +38,7 @@ for sentence level relations preparation (dubbed as contexts);
 ## Installation 
 
 ```bash
-pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.1-rc
 ```
 
 ## Usage
@@ -57,4 +57,4 @@ if you use or extend our work, please cite as follows:
   year={2024},
   organization={Springer}
 }
-```
+```

{arekit-0.25.0 → arekit-0.25.1}/arekit/common/data/storages/base.py

@@ -10,6 +10,9 @@ logger = logging.getLogger(__name__)
 
 class BaseRowsStorage(object):
 
+    def __init__(self, log_out=None):
+        self.__log_out = log_out
+
     # region protected methods
 
     def _begin_filling_row(self, row_ind):
@@ -31,27 +34,12 @@ class BaseRowsStorage(object):
     def _get_rows_count(self):
         raise NotImplemented()
 
-    def find_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def find_first_by_value(self, column_name, value):
-        raise NotImplemented()
-
-    def iter_column_values(self, column_name, dtype=None):
-        raise NotImplemented()
-
     def get_row(self, row_index):
         raise NotImplemented()
 
-    def get_cell(self, row_index, column_name):
-        raise NotImplemented()
-
     def init_empty(self, columns_provider):
         raise NotImplemented()
 
-    def iter_shuffled(self):
-        raise NotImplemented()
-
     def iter_column_names(self):
         raise NotImplemented()
 
@@ -81,6 +69,7 @@ class BaseRowsStorage(object):
             condition_func=lambda item: not isinstance(item[1], MetaEmptyLinkedDataWrapper),
             postfix_func=postfix_func,
             desc="{fmt}".format(fmt=desc),
+            file=self.__log_out,
             total=rows_count)
 
         for row_index, item in enumerate(pbar_it):

{arekit-0.25.0 → arekit-0.25.1}/arekit/common/docs/parser.py

@@ -1,42 +1,14 @@
-from tqdm import tqdm
 from arekit.common.docs.base import Document
 from arekit.common.docs.parsed.base import ParsedDocument
-from arekit.common.pipeline.base import BasePipelineLauncher
 from arekit.common.pipeline.batching import BatchingPipelineLauncher
 from arekit.common.pipeline.context import PipelineContext
 from arekit.common.pipeline.utils import BatchIterator
 from arekit.common.text.parsed import BaseParsedText
+from arekit.common.utils import progress_bar_defined
 
 
 class DocumentParsers(object):
 
-    @staticmethod
-    def parse(doc, pipeline_items, parent_ppl_ctx=None, src_key="input", show_progress=False):
-        """ This document parser is based on single text parts (sentences)
-            that passes sequentially through the pipeline of transformations.
-        """
-        assert(isinstance(doc, Document))
-        assert(isinstance(pipeline_items, list))
-        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
-
-        parsed_sentences = []
-
-        data_it = range(doc.SentencesCount)
-        progress_it = tqdm(data_it, disable=not show_progress)
-
-        for sent_ind in progress_it:
-
-            # Composing the context from a single sentence.
-            ctx = PipelineContext({src_key: doc.get_sentence(sent_ind)}, parent_ctx=parent_ppl_ctx)
-
-            # Apply all the operations.
-            BasePipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key=src_key)
-
-            # Collecting the result.
-            parsed_sentences.append(BaseParsedText(terms=ctx.provide("result")))
-
-        return ParsedDocument(doc_id=doc.ID, parsed_sentences=parsed_sentences)
-
     @staticmethod
     def parse_batch(doc, pipeline_items, batch_size, parent_ppl_ctx=None, src_key="input", show_progress=False):
         """ This document parser is based on batch of sentences.
@@ -49,7 +21,8 @@ class DocumentParsers(object):
         parsed_sentences = []
 
         data_it = BatchIterator(data_iter=iter(range(doc.SentencesCount)), batch_size=batch_size)
-        progress_it = tqdm(data_it, total=round(doc.SentencesCount / batch_size), disable=not show_progress)
+        progress_it = progress_bar_defined(data_it, total=round(doc.SentencesCount / batch_size),
+                                           disable=not show_progress)
 
         for batch in progress_it:
 

{arekit-0.25.0 → arekit-0.25.1}/arekit/common/pipeline/items/base.py

@@ -2,7 +2,7 @@ from arekit.common.pipeline.context import PipelineContext
 
 
 class BasePipelineItem(object):
-    """ Single pipeline item that might be instatiated and embedded into pipeline.
+    """ Single pipeline item that might be instantiated and embedded into pipeline.
     """
 
     def __init__(self, src_key="result", result_key="result", src_func=None):

{arekit-0.25.0 → arekit-0.25.1}/arekit/common/utils.py

@@ -1,4 +1,3 @@
-import sys
 import os
 from tqdm import tqdm
 
@@ -27,14 +26,14 @@ def split_by_whitespaces(text):
     return text.split()
 
 
-def progress_bar(iterable, total, desc="", unit="it"):
+def progress_bar(iterable, total, desc="", unit="it", file=None, disable=False):
     if total is not None:
-        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit)
+        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit, file=file, disable=disable)
     else:
-        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit)
+        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit, file=file, disable=disable)
 
 
-def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it"):
+def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it", file=None):
     """ This progress-bar updates only on the
         specific conditions during the iteration process.
     """
@@ -47,7 +46,7 @@ def progress_bar_conditional(iterable, condition_func, total, postfix_func=None,
             yield 0
 
     pbar_it = progress_bar(iterable=__iter_infinite_placeholder(),
-                           desc=desc, unit=unit, total=total)
+                           desc=desc, unit=unit, total=total, file=file)
     element = iter(pbar_it)
 
     # Initialize with 0.
@@ -65,7 +64,7 @@ def progress_bar_conditional(iterable, condition_func, total, postfix_func=None,
             pbar_it.set_postfix(postfix_func(item))
 
 
-def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
+def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it", file=None, disable=False):
     return tqdm(iterable=iterable,
                 total=total,
                 desc=desc,
@@ -73,13 +72,17 @@ def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
                 position=0,
                 leave=True,
                 unit=unit,
+                file=file,
+                disable=disable,
                 miniters=total / miniters if total is not None else total)
 
 
-def progress_bar_iter(iterable, desc="", unit='it'):
+def progress_bar_iter(iterable, desc="", unit='it', file=None, disable=False):
     return tqdm(iterable=iterable,
                 desc=desc,
                 position=0,
                 leave=True,
                 ncols=120,
+                file=file,
+                disable=disable,
                 unit=unit)

{arekit-0.25.0 → arekit-0.25.1}/arekit/contrib/utils/data/storages/jsonl_based.py

@@ -5,8 +5,9 @@ from arekit.common.data.storages.base import BaseRowsStorage
 
 class JsonlBasedRowsStorage(BaseRowsStorage):
 
-    def __init__(self, rows):
+    def __init__(self, rows, **kwargs):
         assert(isinstance(rows, list))
+        super(JsonlBasedRowsStorage, self).__init__(**kwargs)
         self.__rows = rows
 
     def _iter_rows(self):

{arekit-0.25.0 → arekit-0.25.1}/arekit/contrib/utils/data/storages/pandas_based.py

@@ -12,7 +12,8 @@ class PandasBasedRowsStorage(BaseRowsStorage):
         based on the pandas DataFrames.
     """
 
-    def __init__(self, df=None):
+    def __init__(self, df=None, **kwargs):
+        super(PandasBasedRowsStorage, self).__init__(**kwargs)
         self._df = df
 
     @property
@@ -96,26 +97,10 @@ class PandasBasedRowsStorage(BaseRowsStorage):
     def get_row(self, row_index):
         return self._df.iloc[row_index]
 
-    def get_cell(self, row_index, column_name):
-        return self._df.iloc[row_index][column_name]
-
-    def iter_column_values(self, column_name, dtype=None):
-        values = self._df[column_name]
-        if dtype is None:
-            return values
-        return values.astype(dtype)
-
-    def find_by_value(self, column_name, value):
-        return self.__filter(column_name=column_name, value=value)
-
     def init_empty(self, columns_provider):
         cols_with_types = columns_provider.get_columns_list_with_types()
         self._df = self.__create_empty(cols_with_types)
 
-    def iter_shuffled(self):
-        shuffled_df = self._df.sample(frac=1)
-        return self.__iter_rows_core(shuffled_df)
-
     def free(self):
         del self._df
         super(PandasBasedRowsStorage, self).free()

{arekit-0.25.0 → arekit-0.25.1}/arekit/contrib/utils/data/storages/row_cache.py

@@ -6,13 +6,14 @@ class RowCacheStorage(BaseRowsStorage):
     """ Row Caching storage kernel, based on python dictionary.
     """
 
-    def __init__(self, force_collect_columns=None):
+    def __init__(self, force_collect_columns=None, **kwargs):
         """ This is a particular/related solution for the following issue:
             https://github.com/nicolay-r/AREkit/issues/464
             force_collect_columns: list
                 columns that supposed to be additionally considered in output.
         """
         assert(isinstance(force_collect_columns, list) or force_collect_columns is None)
+        super(RowCacheStorage, self).__init__(**kwargs)
         self.__f = None
         self.__row_cache = {}
         self.__column_names = []

{arekit-0.25.0 → arekit-0.25.1}/arekit/contrib/utils/data/storages/sqlite_based.py

@@ -4,7 +4,8 @@ from arekit.common.data.storages.base import BaseRowsStorage
 
 class SQliteBasedRowsStorage(BaseRowsStorage):
 
-    def __init__(self, path, table_name):
+    def __init__(self, path, table_name, **kwargs):
+        super(SQliteBasedRowsStorage, self).__init__(**kwargs)
         self.__path = path
         self.__table_name = table_name
         self.__conn = None

{arekit-0.25.0 → arekit-0.25.1}/arekit/contrib/utils/pipelines/text_opinion/extraction.py

@@ -15,7 +15,7 @@ from arekit.contrib.utils.pipelines.text_opinion.filters.limitation import Frame
 def __iter_text_opinion_linkages(parsed_doc, annotators, entity_index_func,
                                  text_opinion_filters, use_meta):
     """ use_meta: bool
-            this is mainly for tqdm and other console parameters to stay up-to-date
+            this is mainly for the progress-bar and other console parameters to stay up-to-date
             with the state in the case we do not have that much output results
             across multiple amount of documents.
     """
@@ -62,12 +62,13 @@ def __iter_text_opinion_linkages(parsed_doc, annotators, entity_index_func,
         yield MetaEmptyLinkedDataWrapper(doc_id=parsed_doc.RelatedDocID)
 
 
-def text_opinion_extraction_pipeline(pipeline_items, get_doc_by_id_func, annotators, entity_index_func,
+def text_opinion_extraction_pipeline(pipeline_items, get_doc_by_id_func, annotators, entity_index_func, batch_size,
                                      text_opinion_filters=None, use_meta_between_docs=True):
     assert(callable(get_doc_by_id_func))
     assert(isinstance(annotators, list))
     assert(isinstance(text_opinion_filters, list) or text_opinion_filters is None)
     assert(isinstance(use_meta_between_docs, bool))
+    assert(isinstance(batch_size, int) and batch_size > 0)
 
     extra_filters = [] if text_opinion_filters is None else text_opinion_filters
     actual_text_opinion_filters = [FrameworkLimitationsTextOpinionFilter()] + extra_filters
@@ -77,8 +78,8 @@ def text_opinion_extraction_pipeline(pipeline_items, get_doc_by_id_func, annotat
         MapPipelineItem(map_func=lambda doc_id: get_doc_by_id_func(doc_id)),
 
         # (doc, ppl_ctx) -> (parsed_doc)
-        MapNestedPipelineItem(map_func=lambda doc, ppl_ctx: DocumentParsers.parse(
-            doc=doc, pipeline_items=pipeline_items, parent_ppl_ctx=ppl_ctx)),
+        MapNestedPipelineItem(map_func=lambda doc, ppl_ctx: DocumentParsers.parse_batch(
+            doc=doc, pipeline_items=pipeline_items, parent_ppl_ctx=ppl_ctx, batch_size=batch_size)),
 
         # (parsed_doc) -> (text_opinions)
         MapPipelineItem(map_func=lambda parsed_doc: __iter_text_opinion_linkages(

{arekit-0.25.0 → arekit-0.25.1}/arekit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arekit
-Version: 0.25.0
+Version: 0.25.1
 Summary: Document level Attitude and Relation Extraction toolkit (AREkit) for sampling and prompting mass-media news into datasets for ML-model training
 Home-page: https://github.com/nicolay-r/AREkit
 Author: Nicolay Rusnachenko
@@ -18,9 +18,8 @@ License-File: LICENSE
 Requires-Dist: tqdm
 Requires-Dist: enum34==1.1.10
 Requires-Dist: numpy>=1.14.5
-Requires-Dist: pymystem3==0.2.0
 
-# AREkit 0.25.0
+# AREkit 0.25.1
 
 ![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
 
@@ -34,7 +33,7 @@ is a python toolkit, devoted to document level Attitude and Relation Extraction
 ## Description
 
 
-This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
 
 <p align="center">
     <img src="docs/arekit-pipeline-concept.png"/>
@@ -60,7 +59,7 @@ for sentence level relations preparation (dubbed as contexts);
 ## Installation 
 
 ```bash
-pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.1-rc
 ```
 
 ## Usage

{arekit-0.25.0 → arekit-0.25.1}/arekit.egg-info/SOURCES.txt

@@ -44,13 +44,8 @@ arekit/common/data/input/providers/sample/__init__.py
 arekit/common/data/input/providers/sample/cropped.py
 arekit/common/data/input/providers/text/__init__.py
 arekit/common/data/input/providers/text/single.py
-arekit/common/data/input/repositories/__init__.py
-arekit/common/data/input/repositories/base.py
-arekit/common/data/input/repositories/sample.py
 arekit/common/data/storages/__init__.py
 arekit/common/data/storages/base.py
-arekit/common/data/views/__init__.py
-arekit/common/data/views/samples.py
 arekit/common/docs/__init__.py
 arekit/common/docs/base.py
 arekit/common/docs/entities_grouping.py
@@ -130,8 +125,6 @@ arekit/common/pipeline/items/handle.py
 arekit/common/pipeline/items/iter.py
 arekit/common/pipeline/items/map.py
 arekit/common/pipeline/items/map_nested.py
-arekit/common/service/__init__.py
-arekit/common/service/sqlite.py
 arekit/common/synonyms/__init__.py
 arekit/common/synonyms/base.py
 arekit/common/synonyms/grouping.py
@@ -150,29 +143,9 @@ arekit/contrib/bert/input/providers/cropped_sample.py
 arekit/contrib/bert/input/providers/text_pair.py
 arekit/contrib/bert/terms/__init__.py
 arekit/contrib/bert/terms/mapper.py
-arekit/contrib/networks/__init__.py
-arekit/contrib/networks/embedding.py
-arekit/contrib/networks/embedding_io.py
-arekit/contrib/networks/vectorizer.py
-arekit/contrib/networks/input/__init__.py
-arekit/contrib/networks/input/const.py
-arekit/contrib/networks/input/ctx_serialization.py
-arekit/contrib/networks/input/rows_parser.py
-arekit/contrib/networks/input/term_types.py
-arekit/contrib/networks/input/terms_mapping.py
-arekit/contrib/networks/input/embedding/__init__.py
-arekit/contrib/networks/input/embedding/matrix.py
-arekit/contrib/networks/input/embedding/offsets.py
-arekit/contrib/networks/input/formatters/__init__.py
-arekit/contrib/networks/input/formatters/pos_mapper.py
-arekit/contrib/networks/input/providers/__init__.py
-arekit/contrib/networks/input/providers/sample.py
-arekit/contrib/networks/input/providers/term_connotation.py
-arekit/contrib/networks/input/providers/text.py
 arekit/contrib/prompt/__init__.py
 arekit/contrib/prompt/sample.py
 arekit/contrib/utils/__init__.py
-arekit/contrib/utils/serializer.py
 arekit/contrib/utils/bert/__init__.py
 arekit/contrib/utils/bert/samplers.py
 arekit/contrib/utils/data/__init__.py
@@ -181,13 +154,6 @@ arekit/contrib/utils/data/contents/opinions.py
 arekit/contrib/utils/data/doc_provider/__init__.py
 arekit/contrib/utils/data/doc_provider/dict_based.py
 arekit/contrib/utils/data/doc_provider/dir_based.py
-arekit/contrib/utils/data/readers/__init__.py
-arekit/contrib/utils/data/readers/base.py
-arekit/contrib/utils/data/readers/csv_pd.py
-arekit/contrib/utils/data/readers/jsonl.py
-arekit/contrib/utils/data/readers/sqlite.py
-arekit/contrib/utils/data/service/__init__.py
-arekit/contrib/utils/data/service/balance.py
 arekit/contrib/utils/data/storages/__init__.py
 arekit/contrib/utils/data/storages/jsonl_based.py
 arekit/contrib/utils/data/storages/pandas_based.py
@@ -195,38 +161,19 @@ arekit/contrib/utils/data/storages/row_cache.py
 arekit/contrib/utils/data/storages/sqlite_based.py
 arekit/contrib/utils/data/writers/__init__.py
 arekit/contrib/utils/data/writers/base.py
-arekit/contrib/utils/data/writers/csv_native.py
-arekit/contrib/utils/data/writers/csv_pd.py
-arekit/contrib/utils/data/writers/json_opennre.py
-arekit/contrib/utils/data/writers/sqlite_native.py
-arekit/contrib/utils/embeddings/__init__.py
-arekit/contrib/utils/embeddings/rusvectores.py
-arekit/contrib/utils/embeddings/tokens.py
 arekit/contrib/utils/entities/__init__.py
 arekit/contrib/utils/entities/filter.py
 arekit/contrib/utils/entities/formatters/__init__.py
 arekit/contrib/utils/entities/formatters/str_display.py
 arekit/contrib/utils/entities/formatters/str_simple_sharp_prefixed_fmt.py
 arekit/contrib/utils/io_utils/__init__.py
-arekit/contrib/utils/io_utils/embedding.py
 arekit/contrib/utils/io_utils/utils.py
-arekit/contrib/utils/np_utils/__init__.py
-arekit/contrib/utils/np_utils/embedding.py
-arekit/contrib/utils/np_utils/npz_utils.py
-arekit/contrib/utils/np_utils/vocab.py
 arekit/contrib/utils/pipelines/__init__.py
 arekit/contrib/utils/pipelines/opinion_collections.py
 arekit/contrib/utils/pipelines/items/__init__.py
-arekit/contrib/utils/pipelines/items/sampling/__init__.py
-arekit/contrib/utils/pipelines/items/sampling/base.py
-arekit/contrib/utils/pipelines/items/sampling/networks.py
 arekit/contrib/utils/pipelines/items/text/__init__.py
 arekit/contrib/utils/pipelines/items/text/entities_default.py
 arekit/contrib/utils/pipelines/items/text/frames.py
-arekit/contrib/utils/pipelines/items/text/frames_lemmatized.py
-arekit/contrib/utils/pipelines/items/text/frames_negation.py
-arekit/contrib/utils/pipelines/items/text/tokenizer.py
-arekit/contrib/utils/pipelines/items/text/translator.py
 arekit/contrib/utils/pipelines/text_opinion/__init__.py
 arekit/contrib/utils/pipelines/text_opinion/extraction.py
 arekit/contrib/utils/pipelines/text_opinion/annot/__init__.py
@@ -237,26 +184,6 @@ arekit/contrib/utils/pipelines/text_opinion/filters/distance_based.py
 arekit/contrib/utils/pipelines/text_opinion/filters/entity_based.py
 arekit/contrib/utils/pipelines/text_opinion/filters/limitation.py
 arekit/contrib/utils/processing/__init__.py
-arekit/contrib/utils/processing/languages/__init__.py
-arekit/contrib/utils/processing/languages/mods.py
-arekit/contrib/utils/processing/languages/pos.py
-arekit/contrib/utils/processing/languages/ru/__init__.py
-arekit/contrib/utils/processing/languages/ru/cases.py
-arekit/contrib/utils/processing/languages/ru/constants.py
-arekit/contrib/utils/processing/languages/ru/mods.py
-arekit/contrib/utils/processing/languages/ru/number.py
-arekit/contrib/utils/processing/languages/ru/pos_service.py
-arekit/contrib/utils/processing/lemmatization/__init__.py
-arekit/contrib/utils/processing/lemmatization/mystem.py
-arekit/contrib/utils/processing/pos/__init__.py
-arekit/contrib/utils/processing/pos/base.py
-arekit/contrib/utils/processing/pos/mystem_wrap.py
-arekit/contrib/utils/processing/pos/russian.py
-arekit/contrib/utils/processing/text/__init__.py
-arekit/contrib/utils/processing/text/tokens.py
 arekit/contrib/utils/synonyms/__init__.py
 arekit/contrib/utils/synonyms/simple.py
-arekit/contrib/utils/synonyms/stemmer_based.py
-arekit/contrib/utils/vectorizers/__init__.py
-arekit/contrib/utils/vectorizers/bpe.py
-arekit/contrib/utils/vectorizers/random_norm.py
+arekit/contrib/utils/synonyms/stemmer_based.py

{arekit-0.25.0 → arekit-0.25.1}/arekit.egg-info/requires.txt

@@ -1,4 +1,3 @@
 tqdm
 enum34==1.1.10
 numpy>=1.14.5
-pymystem3==0.2.0

{arekit-0.25.0 → arekit-0.25.1}/setup.py

@@ -15,7 +15,7 @@ def get_requirements(filenames):
 
 setup(
     name='arekit',
-    version='0.25.0',
+    version='0.25.1',
     python_requires=">=3.6",
     description='Document level Attitude and Relation Extraction toolkit (AREkit)'
                 ' for sampling and prompting mass-media news into datasets for ML-model training',
@@ -35,7 +35,6 @@ setup(
     keywords='natural language processing, relation extraction, sentiment analysis',
     packages=find_packages(),
     install_requires=get_requirements([
-        'dependencies.txt',
-        'arekit/contrib/utils/dependencies.txt']),
+        'dependencies.txt']),
     data_files=["logo.png"],
 )

arekit-0.25.0/arekit/common/data/input/repositories/base.py

@@ -1,68 +0,0 @@
-from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
-from arekit.common.data.input.providers.contents import ContentsProvider
-from arekit.common.data.input.providers.rows.base import BaseRowProvider
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-
-class BaseInputRepository(object):
-
-    def __init__(self, columns_provider, rows_provider, storage):
-        assert(isinstance(columns_provider, BaseColumnsProvider))
-        assert(isinstance(rows_provider, BaseRowProvider))
-        assert(isinstance(storage, BaseRowsStorage))
-
-        self._columns_provider = columns_provider
-        self._rows_provider = rows_provider
-        self._storage = storage
-
-        # Do setup operations.
-        self._setup_columns_provider()
-        self._setup_rows_provider()
-
-    # region protected methods
-
-    def _setup_columns_provider(self):
-        pass
-
-    def _setup_rows_provider(self):
-        pass
-
-    # endregion
-
-    def populate(self, contents_provider, doc_ids, desc="", writer=None, target=None):
-        assert(isinstance(contents_provider, ContentsProvider))
-        assert(isinstance(self._storage, BaseRowsStorage))
-        assert(isinstance(doc_ids, list))
-        assert(isinstance(writer, BaseWriter) or writer is None)
-        assert(isinstance(target, str) or target is None)
-
-        def iter_rows(idle_mode):
-            return self._rows_provider.iter_by_rows(
-                contents_provider=contents_provider,
-                doc_ids_iter=doc_ids,
-                idle_mode=idle_mode)
-
-        self._storage.init_empty(columns_provider=self._columns_provider)
-
-        is_async_write_mode_on = writer is not None and target is not None
-
-        if is_async_write_mode_on:
-            writer.open_target(target)
-
-        self._storage.fill(lambda idle_mode: iter_rows(idle_mode),
-                           columns_provider=self._columns_provider,
-                           row_handler=lambda: writer.commit_line(self._storage) if is_async_write_mode_on else None,
-                           desc=desc)
-
-        if is_async_write_mode_on:
-            writer.close_target()
-
-    def push(self, writer, target, free_storage=True):
-        if not isinstance(self._storage, RowCacheStorage):
-            writer.write_all(self._storage, target)
-
-        # After writing we free the contents of the storage.
-        if free_storage:
-            self._storage.free()

arekit-0.25.0/arekit/common/data/input/repositories/sample.py

@@ -1,22 +0,0 @@
-import logging
-
-from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
-from arekit.common.data.input.repositories.base import BaseInputRepository
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class BaseInputSamplesRepository(BaseInputRepository):
-
-    def _setup_rows_provider(self):
-        """ Setup store labels.
-        """
-        assert(isinstance(self._rows_provider, BaseSampleRowProvider))
-        self._rows_provider.set_store_labels(self._columns_provider.StoreLabels)
-
-    def _setup_columns_provider(self):
-        """ Setup text column names.
-        """
-        text_column_names = list(self._rows_provider.TextProvider.iter_columns())
-        self._columns_provider.set_text_column_names(text_column_names)

arekit-0.25.0/arekit/common/data/views/samples.py

@@ -1,26 +0,0 @@
-from arekit.common.data import const
-from arekit.common.data.storages.base import BaseRowsStorage
-
-
-# TODO. This is a particular type of view, and expected to be off the core.
-class LinkedSamplesStorageView(object):
-
-    def iter_from_storage(self, storage):
-        assert(isinstance(storage, BaseRowsStorage))
-        undefined = -1
-
-        linked = []
-        current_opinion_id = undefined
-        for row_index, opinion_id in enumerate(storage.iter_column_values(const.OPINION_ID)):
-            if current_opinion_id != undefined:
-                if opinion_id != current_opinion_id:
-                    yield linked
-                    linked = []
-                    current_opinion_id = opinion_id
-            else:
-                current_opinion_id = opinion_id
-
-            linked.append(storage.get_row(row_index))
-
-        if len(linked) > 0:
-            yield linked