arekit 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

arekit/contrib/networks/input/rows_parser.py

@@ -1,134 +1,47 @@
-from arekit.common.data import const
-from arekit.common.utils import filter_whitespaces, split_by_whitespaces
-
-import arekit.contrib.networks.input.const as network_input_const
-
-empty_list = []
-
-
-def no_value():
-    return None
-
-
-def __process_values_list(value):
-    return value.split(network_input_const.ArgsSep)
-
-
-def __process_indices_list(value):
-    return no_value() if not value else [int(v) for v in str(value).split(network_input_const.ArgsSep)]
-
-
-def __process_int_values_list(value):
-    return __process_indices_list(value)
-
-
-def __handle_text(value):
-    """ The core method of the input text processing.
-    """
-    assert(isinstance(value, str) or isinstance(value, list))
-    return filter_whitespaces([term for term in split_by_whitespaces(value)]
-                              if isinstance(value, str) else value)
-
-
-parse_value = {
-    const.ID: lambda value: value,
-    const.DOC_ID: lambda value: value,
-    const.S_IND: lambda value: int(value),
-    const.T_IND: lambda value: int(value),
-    const.SENT_IND: lambda value: int(value),
-    const.ENTITY_VALUES: lambda value: __process_values_list(value),
-    const.ENTITY_TYPES: lambda value: __process_values_list(value),
-    const.ENTITIES: lambda value: __process_indices_list(value),
-    const.TEXT: lambda value: __handle_text(value),
-    network_input_const.FrameVariantIndices: lambda value:
-        __process_indices_list(value) if isinstance(value, str) else empty_list,
-    network_input_const.FrameConnotations: lambda value:
-        __process_indices_list(value) if isinstance(value, str) else empty_list,
-    network_input_const.SynonymObject: lambda value: __process_indices_list(value),
-    network_input_const.SynonymSubject: lambda value: __process_indices_list(value),
-    network_input_const.PosTags: lambda value: __process_int_values_list(value)
-}
-
-
-class ParsedSampleRow(object):
-    """ Provides a parsed information for a sample row.
-    """
-
-    def __init__(self, row):
-        """ row: dict
-                dict of the pairs ("field_name", value)
-        """
-        assert(isinstance(row, dict))
-
-        self.__uint_label = None
-        self.__params = {}
-
-        for key, value in row.items():
-
-            if key == const.LABEL:
-                self.__uint_label = int(value)
-                # TODO: To be adopted in future instead of __uint_label
-                self.__params[key] = value
-                continue
-
-            if key not in parse_value:
-                continue
-
-            self.__params[key] = parse_value[key](value)
-
-    def __value_or_none(self, key):
-        return self.__params[key] if key in self.__params else no_value()
-
-    @property
-    def SampleID(self):
-        return self.__params[const.ID]
-    
-    @property
-    def Terms(self):
-        return self.__params[const.TEXT]
-
-    @property
-    def SubjectIndex(self):
-        return self.__params[const.S_IND]
-
-    @property
-    def ObjectIndex(self):
-        return self.__params[const.T_IND]
-
-    @property
-    def UintLabel(self):
-        return self.__uint_label
-
-    @property
-    def PartOfSpeechTags(self):
-        return self.__value_or_none(network_input_const.PosTags)
-
-    @property
-    def TextFrameVariantIndices(self):
-        return self.__value_or_none(network_input_const.FrameVariantIndices)
-
-    @property
-    def TextFrameConnotations(self):
-        return self.__value_or_none(network_input_const.FrameConnotations)
-
-    @property
-    def EntityInds(self):
-        return self.__value_or_none(const.ENTITIES)
-
-    @property
-    def SynonymObjectInds(self):
-        return self.__value_or_none(network_input_const.SynonymObject)
-
-    @property
-    def SynonymSubjectInds(self):
-        return self.__value_or_none(network_input_const.SynonymSubject)
-
-    def __getitem__(self, item):
-        assert (isinstance(item, str) or item is None)
-        if item not in self.__params:
-            return no_value()
-        return self.__params[item] if item is not None else no_value()
-
-    @classmethod
-    def parse(cls, row):
-        return cls(row=row)
+import arekit.contrib.networks.input.const as const
+from arekit.common.data.rows_fmt import process_indices_list
+
+
+def create_nn_column_formatters(no_value_func=lambda: None, args_sep=","):
+    assert(callable(no_value_func))
+
+    empty_list = []
+
+    def str_to_list(value):
+        return process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
+
+    def list_to_str(inds_iter):
+        return args_sep.join([str(i) for i in inds_iter])
+
+    return {
+        const.FrameVariantIndices: {
+            "writer": lambda value: list_to_str(value),
+            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
+                if isinstance(value, str) else empty_list
+        },
+        const.FrameConnotations: {
+            "writer": lambda value: list_to_str(value),
+            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
+                if isinstance(value, str) else empty_list
+        },
+        const.SynonymObject: {
+            "writer": lambda value: list_to_str(value),
+            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
+        },
+        const.SynonymSubject: {
+            "writer": lambda value: list_to_str(value),
+            "parser": lambda value: process_indices_list(value, no_value_func=no_value_func, args_sep=args_sep)
+        },
+        const.PosTags: {
+            "writer": lambda value: list_to_str(value),
+            "parser": lambda value: str_to_list(value)
+        }
+    }
+
+
+def create_nn_val_writer_fmt(fmt_type, args_sep=","):
+    assert(isinstance(fmt_type, str))
+    d = create_nn_column_formatters(args_sep=args_sep)
+    for k, v in d.items():
+        d[k] = v[fmt_type]
+    return d

arekit/contrib/prompt/sample.py

@@ -28,32 +28,34 @@ class PromptedSampleRowProvider(CroppedSampleRowProvider):
         self.__labels_fmt = label_fmt
 
     def _fill_row_core(self, row, text_opinion_linkage, index_in_linked, etalon_label,
-                       parsed_news, sentence_ind, s_ind, t_ind):
+                       parsed_doc, sentence_ind, s_ind, t_ind):
 
         super(PromptedSampleRowProvider, self)._fill_row_core(row=row,
                                                               text_opinion_linkage=text_opinion_linkage,
                                                               index_in_linked=index_in_linked,
                                                               etalon_label=etalon_label,
-                                                              parsed_news=parsed_news,
+                                                              parsed_doc=parsed_doc,
                                                               sentence_ind=sentence_ind,
                                                               s_ind=s_ind,
                                                               t_ind=t_ind)
         original_text = row[BaseSingleTextProvider.TEXT_A]
 
         sentence_terms, actual_s_ind, actual_t_ind = self._provide_sentence_terms(
-            parsed_news=parsed_news, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
+            parsed_doc=parsed_doc, sentence_ind=sentence_ind, s_ind=s_ind, t_ind=t_ind)
 
-        label_uint = row[const.LABEL] if const.LABEL in row else None
+        label_uint = row[const.LABEL_UINT] if const.LABEL_UINT in row else None
         label_val = str(label_uint) if label_uint is None or self.__labels_fmt is None else \
-            self.__labels_fmt.label_to_str(self._label_provider.LabelScaler.uint_to_label(row[const.LABEL]))
-
-        row[BaseSingleTextProvider.TEXT_A] = self.__prompt.format(
-            text=original_text,
-            s_ind=row[const.S_IND],
-            t_ind=row[const.T_IND],
-            s_val=sentence_terms[actual_s_ind].DisplayValue,
-            t_val=sentence_terms[actual_t_ind].DisplayValue,
-            label_uint=label_uint,
-            label_val=label_val)
-
-        return row
+            self.__labels_fmt.label_to_str(self._label_provider.LabelScaler.uint_to_label(row[const.LABEL_UINT]))
+
+        vm = {
+           const.TEXT: self.__prompt.format(
+               text=original_text,
+               s_ind=row[const.S_IND],
+               t_ind=row[const.T_IND],
+               s_val=sentence_terms[actual_s_ind].DisplayValue,
+               t_val=sentence_terms[actual_t_ind].DisplayValue,
+               label_uint=label_uint,
+               label_val=label_val)
+        }
+
+        self._apply_row_data(row=row, vm=vm, val_fmt=self._val_fmt)

arekit/contrib/utils/data/contents/opinions.py

@@ -1,7 +1,9 @@
 from arekit.common.data.input.providers.const import IDLE_MODE
 from arekit.common.data.input.providers.contents import ContentsProvider
+from arekit.common.linkage.base import LinkedDataWrapper
 from arekit.common.linkage.text_opinions import TextOpinionsLinkage
-from arekit.common.pipeline.base import BasePipeline
+from arekit.common.pipeline.base import BasePipelineLauncher
+from arekit.common.pipeline.context import PipelineContext
 from arekit.common.text_opinions.base import TextOpinion
 
 
@@ -12,7 +14,7 @@ class InputTextOpinionProvider(ContentsProvider):
             results in a TextOpinionLinkage instances.
             pipeline: id -> ... -> TextOpinionLinkage[]
         """
-        assert(isinstance(pipeline, BasePipeline))
+        assert(isinstance(pipeline, list))
         self.__pipeline = pipeline
         self.__current_id = None
 
@@ -29,7 +31,17 @@ class InputTextOpinionProvider(ContentsProvider):
 
     def from_doc_ids(self, doc_ids, idle_mode=False):
         self.__current_id = 0
-        for linkage in self.__pipeline.run(doc_ids, params_dict={IDLE_MODE: idle_mode}):
-            assert(isinstance(linkage, TextOpinionsLinkage))
-            self.__assign_ids(linkage)
+
+        ctx = PipelineContext(d={
+            "result": doc_ids,
+            IDLE_MODE: idle_mode
+        })
+
+        # Launching pipeline with the passed context
+        BasePipelineLauncher.run(pipeline=self.__pipeline, pipeline_ctx=ctx)
+
+        for linkage in ctx.provide("result"):
+            assert(isinstance(linkage, LinkedDataWrapper))
+            if isinstance(linkage, TextOpinionsLinkage):
+                self.__assign_ids(linkage)
             yield linkage

arekit/contrib/utils/data/doc_provider/dict_based.py

@@ -0,0 +1,13 @@
+from arekit.common.data.doc_provider import DocumentProvider
+
+
+class DictionaryBasedDocumentProvider(DocumentProvider):
+
+    def __init__(self, d):
+        assert(isinstance(d, dict))
+        super(DictionaryBasedDocumentProvider, self).__init__()
+        self.__d = d
+
+    def by_id(self, doc_id):
+        assert(isinstance(doc_id, int))
+        return self.__d[doc_id]

arekit/contrib/utils/data/{doc_ops → doc_provider}/dir_based.py

@@ -1,12 +1,12 @@
 from os.path import join
 
-from arekit.common.experiment.api.ops_doc import DocumentOperations
-from arekit.common.news.base import News
-from arekit.common.news.sentence import BaseNewsSentence
+from arekit.common.data.doc_provider import DocumentProvider
+from arekit.common.docs.base import Document
+from arekit.common.docs.sentence import BaseDocumentSentence
 
 
-class DirectoryFilesDocOperations(DocumentOperations):
-    """ Document Operations based on the list of provided file paths
+class DirectoryFilesDocProvider(DocumentProvider):
+    """ Document Providers based on the list of provided file paths
         for the particular directory.
     """
 
@@ -36,10 +36,10 @@ class DirectoryFilesDocOperations(DocumentOperations):
         """
         # setup input data.
         sentences = self.__sentence_parser(contents)
-        sentences = list(map(lambda text: BaseNewsSentence(text), sentences))
+        sentences = list(map(lambda text: BaseDocumentSentence(text), sentences))
 
         # Parse text.
-        return News(doc_id=doc_id, sentences=sentences)
+        return Document(doc_id=doc_id, sentences=sentences)
 
     def by_id(self, doc_id):
         """ Perform reading operation of the document.

arekit/contrib/utils/data/readers/base.py

@@ -1,4 +1,7 @@
 class BaseReader(object):
 
+    def extension(self):
+        raise NotImplementedError()
+
     def read(self, target):
         raise NotImplementedError()

arekit/contrib/utils/data/readers/csv_pd.py

@@ -1,23 +1,29 @@
 import importlib
+
 from arekit.contrib.utils.data.readers.base import BaseReader
 from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
 
 
 class PandasCsvReader(BaseReader):
-    """ Represents a CSV-based reader, implemented via pandas API.
+    """ Represents a CSV-based reader, implmented via pandas API.
     """
 
-    def __init__(self, sep='\t', header='infer', compression='infer', encoding='utf-8', col_types=None):
+    def __init__(self, sep='\t', header='infer', compression='infer', encoding='utf-8', col_types=None,
+                 custom_extension=None):
         self.__sep = sep
         self.__compression = compression
         self.__encoding = encoding
         self.__header = header
+        self.__custom_extension = custom_extension
 
-        # Speciall assignation of types for certain columns.
+        # Special assignation of types for certain columns.
         self.__col_types = col_types
         if self.__col_types is None:
             self.__col_types = dict()
 
+    def extension(self):
+        return ".tsv.gz" if self.__custom_extension is None else self.__custom_extension
+
     def __from_csv(self, filepath):
         pd = importlib.import_module("pandas")
         return pd.read_csv(filepath,
@@ -29,4 +35,4 @@ class PandasCsvReader(BaseReader):
 
     def read(self, target):
         df = self.__from_csv(filepath=target)
-        return PandasBasedRowsStorage(df)
+        return PandasBasedRowsStorage(df)

arekit/contrib/utils/data/readers/jsonl.py

@@ -4,6 +4,9 @@ from arekit.contrib.utils.data.storages.jsonl_based import JsonlBasedRowsStorage
 
 class JsonlReader(BaseReader):
 
+    def extension(self):
+        return ".jsonl"
+
     def read(self, target):
         rows = []
         with open(target, "r") as f:

arekit/contrib/utils/data/readers/sqlite.py

@@ -0,0 +1,14 @@
+from arekit.contrib.utils.data.readers.base import BaseReader
+from arekit.contrib.utils.data.storages.sqlite_based import SQliteBasedRowsStorage
+
+
+class SQliteReader(BaseReader):
+
+    def __init__(self, table_name):
+        self.__table_name = table_name
+
+    def extension(self):
+        return ".sqlite"
+
+    def read(self, target):
+        return SQliteBasedRowsStorage(path=target, table_name=self.__table_name)

arekit/contrib/utils/data/service/balance.py

@@ -1,6 +1,5 @@
 import gc
 import importlib
-
 from arekit.contrib.utils.data.storages.pandas_based import PandasBasedRowsStorage
 
 

arekit/contrib/utils/data/storages/pandas_based.py

@@ -48,6 +48,9 @@ class PandasBasedRowsStorage(BaseRowsStorage):
     def iter_column_names(self):
         return iter(self._df.columns)
 
+    def iter_column_types(self):
+        return iter(self._df.dtypes)
+
     def _set_row_value(self, row_ind, column, value):
         self._df.at[row_ind, column] = value
 
@@ -105,11 +108,6 @@ class PandasBasedRowsStorage(BaseRowsStorage):
     def find_by_value(self, column_name, value):
         return self.__filter(column_name=column_name, value=value)
 
-    def find_first_by_value(self, column_name, value):
-        # TODO. Return new storage. (Encapsulation)
-        rows = self.__filter(column_name=column_name, value=value)
-        return rows.iloc[0]
-
     def init_empty(self, columns_provider):
         cols_with_types = columns_provider.get_columns_list_with_types()
         self._df = self.__create_empty(cols_with_types)

arekit/contrib/utils/data/storages/row_cache.py

@@ -15,7 +15,8 @@ class RowCacheStorage(BaseRowsStorage):
         assert(isinstance(force_collect_columns, list) or force_collect_columns is None)
         self.__f = None
         self.__row_cache = {}
-        self.__columns = []
+        self.__column_names = []
+        self.__column_types = []
         self.__force_collect_columns = [] if force_collect_columns is None else force_collect_columns
 
     @property
@@ -24,15 +25,26 @@ class RowCacheStorage(BaseRowsStorage):
 
     def init_empty(self, columns_provider):
         assert (isinstance(columns_provider, BaseColumnsProvider))
-        for col_name, _ in columns_provider.get_columns_list_with_types():
-            self.__columns.append(col_name)
+
+        self.__column_names.clear()
+        for col_name, col_type in columns_provider.get_columns_list_with_types():
+            self.__column_names.append(col_name)
+            self.__column_types.append(col_type)
 
         # Expand with columns that are forced to be provided.
-        existed_set = set(self.__columns)
-        self.__columns += [c for c in self.__force_collect_columns if c not in existed_set]
+        existed_set = set(self.__column_names)
+
+        # Calculate extension: columns that were not mentioned in column names list.
+        extension = [c for c in self.__force_collect_columns if c not in existed_set]
+
+        self.__column_names += extension
+        self.__column_types += [str] * len(extension)
 
     def iter_column_names(self):
-        return iter(self.__columns)
+        return iter(self.__column_names)
+
+    def iter_column_types(self):
+        return iter(self.__column_types)
 
     def _set_row_value(self, row_ind, column, value):
         self.__row_cache[column] = value

arekit/contrib/utils/data/storages/sqlite_based.py

@@ -0,0 +1,17 @@
+import sqlite3
+from arekit.common.data.storages.base import BaseRowsStorage
+
+
+class SQliteBasedRowsStorage(BaseRowsStorage):
+
+    def __init__(self, path, table_name):
+        self.__path = path
+        self.__table_name = table_name
+        self.__conn = None
+
+    def _iter_rows(self):
+        with sqlite3.connect(self.__path) as conn:
+            cursor = conn.execute(f"select * from {self.__table_name}")
+            for row_index, row in enumerate(cursor.fetchall()):
+                row_dict = {cursor.description[i][0]: value for i, value in enumerate(row)}
+                yield row_index, row_dict

arekit/contrib/utils/data/writers/base.py

@@ -1,5 +1,10 @@
 class BaseWriter(object):
 
+    def extension(self):
+        """ Expected output extension type.
+        """
+        raise NotImplementedError()
+
     def open_target(self, target):
         pass
 

arekit/contrib/utils/data/writers/csv_native.py

@@ -17,6 +17,9 @@ class NativeCsvWriter(BaseWriter):
         self.__header = header
         self.__header_written = None
 
+    def extension(self):
+        return ".csv"
+
     @staticmethod
     def __iter_storage_column_names(storage):
         """ Iter only those columns that existed in storage.

arekit/contrib/utils/data/writers/csv_pd.py

@@ -15,6 +15,9 @@ class PandasCsvWriter(BaseWriter):
         super(PandasCsvWriter, self).__init__()
         self.__write_header = write_header
 
+    def extension(self):
+        return ".tsv.gz"
+
     def write_all(self, storage, target):
         assert(isinstance(storage, PandasBasedRowsStorage))
         assert(isinstance(target, str))

arekit/contrib/utils/data/writers/json_opennre.py

@@ -27,9 +27,8 @@ class OpenNREJsonWriter(BaseWriter):
         During the dataset reading stage via OpenNRE, these linkages automaticaly groups into bags.
     """
 
-    EXTRA_KEYS_TEMPLATE = "_{}"
-
-    def __init__(self, text_columns, encoding="utf-8"):
+    def __init__(self, text_columns, encoding="utf-8", na_value="NA", keep_extra_columns=True,
+                 skip_extra_existed=True):
         """ text_columns: list
                 column names that expected to be joined into a single (token) column.
         """
@@ -38,16 +37,23 @@ class OpenNREJsonWriter(BaseWriter):
         self.__text_columns = text_columns
         self.__encoding = encoding
         self.__target_f = None
+        self.__keep_extra_columns = keep_extra_columns
+        self.__na_value = na_value
+        self.__skip_extra_existed = skip_extra_existed
+
+    def extension(self):
+        return ".jsonl"
 
     @staticmethod
-    def __format_row(row, text_columns):
+    def __format_row(row, na_value, text_columns, keep_extra_columns, skip_extra_existed):
         """ Formatting that is compatible with the OpenNRE.
         """
+        assert(isinstance(na_value, str))
 
         sample_id = row[const.ID]
         s_ind = int(row[const.S_IND])
         t_ind = int(row[const.T_IND])
-        bag_id = sample_id[0:sample_id.find('_i')]
+        bag_id = str(row[const.OPINION_ID])
 
         # Gather tokens.
         tokens = []
@@ -62,13 +68,18 @@ class OpenNREJsonWriter(BaseWriter):
             "token": tokens,
             "h": {"pos": [s_ind, s_ind + 1], "id": str(bag_id + "s")},
             "t": {"pos": [t_ind, t_ind + 1], "id": str(bag_id + "t")},
-            "relation": str(int(row[const.LABEL])) if const.LABEL in row else "NA"
+            "relation": str(int(row[const.LABEL_UINT])) if const.LABEL_UINT in row else na_value
         }
 
-        # Register extra fields.
-        for key, value in row.items():
-            if key not in formatted_data and key not in text_columns:
-                formatted_data[OpenNREJsonWriter.EXTRA_KEYS_TEMPLATE.format(key)] = value
+        # Register extra fields (optionally).
+        if keep_extra_columns:
+            for key, value in row.items():
+                if key not in formatted_data and key not in text_columns:
+                    formatted_data[key] = value
+                else:
+                    if not skip_extra_existed:
+                        raise Exception(f"key `{key}` is already exist in formatted data "
+                                        f"or a part of the text columns list: {text_columns}")
 
         return formatted_data
 
@@ -90,8 +101,12 @@ class OpenNREJsonWriter(BaseWriter):
                 continue
             row_data[col_name] = storage.RowCache[col_name]
 
-        self.__write_bag(bag=self.__format_row(row_data, text_columns=self.__text_columns),
-                         json_file=self.__target_f)
+        bag = self.__format_row(row_data, text_columns=self.__text_columns,
+                                keep_extra_columns=self.__keep_extra_columns,
+                                na_value=self.__na_value,
+                                skip_extra_existed=self.__skip_extra_existed)
+
+        self.__write_bag(bag=bag, json_file=self.__target_f)
 
     @staticmethod
     def __write_bag(bag, json_file):
@@ -108,7 +123,10 @@ class OpenNREJsonWriter(BaseWriter):
         os.makedirs(os.path.dirname(target), exist_ok=True)
         with open(target, "w", encoding=self.__encoding) as json_file:
             for row_index, row in storage:
-                self.__write_bag(bag=self.__format_row(row, text_columns=self.__text_columns),
+                self.__write_bag(bag=self.__format_row(row, text_columns=self.__text_columns,
+                                                       keep_extra_columns=self.__keep_extra_columns,
+                                                       na_value=self.__na_value,
+                                                       skip_extra_existed=self.__skip_extra_existed),
                                  json_file=json_file)
 
         logger.info("Saving completed!")