arekit 0.24.0__tar.gz → 0.25.0__tar.gz

arekit-0.25.0/PKG-INFO

@@ -0,0 +1,82 @@
+Metadata-Version: 2.1
+Name: arekit
+Version: 0.25.0
+Summary: Document level Attitude and Relation Extraction toolkit (AREkit) for sampling and prompting mass-media news into datasets for ML-model training
+Home-page: https://github.com/nicolay-r/AREkit
+Author: Nicolay Rusnachenko
+Author-email: rusnicolay@gmail.com
+License: MIT License
+Keywords: natural language processing,relation extraction,sentiment analysis
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: tqdm
+Requires-Dist: enum34==1.1.10
+Requires-Dist: numpy>=1.14.5
+Requires-Dist: pymystem3==0.2.0
+
+# AREkit 0.25.0
+
+![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+
+<p align="center">
+    <img src="logo.png"/>
+</p>
+
+**AREkit** (Attitude and Relation Extraction Toolkit) --
+is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+
+## Description
+
+
+This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+
+<p align="center">
+    <img src="docs/arekit-pipeline-concept.png"/>
+</p>
+
+> Figure: AREkit pipelines design. More on 
+> **[ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction](https://link.springer.com/chapter/10.1007/978-3-031-56069-9_23)** paper
+
+In particular, this framework serves the following features: 
+* ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+* 🔗 EL (entity-linking) API support for objects, 
+* ➰ avoidance of cyclic connections,
+* :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+* 📑 relations annotations and filtering rules,
+* *️⃣ entities formatting or masking, and more.
+
+The core functionality includes: 
+* API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+for sentence level relations preparation (dubbed as contexts);
+* API for contexts extraction;
+* Relations transferring from sentence-level onto document-level, and more.
+
+## Installation 
+
+```bash
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+```
+
+## Usage
+
+Please follow the **[tutorial section on project Wiki](https://github.com/nicolay-r/AREkit/wiki/Tutorials)** for mode details.
+
+## How to cite
+A great research is also accompanied by the faithful reference. 
+if you use or extend our work, please cite as follows:
+
+```bibtex
+@inproceedings{rusnachenko2024arelight,
+  title={ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction},
+  author={Rusnachenko, Nicolay and Liang, Huizhi and Kolomeets, Maxim and Shi, Lei},
+  booktitle={European Conference on Information Retrieval},
+  year={2024},
+  organization={Springer}
+}
+```

arekit-0.25.0/README.md

@@ -0,0 +1,60 @@
+# AREkit 0.25.0
+
+![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+
+<p align="center">
+    <img src="logo.png"/>
+</p>
+
+**AREkit** (Attitude and Relation Extraction Toolkit) --
+is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+
+## Description
+
+
+This toolkit aims at memory-effective data processing in Relation Extraction (RE) related tasks.
+
+<p align="center">
+    <img src="docs/arekit-pipeline-concept.png"/>
+</p>
+
+> Figure: AREkit pipelines design. More on 
+> **[ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction](https://link.springer.com/chapter/10.1007/978-3-031-56069-9_23)** paper
+
+In particular, this framework serves the following features: 
+* ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+* 🔗 EL (entity-linking) API support for objects, 
+* ➰ avoidance of cyclic connections,
+* :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+* 📑 relations annotations and filtering rules,
+* *️⃣ entities formatting or masking, and more.
+
+The core functionality includes: 
+* API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+for sentence level relations preparation (dubbed as contexts);
+* API for contexts extraction;
+* Relations transferring from sentence-level onto document-level, and more.
+
+## Installation 
+
+```bash
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.0-rc
+```
+
+## Usage
+
+Please follow the **[tutorial section on project Wiki](https://github.com/nicolay-r/AREkit/wiki/Tutorials)** for mode details.
+
+## How to cite
+A great research is also accompanied by the faithful reference. 
+if you use or extend our work, please cite as follows:
+
+```bibtex
+@inproceedings{rusnachenko2024arelight,
+  title={ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction},
+  author={Rusnachenko, Nicolay and Liang, Huizhi and Kolomeets, Maxim and Shi, Lei},
+  booktitle={European Conference on Information Retrieval},
+  year={2024},
+  organization={Springer}
+}
+```

{arekit-0.24.0 → arekit-0.25.0}/arekit/common/docs/entities_grouping.py

@@ -4,8 +4,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class EntitiesGroupingPipelineItem(BasePipelineItem):
 
-    def __init__(self, value_to_group_id_func):
+    def __init__(self, value_to_group_id_func, **kwargs):
         assert(callable(value_to_group_id_func))
+        super(EntitiesGroupingPipelineItem, self).__init__(**kwargs)
         self.__value_to_group_id_func = value_to_group_id_func
 
     def apply_core(self, input_data, pipeline_ctx):

arekit-0.25.0/arekit/common/docs/parser.py

@@ -0,0 +1,66 @@
+from tqdm import tqdm
+from arekit.common.docs.base import Document
+from arekit.common.docs.parsed.base import ParsedDocument
+from arekit.common.pipeline.base import BasePipelineLauncher
+from arekit.common.pipeline.batching import BatchingPipelineLauncher
+from arekit.common.pipeline.context import PipelineContext
+from arekit.common.pipeline.utils import BatchIterator
+from arekit.common.text.parsed import BaseParsedText
+
+
+class DocumentParsers(object):
+
+    @staticmethod
+    def parse(doc, pipeline_items, parent_ppl_ctx=None, src_key="input", show_progress=False):
+        """ This document parser is based on single text parts (sentences)
+            that passes sequentially through the pipeline of transformations.
+        """
+        assert(isinstance(doc, Document))
+        assert(isinstance(pipeline_items, list))
+        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
+
+        parsed_sentences = []
+
+        data_it = range(doc.SentencesCount)
+        progress_it = tqdm(data_it, disable=not show_progress)
+
+        for sent_ind in progress_it:
+
+            # Composing the context from a single sentence.
+            ctx = PipelineContext({src_key: doc.get_sentence(sent_ind)}, parent_ctx=parent_ppl_ctx)
+
+            # Apply all the operations.
+            BasePipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key=src_key)
+
+            # Collecting the result.
+            parsed_sentences.append(BaseParsedText(terms=ctx.provide("result")))
+
+        return ParsedDocument(doc_id=doc.ID, parsed_sentences=parsed_sentences)
+
+    @staticmethod
+    def parse_batch(doc, pipeline_items, batch_size, parent_ppl_ctx=None, src_key="input", show_progress=False):
+        """ This document parser is based on batch of sentences.
+        """
+        assert(isinstance(batch_size, int) and batch_size > 0)
+        assert(isinstance(doc, Document))
+        assert(isinstance(pipeline_items, list))
+        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
+
+        parsed_sentences = []
+
+        data_it = BatchIterator(data_iter=iter(range(doc.SentencesCount)), batch_size=batch_size)
+        progress_it = tqdm(data_it, total=round(doc.SentencesCount / batch_size), disable=not show_progress)
+
+        for batch in progress_it:
+
+            # Composing the context from a single sentence.
+            ctx = PipelineContext({src_key: [doc.get_sentence(s_ind) for s_ind in batch]},
+                                  parent_ctx=parent_ppl_ctx)
+
+            # Apply all the operations.
+            BatchingPipelineLauncher.run(pipeline=pipeline_items, pipeline_ctx=ctx, src_key=src_key)
+
+            # Collecting the result.
+            parsed_sentences += [BaseParsedText(terms=result) for result in ctx.provide("result")]
+
+        return ParsedDocument(doc_id=doc.ID, parsed_sentences=parsed_sentences)

arekit-0.25.0/arekit/common/pipeline/base.py

@@ -0,0 +1,21 @@
+from arekit.common.pipeline.context import PipelineContext
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class BasePipelineLauncher:
+
+    @staticmethod
+    def run(pipeline, pipeline_ctx, src_key=None, has_input=True):
+        assert(isinstance(pipeline, list))
+        assert(isinstance(pipeline_ctx, PipelineContext))
+        assert(isinstance(src_key, str) or src_key is None)
+
+        for ind, item in enumerate(filter(lambda itm: itm is not None, pipeline)):
+            assert(isinstance(item, BasePipelineItem))
+            do_force_key = src_key is not None and ind == 0
+            input_data = item.get_source(pipeline_ctx, force_key=src_key if do_force_key else None) \
+                if has_input or ind > 0 else None
+            item_result = item.apply(input_data=input_data, pipeline_ctx=pipeline_ctx)
+            pipeline_ctx.update(param=item.ResultKey, value=item_result, is_new_key=False)
+
+        return pipeline_ctx

arekit-0.25.0/arekit/common/pipeline/batching.py

@@ -0,0 +1,28 @@
+from arekit.common.pipeline.context import PipelineContext
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class BatchingPipelineLauncher:
+
+    @staticmethod
+    def run(pipeline, pipeline_ctx, src_key=None):
+        assert(isinstance(pipeline, list))
+        assert(isinstance(pipeline_ctx, PipelineContext))
+        assert(isinstance(src_key, str) or src_key is None)
+
+        for ind, item in enumerate(filter(lambda itm: itm is not None, pipeline)):
+            assert (isinstance(item, BasePipelineItem))
+
+            # Handle the content of the batch or batch itself.
+            content = item.get_source(pipeline_ctx, call_func=False, force_key=src_key if ind == 0 else None)
+            handled_batch = [item._src_func(i) if item._src_func is not None else i for i in content]
+
+            if item.SupportBatching:
+                batch_result = list(item.apply(input_data=handled_batch, pipeline_ctx=pipeline_ctx))
+            else:
+                batch_result = [item.apply(input_data=input_data, pipeline_ctx=pipeline_ctx)
+                                for input_data in handled_batch]
+
+            pipeline_ctx.update(param=item.ResultKey, value=batch_result, is_new_key=False)
+
+        return pipeline_ctx

{arekit-0.24.0 → arekit-0.25.0}/arekit/common/pipeline/context.py

@@ -13,6 +13,8 @@ class PipelineContext(object):
         self._d[PARENT_CTX] = parent_ctx
 
     def __provide(self, param):
+        if param not in self._d:
+            raise Exception(f"Key `{param}` is not in dictionary.\n{self._d}")
         return self._d[param]
 
     # region public
@@ -23,7 +25,9 @@ class PipelineContext(object):
     def provide_or_none(self, param):
         return self.__provide(param) if param in self._d else None
 
-    def update(self, param, value):
+    def update(self, param, value, is_new_key=False):
+        if is_new_key and param in self._d:
+            raise Exception(f"Key `{param}` is already presented in pipeline context dictionary.")
         self._d[param] = value
 
     # endregion

arekit-0.25.0/arekit/common/pipeline/items/base.py

@@ -0,0 +1,49 @@
+from arekit.common.pipeline.context import PipelineContext
+
+
+class BasePipelineItem(object):
+    """ Single pipeline item that might be instatiated and embedded into pipeline.
+    """
+
+    def __init__(self, src_key="result", result_key="result", src_func=None):
+        assert(isinstance(src_key, str) or src_key is None)
+        assert(callable(src_func) or src_func is None)
+        self.__src_key = src_key
+        self._src_func = src_func
+        self.__result_key = result_key
+
+    @property
+    def ResultKey(self):
+        return self.__result_key
+
+    @property
+    def SupportBatching(self):
+        """ By default pipeline item is not designed for batching.
+        """
+        return False
+
+    def get_source(self, src_ctx, call_func=True, force_key=None):
+        """ Extract input element for processing.
+        """
+        assert(isinstance(src_ctx, PipelineContext))
+
+        # If there is no information about key, then we consider absence of the source.
+        if self.__src_key is None:
+            return None
+
+        # Extracting actual source.
+        src_data = src_ctx.provide(self.__src_key if force_key is None else force_key)
+        if self._src_func is not None and call_func:
+            src_data = self._src_func(src_data)
+
+        return src_data
+
+    def apply_core(self, input_data, pipeline_ctx):
+        """By default we do nothing."""
+        pass
+
+    def apply(self, input_data, pipeline_ctx=None):
+        """ Performs input processing an update it for a further pipeline items.
+        """
+        output_data = self.apply_core(input_data=input_data, pipeline_ctx=pipeline_ctx)
+        return output_data if output_data is not None else input_data

{arekit-0.24.0 → arekit-0.25.0}/arekit/common/pipeline/items/flatten.py

@@ -5,10 +5,14 @@ class FlattenIterPipelineItem(BasePipelineItem):
     """ Considered to flat iterations of items that represent iterations.
     """
 
+    def __init__(self, **kwargs):
+        super(FlattenIterPipelineItem, self).__init__(**kwargs)
+        pass
+
     def __flat_iter(self, iter_data):
         for iter_item in iter_data:
             for item in iter_item:
                 yield item
 
     def apply_core(self, input_data, pipeline_ctx):
-        return self.__flat_iter(input_data)
+        return self.__flat_iter(input_data)

{arekit-0.24.0 → arekit-0.25.0}/arekit/common/pipeline/items/handle.py

@@ -3,8 +3,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class HandleIterPipelineItem(BasePipelineItem):
 
-    def __init__(self, handle_func=None):
+    def __init__(self, handle_func=None, **kwargs):
         assert(callable(handle_func))
+        super(HandleIterPipelineItem, self).__init__(**kwargs)
         self.__handle_func = handle_func
 
     def __updated_data(self, items_iter):

{arekit-0.24.0 → arekit-0.25.0}/arekit/common/pipeline/items/iter.py

@@ -3,8 +3,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class FilterPipelineItem(BasePipelineItem):
 
-    def __init__(self, filter_func=None):
+    def __init__(self, filter_func=None, **kwargs):
         assert(callable(filter_func))
+        super(FilterPipelineItem, self).__init__(**kwargs)
         self.__filter_func = filter_func
 
     def apply_core(self, input_data, pipeline_ctx):

{arekit-0.24.0 → arekit-0.25.0}/arekit/common/pipeline/items/map.py

@@ -3,8 +3,9 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class MapPipelineItem(BasePipelineItem):
 
-    def __init__(self, map_func=None):
+    def __init__(self, map_func=None, **kwargs):
         assert(callable(map_func))
+        super(MapPipelineItem, self).__init__(**kwargs)
         self._map_func = map_func
 
     def apply_core(self, input_data, pipeline_ctx):

{arekit-0.24.0 → arekit-0.25.0}/arekit/common/pipeline/items/map_nested.py

@@ -9,5 +9,9 @@ class MapNestedPipelineItem(MapPipelineItem):
         suppose to be mapped with the passed pipeline context.
     """
 
+    def __init__(self, **kwargs):
+        super(MapNestedPipelineItem, self).__init__(**kwargs)
+        pass
+
     def apply_core(self, input_data, pipeline_ctx):
         return map(lambda item: self._map_func(item, pipeline_ctx), input_data)

arekit-0.25.0/arekit/common/pipeline/utils.py

@@ -0,0 +1,32 @@
+class BatchIterator:
+
+    def __init__(self, data_iter, batch_size, end_value=None):
+        assert(isinstance(batch_size, int) and batch_size > 0)
+        assert(callable(end_value) or end_value is None)
+        self.__data_iter = data_iter
+        self.__index = 0
+        self.__batch_size = batch_size
+        self.__end_value = end_value
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        buffer = []
+        while True:
+            try:
+                data = next(self.__data_iter)
+            except StopIteration:
+                break
+            buffer.append(data)
+            if len(buffer) == self.__batch_size:
+                break
+
+        if len(buffer) > 0:
+            self.__index += 1
+            return buffer
+
+        if self.__end_value is None:
+            raise StopIteration
+        else:
+            return self.__end_value()

arekit-0.25.0/arekit/common/service/sqlite.py

@@ -0,0 +1,36 @@
+import sqlite3
+
+
+class SQLiteProvider(object):
+
+    @staticmethod
+    def write(data_it, target, data2col_func, table_name, columns, sqlite3_column_types,
+              id_column_name="id", id_column_type="TEXT"):
+        assert(callable(data2col_func))
+        assert(isinstance(columns, list))
+        assert(isinstance(sqlite3_column_types, list))
+        assert(len(columns) == len(sqlite3_column_types))
+
+        with sqlite3.connect(target) as con:
+            cur = con.cursor()
+
+            # Provide the ID column.
+            columns = [id_column_name] + columns
+            sqlite3_column_types = [id_column_type] + sqlite3_column_types
+
+            # Compose the whole columns list.
+            content = ", ".join([" ".join(item) for item in zip(columns, sqlite3_column_types)])
+            cur.execute(f"CREATE TABLE IF NOT EXISTS {table_name}({content})")
+            cur.execute(f"CREATE INDEX IF NOT EXISTS i_id ON {table_name}({id_column_name})")
+
+            for uid, data in data_it:
+                r = cur.execute(f"SELECT EXISTS(SELECT 1 FROM {table_name} WHERE {id_column_name}='{uid}');")
+                ans = r.fetchone()[0]
+                if ans == 1:
+                    continue
+
+                params = ", ".join(tuple(['?'] * (len(columns))))
+                cur.execute(f"INSERT INTO {table_name} VALUES ({params})", [str(uid)] + data2col_func(data))
+                con.commit()
+
+            cur.close()

arekit-0.24.0/arekit/common/text/partitioning/str.py → arekit-0.25.0/arekit/common/text/partitioning.py

@@ -1,28 +1,34 @@
 from collections.abc import Iterable
 
 from arekit.common.bound import Bound
-from arekit.common.text.partitioning.base import BasePartitioning
 
 
-class StringPartitioning(BasePartitioning):
+class Partitioning(object):
     """ NOTE: considering that provided parts
         has no intersections between each other
     """
 
+    list_reg_types = {
+        "str": lambda p, item: p.append(item),
+        "list": lambda p, item: p.extend(item)
+    }
+
+    def __init__(self, text_fmt):
+        assert(isinstance(text_fmt, str) and text_fmt in self.list_reg_types)
+        self.__reg_part = self.list_reg_types[text_fmt]
+
     def provide(self, text, parts_it):
-        assert(isinstance(text, str))
         assert(isinstance(parts_it, Iterable))
 
-        start = 0
         parts = []
+        start = 0
+
         for value, bound in parts_it:
             assert(isinstance(bound, Bound))
             assert(bound.Position >= start)
 
             # Release everything till the current value position.
-            part = text[start:bound.Position]
-
-            parts.append(part)
+            self.__reg_part(p=parts, item=text[start:bound.Position])
 
             # Release the entity value.
             parts.extend([value])
@@ -30,7 +36,6 @@ class StringPartitioning(BasePartitioning):
             start = bound.Position + bound.Length
 
         # Release everything after the last entity.
-        last_part = text[start:len(text)]
-        parts.extend([last_part])
+        self.__reg_part(p=parts, item=text[start:len(text)])
 
         return parts

{arekit-0.24.0 → arekit-0.25.0}/arekit/common/utils.py

@@ -1,6 +1,5 @@
 import sys
 import os
-import requests
 from tqdm import tqdm
 
 
@@ -84,46 +83,3 @@ def progress_bar_iter(iterable, desc="", unit='it'):
                 leave=True,
                 ncols=120,
                 unit=unit)
-
-
-def get_default_download_dir():
-    """ Refered to NLTK toolkit approach
-        https://github.com/nltk/nltk/blob/8e771679cee1b4a9540633cc3ea17f4421ffd6c0/nltk/downloader.py#L1051
-    """
-
-    # On Windows, use %APPDATA%
-    if sys.platform == "win32" and "APPDATA" in os.environ:
-        homedir = os.environ["APPDATA"]
-
-    # Otherwise, install in the user's home directory.
-    else:
-        homedir = os.path.expanduser("~/")
-        if homedir == "~/":
-            raise ValueError("Could not find a default download directory")
-
-    return os.path.join(homedir, ".arekit")
-
-
-def download(dest_file_path, source_url):
-    """ Refered to https://github.com/nicolay-r/ner-bilstm-crf-tensorflow/blob/master/ner/utils.py
-        Simple http file downloader
-    """
-    print(('Downloading from {src} to {dest}'.format(src=source_url, dest=dest_file_path)))
-
-    sys.stdout.flush()
-    datapath = os.path.dirname(dest_file_path)
-
-    if not os.path.exists(datapath):
-        os.makedirs(datapath, mode=0o755)
-
-    dest_file_path = os.path.abspath(dest_file_path)
-
-    r = requests.get(source_url, stream=True)
-    total_length = int(r.headers.get('content-length', 0))
-
-    with open(dest_file_path, 'wb') as f:
-        pbar = tqdm(total=total_length, unit='B', unit_scale=True)
-        for chunk in r.iter_content(chunk_size=32 * 1024):
-            if chunk:  # filter out keep-alive new chunks
-                pbar.update(len(chunk))
-                f.write(chunk)

{arekit-0.24.0 → arekit-0.25.0}/arekit/contrib/utils/data/contents/opinions.py

@@ -2,7 +2,8 @@ from arekit.common.data.input.providers.const import IDLE_MODE
 from arekit.common.data.input.providers.contents import ContentsProvider
 from arekit.common.linkage.base import LinkedDataWrapper
 from arekit.common.linkage.text_opinions import TextOpinionsLinkage
-from arekit.common.pipeline.base import BasePipeline
+from arekit.common.pipeline.base import BasePipelineLauncher
+from arekit.common.pipeline.context import PipelineContext
 from arekit.common.text_opinions.base import TextOpinion
 
 
@@ -13,7 +14,7 @@ class InputTextOpinionProvider(ContentsProvider):
             results in a TextOpinionLinkage instances.
             pipeline: id -> ... -> TextOpinionLinkage[]
         """
-        assert(isinstance(pipeline, BasePipeline))
+        assert(isinstance(pipeline, list))
         self.__pipeline = pipeline
         self.__current_id = None
 
@@ -30,7 +31,16 @@ class InputTextOpinionProvider(ContentsProvider):
 
     def from_doc_ids(self, doc_ids, idle_mode=False):
         self.__current_id = 0
-        for linkage in self.__pipeline.run(doc_ids, params_dict={IDLE_MODE: idle_mode}):
+
+        ctx = PipelineContext(d={
+            "result": doc_ids,
+            IDLE_MODE: idle_mode
+        })
+
+        # Launching pipeline with the passed context
+        BasePipelineLauncher.run(pipeline=self.__pipeline, pipeline_ctx=ctx)
+
+        for linkage in ctx.provide("result"):
             assert(isinstance(linkage, LinkedDataWrapper))
             if isinstance(linkage, TextOpinionsLinkage):
                 self.__assign_ids(linkage)

arekit-0.25.0/arekit/contrib/utils/data/readers/sqlite.py

@@ -0,0 +1,14 @@
+from arekit.contrib.utils.data.readers.base import BaseReader
+from arekit.contrib.utils.data.storages.sqlite_based import SQliteBasedRowsStorage
+
+
+class SQliteReader(BaseReader):
+
+    def __init__(self, table_name):
+        self.__table_name = table_name
+
+    def extension(self):
+        return ".sqlite"
+
+    def read(self, target):
+        return SQliteBasedRowsStorage(path=target, table_name=self.__table_name)

{arekit-0.24.0 → arekit-0.25.0}/arekit/contrib/utils/data/storages/row_cache.py

@@ -33,7 +33,12 @@ class RowCacheStorage(BaseRowsStorage):
 
         # Expand with columns that are forced to be provided.
         existed_set = set(self.__column_names)
-        self.__column_names += [c for c in self.__force_collect_columns if c not in existed_set]
+
+        # Calculate extension: columns that were not mentioned in column names list.
+        extension = [c for c in self.__force_collect_columns if c not in existed_set]
+
+        self.__column_names += extension
+        self.__column_types += [str] * len(extension)
 
     def iter_column_names(self):
         return iter(self.__column_names)

arekit-0.25.0/arekit/contrib/utils/data/storages/sqlite_based.py

@@ -0,0 +1,17 @@
+import sqlite3
+from arekit.common.data.storages.base import BaseRowsStorage
+
+
+class SQliteBasedRowsStorage(BaseRowsStorage):
+
+    def __init__(self, path, table_name):
+        self.__path = path
+        self.__table_name = table_name
+        self.__conn = None
+
+    def _iter_rows(self):
+        with sqlite3.connect(self.__path) as conn:
+            cursor = conn.execute(f"select * from {self.__table_name}")
+            for row_index, row in enumerate(cursor.fetchall()):
+                row_dict = {cursor.description[i][0]: value for i, value in enumerate(row)}
+                yield row_index, row_dict