arekit 0.24.0__py3-none-any.whl → 0.25.1__py3-none-any.whl

arekit/contrib/utils/io_utils/utils.py

@@ -1,29 +1,12 @@
 from collections.abc import Iterable
 import logging
-from os.path import join, exists
-
-from arekit.common.experiment.data_type import DataType
+from os.path import exists
 
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
 
-def join_dir_with_subfolder_name(subfolder_name, dir):
-    """ Returns subfolder in in directory
-    """
-    assert(isinstance(subfolder_name, str))
-    assert(isinstance(dir, str))
-
-    target_dir = join(dir, "{}/".format(subfolder_name))
-    return target_dir
-
-
-def filename_template(data_type):
-    assert(isinstance(data_type, DataType))
-    return "{data_type}-0".format(data_type=data_type.name.lower())
-
-
 def check_targets_existence(targets):
     assert (isinstance(targets, Iterable))
 

arekit/contrib/utils/pipelines/items/text/entities_default.py

@@ -4,8 +4,8 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class TextEntitiesParser(BasePipelineItem):
 
-    def __init__(self):
-        super(TextEntitiesParser, self).__init__()
+    def __init__(self, **kwargs):
+        super(TextEntitiesParser, self).__init__(**kwargs)
 
     @staticmethod
     def __process_word(word):

arekit/contrib/utils/pipelines/items/text/frames.py

@@ -6,11 +6,10 @@ from arekit.common.pipeline.items.base import BasePipelineItem
 
 class FrameVariantsParser(BasePipelineItem):
 
-    def __init__(self, frame_variants):
+    def __init__(self, frame_variants, **kwargs):
         assert(isinstance(frame_variants, FrameVariantsCollection))
         assert(len(frame_variants) > 0)
-
-        super(FrameVariantsParser, self).__init__()
+        super(FrameVariantsParser, self).__init__(**kwargs)
 
         self.__frame_variants = frame_variants
         self.__max_variant_len = max([len(variant) for _, variant in frame_variants.iter_variants()])

arekit/contrib/utils/pipelines/text_opinion/extraction.py

@@ -3,12 +3,10 @@ from arekit.common.linkage.text_opinions import TextOpinionsLinkage
 from arekit.common.docs.parsed.base import ParsedDocument
 from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider
 from arekit.common.docs.parsed.service import ParsedDocumentService
-from arekit.common.docs.parser import DocumentParser
-from arekit.common.pipeline.base import BasePipeline
+from arekit.common.docs.parser import DocumentParsers
 from arekit.common.pipeline.items.flatten import FlattenIterPipelineItem
 from arekit.common.pipeline.items.map import MapPipelineItem
 from arekit.common.pipeline.items.map_nested import MapNestedPipelineItem
-from arekit.common.text.parser import BaseTextParser
 from arekit.common.text_opinions.base import TextOpinion
 from arekit.contrib.utils.pipelines.text_opinion.filters.base import TextOpinionFilter
 from arekit.contrib.utils.pipelines.text_opinion.filters.limitation import FrameworkLimitationsTextOpinionFilter
@@ -17,7 +15,7 @@ from arekit.contrib.utils.pipelines.text_opinion.filters.limitation import Frame
 def __iter_text_opinion_linkages(parsed_doc, annotators, entity_index_func,
                                  text_opinion_filters, use_meta):
     """ use_meta: bool
-            this is mainly for tqdm and other console parameters to stay up-to-date
+            this is mainly for the progress-bar and other console parameters to stay up-to-date
             with the state in the case we do not have that much output results
             across multiple amount of documents.
     """
@@ -64,24 +62,24 @@ def __iter_text_opinion_linkages(parsed_doc, annotators, entity_index_func,
         yield MetaEmptyLinkedDataWrapper(doc_id=parsed_doc.RelatedDocID)
 
 
-def text_opinion_extraction_pipeline(text_parser, get_doc_by_id_func, annotators, entity_index_func,
+def text_opinion_extraction_pipeline(pipeline_items, get_doc_by_id_func, annotators, entity_index_func, batch_size,
                                      text_opinion_filters=None, use_meta_between_docs=True):
-    assert(isinstance(text_parser, BaseTextParser))
     assert(callable(get_doc_by_id_func))
     assert(isinstance(annotators, list))
     assert(isinstance(text_opinion_filters, list) or text_opinion_filters is None)
     assert(isinstance(use_meta_between_docs, bool))
+    assert(isinstance(batch_size, int) and batch_size > 0)
 
     extra_filters = [] if text_opinion_filters is None else text_opinion_filters
     actual_text_opinion_filters = [FrameworkLimitationsTextOpinionFilter()] + extra_filters
 
-    return BasePipeline([
+    return [
         # (doc_id) -> (doc)
         MapPipelineItem(map_func=lambda doc_id: get_doc_by_id_func(doc_id)),
 
         # (doc, ppl_ctx) -> (parsed_doc)
-        MapNestedPipelineItem(map_func=lambda doc, ppl_ctx: DocumentParser.parse(
-            doc=doc, text_parser=text_parser, parent_ppl_ctx=ppl_ctx)),
+        MapNestedPipelineItem(map_func=lambda doc, ppl_ctx: DocumentParsers.parse_batch(
+            doc=doc, pipeline_items=pipeline_items, parent_ppl_ctx=ppl_ctx, batch_size=batch_size)),
 
         # (parsed_doc) -> (text_opinions)
         MapPipelineItem(map_func=lambda parsed_doc: __iter_text_opinion_linkages(
@@ -90,4 +88,4 @@ def text_opinion_extraction_pipeline(text_parser, get_doc_by_id_func, annotators
 
         # linkages[] -> linkages
         FlattenIterPipelineItem()
-    ])
+    ]

arekit-0.25.1.dist-info/METADATA

@@ -0,0 +1,81 @@
+Metadata-Version: 2.1
+Name: arekit
+Version: 0.25.1
+Summary: Document level Attitude and Relation Extraction toolkit (AREkit) for sampling and prompting mass-media news into datasets for ML-model training
+Home-page: https://github.com/nicolay-r/AREkit
+Author: Nicolay Rusnachenko
+Author-email: rusnicolay@gmail.com
+License: MIT License
+Keywords: natural language processing,relation extraction,sentiment analysis
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: tqdm
+Requires-Dist: enum34==1.1.10
+Requires-Dist: numpy>=1.14.5
+
+# AREkit 0.25.1
+
+![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+
+<p align="center">
+    <img src="logo.png"/>
+</p>
+
+**AREkit** (Attitude and Relation Extraction Toolkit) --
+is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+
+## Description
+
+
+This toolkit aims at memory-effective data processing in [Relation Extraction (RE)](https://nlpprogress.com/english/relationship_extraction.html) related tasks.
+
+<p align="center">
+    <img src="docs/arekit-pipeline-concept.png"/>
+</p>
+
+> Figure: AREkit pipelines design. More on 
+> **[ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction](https://link.springer.com/chapter/10.1007/978-3-031-56069-9_23)** paper
+
+In particular, this framework serves the following features: 
+* ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+* 🔗 EL (entity-linking) API support for objects, 
+* ➰ avoidance of cyclic connections,
+* :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+* 📑 relations annotations and filtering rules,
+* *️⃣ entities formatting or masking, and more.
+
+The core functionality includes: 
+* API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+for sentence level relations preparation (dubbed as contexts);
+* API for contexts extraction;
+* Relations transferring from sentence-level onto document-level, and more.
+
+## Installation 
+
+```bash
+pip install git+https://github.com/nicolay-r/AREkit.git@0.25.1-rc
+```
+
+## Usage
+
+Please follow the **[tutorial section on project Wiki](https://github.com/nicolay-r/AREkit/wiki/Tutorials)** for mode details.
+
+## How to cite
+A great research is also accompanied by the faithful reference. 
+if you use or extend our work, please cite as follows:
+
+```bibtex
+@inproceedings{rusnachenko2024arelight,
+  title={ARElight: Context Sampling of Large Texts for Deep Learning Relation Extraction},
+  author={Rusnachenko, Nicolay and Liang, Huizhi and Kolomeets, Maxim and Shi, Lei},
+  booktitle={European Conference on Information Retrieval},
+  year={2024},
+  organization={Springer}
+}
+```

arekit-0.25.1.dist-info/RECORD

@@ -0,0 +1,186 @@
+arekit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/bound.py,sha256=lPpHY6ct_CU9e4qXeYjhJfWbTj6Sb_NVtZ1CJheQPNE,1402
+arekit/common/log_utils.py,sha256=OfEQxbExkuRAl9dxlgFEqcFhI4HHoMYT7WE8ud0IPOM,924
+arekit/common/utils.py,sha256=N061ENJJgvsB338Q9cixc6RWyuikSPQq4Tc8mmgwy9s,2659
+arekit/common/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/context/terms_mapper.py,sha256=QA02Cv7D2JKTlXkez_0w0J8HuvNziNF2vrqLgy4Bwc8,1447
+arekit/common/context/token.py,sha256=CpWAlvprUnJfCtYvO8lwdfU_ofSKAOGOudXTwppyzSk,459
+arekit/common/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/const.py,sha256=J74zim3CGJlLJp-AVn5z9TTuBfmttjiM_8sRW1Pc-iE,457
+arekit/common/data/doc_provider.py,sha256=KU6Q2-B8_cUuFhSBHYp-cDI8OCwFk3fwOahv2QLIR2c,149
+arekit/common/data/rows_fmt.py,sha256=klq9HdzSnhbRBhOw7O4ctp3PZ5L6ZVy-0eIV2vLLYY8,2694
+arekit/common/data/rows_parser.py,sha256=qYSEETvhX_0_JuAqm0bjK_V28_53qq7OY9JAnBdRC78,1513
+arekit/common/data/input/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/input/sample.py,sha256=6JeGxsLbEUXVKPWA1hIlkTDNOaYg4bHCJWw0ULrLByg,2143
+arekit/common/data/input/terms_mapper.py,sha256=DUOMbGwiQETY7qhztoU8uU30d1cQPsIsgNLldpjcufg,3197
+arekit/common/data/input/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/input/providers/const.py,sha256=GDvPkgP7hllHW3QiueMBQgQyu2CtNFI4JYNNja2Im6Q,187
+arekit/common/data/input/providers/contents.py,sha256=jT1LJE_5Igw5H2e1jKsWWciHSbPVg649phT177SzhEA,261
+arekit/common/data/input/providers/columns/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/input/providers/columns/base.py,sha256=Ar4GkC1L8YFMgeVNM-pIkOOUvKqf2CgIIdh5DA0V8uI,225
+arekit/common/data/input/providers/columns/sample.py,sha256=3onDT6LGkFwU3GOAm6M1MvgjD3fEgapTslAV6-9gvIE,1756
+arekit/common/data/input/providers/instances/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/input/providers/instances/base.py,sha256=ybaHQNRpuebdHGU937yzkgZ0E7mO-S7Dm41NwFj44ew,420
+arekit/common/data/input/providers/instances/multiple.py,sha256=6agaTA3srLiLEhBTU0RnD01GUFqMcsITV5NjVkUgR10,1144
+arekit/common/data/input/providers/instances/single.py,sha256=bZKIn_Kw79c8pH1a3aUq1dmOsDu__BoFwQDLGjEtg5I,253
+arekit/common/data/input/providers/label/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/input/providers/label/base.py,sha256=1MOCKw_OP_IbYT5OR5C3b9VZdYnLGg-TxPc_qHpuZJs,620
+arekit/common/data/input/providers/label/binary.py,sha256=jPD6Jn8DYMrdI3jN8ueoWvuGMouUKbelmI07sP9Wau4,337
+arekit/common/data/input/providers/label/multiple.py,sha256=HWbHF_CwwbiLQbYm5dgvnXAm0b6tJOyFYFEUBxuWAqI,492
+arekit/common/data/input/providers/rows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/input/providers/rows/base.py,sha256=syH7ZEW3Agwfb1IR0G7n_Amy3Kkg0EZk2V7kH3r7ADg,2517
+arekit/common/data/input/providers/rows/samples.py,sha256=uqLTP8fnz-0wC7ALLlIDUYtXTG4OpnRqp70Fgv_1Iiw,9427
+arekit/common/data/input/providers/sample/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/input/providers/sample/cropped.py,sha256=jJSos4Si-qy-wb-QmomXxxgURR1UhJnvY0tZoowlfVc,1885
+arekit/common/data/input/providers/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/input/providers/text/single.py,sha256=vm3sShIYZcmses-hmZX9cOfveWXCYGwvKLgQ0qs3VXQ,1604
+arekit/common/data/storages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/data/storages/base.py,sha256=psxo5uIc3hUDi5Cgf4j3Cm-935Fy1VQBYzcBzCcCFZE,2661
+arekit/common/docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/docs/base.py,sha256=uXUOtpR9BEsDBfDHg4eLqOjfSVOV_o9VPii3nSxLZuY,734
+arekit/common/docs/entities_grouping.py,sha256=_r254fNr0j6BjHuLZBLjj21yWm4_k__5aOcBXcAaQUQ,704
+arekit/common/docs/entity.py,sha256=TxrZMdIEgjk-PgCyskCkVis2KAw_M7vTBp3ppP6G05M,662
+arekit/common/docs/parser.py,sha256=dzWjpbbYt-C9UU9sSy_Holnm0kQxJqtz1_6va6kS_L4,1780
+arekit/common/docs/sentence.py,sha256=nZCCFj2yk71POoXCBfEMN3pteM2qQdj60eEzxMVY_3k,302
+arekit/common/docs/parsed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/docs/parsed/base.py,sha256=WPstqOpBuLKjtz6UO_bI0DpOPF3Sm0wYEVwjtldbPXE,3175
+arekit/common/docs/parsed/service.py,sha256=fSzwtRcSvmvlW8LyK6XPf7wJAx66GWlbRgH_3oQf-BU,1029
+arekit/common/docs/parsed/term_position.py,sha256=H9eQQeanLxwP6og30TQUnpcXymGEPwXClRpaE8VnpLs,1040
+arekit/common/docs/parsed/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/docs/parsed/providers/base.py,sha256=IjnG7c6Q78cYYAPTrwuZCOiMQDfMaujDQ6U0gK7JCcw,2587
+arekit/common/docs/parsed/providers/base_pairs.py,sha256=RDYjspkENPQU2pn7Jp5mFrL9566eVWgXMEzWBQlMdRo,2195
+arekit/common/docs/parsed/providers/entity_service.py,sha256=oaBfferpkDXfAFL17vpecSZUsV1Pjvq6lqgHDHsIEZY,6657
+arekit/common/docs/parsed/providers/opinion_pairs.py,sha256=ibeFmvpMBBARtqQ3EKEocIOulgzavv0DeYxePGQK5-U,633
+arekit/common/docs/parsed/providers/text_opinion_pairs.py,sha256=BC4uVgFxy3oZTkCq9VgOlqoqhODia2Z3anoGyGoy0ao,3139
+arekit/common/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/entities/base.py,sha256=kpJFo4pCRVBQX6T8PibLKspp9UwoIrkHDoFMTM9KkUs,1646
+arekit/common/entities/collection.py,sha256=ySSriMYP6zzdto1mC0V9VPXmkAqyJN3mmGoqoNValGI,1931
+arekit/common/entities/str_fmt.py,sha256=gAPeS8RXdhh8Px_u5eOAPbtLREiiyMueid0lQoa4EbQ,250
+arekit/common/entities/types.py,sha256=pxFB0gsevdsmnduN_Ffk7_P2TRiMt6NAHyrutuKOFvs,145
+arekit/common/experiment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/experiment/data_type.py,sha256=DezUkfwLTf6XLYheqPiaWyx3ZwcldsJ8wDV8aNgJtDk,227
+arekit/common/experiment/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/experiment/api/base_samples_io.py,sha256=SN8CnbEYaazE3SldvnENfjoNRHsTejtrg4jJfqfZLMs,516
+arekit/common/frames/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/frames/text_variant.py,sha256=TlWR4jnuF7HW9BMHhOTKkr768V_Ub0wd0E5A4YTwD0c,875
+arekit/common/frames/connotations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/frames/connotations/descriptor.py,sha256=yow1Wo-Hf52rx2hiQlpeSkpP4WFFcFB25ewgXtwm588,408
+arekit/common/frames/connotations/provider.py,sha256=Zm-NFL-aVKJM_NhvTWizIAiNENt6B1tegTrj0k2afoc,114
+arekit/common/frames/variants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/frames/variants/base.py,sha256=PhFxJZl-g9bGLfg1OlPKIUETAsTx4wwSPuBS5yOEPg8,489
+arekit/common/frames/variants/collection.py,sha256=28_DRBny_iAWMdHpupdCnLvBp0FtF2tjz-uUctyrmhY,1935
+arekit/common/labels/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/labels/base.py,sha256=m7EjvPcQPHtzZ0txVqNXIQPUzgNuaU2FmDyND7K4yTE,412
+arekit/common/labels/str_fmt.py,sha256=ecDsP1-7NNHk_aEaBlPaNaNoA_aqy28QBOHoIxtEnDk,1707
+arekit/common/labels/provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/labels/provider/base.py,sha256=KIWvRwXGWNWYhrzEV8A0g9r0Yk7N2E0qQpf9-UpVnbw,151
+arekit/common/labels/provider/constant.py,sha256=bU6DCm1iuk_W2fMkg-NxABMJqgS9DtwxnoHpD_vSnLc,462
+arekit/common/labels/scaler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/labels/scaler/base.py,sha256=FTZ7eTPTVK9IXLcZaXbpidsTTqTjX0-l1Qt-N1bpqWg,2349
+arekit/common/labels/scaler/sentiment.py,sha256=TbYdM9mdtFTQL_fgh9rS9TEc-7U4Fpskp8JvnvN8TAA,180
+arekit/common/labels/scaler/single.py,sha256=tybF3-fO4CHd_QUFnDCEmTbfbljfJA9aZEv9MtpM5Ss,308
+arekit/common/linkage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/linkage/base.py,sha256=toZmKjTr444eHhvNLMSNU23KXtfH5DtOGtS99qGNcOo,1014
+arekit/common/linkage/meta.py,sha256=LFHHhAkzQzym8rha4uuXb0BKwIb61SVGtxnU4iF_Nuk,692
+arekit/common/linkage/opinions.py,sha256=8OQscnh1-5JJL3KX_lCm_6ayGCezDuFnvidfuwkjClI,255
+arekit/common/linkage/text_opinions.py,sha256=qR1-zGEYaVPSpNISnGGXnABpdP6Qx8tc1i5DsEyn9wo,571
+arekit/common/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/model/labeling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/model/labeling/base.py,sha256=uj7_igCWEU23OjnzabNy0LyxoZ6S_qSfCA-ZaoL1erA,727
+arekit/common/model/labeling/modes.py,sha256=DiwC6Aomke-ojwwpR2pcd4qgQSwmRdGCvQlyHHhN3YY,127
+arekit/common/model/labeling/single.py,sha256=Eggi0obocjiT9ofv_U0zLiFoEIeUQhaMCqjCWn14Fh8,773
+arekit/common/opinions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/opinions/base.py,sha256=eIx1RzsngCkpnF2Utju5i_Qp7gqF_rDIe_UDeMGXtmo,2112
+arekit/common/opinions/collection.py,sha256=bdx-CIYYdE-DrjyB1mRTGtkLb-lrGPTSLl25xv5EHnM,4938
+arekit/common/opinions/enums.py,sha256=TE5AGN_xb0NdZ636UtHuYFRMNl24iwXzmyf8WUfvr6w,83
+arekit/common/opinions/provider.py,sha256=q4hXRFDuGoo9fGOf_L9CM048YBtel1v3__ZqfSXL8Xc,168
+arekit/common/opinions/writer.py,sha256=-IbWTIVlX2rhLpSP_8iuQ3_WyzzGwhto7ujfnNL6jhA,173
+arekit/common/opinions/annot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/opinions/annot/algo_based.py,sha256=cvDGDmUoUaQ1Xcbyouxrjs0CkHRfRogW8Mfs5O5cOlc,2240
+arekit/common/opinions/annot/base.py,sha256=IvwrwT8O3s6b2_R0arpMR4Uog7kuWQZUAyRP5cq_27A,382
+arekit/common/opinions/annot/algo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/opinions/annot/algo/base.py,sha256=ymll-4-SplCY7CLswjOZEC1vsVHIEzUP0JMYgvL8hbo,124
+arekit/common/opinions/annot/algo/pair_based.py,sha256=HbYn1mAsn5g11NiC9pfrMqNtJn_GzvqPFGpafMqqB2o,4419
+arekit/common/opinions/annot/algo/predefined.py,sha256=zU39SADPKnykHCNB-Bmn_0bvd6gYWWYmfgfi-68hHSs,741
+arekit/common/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/pipeline/base.py,sha256=8TgWNy5QrnKEp1bq3lhyGSgIfYe5ZIZU3c-DYBJ9LPA,957
+arekit/common/pipeline/batching.py,sha256=DdOvOladOo2aEv3JZ8NQnCvsNGcWk4TFzENrZqTGyXk,1239
+arekit/common/pipeline/context.py,sha256=Fw25lBVakHNAXjtkdEqopR-Jh59cDKGWD2jCJxBrj7Y,1126
+arekit/common/pipeline/conts.py,sha256=NAQNsHt1kK3HnxWv3M6yXi0c7C6Mx6ZZ6KZc0yE0eas,70
+arekit/common/pipeline/utils.py,sha256=5VqH1LtRa4tYUbyiRvWdBmP4biFhTKq9vhr8QiRFFkY,882
+arekit/common/pipeline/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/pipeline/items/base.py,sha256=15-z8ERQ0QxaRszs7sHQduU0KIBJIm8B0V2nwCva6d0,1695
+arekit/common/pipeline/items/flatten.py,sha256=9T4jWqPGv4UDxajlM0Nm0-gvwUgqqYB8XH0efTum9a0,542
+arekit/common/pipeline/items/handle.py,sha256=QS5Byj7-o5jmFi0ag58NE3zm2-JzVIunIgc3Pn1ij6g,578
+arekit/common/pipeline/items/iter.py,sha256=Tk9WdUMPOq20s7jEWEpU4PmillnVtQ8nIa2ct7iw-3s,406
+arekit/common/pipeline/items/map.py,sha256=G5wBdjaaxePD0pijrxsfpJACeP7kzj7HerjCkNIhmII,381
+arekit/common/pipeline/items/map_nested.py,sha256=vs0GdJNr3qSF9p2yd1nWji5E1HGzECbvOfN2MqoHc2A,630
+arekit/common/synonyms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/synonyms/base.py,sha256=YxD-CKCjlEtar1zTdumnfC3vKgbP2wLODR9mMEwbbnA,4237
+arekit/common/synonyms/grouping.py,sha256=fi7QQbBvsTvvP2CPTesSPEsPNmGfc6euqj-HPhVvtlg,698
+arekit/common/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/text/enums.py,sha256=nelEI7B-szLUtl8xds8Kw_vgK5JWg_Aj7IadEj2q_1Y,141
+arekit/common/text/parsed.py,sha256=YxGRHtozDd3sDVI3hMT_hOO7Wmsy7_zLkblfnSXeJ9g,1104
+arekit/common/text/partitioning.py,sha256=OL8r3-xaMafnT7FuPXDHINlA-BQgx6cLaMqm366WKCU,1153
+arekit/common/text/stemmer.py,sha256=OJ5XelxLN-7m3uLPDU9C7CWdkXDeK-xieexQN6RYLXc,341
+arekit/common/text_opinions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/common/text_opinions/base.py,sha256=KootNvGAbUVCV5uFgLjK-bm9bbQSIvZUz0q9CBToGa8,3447
+arekit/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/bert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/bert/input/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/bert/input/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/bert/input/providers/cropped_sample.py,sha256=46uHHhAe8cGxV2JlfO3thog5XV6T2niUIflFghfUSBM,866
+arekit/contrib/bert/input/providers/text_pair.py,sha256=_1d-he0n42y3ksj8RjJlNHgHnaQUEq0aQhUdTPRMKgg,2817
+arekit/contrib/bert/terms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/bert/terms/mapper.py,sha256=oHX-lsaZYjBFLjngzSKT5z_JPJCHbclUsEe4i4fup_8,992
+arekit/contrib/prompt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/prompt/sample.py,sha256=MxpbDR0ww7WmdtuPu74B8R6QKVXeuzO0CKGOJIYwbRk,3164
+arekit/contrib/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/bert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/bert/samplers.py,sha256=ZVe3rbUAH0Jw1xR_yHE1DoUJf3CI0pDgbBQQzlLWevc,989
+arekit/contrib/utils/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/data/contents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/data/contents/opinions.py,sha256=MSV7NytEe15adKhhHCq5KiCj6ZBq31nV-u2rcSfFCgE,1738
+arekit/contrib/utils/data/doc_provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/data/doc_provider/dict_based.py,sha256=zUOiiIbj5zby4xqMb0m9N-a6enavJJ7wFmPaGErykWU,371
+arekit/contrib/utils/data/doc_provider/dir_based.py,sha256=FTw3kLV_CYtPoUoHl39IrP6RjLvTecCno9May95jVXw,1916
+arekit/contrib/utils/data/storages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/data/storages/jsonl_based.py,sha256=dz8uizu9t1C215o0HEL8y4LiDKR4aC_-OwDu_xF0xIM,522
+arekit/contrib/utils/data/storages/pandas_based.py,sha256=gMkWUFHZE9Oe1Uy04vEBcUfTIAdh46r5zpjlPAwwG2g,3842
+arekit/contrib/utils/data/storages/row_cache.py,sha256=MRK0uJFvw6O99k2aFb3JLZhLUBo2JUO-WYQ4EeRRu6M,2051
+arekit/contrib/utils/data/storages/sqlite_based.py,sha256=cIYAHyiB4CMftKgrgLqw-L4F1WnhbspjwWLSPqH5NHk,682
+arekit/contrib/utils/data/writers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/data/writers/base.py,sha256=JLwf5WVl_U319sdMev8YOn4OoCcrgNIUZtrOuG1JLjI,766
+arekit/contrib/utils/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/entities/filter.py,sha256=aHTExIMFaMdy4QL8iYE23eiby3qLImAakXR6gNqG6fs,145
+arekit/contrib/utils/entities/formatters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/entities/formatters/str_display.py,sha256=N8igv7EVaTFayvLXkyBGtm67KwHaeP-M-L8d7oqBG9Q,401
+arekit/contrib/utils/entities/formatters/str_simple_sharp_prefixed_fmt.py,sha256=rEUIma9O3kOBWIguGtJ69JH-00Dhm0vUBOd5yNcKweY,653
+arekit/contrib/utils/io_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/io_utils/utils.py,sha256=310SIJTsNLn2OZrGPer9W4ZP52PHkjBK3zsyqxVs3h0,537
+arekit/contrib/utils/pipelines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/pipelines/opinion_collections.py,sha256=y9-klVJGCN9mPd7t1ECllAiCnAb3MKVXC1PnYddp5sQ,3195
+arekit/contrib/utils/pipelines/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/pipelines/items/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/pipelines/items/text/entities_default.py,sha256=vNx5ir2mf7a1gg_OeqUsf_p1Fu2k7QIFxVpe-CuwZ84,727
+arekit/contrib/utils/pipelines/items/text/frames.py,sha256=pZQybYfgEQB1DM3PtmsgrtB2Xl0HejmP4rhT0nR_YKE,2586
+arekit/contrib/utils/pipelines/text_opinion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/pipelines/text_opinion/extraction.py,sha256=MT1WMlvVI25JRL0g7W83bV8BGUr7_MNOQBj7ZAHgrnU,4245
+arekit/contrib/utils/pipelines/text_opinion/annot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/pipelines/text_opinion/annot/algo_based.py,sha256=bwS-UR2x3rgp_xqnf6z-73T-eIZE_kltRSGYxgd_WpU,1751
+arekit/contrib/utils/pipelines/text_opinion/filters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/pipelines/text_opinion/filters/base.py,sha256=GnKnJB4MKqiMSJny3a9Na7l7Csm7abbt6GADBCY18Mw,143
+arekit/contrib/utils/pipelines/text_opinion/filters/distance_based.py,sha256=3Pjq4IJJMT7dYpK266lN66WQJUnQO3P0rG6wcAvJOOA,649
+arekit/contrib/utils/pipelines/text_opinion/filters/entity_based.py,sha256=pdWFJaKh4kKIsUuBNp3WNy5Rj80CjWEy2wp-0axFnrI,1254
+arekit/contrib/utils/pipelines/text_opinion/filters/limitation.py,sha256=4AFS5zhocJuYphGO2ZMWmYTtIhGItKDTkB0--AmjgnA,1151
+arekit/contrib/utils/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/synonyms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arekit/contrib/utils/synonyms/simple.py,sha256=ST9EwuWP88FzbyV8Gi0-biTPgGOsZ7OWyaBWHL_U_eo,557
+arekit/contrib/utils/synonyms/stemmer_based.py,sha256=q19P_XOCWN2_JrBtybAt7ToMIr1ambw4ahr0fSEEHmQ,1400
+arekit-0.25.1.data/data/logo.png,sha256=S8OZ4MGGD72Pf5co7ngYbXKkJH1EUhbErUXv1ZjUWiU,45718
+arekit-0.25.1.dist-info/LICENSE,sha256=JO9tIbxAvhwDv73cX-gUStr9yA-TY7wusUeLHRx7JuY,1076
+arekit-0.25.1.dist-info/METADATA,sha256=ryWGTL4fYqR36z2qh1UuYBg6UIU6n7_U9Y09KPRS6xk,3177
+arekit-0.25.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+arekit-0.25.1.dist-info/top_level.txt,sha256=4pXuFE8IE0lBsqi6ZsR7figx0H939VIX4_-76YIbkOQ,7
+arekit-0.25.1.dist-info/RECORD,,

{arekit-0.24.0.dist-info → arekit-0.25.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.1)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

arekit/common/data/input/repositories/base.py

@@ -1,68 +0,0 @@
-from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
-from arekit.common.data.input.providers.contents import ContentsProvider
-from arekit.common.data.input.providers.rows.base import BaseRowProvider
-from arekit.common.data.storages.base import BaseRowsStorage
-from arekit.contrib.utils.data.storages.row_cache import RowCacheStorage
-from arekit.contrib.utils.data.writers.base import BaseWriter
-
-
-class BaseInputRepository(object):
-
-    def __init__(self, columns_provider, rows_provider, storage):
-        assert(isinstance(columns_provider, BaseColumnsProvider))
-        assert(isinstance(rows_provider, BaseRowProvider))
-        assert(isinstance(storage, BaseRowsStorage))
-
-        self._columns_provider = columns_provider
-        self._rows_provider = rows_provider
-        self._storage = storage
-
-        # Do setup operations.
-        self._setup_columns_provider()
-        self._setup_rows_provider()
-
-    # region protected methods
-
-    def _setup_columns_provider(self):
-        pass
-
-    def _setup_rows_provider(self):
-        pass
-
-    # endregion
-
-    def populate(self, contents_provider, doc_ids, desc="", writer=None, target=None):
-        assert(isinstance(contents_provider, ContentsProvider))
-        assert(isinstance(self._storage, BaseRowsStorage))
-        assert(isinstance(doc_ids, list))
-        assert(isinstance(writer, BaseWriter) or writer is None)
-        assert(isinstance(target, str) or target is None)
-
-        def iter_rows(idle_mode):
-            return self._rows_provider.iter_by_rows(
-                contents_provider=contents_provider,
-                doc_ids_iter=doc_ids,
-                idle_mode=idle_mode)
-
-        self._storage.init_empty(columns_provider=self._columns_provider)
-
-        is_async_write_mode_on = writer is not None and target is not None
-
-        if is_async_write_mode_on:
-            writer.open_target(target)
-
-        self._storage.fill(lambda idle_mode: iter_rows(idle_mode),
-                           columns_provider=self._columns_provider,
-                           row_handler=lambda: writer.commit_line(self._storage) if is_async_write_mode_on else None,
-                           desc=desc)
-
-        if is_async_write_mode_on:
-            writer.close_target()
-
-    def push(self, writer, target, free_storage=True):
-        if not isinstance(self._storage, RowCacheStorage):
-            writer.write_all(self._storage, target)
-
-        # After writing we free the contents of the storage.
-        if free_storage:
-            self._storage.free()

arekit/common/data/input/repositories/sample.py

@@ -1,22 +0,0 @@
-import logging
-
-from arekit.common.data.input.providers.rows.samples import BaseSampleRowProvider
-from arekit.common.data.input.repositories.base import BaseInputRepository
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class BaseInputSamplesRepository(BaseInputRepository):
-
-    def _setup_rows_provider(self):
-        """ Setup store labels.
-        """
-        assert(isinstance(self._rows_provider, BaseSampleRowProvider))
-        self._rows_provider.set_store_labels(self._columns_provider.StoreLabels)
-
-    def _setup_columns_provider(self):
-        """ Setup text column names.
-        """
-        text_column_names = list(self._rows_provider.TextProvider.iter_columns())
-        self._columns_provider.set_text_column_names(text_column_names)

arekit/common/data/views/samples.py

@@ -1,26 +0,0 @@
-from arekit.common.data import const
-from arekit.common.data.storages.base import BaseRowsStorage
-
-
-# TODO. This is a particular type of view, and expected to be off the core.
-class LinkedSamplesStorageView(object):
-
-    def iter_from_storage(self, storage):
-        assert(isinstance(storage, BaseRowsStorage))
-        undefined = -1
-
-        linked = []
-        current_opinion_id = undefined
-        for row_index, opinion_id in enumerate(storage.iter_column_values(const.OPINION_ID)):
-            if current_opinion_id != undefined:
-                if opinion_id != current_opinion_id:
-                    yield linked
-                    linked = []
-                    current_opinion_id = opinion_id
-            else:
-                current_opinion_id = opinion_id
-
-            linked.append(storage.get_row(row_index))
-
-        if len(linked) > 0:
-            yield linked

arekit/common/docs/objects_parser.py

@@ -1,37 +0,0 @@
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.text.partitioning.base import BasePartitioning
-from arekit.common.pipeline.context import PipelineContext
-
-
-class SentenceObjectsParserPipelineItem(BasePipelineItem):
-
-    def __init__(self, partitioning):
-        assert(isinstance(partitioning, BasePartitioning))
-        self.__partitioning = partitioning
-
-    # region protected
-
-    def _get_text(self, pipeline_ctx):
-        return None
-
-    def _get_parts_provider_func(self, input_data, pipeline_ctx):
-        raise NotImplementedError()
-
-    # endregion
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        external_input = self._get_text(pipeline_ctx)
-        actual_input = input_data if external_input is None else external_input
-        parts_it = self._get_parts_provider_func(input_data=actual_input, pipeline_ctx=pipeline_ctx)
-        return self.__partitioning.provide(text=actual_input, parts_it=parts_it)
-
-    # region base
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
-
-    # endregion

arekit/common/text/parser.py

@@ -1,12 +0,0 @@
-from arekit.common.pipeline.base import BasePipeline
-from arekit.common.text.parsed import BaseParsedText
-
-
-class BaseTextParser(BasePipeline):
-
-    def run(self, input_data, params_dict=None, parent_ctx=None):
-        output_data = super(BaseTextParser, self).run(input_data=input_data,
-                                                      params_dict=params_dict,
-                                                      parent_ctx=parent_ctx)
-
-        return BaseParsedText(terms=output_data)

arekit/common/text/partitioning/base.py

@@ -1,4 +0,0 @@
-class BasePartitioning(object):
-
-    def provide(self, text, parts_it):
-        raise NotImplementedError()

arekit/common/text/partitioning/terms.py

@@ -1,35 +0,0 @@
-from collections.abc import Iterable
-
-from arekit.common.bound import Bound
-from arekit.common.text.partitioning.base import BasePartitioning
-
-
-class TermsPartitioning(BasePartitioning):
-    """ NOTE: considering that provided parts
-        has no intersections between each other
-    """
-
-    def provide(self, text, parts_it):
-        assert(isinstance(text, list))
-        assert(isinstance(parts_it, Iterable))
-
-        start = 0
-        parts = []
-        for value, bound in parts_it:
-            assert(isinstance(bound, Bound))
-            assert(bound.Position >= start)
-
-            # Release everythig till the current value position.
-            part = text[start:bound.Position]
-
-            parts.extend(part)
-
-            # Release the entity value.
-            parts.extend([value])
-
-            start = bound.Position + bound.Length
-
-        # Release everything after the last entity.
-        parts.extend(text[start:len(text)])
-
-        return parts