arekit 0.24.0__tar.gz

arekit-0.24.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Nicolay Rusnachenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

arekit-0.24.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.1
+Name: arekit
+Version: 0.24.0
+Summary: Library devoted to Document level Attitude and Relation Extraction for text objects with entity-linking (EL) API support
+Home-page: https://github.com/nicolay-r/AREkit
+Author: Nicolay Rusnachenko
+Author-email: rusnicolay@gmail.com
+License: MIT License
+Keywords: natural language processing,relation extraction,sentiment analysis
+Platform: UNKNOWN
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+License-File: LICENSE
+
+UNKNOWN
+

arekit-0.24.0/README.md

@@ -0,0 +1,56 @@
+# AREkit 0.24.0
+
+![](https://img.shields.io/badge/Python-3.9+-brightgreen.svg)
+
+<p align="center">
+    <img src="logo.png"/>
+</p>
+
+**AREkit** (Attitude and Relation Extraction Toolkit) --
+is a python toolkit, devoted to document level Attitude and Relation Extraction between text objects from mass-media news. 
+
+## Description
+
+This toolkit aims to solve data preparation problems in Relation Extraction related taks, considiering such factors as:
+* 🔗 EL (entity-linking) API support for objects, 
+* ➰ avoidance of cyclic connections,
+* :straight_ruler: distance consideration between relation participants (in `terms` or `sentences`),
+* 📑 relations annotations and filtering rules,
+* *️⃣ entities formatting or masking, and more.
+
+Using AREkit you may focus on preparation and experiments with your ML-models by shift all the data-preparation part onto toolset of this project for:
+[neural-networks](https://github.com/nicolay-r/AREkit/wiki/Sampling-for-Neural-Network), 
+[language-models](https://github.com/nicolay-r/AREkit/wiki/Sampling-for-BERT), 
+[ChatGPT](https://github.com/nicolay-r/AREkit/wiki/Sampling-for-ChatGPT).
+
+In order to do so, we provide:
+* :file_folder: API for external [collection binding](https://github.com/nicolay-r/AREkit/wiki/Binding-a-Custom-Source) (native support of [BRAT](https://brat.nlplab.org/)-based exported annotations)
+* ➿ [pipelines](https://github.com/nicolay-r/AREkit/wiki/Pipelines:-Text-Opinion-Annotation) and iterators for handling large-scale collections serialization without out-of-memory issues.
+* evaluators which allows you to assess your trained model.
+
+AREkit is a very close to opensource framework [SeqIO](https://github.com/google/seqio) proposed by [Google](https://github.com/google) 
+for data-preprocessing, evaluation, for sequence models.
+While SeqIO dedicated for conversion/pre-processing of datasets of any type, 
+this project proposes pipelines creation from the very raw or preannotated (BRAT-based) texts, including the solutions for problems mentioned above.
+
+The core functionality includes 
+(1) API for document presentation with EL (Entity Linking, i.e. Object Synonymy) support 
+for sentence level relations preparation (dubbed as contexts)
+(2) API for contexts extraction
+(3) relations transferring from sentence-level onto document-level, and more.
+
+## Installation 
+
+1. Install required dependencies
+```bash
+pip install git+https://github.com/nicolay-r/AREkit.git@0.24.0-rc
+```
+
+2. Download Resources
+```bash
+python -m arekit.download_data
+```
+
+## Usage
+Please follow the wiki page
+[Tutorials List](https://github.com/nicolay-r/AREkit/wiki/Tutorials).

arekit-0.24.0/arekit/common/bound.py

@@ -0,0 +1,48 @@
+class Bound:
+
+    def __init__(self, pos, length):
+        assert(isinstance(pos, int))
+        assert(isinstance(length, int))
+        self.__pos = pos
+        self.__length = length
+
+    # region properties
+
+    @property
+    def Position(self):
+        return self.__pos
+
+    @property
+    def Length(self):
+        return self.__length
+
+    # endregion
+
+    def itersects_with(self, other):
+        begin = self.__pos
+        end = self.__pos + self.__length
+        other_begin = other.Position
+        other_end_included = other.Position + other.Length - 1
+        if end > other_begin >= begin:
+            return True
+        if end > other_end_included >= begin:
+            return True
+        if other_begin < begin and end <= other_end_included:
+            return True
+        return False
+
+    def intersect(self, other):
+        begin = self.__pos
+        end = self.__pos + self.__length
+        other_begin = other.Position
+        other_end = other.Position + other.Length
+        actual_begin = min(begin, other_begin)
+        actual_length = max(end, other_end) - actual_begin
+        return Bound(pos=actual_begin, length=actual_length)
+
+    def contains(self, other):
+        begin = self.__pos
+        end = self.__pos + self.__length
+        other_begin = other.Position
+        other_end = other.Position + other.Length
+        return begin <= other_begin and end >= other_end

arekit-0.24.0/arekit/common/context/terms_mapper.py

@@ -0,0 +1,51 @@
+from collections.abc import Iterable
+
+from arekit.common.context.token import Token
+from arekit.common.entities.base import Entity
+from arekit.common.frames.text_variant import TextFrameVariant
+
+
+class TextTermsMapper(object):
+
+    def iter_mapped(self, terms):
+        """ Performs mapping operation of each terms in a sequence
+        """
+        assert(isinstance(terms, Iterable))
+
+        self._before_mapping()
+
+        for i, term in enumerate(terms):
+
+            if isinstance(term, str):
+                m_term = self.map_word(i, term)
+            elif isinstance(term, Token):
+                m_term = self.map_token(i, term)
+            elif isinstance(term, TextFrameVariant):
+                m_term = self.map_text_frame_variant(i, term)
+            elif isinstance(term, Entity):
+                m_term = self.map_entity(i, term)
+            else:
+                raise Exception("Unsupported type {}".format(term))
+
+            if m_term is not None:
+                yield m_term
+
+        self._after_mapping()
+
+    def _before_mapping(self):
+        pass
+
+    def _after_mapping(self):
+        pass
+
+    def map_word(self, w_ind, word):
+        raise NotImplementedError()
+
+    def map_token(self, t_ind, token):
+        raise NotImplementedError()
+
+    def map_text_frame_variant(self, fv_ind, text_frame_variant):
+        raise NotImplementedError()
+
+    def map_entity(self, e_ind, entity):
+        raise NotImplementedError()

arekit-0.24.0/arekit/common/context/token.py

@@ -0,0 +1,16 @@
+class Token:
+    """
+    Token that stores original and resulted token values
+    i.e.: term=',', token_value='<[COMMA]>'
+    """
+    def __init__(self, term, token_value):
+        assert(isinstance(term, str))
+        assert(isinstance(token_value, str))
+        self.__meta_value = term
+        self.__token_value = token_value
+
+    def get_meta_value(self):
+        return self.__meta_value
+
+    def get_token_value(self):
+        return self.__token_value

arekit-0.24.0/arekit/common/data/const.py

@@ -0,0 +1,21 @@
+ID = 'id'
+DOC_ID = 'doc_id'
+TEXT = 'text_a'
+LABEL_UINT = 'label_uint'
+LABEL_STR = 'label_str'
+
+# Global identifier of the opinion in the sampled data.
+OPINION_ID = "opinion_id"
+OPINION_LINKAGE_ID = "linkage_id"
+
+# Corresponds to fields with attitude ends. (indices, INT)
+S_IND = 's_ind'
+T_IND = 't_ind'
+
+# Provide sentence index.
+SENT_IND = 'sent_ind'
+
+# Entity parameters
+ENTITY_VALUES = 'entity_values'
+ENTITY_TYPES = 'entity_types'
+ENTITIES = 'entities'

arekit-0.24.0/arekit/common/data/doc_provider.py

@@ -0,0 +1,6 @@
+class DocumentProvider(object):
+    """ Provides operations with documents
+    """
+
+    def by_id(self, doc_id):
+        raise NotImplementedError()

arekit-0.24.0/arekit/common/data/input/providers/columns/base.py

@@ -0,0 +1,9 @@
+
+class BaseColumnsProvider(object):
+
+    ROW_ID = 'row_id'
+
+    def get_columns_list_with_types(self):
+        dtypes_list = list()
+        dtypes_list.append((BaseColumnsProvider.ROW_ID, 'int32'))
+        return dtypes_list

arekit-0.24.0/arekit/common/data/input/providers/columns/sample.py

@@ -0,0 +1,59 @@
+from arekit.common.data import const
+from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
+
+
+class SampleColumnsProvider(BaseColumnsProvider):
+    """
+    [id, label, text_a] -- for train
+    [id, text_a] -- for test
+    """
+
+    def __init__(self, store_labels):
+        super(SampleColumnsProvider, self).__init__()
+        self.__store_labels = store_labels
+        self.__text_column_names = None
+
+    # region properties
+
+    @property
+    def StoreLabels(self):
+        return self.__store_labels
+
+    @property
+    def TextColumnNames(self):
+        return self.__text_column_names
+
+    # endregion
+
+    def get_columns_list_with_types(self):
+        """
+        Composing df with the following columns:
+            [id, label, type, text_a]
+        """
+        dtypes_list = super(SampleColumnsProvider, self).get_columns_list_with_types()
+
+        dtypes_list.append((const.ID, str))
+        dtypes_list.append((const.DOC_ID, str))
+
+        # insert labels
+        if self.__store_labels:
+            dtypes_list.append((const.LABEL_UINT, 'int32'))
+            dtypes_list.append((const.LABEL_STR, str))
+
+        # insert text columns
+        for col_name in self.__text_column_names:
+            dtypes_list.append((col_name, str))
+
+        # insert indices
+        dtypes_list.append((const.S_IND, 'int32'))
+        dtypes_list.append((const.T_IND, 'int32'))
+
+        # opinion-extraction task related fields
+        dtypes_list.append((const.OPINION_ID, 'int32'))
+        dtypes_list.append((const.OPINION_LINKAGE_ID, 'int32'))
+
+        return dtypes_list
+
+    def set_text_column_names(self, text_column_names):
+        assert(isinstance(text_column_names, list))
+        self.__text_column_names = text_column_names

arekit-0.24.0/arekit/common/data/input/providers/const.py

@@ -0,0 +1,3 @@
+# Idle mode related to iteration format, which is considered only for assessing the expected
+# amount of sampled data, while its contents could be avoided at all.
+IDLE_MODE = "idle_mode"

arekit-0.24.0/arekit/common/data/input/providers/contents.py

@@ -0,0 +1,9 @@
+
+class ContentsProvider(object):
+    """ This is a main provider of the contents of further sampled output.
+    """
+
+    def from_doc_ids(self, doc_ids, idle_mode=False):
+        """ This is a main method is expected to be implemented.
+        """
+        pass

arekit-0.24.0/arekit/common/data/input/providers/instances/base.py

@@ -0,0 +1,14 @@
+from arekit.common.linkage.base import LinkedDataWrapper
+
+
+class BaseLinkedDataInstancesProvider(object):
+
+    def iter_instances(self, linked_data):
+        raise NotImplementedError()
+
+    @staticmethod
+    def provide_label(linked_data):
+        """ Implementation based on the first element of the linkage.
+        """
+        assert(isinstance(linked_data, LinkedDataWrapper))
+        return linked_data.First.Label

arekit-0.24.0/arekit/common/data/input/providers/instances/multiple.py

@@ -0,0 +1,27 @@
+from arekit.common.data.input.providers.instances.base import BaseLinkedDataInstancesProvider
+from arekit.common.linkage.text_opinions import TextOpinionsLinkage
+from arekit.common.text_opinions.base import TextOpinion
+
+
+class MultipleInstancesLinkedTextOpinionsProvider(BaseLinkedDataInstancesProvider):
+
+    def __init__(self, supported_labels):
+        assert(isinstance(supported_labels, list))
+        self.__supported_labels = supported_labels
+
+    def iter_instances(self, linked_data):
+        """ Enumerate all opinions as if it would be with the different label types.
+        """
+        for label in self.__supported_labels:
+            yield self.__modify_first_and_copy_linked_wrap(linked_data, label)
+
+    @staticmethod
+    def __modify_first_and_copy_linked_wrap(text_opinions_linkage, label):
+        assert (isinstance(text_opinions_linkage, TextOpinionsLinkage))
+
+        linkage = list(text_opinions_linkage)
+        text_opinion_copy = TextOpinion.create_copy(other=linkage[0])
+        text_opinion_copy.set_label(label=label)
+        linkage[0] = text_opinion_copy
+
+        return TextOpinionsLinkage(linked_data=linkage)

arekit-0.24.0/arekit/common/data/input/providers/instances/single.py

@@ -0,0 +1,8 @@
+from arekit.common.data.input.providers.instances.base import BaseLinkedDataInstancesProvider
+
+
+class SingleInstanceLinkedDataProvider(BaseLinkedDataInstancesProvider):
+
+    def iter_instances(self, linked_data):
+        yield linked_data
+        return

arekit-0.24.0/arekit/common/data/input/providers/label/base.py

@@ -0,0 +1,24 @@
+from arekit.common.labels.scaler.base import BaseLabelScaler
+
+
+class LabelProvider(object):
+
+    def __init__(self, label_scaler):
+        assert(isinstance(label_scaler, BaseLabelScaler))
+        self.__label_scaler = label_scaler
+
+    @property
+    def LabelScaler(self):
+        return self.__label_scaler
+
+    @property
+    def SupportedLabels(self):
+        return self.__label_scaler.ordered_suppoted_labels()
+
+    @property
+    def OutputLabelsUint(self):
+        raise NotImplementedError()
+
+    def calculate_output_uint_label(self, expected_uint_label, etalon_uint_label):
+        raise NotImplementedError()
+

arekit-0.24.0/arekit/common/data/input/providers/label/binary.py

@@ -0,0 +1,11 @@
+from arekit.common.data.input.providers.label.base import LabelProvider
+
+
+class BinaryLabelProvider(LabelProvider):
+
+    def calculate_output_uint_label(self, expected_uint_label, etalon_uint_label):
+        return 1 if expected_uint_label == etalon_uint_label else 0
+
+    @property
+    def OutputLabelsUint(self):
+        return [0, 1]

arekit-0.24.0/arekit/common/data/input/providers/label/multiple.py

@@ -0,0 +1,15 @@
+from arekit.common.data.input.providers.label.base import LabelProvider
+
+
+class MultipleLabelProvider(LabelProvider):
+
+    def __init__(self, label_scaler):
+        super(MultipleLabelProvider, self).__init__(label_scaler=label_scaler)
+
+    def calculate_output_uint_label(self, expected_uint_label, etalon_uint_label):
+        return expected_uint_label
+
+    @property
+    def OutputLabelsUint(self):
+        return [self.LabelScaler.label_to_uint(label) for label in self.SupportedLabels]
+

arekit-0.24.0/arekit/common/data/input/providers/rows/base.py

@@ -0,0 +1,64 @@
+from collections import Counter
+from collections.abc import Iterable
+import logging
+
+from arekit.common.data.input.providers.contents import ContentsProvider
+from arekit.common.linkage.base import LinkedDataWrapper
+from arekit.common.docs.parsed.providers.entity_service import EntityServiceProvider
+from arekit.common.linkage.meta import MetaEmptyLinkedDataWrapper
+
+logger = logging.getLogger(__name__)
+
+
+class BaseRowProvider(object):
+    """ Base provider for rows that suppose to be filled into BaseRowsStorage.
+    """
+
+    def __init__(self):
+        self.__rows_counter = None
+
+    # region protected methods
+
+    # TODO. This might be also generalized.
+    # TODO. Idle-mode is also a implementation and task specific parameter, i.e. might be removed from here.
+    def _provide_rows(self, parsed_doc, entity_service, text_opinion_linkage, idle_mode):
+        raise NotImplementedError()
+
+    def _count_row(self):
+        index = self.__rows_counter["rows_iterated"]
+        self.__rows_counter["rows_iterated"] += 1
+        return index
+
+    # endregion
+
+    def __iter_rows(self, linked_data, idle_mode):
+        parsed_doc_service = linked_data.Tag
+        return self._provide_rows(parsed_doc=parsed_doc_service.ParsedDocument,
+                                  entity_service=parsed_doc_service.get_provider(EntityServiceProvider.NAME),
+                                  text_opinion_linkage=linked_data,
+                                  idle_mode=idle_mode)
+
+    def iter_by_rows(self, contents_provider, doc_ids_iter, idle_mode):
+        assert(isinstance(contents_provider, ContentsProvider))
+        assert(isinstance(doc_ids_iter, Iterable))
+
+        self.__rows_counter = Counter()
+
+        for linked_data in contents_provider.from_doc_ids(doc_ids=doc_ids_iter, idle_mode=idle_mode):
+            assert(isinstance(linked_data, LinkedDataWrapper))
+
+            if isinstance(linked_data, MetaEmptyLinkedDataWrapper):
+                if idle_mode:
+                    # In the case of the IDLE mode we do not consider the meta-data.
+                    data_it = []
+                else:
+                    # Consider the actual linked data instance.
+                    data_it = [linked_data]
+            else:
+                # Consider the actual rows of the related linked data.
+                data_it = self.__iter_rows(linked_data=linked_data, idle_mode=idle_mode)
+
+            for data in data_it:
+                yield linked_data.RelatedDocID, data
+
+        self.__rows_counter = None