arekit 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

{arekit-0.23.1.dist-info → arekit-0.25.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.1)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

arekit/common/data/row_ids/base.py

@@ -1,79 +0,0 @@
-from arekit.common.linkage.text_opinions import TextOpinionsLinkage
-
-
-class BaseIDProvider(object):
-    """
-    Opinion in text is a sequence of opinions in context
-    o1, o2, o3, ..., on
-
-    o1 -- first_text_opinion
-    i -- index in lined (for example: i=3 => 03)
-
-    # TODO. #376. This should be definitely refactored. This implementation
-      TODO. combines opinion-based and sample-based data sources, which allows
-      TODO. us to bypass such connection via external foreign keys.
-
-      Since we are head to remove opinions, there is a need to refactor so in a
-      way of an additional column that provides such information for further connection
-      between rows of different storages.
-    """
-
-    SEPARATOR = '_'
-    OPINION = "o{}" + SEPARATOR
-    INDEX = "i{}" + SEPARATOR
-
-    # region 'create' methods
-
-    @staticmethod
-    def create_opinion_id(text_opinions_linkage, index_in_linked):
-        assert(isinstance(text_opinions_linkage, TextOpinionsLinkage))
-        assert(isinstance(index_in_linked, int))
-
-        template = ''.join([BaseIDProvider.OPINION,
-                            BaseIDProvider.INDEX])
-
-        text_opinion_id = text_opinions_linkage.First.TextOpinionID
-        assert(isinstance(text_opinion_id, int))
-
-        return template.format(text_opinion_id,
-                               index_in_linked)
-
-    @staticmethod
-    def create_sample_id(linked_opinions, index_in_linked, label_scaler):
-        raise NotImplementedError()
-
-    @staticmethod
-    def create_pattern(id_value, p_type):
-        assert(isinstance(id_value, int))
-        assert(isinstance(p_type, str))
-        return p_type.format(id_value)
-
-    # endregion
-
-    @staticmethod
-    def convert_sample_id_to_opinion_id(sample_id):
-        assert(isinstance(sample_id, str))
-        return sample_id[:sample_id.index(BaseIDProvider.INDEX[0])] + BaseIDProvider.INDEX.format(0)
-
-    # region 'parse' methods
-
-    @staticmethod
-    def _parse(row_id, pattern):
-        assert(isinstance(pattern, str))
-
-        _from = row_id.index(pattern[0]) + 1
-        _to = row_id.index(BaseIDProvider.SEPARATOR, _from, len(row_id))
-
-        return int(row_id[_from:_to])
-
-    @staticmethod
-    def parse_opinion_in_opinion_id(opinion_id):
-        assert(isinstance(opinion_id, str))
-        return BaseIDProvider._parse(opinion_id, BaseIDProvider.OPINION)
-
-    @staticmethod
-    def parse_opinion_in_sample_id(sample_id):
-        assert(isinstance(sample_id, str))
-        return BaseIDProvider._parse(sample_id, BaseIDProvider.OPINION)
-
-    # endregion

arekit/common/data/row_ids/binary.py

@@ -1,38 +0,0 @@
-from arekit.common.data.row_ids.base import BaseIDProvider
-from arekit.common.labels.scaler.base import BaseLabelScaler
-from arekit.common.linkage.text_opinions import TextOpinionsLinkage
-
-
-class BinaryIDProvider(BaseIDProvider):
-    """
-    Considered that label of opinion IS A PART OF id.
-    # TODO. #376 related. This should be removed after refactoring, because
-    # TODO. we consider an ordinary IDs, that not based on the other data.
-    """
-
-    LABEL = 'l{}' + BaseIDProvider.SEPARATOR
-
-    @staticmethod
-    def create_sample_id(linked_opinions, index_in_linked, label_scaler):
-        assert(isinstance(linked_opinions, TextOpinionsLinkage))
-        assert(isinstance(index_in_linked, int))
-        assert(isinstance(label_scaler, BaseLabelScaler))
-
-        o_id = BaseIDProvider.create_opinion_id(text_opinions_linkage=linked_opinions,
-                                                index_in_linked=index_in_linked)
-
-        template = ''.join(["{}", BinaryIDProvider.LABEL])
-
-        return template.format(o_id,
-                               label_scaler.label_to_uint(linked_opinions.get_linked_label()))
-
-    @staticmethod
-    def parse_label_in_sample_id(sample_id):
-        assert(isinstance(sample_id, str))
-        return BinaryIDProvider._parse(row_id=sample_id, pattern=BinaryIDProvider.LABEL)
-
-    @staticmethod
-    def parse_index_in_sample_id(sample_id):
-        assert(isinstance(sample_id, str))
-        return BinaryIDProvider._parse(row_id=sample_id, pattern=BinaryIDProvider.INDEX)
-

arekit/common/data/row_ids/multiple.py

@@ -1,14 +0,0 @@
-from arekit.common.data.row_ids.base import BaseIDProvider
-
-
-class MultipleIDProvider(BaseIDProvider):
-    """
-    Considered that label of opinion is not a part of id.
-    # TODO. #376 related. This should be removed after refactoring, because
-    # TODO. we consider an ordinary IDs, that not based on the other data.
-    """
-
-    @staticmethod
-    def create_sample_id(linked_opinions, index_in_linked, label_scaler):
-        return BaseIDProvider.create_opinion_id(text_opinions_linkage=linked_opinions,
-                                                index_in_linked=index_in_linked)

arekit/common/folding/base.py

@@ -1,36 +0,0 @@
-import collections
-
-
-class BaseDataFolding(object):
-    """ Describes and provides API on how to handle doc_ids during experiment,
-        i.e. how many states does nested folding algorithm supports,
-        how to perform folding for a particular state (current),
-        and how to such state into string.
-    """
-
-    def __init__(self, doc_ids_to_fold, supported_data_types):
-        assert(isinstance(doc_ids_to_fold, collections.Iterable))
-        assert(isinstance(supported_data_types, list))
-        self._doc_ids_to_fold_set = set(doc_ids_to_fold)
-        self._supported_data_types = supported_data_types
-
-    def contains_doc_id(self, doc_id):
-        assert(isinstance(doc_id, int))
-        return doc_id in self._doc_ids_to_fold_set
-
-    def iter_doc_ids(self):
-        return iter(self._doc_ids_to_fold_set)
-
-    def iter_supported_data_types(self):
-        """ Iterates through data_types, supported in a related experiment
-            Note:
-            In CV-split algorithm, the first part corresponds to a LARGE split,
-            Jand second to small; therefore, the correct sequence is as follows:
-            DataType.Train, DataType.Test.
-        """
-        return iter(self._supported_data_types)
-
-    def fold_doc_ids_set(self):
-        """ Perform the doc_ids folding process onto provided data_types
-        """
-        raise NotImplementedError()

arekit/common/folding/fixed.py

@@ -1,42 +0,0 @@
-from arekit.common.folding.base import BaseDataFolding
-
-
-class FixedFolding(BaseDataFolding):
-
-    def __init__(self, doc_to_datatypes_func, doc_ids_to_fold, supported_data_types):
-        assert(callable(doc_to_datatypes_func))
-
-        super(FixedFolding, self).__init__(doc_ids_to_fold=doc_ids_to_fold,
-                                           supported_data_types=supported_data_types)
-
-        self.__doc_to_datatypes_func = doc_to_datatypes_func
-
-    @classmethod
-    def from_parts(cls, parts):
-        """ parts: dict
-                dictionary of {data_type: [doc_ids]}
-        """
-        assert(isinstance(parts, dict))
-
-        doc_to_datatypes = {}
-        for data_type, doc_ids in parts.items():
-            for doc_id in doc_ids:
-                if doc_id not in doc_to_datatypes:
-                    doc_to_datatypes[doc_id] = []
-                doc_to_datatypes[doc_id].append(data_type)
-
-        return cls(doc_to_datatypes_func=lambda doc_id: doc_to_datatypes[doc_id],
-                   doc_ids_to_fold=doc_to_datatypes.keys(),
-                   supported_data_types=list(parts.keys()))
-
-    def fold_doc_ids_set(self):
-
-        folded = {}
-        for data_type in self._supported_data_types:
-            folded[data_type] = []
-
-        for doc_id in self._doc_ids_to_fold_set:
-            for data_type in self.__doc_to_datatypes_func(doc_id):
-                folded[data_type].append(doc_id)
-
-        return folded

arekit/common/folding/nofold.py

@@ -1,15 +0,0 @@
-from arekit.common.folding.base import BaseDataFolding
-
-
-class NoFolding(BaseDataFolding):
-    """ The case of absent folding in experiment.
-    """
-
-    def __init__(self, doc_ids, supported_data_type):
-        super(NoFolding, self).__init__(doc_ids_to_fold=doc_ids,
-                                        supported_data_types=[supported_data_type])
-
-    def fold_doc_ids_set(self):
-        return {
-            self._supported_data_types[0]: list(self._doc_ids_to_fold_set)
-        }

arekit/common/folding/united.py

@@ -1,46 +0,0 @@
-from arekit.common.folding.base import BaseDataFolding
-
-
-class UnitedFolding(BaseDataFolding):
-
-    def __init__(self, foldings):
-        assert(isinstance(foldings, list))
-        self.__foldings = foldings
-        super(UnitedFolding, self).__init__(
-            doc_ids_to_fold=UnitedFolding.__iter_all_doc_ids(foldings),
-            supported_data_types=list(set(UnitedFolding.__iter_all_data_types(foldings))))
-
-    @staticmethod
-    def __iter_all_doc_ids(foldings):
-        for folding in foldings:
-            assert(isinstance(folding, BaseDataFolding))
-            for doc_id in folding.iter_doc_ids():
-                yield doc_id
-
-    @staticmethod
-    def __iter_all_data_types(foldings):
-        for folding in foldings:
-            assert(isinstance(folding, BaseDataFolding))
-            for d_type in folding.iter_supported_data_types():
-                yield d_type
-
-    @staticmethod
-    def __merge(origin, new_data):
-        assert(isinstance(origin, dict))
-        assert(isinstance(new_data, dict))
-        for key, value in new_data.items():
-            if key not in origin:
-                # Assign list
-                origin[key] = value
-            else:
-                # Combine lists
-                origin[key] += value
-
-    def fold_doc_ids_set(self):
-        origin = {}
-        for folding in self.__foldings:
-            assert(isinstance(folding, BaseDataFolding))
-            new_data = folding.fold_doc_ids_set()
-            self.__merge(origin=origin, new_data=new_data)
-
-        return origin

arekit/common/news/objects_parser.py

@@ -1,37 +0,0 @@
-from arekit.common.pipeline.items.base import BasePipelineItem
-from arekit.common.text.partitioning.base import BasePartitioning
-from arekit.common.pipeline.context import PipelineContext
-
-
-class SentenceObjectsParserPipelineItem(BasePipelineItem):
-
-    def __init__(self, partitioning):
-        assert(isinstance(partitioning, BasePartitioning))
-        self.__partitioning = partitioning
-
-    # region protected
-
-    def _get_text(self, pipeline_ctx):
-        return None
-
-    def _get_parts_provider_func(self, input_data, pipeline_ctx):
-        raise NotImplementedError()
-
-    # endregion
-
-    def apply_core(self, input_data, pipeline_ctx):
-        assert(isinstance(pipeline_ctx, PipelineContext))
-        external_input = self._get_text(pipeline_ctx)
-        actual_input = input_data if external_input is None else external_input
-        parts_it = self._get_parts_provider_func(input_data=actual_input, pipeline_ctx=pipeline_ctx)
-        return self.__partitioning.provide(text=actual_input, parts_it=parts_it)
-
-    # region base
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
-
-    # endregion

arekit/common/news/parsed/providers/base.py

@@ -1,48 +0,0 @@
-from arekit.common.entities.base import Entity
-from arekit.common.news.entity import DocumentEntity
-from arekit.common.news.parsed.base import ParsedNews
-
-
-class BaseParsedNewsServiceProvider(object):
-
-    def __init__(self, entity_index_func=None):
-        """ Outside enity indexing function
-            entity_index_func: provides id for a given entity, i.e.
-                func(entity) -> int (id)
-        """
-        assert(callable(entity_index_func) or entity_index_func is None)
-        self._doc_entities = None
-        self.__entity_map = {}
-        self.__entity_index_func = entity_index_func
-
-    @property
-    def Name(self):
-        raise NotImplementedError()
-
-    def init_parsed_news(self, parsed_news):
-        assert(isinstance(parsed_news, ParsedNews))
-
-        self._doc_entities = []
-        self.__entity_map.clear()
-
-        for index, entity in enumerate(parsed_news.iter_entities()):
-
-            doc_entity = DocumentEntity(id_in_doc=index,
-                                        value=entity.Value,
-                                        e_type=entity.Type,
-                                        display_value=entity.DisplayValue,
-                                        group_index=entity.GroupIndex)
-
-            self._doc_entities.append(doc_entity)
-
-            if self.__entity_index_func is not None:
-                self.__entity_map[self.__entity_index_func(entity)] = doc_entity
-
-    def get_document_entity(self, entity):
-        """ Maps entity to the related one with DocumentEntity type
-        """
-        assert(isinstance(entity, Entity))
-        return self.__entity_map[self.__entity_index_func(entity)]
-
-    def contains_entity(self, entity):
-        return self.__entity_index_func(entity) in self.__entity_map

arekit/common/news/parsed/service.py

@@ -1,31 +0,0 @@
-from arekit.common.news.parsed.base import ParsedNews
-from arekit.common.news.parsed.providers.base import BaseParsedNewsServiceProvider
-
-
-class ParsedNewsService(object):
-    """ Represents a collection of providers, combined with the parsed news.
-    """
-
-    def __init__(self, parsed_news, providers):
-        assert(isinstance(parsed_news, ParsedNews))
-        assert(isinstance(providers, list))
-        self.__parsed_news = parsed_news
-        self.__providers = {}
-
-        for provider in providers:
-            assert(isinstance(provider, BaseParsedNewsServiceProvider))
-            assert(provider.Name not in self.__providers)
-
-            # Link provider with the related name.
-            self.__providers[provider.Name] = provider
-
-            # Post initialize with the related parsed news.
-            provider.init_parsed_news(self.__parsed_news)
-
-
-    @property
-    def ParsedNews(self):
-        return self.__parsed_news
-
-    def get_provider(self, name):
-        return self.__providers[name]

arekit/common/news/parser.py

@@ -1,34 +0,0 @@
-from arekit.common.news.base import News
-from arekit.common.news.parsed.base import ParsedNews
-from arekit.common.pipeline.context import PipelineContext
-from arekit.common.text.parser import BaseTextParser
-
-
-class NewsParser(object):
-
-    @staticmethod
-    def __get_sent(news, sent_ind):
-        return news.get_sentence(sent_ind)
-
-    @staticmethod
-    def parse(news, text_parser, parent_ppl_ctx=None):
-        assert(isinstance(news, News))
-        assert(isinstance(text_parser, BaseTextParser))
-        assert(isinstance(parent_ppl_ctx, PipelineContext) or parent_ppl_ctx is None)
-
-        parsed_sentences = [text_parser.run(input_data=NewsParser.__get_sent(news, sent_ind).Text,
-                                            params_dict=NewsParser.__create_ppl_params(news=news, sent_ind=sent_ind),
-                                            parent_ctx=parent_ppl_ctx)
-                            for sent_ind in range(news.SentencesCount)]
-
-        return ParsedNews(doc_id=news.ID,
-                          parsed_sentences=parsed_sentences)
-
-    @staticmethod
-    def __create_ppl_params(news, sent_ind):
-        assert(isinstance(news, News))
-        return {
-            "s_ind": sent_ind,                                  # sentence index. (as Metadata)
-            "doc_id": news.ID,                                  # document index. (as Metadata)
-            "sentence": NewsParser.__get_sent(news, sent_ind),  # Required for special sources.
-        }

arekit/common/text/parser.py

@@ -1,12 +0,0 @@
-from arekit.common.pipeline.base import BasePipeline
-from arekit.common.text.parsed import BaseParsedText
-
-
-class BaseTextParser(BasePipeline):
-
-    def run(self, input_data, params_dict=None, parent_ctx=None):
-        output_data = super(BaseTextParser, self).run(input_data=input_data,
-                                                      params_dict=params_dict,
-                                                      parent_ctx=parent_ctx)
-
-        return BaseParsedText(terms=output_data)

arekit/common/text/partitioning/base.py

@@ -1,4 +0,0 @@
-class BasePartitioning(object):
-
-    def provide(self, text, parts_it):
-        raise NotImplementedError()

arekit/common/text/partitioning/terms.py

@@ -1,35 +0,0 @@
-import collections
-
-from arekit.common.bound import Bound
-from arekit.common.text.partitioning.base import BasePartitioning
-
-
-class TermsPartitioning(BasePartitioning):
-    """ NOTE: considering that provided parts
-        has no intersections between each other
-    """
-
-    def provide(self, text, parts_it):
-        assert(isinstance(text, list))
-        assert(isinstance(parts_it, collections.Iterable))
-
-        start = 0
-        parts = []
-        for value, bound in parts_it:
-            assert(isinstance(bound, Bound))
-            assert(bound.Position >= start)
-
-            # Release everythig till the current value position.
-            part = text[start:bound.Position]
-
-            parts.extend(part)
-
-            # Release the entity value.
-            parts.extend([value])
-
-            start = bound.Position + bound.Length
-
-        # Release everything after the last entity.
-        parts.extend(text[start:len(text)])
-
-        return parts

arekit/contrib/source/brat/annot.py

@@ -1,83 +0,0 @@
-from arekit.contrib.source.brat.entities.entity import BratEntity
-from arekit.contrib.source.brat.relation import BratRelation
-
-
-class BratAnnotationParser:
-
-    ENTITIES = "entities"
-    RELATIONS = "relations"
-
-    @staticmethod
-    def __non_prefixed_id(value):
-        assert (isinstance(value, str))
-        return value[1:]
-
-    @staticmethod
-    def handle_entity(args):
-        """ T2	Location 10 23	South America
-            T1	Location 0 5;16 23	North America
-        """
-        assert(len(args) == 3)
-
-        e_id = int(BratAnnotationParser.__non_prefixed_id(args[0]))
-        entity_params = args[1].split()
-
-        if len(entity_params) != 3:
-            # We do not support the case of a non-continuous entity mentions.
-            return None
-
-        e_str_type, e_begin, e_end = entity_params
-
-        return BratEntity(id_in_doc=e_id,
-                          e_type=e_str_type,
-                          index_begin=int(e_begin),
-                          index_end=int(e_end),
-                          value=args[2].strip())
-
-    @staticmethod
-    def handle_relation(args):
-        """ Example:
-            R1	Origin Arg1:T3 Arg2:T4
-        """
-
-        # Parse identifier index.
-        e_id = args[0][1:]
-
-        # Parse relation arguments.
-        rel_type, source, target = args[1].split()
-
-        source_id = source.split(':')[1]
-        target_id = target.split(':')[1]
-
-        return BratRelation(id_in_doc=e_id,
-                            source_id=int(BratAnnotationParser.__non_prefixed_id(source_id)),
-                            target_id=int(BratAnnotationParser.__non_prefixed_id(target_id)),
-                            rel_type=rel_type)
-
-    @staticmethod
-    def parse_annotations(input_file, encoding='utf-8'):
-        """ Read annotation collection from file
-        """
-        entities = []
-        relations = []
-
-        for line in input_file.readlines():
-            line = line.decode(encoding)
-
-            args = line.split('\t')
-
-            record_type = args[0][0]
-
-            # Entities (objects) are prefixed with `T`
-            if record_type == "T":
-                entity = BratAnnotationParser.handle_entity(args)
-                if entity is not None:
-                    entities.append(entity)
-
-            elif record_type == "R":
-                relations.append(BratAnnotationParser.handle_relation(args))
-
-        return {
-            BratAnnotationParser.ENTITIES: entities,
-            BratAnnotationParser.RELATIONS: relations
-        }

arekit/contrib/source/brat/entities/compound.py

@@ -1,33 +0,0 @@
-from arekit.contrib.source.brat.entities.entity import BratEntity
-
-
-class BratCompoundEntity(BratEntity):
-    """ Entity which contains the hierarchy of the other entities.
-    """
-
-    def __init__(self, id_in_doc, value, e_type, root, entities, index_begin, index_end,
-                 display_value=None, group_index=None):
-        assert(isinstance(entities, list))
-        assert(isinstance(root, BratCompoundEntity) or root is None)
-        super(BratCompoundEntity, self).__init__(value=value, e_type=e_type,
-                                                 id_in_doc=id_in_doc,
-                                                 index_begin=index_begin,
-                                                 index_end=index_end,
-                                                 display_value=display_value,
-                                                 group_index=group_index)
-        self.__entities = entities
-        self.__root = root
-
-    @classmethod
-    def from_list(cls, root, childs):
-        assert(isinstance(root, BratEntity))
-        assert(isinstance(childs, list) and len(childs) > 0)
-        return cls(id_in_doc=root.ID, value=root.Value, e_type=root.Type, root=None,
-                   entities=childs, index_begin=root.IndexBegin, index_end=root.IndexEnd)
-
-    @property
-    def Root(self):
-        return self.__root
-
-    def iter_childs(self):
-        return iter(self.__entities)

arekit/contrib/source/brat/entities/entity.py

@@ -1,42 +0,0 @@
-from arekit.common.entities.base import Entity
-
-
-class BratEntity(Entity):
-    """ Annotated entity in Brat-based collection corpus.
-        Provides bounds, i.e. char indices in related sentence.
-    """
-
-    def __init__(self, id_in_doc, e_type, index_begin, index_end, value, display_value=None, group_index=None):
-        """ index_begin: int
-                - char index (in case of string type of `text`)
-                - term index (in case of list type of `text`)
-            index_end: int
-                - char index (in case of string type of `text`)
-                - term index (in case of list type of `text`)
-        """
-        assert(isinstance(e_type, str))
-        assert(isinstance(index_begin, int))
-        assert(isinstance(index_end, int))
-        super(BratEntity, self).__init__(value=value, e_type=e_type,
-                                         display_value=display_value, group_index=group_index)
-
-        self.__e_type = e_type
-        self.__begin = index_begin
-        self.__end = index_end
-        self.__id = id_in_doc
-
-    @property
-    def IndexBegin(self):
-        return self.__begin
-
-    @property
-    def IndexEnd(self):
-        return self.__end
-
-    @property
-    def Type(self):
-        return self.__e_type
-
-    @property
-    def ID(self):
-        return self.__id