arekit 0.24.0__py3-none-any.whl

arekit/common/pipeline/items/map.py

@@ -0,0 +1,11 @@
+from arekit.common.pipeline.items.base import BasePipelineItem
+
+
+class MapPipelineItem(BasePipelineItem):
+
+    def __init__(self, map_func=None):
+        assert(callable(map_func))
+        self._map_func = map_func
+
+    def apply_core(self, input_data, pipeline_ctx):
+        return map(self._map_func, input_data)

arekit/common/pipeline/items/map_nested.py

@@ -0,0 +1,13 @@
+from arekit.common.pipeline.items.map import MapPipelineItem
+
+
+class MapNestedPipelineItem(MapPipelineItem):
+    """ This type is considered for describing nested pipelines,
+        which might be required in parameters of the parent pipeline-contexts.
+
+        Data treated as a sequence, in which every element is
+        suppose to be mapped with the passed pipeline context.
+    """
+
+    def apply_core(self, input_data, pipeline_ctx):
+        return map(lambda item: self._map_func(item, pipeline_ctx), input_data)

arekit/common/synonyms/base.py

@@ -0,0 +1,151 @@
+from collections.abc import Iterable
+
+from arekit.common import log_utils
+
+
+class SynonymsCollection(object):
+
+    def __init__(self, iter_group_values_lists=None, is_read_only=True, debug=False):
+        """ iter_group_values_lists: iterable or None
+            is_read_only: bool
+                whether the relation collection could be expanded or not
+            debug: bool
+                utilized for logging the salient information during usage.
+        """
+        assert(isinstance(iter_group_values_lists, Iterable) or iter_group_values_lists is None)
+        assert(isinstance(is_read_only, bool))
+        assert(isinstance(debug, bool))
+
+        # Assumes to be filled
+        self.__by_sid = {}
+        self.__by_index = []
+
+        self.__is_read_only = is_read_only
+        self.__debug = debug
+        self.__fill(iter_group_values_lists=[] if iter_group_values_lists is None else iter_group_values_lists)
+
+    # region properties
+
+    @property
+    def IsReadOnly(self):
+        return self.__is_read_only
+
+    # endregion
+
+    # region public 'add' methods
+
+    def add_synonym_value(self, value):
+        assert(isinstance(value, str))
+
+        if self.__contains_synonym_value(value):
+            raise Exception(("Collection already contains synonyms '{}'".format(value)).encode('utf-8'))
+
+        if self.__is_read_only:
+            raise Exception(("Failed to add '{}'. Synonym collection is read only!".format(value)).encode('utf-8'))
+
+        sid = self._create_external_sid(value)
+        self.__by_sid[sid] = self.__get_groups_count()
+        self.__by_index.append([value])
+
+    # endregion
+
+    # region public 'contains' methods
+
+    def contains_synonym_value(self, value):
+        return self.__contains_synonym_value(value)
+
+    # endregion
+
+    # region public 'get' methods
+
+    def get_synonym_group_index(self, value):
+        """ NOTE: Before use this, please take a look at the grouping (see #327 issue).
+            It is better to use that class API rather than pass that method for `value_to_group_id_func`
+        """
+        assert(isinstance(value, str))
+        return self.__get_group_index(value)
+
+    # endregion
+
+    # region public 'create' methods
+
+    def create_synonym_id(self, value):
+        return self._create_external_sid(value)
+
+    # endregion
+
+    # region protected methods
+
+    def _contains_sid(self, v_id):
+        return v_id in self.__by_sid
+
+    def _create_internal_sid(self, value):
+        """ Utilized during filling stage.
+        """
+        raise NotImplementedError()
+
+    def _create_external_sid(self, value):
+        raise NotImplementedError()
+
+    # endregion
+
+    # region public 'iter' methods
+
+    def iter_synonym_values(self, value):
+        assert(isinstance(value, str))
+        sid = self._create_external_sid(value)
+        index = self.__by_sid[sid]
+        return iter(self.__by_index[index])
+
+    def iter_by_index(self):
+        return iter(self.__by_index)
+
+    def iter_group(self, group_index):
+        assert(isinstance(group_index, int))
+        return iter(self.__by_index[group_index])
+
+    # endregion
+
+    # region private methods
+
+    def __fill(self, iter_group_values_lists):
+        for group in iter_group_values_lists:
+            self.__process_group(group)
+
+    def __process_group(self, group_values_list):
+        group_index = len(self.__by_index)
+        synonym_list = []
+
+        for synonym_value in group_values_list:
+
+            value = synonym_value.strip()
+
+            sid = self._create_internal_sid(value)
+
+            if self._contains_sid(sid) and self.__debug:
+                log_utils.log_synonym_existed(value)
+                continue
+
+            synonym_list.append(value)
+            self.__by_sid[sid] = group_index
+
+        self.__by_index.append(synonym_list)
+
+    def __get_groups_count(self):
+        return len(self.__by_index)
+
+    def __get_group_index(self, value):
+        sid = self._create_external_sid(value)
+        return self.__by_sid[sid]
+
+    def __contains_synonym_value(self, value):
+        return self._contains_sid(self._create_external_sid(value))
+
+    # endregion
+
+    # region overridden methods
+
+    def __len__(self):
+        return len(self.__by_index)
+
+    # endregion

arekit/common/synonyms/grouping.py

@@ -0,0 +1,21 @@
+from arekit.common.synonyms.base import SynonymsCollection
+
+
+class SynonymsCollectionValuesGroupingProviders:
+    """ Providers for the grouping.
+    """
+
+    @staticmethod
+    def provide_existed_or_register_missed_value(synonyms, value):
+        """ grouping with a potential expansion.
+        """
+        assert(isinstance(synonyms, SynonymsCollection))
+        if not synonyms.contains_synonym_value(value):
+            synonyms.add_synonym_value(value)
+        return synonyms.get_synonym_group_index(value)
+
+    @staticmethod
+    def provide_existed_value(synonyms, value):
+        """ grouping by using only existed value.
+        """
+        return synonyms.get_synonym_group_index(value)

arekit/common/text/enums.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+
+class TermFormat(Enum):
+    """
+    Supported types of terms
+    """
+
+    """
+    Original value
+    """
+    Raw = 1

arekit/common/text/parsed.py

@@ -0,0 +1,42 @@
+from arekit.common.text.enums import TermFormat
+
+
+class BaseParsedText(object):
+    """
+    Represents a processed text with extra parameters
+    that were used during parsing.
+    """
+
+    # region constructors
+
+    def __init__(self, terms):
+        assert(isinstance(terms, list))
+        self._terms = terms
+
+    # endregion
+
+    def get_term(self, index, term_format):
+        assert(isinstance(term_format, TermFormat))
+        terms = self._get_terms(term_format)
+        return terms[index]
+
+    def iter_terms(self, term_format, filter=None):
+        assert(isinstance(term_format, TermFormat))
+        assert(callable(filter) or filter is None)
+        terms = self._get_terms(term_format)
+        for term in terms:
+            if filter is not None and not list(filter(term)):
+                continue
+            yield term
+
+    # region private methods
+
+    def _get_terms(self, term_format):
+        assert(isinstance(term_format, TermFormat))
+        assert(term_format == TermFormat.Raw)
+        return self._terms
+
+    # endregion
+
+    def __len__(self):
+        return len(self._terms)

arekit/common/text/parser.py

@@ -0,0 +1,12 @@
+from arekit.common.pipeline.base import BasePipeline
+from arekit.common.text.parsed import BaseParsedText
+
+
+class BaseTextParser(BasePipeline):
+
+    def run(self, input_data, params_dict=None, parent_ctx=None):
+        output_data = super(BaseTextParser, self).run(input_data=input_data,
+                                                      params_dict=params_dict,
+                                                      parent_ctx=parent_ctx)
+
+        return BaseParsedText(terms=output_data)

arekit/common/text/partitioning/base.py

@@ -0,0 +1,4 @@
+class BasePartitioning(object):
+
+    def provide(self, text, parts_it):
+        raise NotImplementedError()

arekit/common/text/partitioning/str.py

@@ -0,0 +1,36 @@
+from collections.abc import Iterable
+
+from arekit.common.bound import Bound
+from arekit.common.text.partitioning.base import BasePartitioning
+
+
+class StringPartitioning(BasePartitioning):
+    """ NOTE: considering that provided parts
+        has no intersections between each other
+    """
+
+    def provide(self, text, parts_it):
+        assert(isinstance(text, str))
+        assert(isinstance(parts_it, Iterable))
+
+        start = 0
+        parts = []
+        for value, bound in parts_it:
+            assert(isinstance(bound, Bound))
+            assert(bound.Position >= start)
+
+            # Release everything till the current value position.
+            part = text[start:bound.Position]
+
+            parts.append(part)
+
+            # Release the entity value.
+            parts.extend([value])
+
+            start = bound.Position + bound.Length
+
+        # Release everything after the last entity.
+        last_part = text[start:len(text)]
+        parts.extend([last_part])
+
+        return parts

arekit/common/text/partitioning/terms.py

@@ -0,0 +1,35 @@
+from collections.abc import Iterable
+
+from arekit.common.bound import Bound
+from arekit.common.text.partitioning.base import BasePartitioning
+
+
+class TermsPartitioning(BasePartitioning):
+    """ NOTE: considering that provided parts
+        has no intersections between each other
+    """
+
+    def provide(self, text, parts_it):
+        assert(isinstance(text, list))
+        assert(isinstance(parts_it, Iterable))
+
+        start = 0
+        parts = []
+        for value, bound in parts_it:
+            assert(isinstance(bound, Bound))
+            assert(bound.Position >= start)
+
+            # Release everythig till the current value position.
+            part = text[start:bound.Position]
+
+            parts.extend(part)
+
+            # Release the entity value.
+            parts.extend([value])
+
+            start = bound.Position + bound.Length
+
+        # Release everything after the last entity.
+        parts.extend(text[start:len(text)])
+
+        return parts

arekit/common/text/stemmer.py

@@ -0,0 +1,16 @@
+class Stemmer:
+    """
+    Interface
+    """
+
+    def lemmatize_to_list(self, text):
+        raise NotImplementedError()
+
+    def lemmatize_to_str(self, text):
+        raise NotImplementedError()
+
+    def is_adjective(self, pos_type):
+        raise NotImplementedError()
+
+    def is_noun(self, pos_type):
+        raise NotImplementedError()

arekit/common/text_opinions/base.py

@@ -0,0 +1,105 @@
+from arekit.common.labels.base import Label
+
+
+class TextOpinion(object):
+    """
+    Represents a relation which were found in doc article
+    and composed between two named entities
+        (it was found especially by Opinion with predefined label)
+        allows to modify label using set_label
+
+    NOTE: it is important to keep document level IDs. (designed for that)
+    """
+
+    # region constructors
+
+    def __init__(self, doc_id, text_opinion_id, source_id, target_id, label):
+        """ source_id: document level object id
+            target_id: document level object id
+        """
+        self.__doc_id = doc_id
+        self.__source_id = source_id
+        self.__target_id = target_id
+        self.__text_opinion_id = text_opinion_id
+        self.__modifiable_label = None
+        self.__set_label_core(label)
+
+    @classmethod
+    def create_copy(cls, other, keep_text_opinion_id=True):
+        assert(isinstance(other, TextOpinion))
+        assert(isinstance(keep_text_opinion_id, bool))
+        return cls.__try_create_copy_core(other=other, keep_text_opinion_id=keep_text_opinion_id)
+
+    @staticmethod
+    def try_convert(other, convert_entity_id_func):
+        """ Creates a copy of `other` opinion with different id of opinion participants.
+            Use cases: required for BaseParsedDocumentServiceProvider, when we decided to bring the outside
+            opinion into one which is based on DocumentEntities.
+        """
+        assert(isinstance(other, TextOpinion))
+        assert(callable(convert_entity_id_func))
+        return TextOpinion.__try_create_copy_core(other=other,
+                                                  convert_entity_id_func=convert_entity_id_func,
+                                                  keep_text_opinion_id=False)
+
+    @staticmethod
+    def __try_create_copy_core(other, convert_entity_id_func=lambda part_id: part_id, keep_text_opinion_id=True):
+        """ Tries to compose a copy by considering an optional id conversion,
+            and identification keeping.
+            convert_id:
+                func(id) -> id
+        """
+        assert(callable(convert_entity_id_func))
+
+        source_id = convert_entity_id_func(other.SourceId)
+        target_id = convert_entity_id_func(other.TargetId)
+
+        if source_id is None or target_id is None:
+            return None
+
+        return TextOpinion(doc_id=other.__doc_id,
+                           text_opinion_id=other.__text_opinion_id if keep_text_opinion_id else None,
+                           source_id=source_id,
+                           target_id=target_id,
+                           label=other.Label)
+
+    def __set_label_core(self, label):
+        assert(isinstance(label, Label))
+        self.__modifiable_label = label
+
+    # endregion
+
+    # region properties
+
+    @property
+    def Label(self):
+        return self.__modifiable_label
+
+    @property
+    def DocID(self):
+        return self.__doc_id
+
+    @property
+    def TextOpinionID(self):
+        return self.__text_opinion_id
+
+    @property
+    def SourceId(self):
+        return self.__source_id
+
+    @property
+    def TargetId(self):
+        return self.__target_id
+
+    # endregion
+
+    # region public methods
+
+    def set_text_opinion_id(self, text_opinion_id):
+        assert(self.__text_opinion_id is None)
+        self.__text_opinion_id = text_opinion_id
+
+    def set_label(self, label):
+        self.__set_label_core(label)
+
+    # endregion

arekit/common/utils.py

@@ -0,0 +1,129 @@
+import sys
+import os
+import requests
+from tqdm import tqdm
+
+
+def create_dir_if_not_exists(filepath):
+    dir = os.path.dirname(filepath)
+
+    # Check whether string is empty.
+    if not dir:
+        return
+
+    if not os.path.exists(dir):
+        os.makedirs(dir)
+
+
+def filter_whitespaces(terms):
+    return [term.strip() for term in terms if term.strip()]
+
+
+def split_by_whitespaces(text):
+    """
+    Assumes to perform a word separation including a variety of space entries.
+    In terms of the latter we consider any whitespace separator.
+    """
+    assert(isinstance(text, str))
+    return text.split()
+
+
+def progress_bar(iterable, total, desc="", unit="it"):
+    if total is not None:
+        return progress_bar_defined(iterable=iterable, total=total, desc=desc, unit=unit)
+    else:
+        return progress_bar_iter(iterable=iterable, desc=desc, unit=unit)
+
+
+def progress_bar_conditional(iterable, condition_func, total, postfix_func=None, desc="", unit="it"):
+    """ This progress-bar updates only on the
+        specific conditions during the iteration process.
+    """
+    assert(callable(condition_func))
+    assert(callable(postfix_func) or postfix_func is None)
+
+    # We consider artificial function that always iters 0.
+    def __iter_infinite_placeholder():
+        while True:
+            yield 0
+
+    pbar_it = progress_bar(iterable=__iter_infinite_placeholder(),
+                           desc=desc, unit=unit, total=total)
+    element = iter(pbar_it)
+
+    # Initialize with 0.
+    next(element)
+
+    for item in iterable:
+
+        # Optionally Update progress bar with the next state.
+        if condition_func(item):
+            next(element)
+            yield item
+
+        # Optionally provide meta-information.
+        if postfix_func is not None:
+            pbar_it.set_postfix(postfix_func(item))
+
+
+def progress_bar_defined(iterable, total, miniters=200, desc="", unit="it"):
+    return tqdm(iterable=iterable,
+                total=total,
+                desc=desc,
+                ncols=120,
+                position=0,
+                leave=True,
+                unit=unit,
+                miniters=total / miniters if total is not None else total)
+
+
+def progress_bar_iter(iterable, desc="", unit='it'):
+    return tqdm(iterable=iterable,
+                desc=desc,
+                position=0,
+                leave=True,
+                ncols=120,
+                unit=unit)
+
+
+def get_default_download_dir():
+    """ Refered to NLTK toolkit approach
+        https://github.com/nltk/nltk/blob/8e771679cee1b4a9540633cc3ea17f4421ffd6c0/nltk/downloader.py#L1051
+    """
+
+    # On Windows, use %APPDATA%
+    if sys.platform == "win32" and "APPDATA" in os.environ:
+        homedir = os.environ["APPDATA"]
+
+    # Otherwise, install in the user's home directory.
+    else:
+        homedir = os.path.expanduser("~/")
+        if homedir == "~/":
+            raise ValueError("Could not find a default download directory")
+
+    return os.path.join(homedir, ".arekit")
+
+
+def download(dest_file_path, source_url):
+    """ Refered to https://github.com/nicolay-r/ner-bilstm-crf-tensorflow/blob/master/ner/utils.py
+        Simple http file downloader
+    """
+    print(('Downloading from {src} to {dest}'.format(src=source_url, dest=dest_file_path)))
+
+    sys.stdout.flush()
+    datapath = os.path.dirname(dest_file_path)
+
+    if not os.path.exists(datapath):
+        os.makedirs(datapath, mode=0o755)
+
+    dest_file_path = os.path.abspath(dest_file_path)
+
+    r = requests.get(source_url, stream=True)
+    total_length = int(r.headers.get('content-length', 0))
+
+    with open(dest_file_path, 'wb') as f:
+        pbar = tqdm(total=total_length, unit='B', unit_scale=True)
+        for chunk in r.iter_content(chunk_size=32 * 1024):
+            if chunk:  # filter out keep-alive new chunks
+                pbar.update(len(chunk))
+                f.write(chunk)

arekit/contrib/bert/input/providers/cropped_sample.py

@@ -0,0 +1,17 @@
+from arekit.common.data.input.providers.sample.cropped import CroppedSampleRowProvider
+from arekit.common.data.input.providers.text.single import BaseSingleTextProvider
+from arekit.contrib.bert.input.providers.text_pair import PairTextProvider
+
+
+class CroppedBertSampleRowProvider(CroppedSampleRowProvider):
+
+    def __init__(self, crop_window_size, label_scaler, text_terms_mapper, text_b_template):
+
+        text_provider = BaseSingleTextProvider(text_terms_mapper=text_terms_mapper) \
+            if text_b_template is None else PairTextProvider(text_b_prompt=text_b_template,
+                                                             text_terms_mapper=text_terms_mapper)
+
+        super(CroppedBertSampleRowProvider, self).__init__(
+            crop_window_size=crop_window_size,
+            label_scaler=label_scaler,
+            text_provider=text_provider)