pyNlple 0.0.0__tar.gz

pynlple-0.0.0/MANIFEST.in

@@ -0,0 +1,2 @@
+include requirements.txt
+recursive-include pynlple *.txt

pynlple-0.0.0/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.1
+Name: pyNlple
+Version: 0.0.0
+Summary: NLP procedures in python brought to you by YouScan.
+Author-email: "YouScan Data Science team, NLP guild." <ds@youscan.io>
+License: MIT Licence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: emoji>=0.5.0
+Requires-Dist: numpy>=1.18
+Requires-Dist: pandas>=0.19.0
+Requires-Dist: requests>=2.32.3
+Requires-Dist: scikit-learn>=0.22
+
+Overview
+========
+pyNlple - a library with basic NLP utils. Also includes methods and APIs for data accessing/writing.
+
+Requirements
+============
+
+Cat `requirements.txt`
+
+Installation
+============
+
+Run setup.py to install / `pip install pynlple`
+
+Notes
+============
+- name inspired by https://www.youtube.com/watch?v=1W3sslyiUfg
+- yeah, we had "PYthon" and "NLP" procedures, so...
+- and yeah, try to read it as #pineapple http://www.veepy.com/wp-content/uploads/2015/05/1965.jpg
+
+## Licenses
+
+Install from https://pypi.org/project/pip-licenses/
+
+```
+pip install pip-licenses
+```
+
+Run 
+
+```
+make licenses
+```

pynlple-0.0.0/README.md

@@ -0,0 +1,33 @@
+Overview
+========
+pyNlple - a library with basic NLP utils. Also includes methods and APIs for data accessing/writing.
+
+Requirements
+============
+
+Cat `requirements.txt`
+
+Installation
+============
+
+Run setup.py to install / `pip install pynlple`
+
+Notes
+============
+- name inspired by https://www.youtube.com/watch?v=1W3sslyiUfg
+- yeah, we had "PYthon" and "NLP" procedures, so...
+- and yeah, try to read it as #pineapple http://www.veepy.com/wp-content/uploads/2015/05/1965.jpg
+
+## Licenses
+
+Install from https://pypi.org/project/pip-licenses/
+
+```
+pip install pip-licenses
+```
+
+Run 
+
+```
+make licenses
+```

pynlple-0.0.0/pyNlple.egg-info/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.1
+Name: pyNlple
+Version: 0.0.0
+Summary: NLP procedures in python brought to you by YouScan.
+Author-email: "YouScan Data Science team, NLP guild." <ds@youscan.io>
+License: MIT Licence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: emoji>=0.5.0
+Requires-Dist: numpy>=1.18
+Requires-Dist: pandas>=0.19.0
+Requires-Dist: requests>=2.32.3
+Requires-Dist: scikit-learn>=0.22
+
+Overview
+========
+pyNlple - a library with basic NLP utils. Also includes methods and APIs for data accessing/writing.
+
+Requirements
+============
+
+Cat `requirements.txt`
+
+Installation
+============
+
+Run setup.py to install / `pip install pynlple`
+
+Notes
+============
+- name inspired by https://www.youtube.com/watch?v=1W3sslyiUfg
+- yeah, we had "PYthon" and "NLP" procedures, so...
+- and yeah, try to read it as #pineapple http://www.veepy.com/wp-content/uploads/2015/05/1965.jpg
+
+## Licenses
+
+Install from https://pypi.org/project/pip-licenses/
+
+```
+pip install pip-licenses
+```
+
+Run 
+
+```
+make licenses
+```

pynlple-0.0.0/pyNlple.egg-info/SOURCES.txt

@@ -0,0 +1,48 @@
+MANIFEST.in
+README.md
+pyproject.toml
+requirements.txt
+pyNlple.egg-info/PKG-INFO
+pyNlple.egg-info/SOURCES.txt
+pyNlple.egg-info/dependency_links.txt
+pyNlple.egg-info/requires.txt
+pyNlple.egg-info/top_level.txt
+pynlple/__init__.py
+pynlple/exceptions.py
+pynlple/lime.py
+pynlple/module.py
+pynlple/utils.py
+pynlple/data/__init__.py
+pynlple/data/corpus.py
+pynlple/data/datasource.py
+pynlple/data/filesource.py
+pynlple/data/jsonsource.py
+pynlple/data/source.py
+pynlple/ml/__init__.py
+pynlple/ml/classifiers.py
+pynlple/ml/transformers.py
+pynlple/ml/vectorizers.py
+pynlple/ml/supervised/__init__.py
+pynlple/ml/supervised/feature_exploration.py
+pynlple/ml/supervised/training.py
+pynlple/processing/__init__.py
+pynlple/processing/dictionary.py
+pynlple/processing/emojis.py
+pynlple/processing/mention.py
+pynlple/processing/preprocessor.py
+pynlple/processing/stopwords.py
+pynlple/processing/text.py
+pynlple/processing/token.py
+pynlple/processing/data/currency.txt
+pynlple/processing/data/punctuation.txt
+pynlple/processing/data/special_symbols.txt
+pynlple/processing/data/special_tags.txt
+pynlple/processing/data/emoji/emoji_components.txt
+pynlple/processing/data/emoji/emojis.txt
+pynlple/processing/data/emoji/emojis_clustered.txt
+pynlple/processing/data/rus/pos/conjunctions.txt
+pynlple/processing/data/rus/pos/particles.txt
+pynlple/processing/data/rus/pos/prepositions.txt
+pynlple/processing/data/rus/pos/pronouns.txt
+pynlple/processing/data/ukr/stopwords.txt
+tests/test_utils.py

pynlple-0.0.0/pyNlple.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pynlple-0.0.0/pyNlple.egg-info/requires.txt

@@ -0,0 +1,5 @@
+emoji>=0.5.0
+numpy>=1.18
+pandas>=0.19.0
+requests>=2.32.3
+scikit-learn>=0.22

pynlple-0.0.0/pyNlple.egg-info/top_level.txt

@@ -0,0 +1 @@
+pynlple

pynlple-0.0.0/pynlple/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.10.1'

pynlple-0.0.0/pynlple/data/__init__.py

@@ -0,0 +1 @@
+

pynlple-0.0.0/pynlple/data/corpus.py

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+import bz2
+import gzip
+import io
+import logging
+
+from pandas import Series
+
+from pynlple.data.source import Source
+
+
+class StackingSource(Source):
+    logger = logging.getLogger(__name__)
+
+    def __init__(self, list_sources, log=False):
+        self.sources = list_sources
+        self.log = log
+
+    def __iter__(self):
+        # if self.log:
+        # self.logger.info('[%s] Corpus iterator %s started yielding elements.', str(self.__class__.__name__), repr(self))
+        for i_, source in enumerate(self.sources):
+            if self.log and i_ % self.log == 0:
+                self.logger.info('[%s] Corpus iterator started yielding elements from source (%d/%d): %s.',
+                                 str(self.__class__.__name__), i_, len(self.sources), repr(source))
+            for item in source:
+                yield item
+
+
+class DeduplicatingFixedCacheSizeSource(Source):
+    logger = logging.getLogger(__name__)
+
+    def __init__(self, source, ordered_dict_cache=None, cache_size=10000, refresh=False, feature_extractor=lambda x: x,
+                 log=100000):
+        self.source = source
+        self.ordered_dict = ordered_dict_cache
+        self.cache_size = cache_size
+        self.refresh = refresh
+        self.feature_extractor = feature_extractor
+        self.log = log
+
+    @staticmethod
+    def __prop_dict(dict_):
+        for k, v in dict_.items():
+            dict_[k] = v + 1
+
+    def __iter__(self):
+        if self.ordered_dict is None:
+            from collections import OrderedDict
+            self.__cache = OrderedDict()
+        else:
+            self.__cache = self.ordered_dict
+        for i, entry in enumerate(self.source):
+            if self.log and i % self.log == 0:
+                stats = Series(list(self.__cache.values()))
+                self.logger.info('[%s] Lifetime stats on iter %d: %s',
+                                 str(self.__class__.__name__), i, repr(stats.describe()))
+
+            f_entry = self.feature_extractor(entry)
+            if f_entry in self.__cache:
+                self.__prop_dict(self.__cache)
+                if self.refresh:
+                    self.__cache.move_to_end(f_entry)
+            else:
+                self.__cache.__setitem__(f_entry, 0)
+                if self.cache_size is not None:
+                    while len(self.__cache) > self.cache_size:
+                        self.__cache.popitem(last=False)
+                yield entry
+
+
+class JsonFieldSource(Source):
+
+    def __init__(self, json_source, key, default=None):
+        self.json = json_source
+        self.key = key
+        self.default = default
+
+    def __iter__(self):
+        for json_entry in self.json:
+            if self.key in json_entry:
+                yield json_entry[self.key]
+            else:
+                yield self.default
+
+
+class FilteringSource(Source):
+
+    def __init__(self, source, condition):
+        self.source = source
+        self.condition = condition
+
+    def __iter__(self):
+        for entry in self.source:
+            if self.condition(entry):
+                yield entry
+
+
+class MappingSource(Source):
+
+    def __init__(self, source, function):
+        self.source = source
+        self.function = function
+
+    def __iter__(self):
+        for entry in self.source:
+            yield self.function(entry)
+
+
+class SplittingSource(Source):
+
+    def __init__(self, source, splitting_function):
+        self.source = source
+        self.function = splitting_function
+
+    def __iter__(self):
+        for entry in self.source:
+            for item in self.function(entry):
+                yield item
+
+
+class FileLineSource(Source):
+
+    def __init__(self, text_file_path, encoding='utf8'):
+        self.source_file = text_file_path
+        self.encoding = encoding
+
+    def __iter__(self):
+        with io.open(self.source_file, mode='rt', encoding=self.encoding) as in_file:
+            for line in in_file:
+                line = line.strip()
+                if len(line) <= 0:
+                    continue
+                yield line
+
+
+class OpensubtitlesSentenceSource(Source):
+    DEFAULT_SENTENCE_TAG = '<s>'
+
+    def __init__(self, line_source, sentence_tag=None):
+        self.source = line_source
+        if sentence_tag:
+            self.sentence_tag = sentence_tag
+        else:
+            self.sentence_tag = OpensubtitlesSentenceSource.DEFAULT_SENTENCE_TAG
+
+    def __iter__(self):
+        for line in self.source:
+            for sentence in line.split(self.sentence_tag):
+                yield sentence.strip()
+
+
+class BZipDocumentSource(Source):
+
+    def __init__(self, bzip_filepath, text_preprocessor=None):
+        self.source_filepath = bzip_filepath
+        self.text_preprocessor = text_preprocessor
+        super().__init__()
+
+    def __iter__(self):
+        with bz2.BZ2File(self.source_filepath, 'rtU') as in_bz:
+            for line in in_bz:
+                text = line
+                if self.text_preprocessor:
+                    text = self.text_preprocessor.preprocess(text)
+                tokens = text.split()
+                yield tokens
+
+
+class GZipDocumentSource(Source):
+
+    def __init__(self, gzip_filepath, text_preprocessor=None):
+        self.source_filepath = gzip_filepath
+        self.text_preprocessor = text_preprocessor
+        super().__init__()
+
+    def __iter__(self):
+        with gzip.GzipFile(self.source_filepath, 'rU') as in_bz:
+            for line in in_bz:
+                text = line
+                if self.text_preprocessor:
+                    text = self.text_preprocessor.preprocess(text)
+                tokens = text.split()
+                yield tokens

pynlple-0.0.0/pynlple/data/datasource.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+import io
+import json
+import logging
+
+from pandas import DataFrame
+from pandas import read_csv, read_json
+
+from pynlple.data.jsonsource import FileJsonDataSource
+from pynlple.data.source import Source
+
+logger = logging.getLogger(__file__)
+
+
+class DataframeSource(Source):
+
+    def __init__(self, dataframe):
+        self.dataframe = dataframe
+
+    def get_dataframe(self):
+        return self.dataframe
+
+    def set_dataframe(self, dataframe):
+        self.dataframe = dataframe
+
+
+class TsvDataframeSource(Source):
+
+    def __init__(self, dataframe_path, separator='\t', quote=0, escape_char='\\', column_names=None,
+                 index_column_names=None, fill_na_map=None, encoding='utf-8', index_columns=None):
+        self.path = dataframe_path
+        self.separator = separator
+        self.column_names = column_names
+        self.na_map = fill_na_map
+        self.encoding = encoding
+        self.index_columns = index_columns
+        self.index_column_names = index_column_names
+        self.quote = quote
+        self.escape_char = escape_char
+
+    def get_dataframe(self):
+        # TODO: Eats \r\n and spits sole \n in literal value strings instead
+        if self.column_names:
+            header = None
+            names = self.column_names
+        else:
+            header = 'infer'
+            names = None
+
+        dataframe = read_csv(self.path,
+                             sep=self.separator,
+                             header=header,
+                             names=names,
+                             quoting=self.quote,
+                             escapechar=self.escape_char,
+                             encoding=self.encoding)
+        if self.index_columns:
+            dataframe.set_index(keys=self.index_columns, inplace=True)
+        if self.na_map:
+            for key, value in self.na_map.items():
+                dataframe[key].fillna(value, inplace=True)
+        logger.debug('Read: {} rows from {}'.format(str(len(dataframe.index)), self.path))
+        return dataframe
+
+    def set_dataframe(self, dataframe):
+        if self.column_names:
+            names = self.column_names
+        else:
+            names = True
+
+        if self.index_column_names:
+            include_index = True
+            index_names = self.index_column_names
+        else:
+            include_index = False
+            index_names = None
+
+        dataframe.to_csv(self.path,
+                         sep=self.separator,
+                         header=names,
+                         index=include_index,
+                         index_label=index_names,
+                         quoting=self.quote,
+                         escapechar=self.escape_char,
+                         encoding=self.encoding)
+        logger.debug('Written: {} rows from {}'.format(str(len(dataframe.index)), self.path))
+
+
+class JsonFileDataframeSource(Source):
+    FILE_READ_METHOD = 'rt'
+    FILE_WRITE_METHOD = 'wt'
+    DEFAULT_ORIENT = 'records'
+    DEFAULT_ENCODING = 'utf-8'
+
+    def __init__(self, json_file_path, fill_na_map=None, index_columns=None,
+                 encoding=DEFAULT_ENCODING, orient=DEFAULT_ORIENT):
+        self.json_file_path = json_file_path
+        self.na_map = fill_na_map
+        self.index_columns = index_columns
+        self.encoding = encoding
+        self.orient = orient
+
+    def get_dataframe(self):
+        with io.open(self.json_file_path, JsonFileDataframeSource.FILE_READ_METHOD,
+                     encoding=self.encoding) as data_file:
+            # TODO: implement fill_na_map
+            df = read_json(data_file, orient=self.orient, encoding=JsonFileDataframeSource.DEFAULT_ENCODING)
+        return df
+
+    def set_dataframe(self, dataframe):
+        with io.open(self.json_file_path, JsonFileDataframeSource.FILE_WRITE_METHOD,
+                     encoding=self.encoding) as data_file:
+            # TODO: implement fill_na_map
+            json.dump(dataframe.reset_index().to_dict(orient=self.orient), data_file, ensure_ascii=False, indent=1)
+
+
+class JsonNullableFileDataframeSource(Source):
+    DEFAULT_ENCODING = 'utf-8'
+
+    def __init__(self, json_file_path, keys=None, fill_na_map=None, index_columns=None,
+                 encoding=DEFAULT_ENCODING):
+        self.__source = JsonDataframeSource(FileJsonDataSource(file_path=json_file_path, encoding_str=encoding),
+                                            keys=keys, fill_na_map=fill_na_map,
+                                            index_columns=index_columns)
+
+    def get_dataframe(self):
+        return self.__source.get_dataframe()
+
+    def set_dataframe(self, dataframe):
+        self.__source.set_dataframe(dataframe)
+
+
+class JsonDataframeSource(Source):
+
+    def __init__(self, json_source, keys=None, fill_na_map=None, index_columns=None):
+        self.json_source = json_source
+        self.keys = keys
+        self.na_map = fill_na_map
+        self.index_columns = index_columns
+
+    def get_dataframe(self):
+        extracted_entries = list()
+        for json_object in self.json_source.get_data():
+            entry = dict()
+            if self.keys:
+                for key in self.keys:
+                    if key not in json_object:
+                        entry[key] = self.na_map[key]
+                    else:
+                        entry[key] = json_object[key]
+            else:
+                for key in json_object:
+                    entry[key] = json_object[key]
+                if self.na_map:
+                    for key, value in self.na_map:
+                        if key not in entry:
+                            entry[key] = value
+            extracted_entries.append(entry)
+        dataframe = DataFrame(extracted_entries)
+        if self.index_columns:
+            dataframe.set_index(keys=self.index_columns, inplace=True)
+        if self.na_map:
+            for key, value in self.na_map.items():
+                dataframe.loc[:, key].fillna(value, inplace=True)
+        logger.debug('Read: {} rows from {}'.format(str(len(dataframe.index)), repr(self.json_source)))
+        return dataframe
+
+    def set_dataframe(self, dataframe):
+        entries = dataframe.reset_index().to_dict(orient='records')
+        for entry in entries:
+            if self.keys:
+                for key in list(entry.keys()):
+                    if key not in self.keys:
+                        entry.pop(key, None)
+            if self.na_map:
+                for key, value in self.na_map:
+                    if key not in entry:
+                        entry[key] = value
+        self.json_source.set_data(entries)

pynlple-0.0.0/pynlple/data/filesource.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+from pynlple.data.source import Source
+from pynlple.exceptions import DataSourceException
+from pynlple.module import is_folder, is_file, list_dir, append_paths
+
+
+class FilePathSource(Source):
+    """Class for providing filepaths from data folders."""
+
+    def __init__(self, paths, extension_suffix=None):
+        self.paths = paths
+        self.extension = extension_suffix
+
+    def get_files(self):
+        accumulated_paths = list()
+        for path in self.paths:
+            if is_file(path):
+                accumulated_paths.append(path)
+            elif is_folder(path):
+                files = list_dir(path)
+                if self.extension:
+                    for file in filter(lambda f: f.endswith(self.extension), files):
+                        accumulated_paths.append(append_paths(path, file))
+                else:
+                    for file in files:
+                        accumulated_paths.append(append_paths(path, file))
+            else:
+                raise DataSourceException('Path {0} does not exist/is neither file nor folder!'.format(path))
+        return accumulated_paths
+
+    def __iter__(self):
+        for file_ in self.get_files():
+            yield file_