py-stringmatching 0.1.0__zip

py_stringmatching-0.1.0/py_stringmatching/tokenizer/whitespace_tokenizer.py

@@ -0,0 +1,57 @@
+from py_stringmatching import utils
+from py_stringmatching.tokenizer.delimiter_tokenizer import DelimiterTokenizer
+
+
+class WhitespaceTokenizer(DelimiterTokenizer):
+    """Segments the input string using whitespaces then returns the segments as tokens. 
+    
+    Currently using the split function in Python, so whitespace character refers to 
+    the actual whitespace character as well as the tab and newline characters. 
+
+    Args:
+        return_set (boolean): A flag to indicate whether to return a set of
+                              tokens instead of a bag of tokens (defaults to False).
+                              
+    Attributes:
+        return_set (boolean): An attribute to store the flag return_set. 
+    """
+    
+    def __init__(self, return_set=False):
+        super(WhitespaceTokenizer, self).__init__([' ', '\t', '\n'], return_set)
+
+    def tokenize(self, input_string):
+        """Tokenizes input string based on white space.
+
+        Args:
+            input_string (str): The string to be tokenized. 
+
+        Returns:
+            A Python list, which is a set or a bag of tokens, depending on whether return_set is True or False. 
+
+        Raises:
+            TypeError : If the input is not a string.
+
+        Examples:
+            >>> ws_tok = WhitespaceTokenizer() 
+            >>> ws_tok.tokenize('data science')
+            ['data', 'science']
+            >>> ws_tok.tokenize('data        science')
+            ['data', 'science']
+            >>> ws_tok.tokenize('data\tscience')
+            ['data', 'science']
+            >>> ws_tok = WhitespaceTokenizer(return_set=True) 
+            >>> ws_tok.tokenize('data   science data integration')
+            ['data', 'science', 'integration']
+        """
+        utils.tok_check_for_none(input_string)
+        utils.tok_check_for_string_input(input_string)
+
+        token_list =  list(filter(None, input_string.split()))
+
+        if self.return_set:
+            return utils.convert_bag_to_set(token_list)
+
+        return token_list
+
+    def set_delim_set(self, delim_set):
+        raise AttributeError('Delimiters cannot be set for WhitespaceTokenizer')

py_stringmatching-0.1.0/py_stringmatching/utils.py

@@ -0,0 +1,70 @@
+import functools
+
+import six
+
+"""
+This module defines a list of utility and validation functions.
+"""
+
+
+def sim_check_for_none(*args):
+    if len(args) > 0 and args[0] is None:
+        raise TypeError("First argument cannot be None")
+    if len(args) > 1 and args[1] is None:
+        raise TypeError("Second argument cannot be None")
+
+
+def sim_check_for_empty(*args):
+    if len(args[0]) == 0 or len(args[1]) == 0:
+        return True
+
+
+def sim_check_for_same_len(*args):
+    if len(args[0]) != len(args[1]):
+        raise ValueError("Undefined for sequences of unequal length")
+
+
+def sim_check_for_string_inputs(*args):
+    if not isinstance(args[0], six.string_types):
+        raise TypeError('First argument is expected to be a string')
+    if not isinstance(args[1], six.string_types):
+        raise TypeError('Second argument is expected to be a string')
+
+
+def sim_check_for_list_or_set_inputs(*args):
+    if not isinstance(args[0], list):
+        if not isinstance(args[0], set):
+            raise TypeError('First argument is expected to be a python list or set')
+    if not isinstance(args[1], list):
+        if not isinstance(args[1], set):
+            raise TypeError('Second argument is expected to be a python list or set')
+
+
+
+
+def sim_check_for_exact_match(*args):
+    if args[0] == args[1]:
+        return True
+
+
+
+
+def tok_check_for_string_input(*args):
+    for i in range(len(args)):
+        if not isinstance(args[i], six.string_types):
+            raise TypeError('Input is expected to be a string')
+
+
+def tok_check_for_none(*args):
+    if args[0] is None:
+        raise TypeError("First argument cannot be None")
+
+
+def convert_bag_to_set(input_list):
+    seen_tokens = {}
+    output_set =[]
+    for token in input_list:
+        if seen_tokens.get(token) == None:
+            output_set.append(token)
+            seen_tokens[token] = True
+    return output_set 

py_stringmatching-0.1.0/py_stringmatching.egg-info/PKG-INFO

@@ -0,0 +1,57 @@
+Metadata-Version: 1.1
+Name: py-stringmatching
+Version: 0.1.0
+Summary: Python library for string matching.
+Home-page: https://sites.google.com/site/anhaidgroup/projects/py_stringmatching
+Author: UW Magellan Team
+Author-email: uwmagellan@gmail.com
+License: BSD
+Description: py_stringmatching
+        =================
+        
+        py_stringmatching is a software package in Python that consists of a comprehensive set of tokenizers and string similarity measures (
+        such as edit distance, Jaccard, and TF/IDF). It is free, open-source, and BSD-licensed.
+        
+        Important links
+        ===============
+        
+         * Repository: https://github.com/anhaidgroup/py_stringmatching
+         * Documentation: http://anhaidgroup.github.io/py_stringmatching/v0.1.x/index.html 
+         * Tutorial: http://anhaidgroup.github.io/py_stringmatching/v0.1.x/Tutorial.html
+         * Issue Tracker: https://github.com/anhaidgroup/py_stringmatching/issues
+         * Mailing List: https://groups.google.com/forum/#!forum/py_stringmatching
+         
+        Dependencies
+        ============
+        
+        py_stringmatching is tested to work under Python 2.7, Python 3.3, Python 3.4 and Python 3.5.
+        
+        The required dependencies to build the package are NumPy >= 1.7.0,
+        Six and a working C/C++ compiler. For the development version, you will also require Cython.
+        
+        Platforms
+        =========
+        
+        py_stringmatching has been tested on Linux, OSX and Windows.
+        
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: Unix
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Utilities
+Classifier: Topic :: Software Development :: Libraries

py_stringmatching-0.1.0/py_stringmatching.egg-info/SOURCES.txt

@@ -0,0 +1,48 @@
+AUTHORS.rst
+CHANGES.txt
+LICENSE
+MANIFEST.in
+README.rst
+requirements.txt
+setup.py
+LICENSES/NUMPY_LICENSE
+LICENSES/SIX_LICENSE
+py_stringmatching/__init__.py
+py_stringmatching/utils.py
+py_stringmatching.egg-info/PKG-INFO
+py_stringmatching.egg-info/SOURCES.txt
+py_stringmatching.egg-info/dependency_links.txt
+py_stringmatching.egg-info/not-zip-safe
+py_stringmatching.egg-info/requires.txt
+py_stringmatching.egg-info/top_level.txt
+py_stringmatching/similarity_measure/__init__.py
+py_stringmatching/similarity_measure/affine.py
+py_stringmatching/similarity_measure/cosine.py
+py_stringmatching/similarity_measure/cython_levenshtein.c
+py_stringmatching/similarity_measure/dice.py
+py_stringmatching/similarity_measure/hamming_distance.py
+py_stringmatching/similarity_measure/hybrid_similarity_measure.py
+py_stringmatching/similarity_measure/jaccard.py
+py_stringmatching/similarity_measure/jaro.py
+py_stringmatching/similarity_measure/jaro_winkler.py
+py_stringmatching/similarity_measure/levenshtein.py
+py_stringmatching/similarity_measure/monge_elkan.py
+py_stringmatching/similarity_measure/needleman_wunsch.py
+py_stringmatching/similarity_measure/overlap_coefficient.py
+py_stringmatching/similarity_measure/sequence_similarity_measure.py
+py_stringmatching/similarity_measure/similarity_measure.py
+py_stringmatching/similarity_measure/smith_waterman.py
+py_stringmatching/similarity_measure/soft_tfidf.py
+py_stringmatching/similarity_measure/tfidf.py
+py_stringmatching/similarity_measure/token_similarity_measure.py
+py_stringmatching/tests/__init__.py
+py_stringmatching/tests/test_simfunctions.py
+py_stringmatching/tests/test_tokenizers.py
+py_stringmatching/tokenizer/__init__.py
+py_stringmatching/tokenizer/alphabetic_tokenizer.py
+py_stringmatching/tokenizer/alphanumeric_tokenizer.py
+py_stringmatching/tokenizer/definition_tokenizer.py
+py_stringmatching/tokenizer/delimiter_tokenizer.py
+py_stringmatching/tokenizer/qgram_tokenizer.py
+py_stringmatching/tokenizer/tokenizer.py
+py_stringmatching/tokenizer/whitespace_tokenizer.py

py_stringmatching-0.1.0/py_stringmatching.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

py_stringmatching-0.1.0/py_stringmatching.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

py_stringmatching-0.1.0/py_stringmatching.egg-info/requires.txt

@@ -0,0 +1,2 @@
+numpy >= 1.7.0
+six

py_stringmatching-0.1.0/py_stringmatching.egg-info/top_level.txt

@@ -0,0 +1 @@
+py_stringmatching

py_stringmatching-0.1.0/requirements.txt

@@ -0,0 +1,2 @@
+numpy>=1.7.0
+six

py_stringmatching-0.1.0/setup.cfg

@@ -0,0 +1,5 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+tag_svn_revision = 0
+

py_stringmatching-0.1.0/setup.py

@@ -0,0 +1,107 @@
+import subprocess
+import sys
+import os
+
+# check if pip is installed. If not, raise an ImportError
+PIP_INSTALLED = True
+
+try:
+    import pip
+except ImportError:
+    PIP_INSTALLED = False
+
+if not PIP_INSTALLED:
+    raise ImportError('pip is not installed.')
+
+def install_and_import(package):
+    import importlib
+    try:
+        importlib.import_module(package)
+    except ImportError:
+        pip.main(['install', package])
+    finally:
+        globals()[package] = importlib.import_module(package)
+
+# check if setuptools is installed. If not, install setuptools
+# automatically using pip.
+install_and_import('setuptools')
+
+# make sure numpy is installed, as we need numpy to compile the C extensions.
+# If numpy is not installed, automatically install it using pip.
+install_and_import('numpy')
+
+def generate_cython():
+    cwd = os.path.abspath(os.path.dirname(__file__))
+    print("Cythonizing sources")
+    p = subprocess.call([sys.executable, os.path.join(cwd,
+                                                      'build_tools',
+                                                      'cythonize.py'),
+                         'py_stringmatching'],
+                        cwd=cwd)
+    if p != 0:
+        raise RuntimeError("Running cythonize failed!")
+
+if __name__ == "__main__":
+
+    no_frills = (len(sys.argv) >= 2 and ('--help' in sys.argv[1:] or
+                                         sys.argv[1] in ('--help-commands',
+                                                         'egg_info', '--version',
+                                                         'clean')))
+
+    cwd = os.path.abspath(os.path.dirname(__file__))
+    if not os.path.exists(os.path.join(cwd, 'PKG-INFO')) and not no_frills:
+        # Generate Cython sources, unless building from source release
+        generate_cython()
+
+    # specify extensions that need to be compiled
+    extensions = [setuptools.Extension("py_stringmatching.similarity_measure.cython_levenshtein",
+                                       ["py_stringmatching/similarity_measure/cython_levenshtein.c"],
+                                       include_dirs=[numpy.get_include()])]
+
+    # find packages to be included. exclude benchmarks.
+    packages = setuptools.find_packages(exclude=["benchmarks"])
+
+    with open('README.rst') as f:
+        LONG_DESCRIPTION = f.read()
+
+    setuptools.setup(
+        name='py_stringmatching',
+        version='0.1.0',
+        description='Python library for string matching.',
+        long_description=LONG_DESCRIPTION,
+        url='https://sites.google.com/site/anhaidgroup/projects/py_stringmatching',
+        author='UW Magellan Team',
+        author_email='uwmagellan@gmail.com',
+        license='BSD',
+        classifiers=[
+            'Development Status :: 4 - Beta',
+            'Environment :: Console',
+            'Intended Audience :: Developers',
+            'Intended Audience :: Science/Research',
+            'Intended Audience :: Education',
+            'License :: OSI Approved :: BSD License',
+            'Operating System :: POSIX',
+            'Operating System :: Unix',
+            'Operating System :: MacOS',
+            'Operating System :: Microsoft :: Windows',
+            'Programming Language :: Python',
+            'Programming Language :: Python :: 2',
+            'Programming Language :: Python :: 3',
+            'Programming Language :: Python :: 2.7',
+            'Programming Language :: Python :: 3.3',
+            'Programming Language :: Python :: 3.4',
+            'Programming Language :: Python :: 3.5',
+            'Topic :: Scientific/Engineering',
+            'Topic :: Utilities',
+            'Topic :: Software Development :: Libraries',
+        ],
+        packages=packages,
+        install_requires=[
+            'numpy >= 1.7.0',
+            'six'
+        ],
+        ext_modules=extensions,
+        include_package_data=True,
+        zip_safe=False
+    )
+