py-stringmatching 0.1.0__zip

py_stringmatching-0.1.0/AUTHORS.rst

@@ -0,0 +1,6 @@
+The following individuals have contributed code, documentation, or expertise to py_stringmatching:
+
+* `Ali Hitawala <https://github.com/alihitawala>`_
+* `Paul Suganthan G. C. <https://github.com/paulgc>`_
+* `Pradap Konda <https://github.com/kvpradap>`_
+* `AnHai Doan <https://github.com/anhaidgroup>`_

py_stringmatching-0.1.0/CHANGES.txt

@@ -0,0 +1,6 @@
+v0.1.0 - 06/14/2016
+  * Initial release. 
+  * Contains 5 tokenizers - Alphabetic tokenizer, Alphanumeric tokenizer, Delimiter tokenizer, Qgram tokenizer and
+    Whitespace tokenizer.
+  * Contains 14 similarity measures - Affine, Cosine, Dice, Hamming distance, Jaccard, Jaro, Jaro-Winkler, 
+    Levenshtein, Monge-Elkan, Needleman-Wunsch, Overlap coefficient, Smith-Waterman, Soft TF-IDF, and TF-IDF.

py_stringmatching-0.1.0/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2016, anhaidgroup
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of py_stringmatching nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

py_stringmatching-0.1.0/LICENSES/NUMPY_LICENSE

@@ -0,0 +1,30 @@
+Copyright (c) 2005-2016, NumPy Developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+       copyright notice, this list of conditions and the following
+       disclaimer in the documentation and/or other materials provided
+       with the distribution.
+
+    * Neither the name of the NumPy Developers nor the names of any
+       contributors may be used to endorse or promote products derived
+       from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

py_stringmatching-0.1.0/LICENSES/SIX_LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2010-2016 Benjamin Peterson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

py_stringmatching-0.1.0/MANIFEST.in

@@ -0,0 +1,6 @@
+include AUTHORS.rst
+include README.rst
+include CHANGES.txt
+include requirements.txt
+include LICENSE
+recursive-include LICENSES *

py_stringmatching-0.1.0/PKG-INFO

@@ -0,0 +1,57 @@
+Metadata-Version: 1.1
+Name: py_stringmatching
+Version: 0.1.0
+Summary: Python library for string matching.
+Home-page: https://sites.google.com/site/anhaidgroup/projects/py_stringmatching
+Author: UW Magellan Team
+Author-email: uwmagellan@gmail.com
+License: BSD
+Description: py_stringmatching
+        =================
+        
+        py_stringmatching is a software package in Python that consists of a comprehensive set of tokenizers and string similarity measures (
+        such as edit distance, Jaccard, and TF/IDF). It is free, open-source, and BSD-licensed.
+        
+        Important links
+        ===============
+        
+         * Repository: https://github.com/anhaidgroup/py_stringmatching
+         * Documentation: http://anhaidgroup.github.io/py_stringmatching/v0.1.x/index.html 
+         * Tutorial: http://anhaidgroup.github.io/py_stringmatching/v0.1.x/Tutorial.html
+         * Issue Tracker: https://github.com/anhaidgroup/py_stringmatching/issues
+         * Mailing List: https://groups.google.com/forum/#!forum/py_stringmatching
+         
+        Dependencies
+        ============
+        
+        py_stringmatching is tested to work under Python 2.7, Python 3.3, Python 3.4 and Python 3.5.
+        
+        The required dependencies to build the package are NumPy >= 1.7.0,
+        Six and a working C/C++ compiler. For the development version, you will also require Cython.
+        
+        Platforms
+        =========
+        
+        py_stringmatching has been tested on Linux, OSX and Windows.
+        
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: Unix
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Utilities
+Classifier: Topic :: Software Development :: Libraries

py_stringmatching-0.1.0/README.rst

@@ -0,0 +1,27 @@
+py_stringmatching
+=================
+
+py_stringmatching is a software package in Python that consists of a comprehensive set of tokenizers and string similarity measures (
+such as edit distance, Jaccard, and TF/IDF). It is free, open-source, and BSD-licensed.
+
+Important links
+===============
+
+ * Repository: https://github.com/anhaidgroup/py_stringmatching
+ * Documentation: http://anhaidgroup.github.io/py_stringmatching/v0.1.x/index.html 
+ * Tutorial: http://anhaidgroup.github.io/py_stringmatching/v0.1.x/Tutorial.html
+ * Issue Tracker: https://github.com/anhaidgroup/py_stringmatching/issues
+ * Mailing List: https://groups.google.com/forum/#!forum/py_stringmatching
+ 
+Dependencies
+============
+
+py_stringmatching is tested to work under Python 2.7, Python 3.3, Python 3.4 and Python 3.5.
+
+The required dependencies to build the package are NumPy >= 1.7.0,
+Six and a working C/C++ compiler. For the development version, you will also require Cython.
+
+Platforms
+=========
+
+py_stringmatching has been tested on Linux, OSX and Windows.

py_stringmatching-0.1.0/py_stringmatching/__init__.py

@@ -0,0 +1,25 @@
+__version__ = "0.1.0"
+
+# Import tokenizers
+from py_stringmatching.tokenizer.alphabetic_tokenizer import AlphabeticTokenizer
+from py_stringmatching.tokenizer.alphanumeric_tokenizer import AlphanumericTokenizer
+from py_stringmatching.tokenizer.delimiter_tokenizer import DelimiterTokenizer
+from py_stringmatching.tokenizer.qgram_tokenizer import QgramTokenizer
+from py_stringmatching.tokenizer.whitespace_tokenizer import WhitespaceTokenizer
+
+# Import similarity measures
+from py_stringmatching.similarity_measure.affine import Affine
+from py_stringmatching.similarity_measure.cosine import Cosine
+from py_stringmatching.similarity_measure.dice import Dice
+from py_stringmatching.similarity_measure.hamming_distance import HammingDistance
+from py_stringmatching.similarity_measure.jaccard import Jaccard
+from py_stringmatching.similarity_measure.jaro import Jaro
+from py_stringmatching.similarity_measure.jaro_winkler import JaroWinkler
+from py_stringmatching.similarity_measure.levenshtein import Levenshtein
+from py_stringmatching.similarity_measure.monge_elkan import MongeElkan
+from py_stringmatching.similarity_measure.needleman_wunsch import NeedlemanWunsch
+from py_stringmatching.similarity_measure.overlap_coefficient import OverlapCoefficient
+from py_stringmatching.similarity_measure.smith_waterman import SmithWaterman
+from py_stringmatching.similarity_measure.soft_tfidf import SoftTfIdf
+from py_stringmatching.similarity_measure.tfidf import TfIdf
+

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/affine.py

@@ -0,0 +1,155 @@
+import numpy as np
+
+from py_stringmatching import utils
+from six.moves import xrange
+from py_stringmatching.similarity_measure.sequence_similarity_measure import \
+                                                    SequenceSimilarityMeasure
+
+
+def sim_ident(char1, char2):
+    return int(char1 == char2)
+
+
+class Affine(SequenceSimilarityMeasure):
+    """Returns the affine gap score between two strings. 
+
+    The affine gap measure is an extension of the Needleman-Wunsch measure that handles the longer gaps more
+    gracefully. For more information refer to the string matching chapter in the DI book ("Principles of Data Integration").
+
+    Args:
+        gap_start (float): Cost for the gap at the start (defaults to 1).
+        gap_continuation (float): Cost for the gap continuation (defaults to 0.5).
+        sim_func (function): Function computing similarity score between two characters, which are represented as strings (defaults
+                             to an identity function, which returns 1 if the two characters are the same and returns 0 otherwise). 
+
+    Attributes:
+        gap_start (float): An attribute to store the gap cost at the start.
+        gap_continuation (float): An attribute to store the gap continuation cost.
+        sim_func (function): An attribute to store the similarity function.
+    """
+
+    def __init__(self, gap_start=1, gap_continuation=0.5, sim_func=sim_ident):
+        self.gap_start = gap_start
+        self.gap_continuation = gap_continuation
+        self.sim_func = sim_func
+        super(Affine, self).__init__()
+
+    def get_raw_score(self, string1, string2):
+        """Computes the affine gap score between two strings. This score can be outside the range [0,1].
+        
+        Args:
+            string1,string2 (str) : Input strings.
+
+        Returns:
+            Affine gap score betwen the two input strings (float).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+
+        Examples:
+            >>> aff = Affine()
+            >>> aff.get_raw_score('dva', 'deeva')
+            1.5
+            >>> aff = Affine(gap_start=2, gap_continuation=0.5)
+            >>> aff.get_raw_score('dva', 'deeve')
+            -0.5
+            >>> aff = Affine(gap_continuation=0.2, sim_func=lambda s1, s2: (int(1 if s1 == s2 else 0)))
+            >>> aff.get_raw_score('AAAGAATTCA', 'AAATCA')
+            4.4
+        """
+        # input validations
+        utils.sim_check_for_none(string1, string2)
+        utils.tok_check_for_string_input(string1, string2)
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(string1, string2):
+            return 0
+
+        gap_start = -self.gap_start
+        gap_continuation = -self.gap_continuation
+        m = np.zeros((len(string1) + 1, len(string2) + 1), dtype=np.float)
+        x = np.zeros((len(string1) + 1, len(string2) + 1), dtype=np.float)
+        y = np.zeros((len(string1) + 1, len(string2) + 1), dtype=np.float)
+
+        # DP initialization
+        for i in xrange(1, len(string1) + 1):
+            m[i][0] = -float("inf")
+            x[i][0] = gap_start + (i - 1) * gap_continuation
+            y[i][0] = -float("inf")
+
+        # DP initialization
+        for j in xrange(1, len(string2) + 1):
+            m[0][j] = -float("inf")
+            x[0][j] = -float("inf")
+            y[0][j] = gap_start + (j - 1) * gap_continuation
+
+        # affine gap calculation using DP
+        for i in xrange(1, len(string1) + 1):
+            for j in xrange(1, len(string2) + 1):
+                # best score between x_1....x_i and y_1....y_j
+                # given that x_i is aligned to y_j
+                m[i][j] = (self.sim_func(string1[i - 1], string2[j - 1]) +
+                           max(m[i - 1][j - 1], x[i - 1][j - 1],
+                               y[i - 1][j - 1]))
+
+                # the best score given that x_i is aligned to a gap
+                x[i][j] = max(gap_start + m[i - 1][j],
+                              gap_continuation + x[i - 1][j])
+
+                # the best score given that y_j is aligned to a gap
+                y[i][j] = max(gap_start + m[i][j - 1],
+                              gap_continuation + y[i][j - 1])
+
+        return max(m[len(string1)][len(string2)], x[len(string1)][len(string2)],
+                   y[len(string1)][len(string2)])
+
+    def get_gap_start(self):
+        """Get gap start cost.
+
+        Returns:
+            gap start cost (float).
+        """
+        return self.gap_start
+
+    def get_gap_continuation(self):
+        """Get gap continuation cost.
+
+        Returns:
+            gap continuation cost (float).
+        """
+        return self.gap_continuation
+
+    def get_sim_func(self):
+        """Get similarity function.
+
+        Returns:
+            similarity function (function).
+        """
+        return self.sim_func
+
+    def set_gap_start(self, gap_start):
+        """Set gap start cost.
+
+        Args:
+            gap_start (float): Cost for the gap at the start.
+        """
+        self.gap_start = gap_start
+        return True
+
+    def set_gap_continuation(self, gap_continuation):
+        """Set gap continuation cost.
+
+        Args:
+            gap_continuation (float): Cost for the gap continuation.
+        """
+        self.gap_continuation = gap_continuation
+        return True
+
+    def set_sim_func(self, sim_func):
+        """Set similarity function.
+
+        Args:
+            sim_func (function): Function computing similarity score between two characters, represented as strings.
+        """
+        self.sim_func = sim_func
+        return True

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/cosine.py

@@ -0,0 +1,86 @@
+import math
+
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.token_similarity_measure import \
+                                                    TokenSimilarityMeasure
+
+
+class Cosine(TokenSimilarityMeasure):
+    """Computes a variant of cosine measure known as Ochiai coefficient.
+
+    This is not the cosine measure that computes the cosine of the angle between two given vectors. Rather, it computes a variant of cosine measure known as Ochiai coefficient (see the Wikipedia page "Cosine Similarity"). Specifically, for two sets X and Y, this measure computes:
+
+        :math:`cosine(X, Y) = \\frac{|X \\cap Y|}{\\sqrt{|X| \\cdot |Y|}}`
+    """
+
+    def __init__(self):
+        super(Cosine, self).__init__()
+
+    def get_raw_score(self, set1, set2):
+        """Computes the raw cosine score between two sets.
+
+        Args:
+            set1,set2 (set or list): Input sets (or lists). Input lists are converted to sets.
+
+        Returns:
+            Cosine similarity (float)
+
+        Raises:
+            TypeError : If the inputs are not sets (or lists) or if one of the inputs is None.
+
+        Examples:
+            >>> cos = Cosine()
+            >>> cos.get_raw_score(['data', 'science'], ['data'])
+            0.7071067811865475
+            >>> cos.get_raw_score(['data', 'data', 'science'], ['data', 'management'])
+            0.4999999999999999
+            >>> cos.get_raw_score([], ['data'])
+            0.0
+
+        References:
+            * String similarity joins: An Experimental Evaluation (a paper appearing in the VLDB 2014 Conference).
+            * Project Flamingo at http://flamingo.ics.uci.edu.
+        """
+        # input validations
+        utils.sim_check_for_none(set1, set2)
+        utils.sim_check_for_list_or_set_inputs(set1, set2)
+
+        # if exact match return 1.0
+        if utils.sim_check_for_exact_match(set1, set2):
+            return 1.0
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(set1, set2):
+            return 0
+
+        if not isinstance(set1, set):
+            set1 = set(set1)
+        if not isinstance(set2, set):
+            set2 = set(set2)
+
+        return float(len(set1 & set2)) / (math.sqrt(float(len(set1))) *
+                                          math.sqrt(float(len(set2))))
+
+    def get_sim_score(self, set1, set2):
+        """Computes the normalized cosine similarity between two sets.
+
+        Args:
+            set1,set2 (set or list): Input sets (or lists). Input lists are converted to sets.
+
+        Returns:
+            Normalized cosine similarity (float)
+
+        Raises:
+            TypeError : If the inputs are not sets (or lists) or if one of the inputs is None.
+
+        Examples:
+            >>> cos = Cosine()
+            >>> cos.get_sim_score(['data', 'science'], ['data'])
+            0.7071067811865475
+            >>> cos.get_sim_score(['data', 'data', 'science'], ['data', 'management'])
+            0.4999999999999999
+            >>> cos.get_sim_score([], ['data'])
+            0.0
+
+        """
+        return self.get_raw_score(set1, set2)