py-stringmatching 0.1.0__zip

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/needleman_wunsch.py

@@ -0,0 +1,121 @@
+import numpy as np
+
+from py_stringmatching import utils
+from six.moves import xrange
+from py_stringmatching.similarity_measure.sequence_similarity_measure import \
+                                                    SequenceSimilarityMeasure
+
+
+def sim_ident(char1, char2):
+    return int(char1 == char2)
+
+
+class NeedlemanWunsch(SequenceSimilarityMeasure):
+    """Computes Needleman-Wunsch measure.
+
+    The Needleman-Wunsch distance generalizes the Levenshtein distance and considers global alignment between two strings.
+    Specifically, it is computed by assigning a score to each alignment between the two input strings and choosing the
+    score of the best alignment, that is, the maximal score. An alignment between two strings is a set of correspondences
+    between their characters, allowing for gaps.
+
+    Args:
+        gap_cost (float): Cost of gap (defaults to 1.0).
+        sim_func (function): Similarity function to give a score for each correspondence between the characters (defaults
+                             to an identity function, which returns 1 if the two characters are the same and 0 otherwise.  
+                             
+    Attributes:
+        gap_cost (float): An attribute to store the gap cost.
+        sim_func (function): An attribute to store the similarity function.
+    """
+
+    def __init__(self, gap_cost=1.0, sim_func=sim_ident):
+        self.gap_cost = gap_cost
+        self.sim_func = sim_func
+        super(NeedlemanWunsch, self).__init__()
+
+    def get_raw_score(self, string1, string2):
+        """Computes the raw Needleman-Wunsch score between two strings.
+
+        Args:
+            string1,string2 (str) : Input strings.
+
+        Returns:
+            Needleman-Wunsch similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+
+        Examples:
+            >>> nw = NeedlemanWunsch()
+            >>> nw.get_raw_score('dva', 'deeva')
+            1.0
+            >>> nw = NeedlemanWunsch(gap_cost=0.0)
+            >>> nw.get_raw_score('dva', 'deeve')
+            2.0
+            >>> nw = NeedlemanWunsch(gap_cost=1.0, sim_func=lambda s1, s2 : (2.0 if s1 == s2 else -1.0))
+            >>> nw.get_raw_score('dva', 'deeve')
+            1.0
+            >>> nw = NeedlemanWunsch(gap_cost=0.5, sim_func=lambda s1, s2 : (1.0 if s1 == s2 else -1.0))
+            >>> nw.get_raw_score('GCATGCUA', 'GATTACA')
+            2.5
+        """
+        
+        # input validations
+        utils.sim_check_for_none(string1, string2)
+        utils.sim_check_for_string_inputs(string1, string2)
+
+        dist_mat = np.zeros((len(string1) + 1, len(string2) + 1),
+                            dtype=np.float)
+
+        # DP initialization
+        for i in xrange(len(string1) + 1):
+            dist_mat[i, 0] = -(i * self.gap_cost)
+
+        # DP initialization
+        for j in xrange(len(string2) + 1):
+            dist_mat[0, j] = -(j * self.gap_cost)
+
+        # Needleman-Wunsch DP calculation
+        for i in xrange(1, len(string1) + 1):
+            for j in xrange(1, len(string2) + 1):
+                match = dist_mat[i - 1, j - 1] + self.sim_func(string1[i - 1],
+                                                               string2[j - 1])
+                delete = dist_mat[i - 1, j] - self.gap_cost
+                insert = dist_mat[i, j - 1] - self.gap_cost
+                dist_mat[i, j] = max(match, delete, insert)
+
+        return dist_mat[dist_mat.shape[0] - 1, dist_mat.shape[1] - 1]
+
+    def get_gap_cost(self):
+        """Get gap cost.
+
+        Returns:
+            Gap cost (float).
+        """
+        return self.gap_cost
+
+    def get_sim_func(self):
+        """Get the similarity function.
+
+        Returns:
+            similarity function (function).
+        """
+        return self.sim_func
+
+    def set_gap_cost(self, gap_cost):
+        """Set gap cost.
+
+        Args:
+            gap_cost (float): Cost of gap.
+        """
+        self.gap_cost = gap_cost
+        return True
+
+    def set_sim_func(self, sim_func):
+        """Set similarity function.
+
+        Args:
+            sim_func (function): Similarity function to give a score for the correspondence between characters.
+        """
+        self.sim_func = sim_func
+        return True

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/overlap_coefficient.py

@@ -0,0 +1,86 @@
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.token_similarity_measure import \
+                                                    TokenSimilarityMeasure
+
+
+class OverlapCoefficient(TokenSimilarityMeasure):
+    """Computes overlap coefficient measure.
+
+    The overlap coefficient is a similarity measure related to the Jaccard
+    measure  that measures the overlap between two sets, and is defined as the size of the intersection divided by
+    the smaller of the size of the two sets. For two sets X and Y, the overlap coefficient is:
+
+        :math:`overlap\\_coefficient(X, Y) = \\frac{|X \\cap Y|}{\\min(|X|, |Y|)}`
+    """
+
+    def __init__(self):
+        super(OverlapCoefficient, self).__init__()
+
+    def get_raw_score(self, set1, set2):
+        """Computes the raw overlap coefficient score between two sets.
+
+        Args:
+            set1,set2 (set or list): Input sets (or lists). Input lists are converted to sets.
+
+        Returns:
+            Overlap coefficient (float).
+
+        Raises:
+            TypeError : If the inputs are not sets (or lists) or if one of the inputs is None.
+
+        Examples:
+            >>> oc = OverlapCoefficient()
+            >>> oc.get_raw_score(['data', 'science'], ['data'])
+            1.0
+            >>> oc.get_raw_score([], [])
+            1.0
+            >>> oc.get_raw_score([], ['data'])
+            0
+
+        References:
+            * Wikipedia article : https://en.wikipedia.org/wiki/Overlap_coefficient
+            * SimMetrics library
+        """
+        
+        # input validations
+        utils.sim_check_for_none(set1, set2)
+        utils.sim_check_for_list_or_set_inputs(set1, set2)
+
+        # if exact match return 1.0
+        if utils.sim_check_for_exact_match(set1, set2):
+            return 1.0
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(set1, set2):
+            return 0
+
+        if not isinstance(set1, set):
+            set1 = set(set1)
+        if not isinstance(set2, set):
+            set2 = set(set2)
+
+        return float(len(set1 & set2)) / min(len(set1), len(set2))
+
+    def get_sim_score(self, set1, set2):
+        """Computes the normalized overlap coefficient between two sets. Simply call get_raw_score. 
+
+        Args:
+            set1,set2 (set or list): Input sets (or lists). Input lists are converted to sets.
+
+        Returns:
+            Normalized overlap coefficient (float).
+
+        Raises:
+            TypeError : If the inputs are not sets (or lists) or if one of the inputs is None.
+
+        Examples:
+            >>> oc = OverlapCoefficient()
+            >>> oc.get_sim_score(['data', 'science'], ['data'])
+            1.0
+            >>> oc.get_sim_score([], [])
+            1.0
+            >>> oc.get_sim_score([], ['data'])
+            0
+
+        """
+        return self.get_raw_score(set1, set2)

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/sequence_similarity_measure.py

@@ -0,0 +1,7 @@
+"""Sequence based similarity measure"""
+
+from py_stringmatching.similarity_measure.similarity_measure import \
+                                                        SimilarityMeasure
+
+class SequenceSimilarityMeasure(SimilarityMeasure):
+    pass

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/similarity_measure.py

@@ -0,0 +1,4 @@
+"""Similarity measure"""
+
+class SimilarityMeasure(object):
+    pass

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/smith_waterman.py

@@ -0,0 +1,115 @@
+# coding=utf-8
+
+import numpy as np
+
+from py_stringmatching import utils
+from six.moves import xrange
+from py_stringmatching.similarity_measure.sequence_similarity_measure import \
+                                                    SequenceSimilarityMeasure
+
+
+def sim_ident(char1, char2):
+    return int(char1 == char2)
+
+
+class SmithWaterman(SequenceSimilarityMeasure):
+    """Computes Smith-Waterman measure.
+
+    The Smith-Waterman algorithm performs local sequence alignment; that is, for determining similar regions
+    between two strings. Instead of looking at the total sequence, the Smith–Waterman algorithm compares segments of
+    all possible lengths and optimizes the similarity measure. See the string matching chapter in the DI book (Principles of Data Integration). 
+
+    Args:
+        gap_cost (float): Cost of gap (defaults to 1.0).
+        sim_func (function): Similarity function to give a score for the correspondence between the characters (defaults
+                             to an identity function, which returns 1 if the two characters are the same and 0 otherwise).
+
+    Attributes:
+        gap_cost (float): An attribute to store the gap cost.
+        sim_func (function): An attribute to store the similarity function.
+    """
+
+    def __init__(self, gap_cost=1.0, sim_func=sim_ident):
+        self.gap_cost = gap_cost
+        self.sim_func = sim_func
+        super(SmithWaterman, self).__init__()
+
+    def get_raw_score(self, string1, string2):
+        """Computes the raw Smith-Waterman score between two strings.
+
+        Args:
+            string1,string2 (str) : Input strings.
+
+        Returns:
+            Smith-Waterman similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+
+        Examples:
+            >>> sw = SmithWaterman()
+            >>> sw.get_raw_score('cat', 'hat')
+            2.0
+            >>> sw = SmithWaterman(gap_cost=2.2)
+            >>> sw.get_raw_score('dva', 'deeve')
+            1.0
+            >>> sw = SmithWaterman(gap_cost=1, sim_func=lambda s1, s2 : (2 if s1 == s2 else -1))
+            >>> sw.get_raw_score('dva', 'deeve')
+            2.0
+            >>> sw = SmithWaterman(gap_cost=1.4, sim_func=lambda s1, s2 : (1.5 if s1 == s2 else 0.5))
+            >>> sw.get_raw_score('GCATAGCU', 'GATTACA')
+            6.5
+        """
+        
+        # input validations
+        utils.sim_check_for_none(string1, string2)
+        utils.sim_check_for_string_inputs(string1, string2)
+
+        dist_mat = np.zeros((len(string1) + 1, len(string2) + 1),
+                            dtype=np.float)
+        max_value = 0
+        # Smith Waterman DP calculations
+        for i in xrange(1, len(string1) + 1):
+            for j in xrange(1, len(string2) + 1):
+                match = dist_mat[i - 1, j - 1] + self.sim_func(string1[i - 1],
+                                                               string2[j - 1])
+                delete = dist_mat[i - 1, j] - self.gap_cost
+                insert = dist_mat[i, j - 1] - self.gap_cost
+                dist_mat[i, j] = max(0, match, delete, insert)
+                max_value = max(max_value, dist_mat[i, j])
+
+        return max_value
+
+    def get_gap_cost(self):
+        """Get gap cost.
+
+        Returns:
+            Gap cost (float).
+        """
+        return self.gap_cost
+
+    def get_sim_func(self):
+        """Get similarity function.
+
+        Returns:
+            Similarity function (function).
+        """
+        return self.sim_func
+
+    def set_gap_cost(self, gap_cost):
+        """Set gap cost.
+
+        Args:
+            gap_cost (float): Cost of gap.
+        """
+        self.gap_cost = gap_cost
+        return True
+
+    def set_sim_func(self, sim_func):
+        """Set similarity function.
+
+        Args:
+            sim_func (function): Similarity function to give a score for the correspondence between the characters.
+        """
+        self.sim_func = sim_func
+        return True

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/soft_tfidf.py

@@ -0,0 +1,198 @@
+from __future__ import division
+from math import sqrt
+import collections
+
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.jaro import Jaro
+from py_stringmatching.similarity_measure.hybrid_similarity_measure import \
+                                                    HybridSimilarityMeasure
+
+
+class SoftTfIdf(HybridSimilarityMeasure):
+    """Computes soft TF/IDF measure.
+
+    Args:
+        corpus_list (list of lists): Corpus list (default is set to None) of strings. If set to None,
+                                     the input list are considered the only corpus.
+        sim_func (function): Secondary similarity function. This should return a similarity score between two strings (optional),
+                             default is the Jaro similarity measure.
+        threshold (float): Threshold value for the secondary similarity function (defaults to 0.5). If the similarity
+                           of a token pair exceeds the threshold, then the token pair is considered a match.
+
+    Attributes:
+        sim_func (function): An attribute to store the secondary similarity function.
+        threshold (float): An attribute to store the threshold value for the secondary similarity function.
+    
+    Note:
+        Currently, this measure is implemented without dampening. This is similar to setting dampen flag to be False in TF-IDF.
+        We plan to add the dampen flag in the next release.    
+    """
+
+    def __init__(self, corpus_list=None, sim_func=Jaro().get_raw_score,
+                 threshold=0.5):
+        self.__corpus_list = corpus_list
+        self.__document_frequency = {}
+        self.__compute_document_frequency()
+        self.__corpus_size = 0 if self.__corpus_list is None else (
+                                                         len(self.__corpus_list))
+        self.sim_func = sim_func
+        self.threshold = threshold
+        super(SoftTfIdf, self).__init__()
+
+    def get_raw_score(self, bag1, bag2):
+        """Computes the raw soft TF/IDF score between two lists given the corpus information.
+
+        Args:
+            bag1,bag2 (list): Input lists
+
+        Returns:
+            Soft TF/IDF score between the input lists (float).
+
+        Raises:
+            TypeError : If the inputs are not lists or if one of the inputs is None.
+
+        Examples:
+            >>> soft_tfidf = SoftTfIdf([['a', 'b', 'a'], ['a', 'c'], ['a']], sim_func=Jaro().get_raw_score, threshold=0.8)
+            >>> soft_tfidf.get_raw_score(['a', 'b', 'a'], ['a', 'c'])
+            0.17541160386140586
+            >>> soft_tfidf = SoftTfIdf([['a', 'b', 'a'], ['a', 'c'], ['a']], threshold=0.9)
+            >>> soft_tfidf.get_raw_score(['a', 'b', 'a'], ['a'])
+            0.5547001962252291
+            >>> soft_tfidf = SoftTfIdf([['x', 'y'], ['w'], ['q']])
+            >>> soft_tfidf.get_raw_score(['a', 'b', 'a'], ['a'])
+            0.0
+            >>> soft_tfidf = SoftTfIdf(sim_func=Affine().get_raw_score, threshold=0.6)
+            >>> soft_tfidf.get_raw_score(['aa', 'bb', 'a'], ['ab', 'ba'])
+            0.81649658092772592
+
+        References:
+            * the string matching chapter of the "Principles of Data Integration" book.
+        """
+        
+        # input validations
+        utils.sim_check_for_none(bag1, bag2)
+        utils.sim_check_for_list_or_set_inputs(bag1, bag2)
+
+        # if the strings match exactly return 1.0
+        if utils.sim_check_for_exact_match(bag1, bag2):
+            return 1.0
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(bag1, bag2):
+            return 0
+
+        # term frequency for input strings
+        tf_x, tf_y = collections.Counter(bag1), collections.Counter(bag2)
+         
+        # find unique elements in the input lists and their document frequency 
+        local_df = {}
+        for element in tf_x:
+            local_df[element] = local_df.get(element, 0) + 1
+        for element in tf_y:
+            local_df[element] = local_df.get(element, 0) + 1
+
+        # if corpus is not provided treat input string as corpus
+        curr_df, corpus_size = (local_df, 2) if self.__corpus_list is None else (
+                                   (self.__document_frequency, self.__corpus_size))
+
+        # calculating the term sim score against the input string 2,
+        # construct similarity map
+        similarity_map = {}
+        for term_x in tf_x:
+            max_score = 0.0
+            for term_y in tf_y:
+                score = self.sim_func(term_x, term_y)
+                # adding sim only if it is above threshold and
+                # highest for this element
+                if score > self.threshold and score > max_score:
+                    similarity_map[term_x] = (term_x, term_y, score)
+                    max_score = score
+
+        # position of first string, second string and sim score
+        # in the tuple
+        first_string_pos = 0
+        second_string_pos = 1
+        sim_score_pos = 2
+
+        result, v_x_2, v_y_2 = 0.0, 0.0, 0.0
+        # soft-tfidf calculation
+        for element in local_df.keys():
+            if curr_df.get(element) is None:
+                continue
+            # numerator
+            if element in similarity_map:
+                sim = similarity_map[element]
+                idf_first = corpus_size / curr_df.get(sim[first_string_pos], 1)
+                idf_second = corpus_size / curr_df.get(sim[second_string_pos], 1)
+                v_x = idf_first * tf_x.get(sim[first_string_pos], 0)
+                v_y = idf_second * tf_y.get(sim[second_string_pos], 0)
+                result += v_x * v_y * sim[sim_score_pos]
+            # denominator
+            idf = corpus_size / curr_df[element]
+            v_x = idf * tf_x.get(element, 0)
+            v_x_2 += v_x * v_x
+            v_y = idf * tf_y.get(element, 0)
+            v_y_2 += v_y * v_y
+        return result if v_x_2 == 0 else result / (sqrt(v_x_2) * sqrt(v_y_2))
+
+    def get_corpus_list(self):
+        """Get corpus list.
+
+        Returns:
+            corpus list (list of lists).
+        """
+        return self.__corpus_list
+
+    def get_sim_func(self):
+        """Get secondary similarity function.
+
+        Returns:
+            secondary similarity function (function).
+        """
+        return self.sim_func
+
+    def get_threshold(self):
+        """Get threshold used for the secondary similarity function.
+
+        Returns:
+            threshold (float).
+        """
+        return self.threshold
+
+    def set_threshold(self, threshold):
+        """Set threshold value for the secondary similarity function.
+
+        Args:
+            threshold (float): threshold value.
+        """
+        self.threshold = threshold
+        return True
+
+    def set_sim_func(self, sim_func):
+        """Set secondary similarity function.
+
+        Args:
+            sim_func (function): Secondary similarity function.
+        """
+        self.sim_func = sim_func
+        return True
+
+    def set_corpus_list(self, corpus_list):
+        """Set corpus list.
+
+        Args:
+            corpus_list (list of lists): Corpus list.
+        """
+        self.__corpus_list = corpus_list
+        self.__document_frequency = {}
+        self.__compute_document_frequency()
+        self.__corpus_size = 0 if self.__corpus_list is None else (
+                                                         len(self.__corpus_list))
+        return True
+
+    def __compute_document_frequency(self):
+        if self.__corpus_list != None:
+            for document in self.__corpus_list:
+                for element in set(document):
+                    self.__document_frequency[element] = (
+                        self.__document_frequency.get(element, 0) + 1)