py-stringmatching 0.1.0__zip

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/dice.py

@@ -0,0 +1,85 @@
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.token_similarity_measure import \
+                                                    TokenSimilarityMeasure
+
+
+class Dice(TokenSimilarityMeasure):
+    """Returns the Dice score between two strings.
+
+    The Dice similarity score is defined as twice the shared information (intersection) divided by sum of cardinalities.
+    For two sets X and Y, the Dice similarity score is:
+
+        :math:`dice(X, Y) = \\frac{2 * |X \\cap Y|}{|X| + |Y|}`
+    """
+
+    def __init__(self):
+        super(Dice, self).__init__()
+
+    def get_raw_score(self, set1, set2):
+        """Computes the raw Dice score between two sets. This score is already in [0,1].
+
+        Args:
+            set1,set2 (set or list): Input sets (or lists). Input lists are converted to sets.
+
+        Returns:
+            Dice similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not sets (or lists) or if one of the inputs is None.
+
+        Examples:
+            >>> dice = Dice()
+            >>> dice.get_raw_score(['data', 'science'], ['data'])
+            0.6666666666666666
+            >>> dice.get_raw_score({1, 1, 2, 3, 4}, {2, 3, 4, 5, 6, 7, 7, 8})
+            0.5454545454545454
+            >>> dice.get_raw_score(['data', 'management'], ['data', 'data', 'science'])
+            0.5
+
+        References:
+            * Wikipedia article : https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Dice%27s_coefficient
+            * SimMetrics library.
+        """
+        
+        # input validations
+        utils.sim_check_for_none(set1, set2)
+        utils.sim_check_for_list_or_set_inputs(set1, set2)
+
+        # if exact match return 1.0
+        if utils.sim_check_for_exact_match(set1, set2):
+            return 1.0
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(set1, set2):
+            return 0
+
+        if not isinstance(set1, set):
+            set1 = set(set1)
+        if not isinstance(set2, set):
+            set2 = set(set2)
+
+        return 2.0 * float(len(set1 & set2)) / float(len(set1) + len(set2))
+
+    def get_sim_score(self, set1, set2):
+        """Computes the normalized dice similarity score between two sets. Simply call get_raw_score.
+
+        Args:
+            set1,set2 (set or list): Input sets (or lists). Input lists are converted to sets.
+
+        Returns:
+            Normalized dice similarity (float).
+
+        Raises:
+            TypeError : If the inputs are not sets (or lists) or if one of the inputs is None.
+
+        Examples:
+            >>> dice = Dice()
+            >>> dice.get_sim_score(['data', 'science'], ['data'])
+            0.6666666666666666
+            >>> dice.get_sim_score({1, 1, 2, 3, 4}, {2, 3, 4, 5, 6, 7, 7, 8})
+            0.5454545454545454
+            >>> dice.get_sim_score(['data', 'management'], ['data', 'data', 'science'])
+            0.5
+
+        """
+        return self.get_raw_score(set1, set2)

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/hamming_distance.py

@@ -0,0 +1,84 @@
+from __future__ import division
+
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.sequence_similarity_measure import \
+                                                    SequenceSimilarityMeasure
+
+
+class HammingDistance(SequenceSimilarityMeasure):
+    """Computes Hamming distance.
+
+    The Hamming distance between two strings of equal length is the number of positions at which the corresponding
+    symbols are different. Thus, it measures the minimum number of substitutions required to change
+    one string into the other, or the minimum number of errors that could have transformed one string into the other.
+    """
+
+    def __init__(self):
+        super(HammingDistance, self).__init__()
+
+    def get_raw_score(self, string1, string2):
+        """Computes the raw hamming distance between two strings.
+
+        Args:
+            string1,string2 (str): Input strings.
+
+        Returns:
+            Hamming distance (int).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+            ValueError : If the input strings are not of same length.
+
+        Examples:
+            >>> hd = HammingDistance()
+            >>> hd.get_raw_score('', '')
+            0
+            >>> hd.get_raw_score('alex', 'john')
+            4
+            >>> hd.get_raw_score(' ', 'a')
+            1
+            >>> hd.get_raw_score('JOHN', 'john')
+            4
+        """
+        
+        # input validations
+        utils.sim_check_for_none(string1, string2)
+        utils.tok_check_for_string_input(string1, string2)
+
+        # for Hamming Distance string length should be same
+        utils.sim_check_for_same_len(string1, string2)
+
+        # sum all the mismatch characters at the corresponding index of
+        # input strings
+        return sum(bool(ord(c1) - ord(c2)) for c1, c2 in zip(string1, string2))
+
+    def get_sim_score(self, string1, string2):
+        """Computes the normalized Hamming similarity score between two strings.
+
+        Args:
+            string1,string2 (str): Input strings.
+
+        Returns:
+            Normalized Hamming similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+            ValueError : If the input strings are not of same length.
+
+        Examples:
+            >>> hd = HammingDistance()
+            >>> hd.get_sim_score('', '')
+            1.0
+            >>> hd.get_sim_score('alex', 'john')
+            0.0
+            >>> hd.get_sim_score(' ', 'a')
+            0.0
+            >>> hd.get_sim_score('JOHN', 'john')
+            0.0
+        """
+        
+        raw_score = self.get_raw_score(string1, string2)
+        common_len = len(string1)
+        if common_len == 0:
+            return 1.0
+        return 1 - (raw_score / common_len)

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/hybrid_similarity_measure.py

@@ -0,0 +1,7 @@
+"""Hybrid similarity measure"""
+
+from py_stringmatching.similarity_measure.similarity_measure import \
+                                                        SimilarityMeasure
+
+class HybridSimilarityMeasure(SimilarityMeasure):
+    pass

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/jaccard.py

@@ -0,0 +1,79 @@
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.token_similarity_measure import \
+                                                    TokenSimilarityMeasure
+
+
+class Jaccard(TokenSimilarityMeasure):
+    """Computes Jaccard measure.
+
+     For two sets X and Y, the Jaccard similarity score is:
+
+        :math:`jaccard(X, Y) = \\frac{|X \\cap Y|}{|X \\cup Y|}`
+    """
+
+    def __init__(self):
+        super(Jaccard, self).__init__()
+
+    def get_raw_score(self, set1, set2):
+        """Computes the raw Jaccard score between two sets.
+
+        Args:
+            set1,set2 (set or list): Input sets (or lists). Input lists are converted to sets.
+
+        Returns:
+            Jaccard similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not sets (or lists) or if one of the inputs is None.
+
+        Examples:
+            >>> jac = Jaccard()
+            >>> jac.get_raw_score(['data', 'science'], ['data'])
+            0.5
+            >>> jac.get_raw_score({1, 1, 2, 3, 4}, {2, 3, 4, 5, 6, 7, 7, 8})
+            0.375
+            >>> jac.get_raw_score(['data', 'management'], ['data', 'data', 'science'])
+            0.3333333333333333
+        """
+        
+        # input validations
+        utils.sim_check_for_none(set1, set2)
+        utils.sim_check_for_list_or_set_inputs(set1, set2)
+
+        # if exact match return 1.0
+        if utils.sim_check_for_exact_match(set1, set2):
+            return 1.0
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(set1, set2):
+            return 0
+
+        if not isinstance(set1, set):
+            set1 = set(set1)
+        if not isinstance(set2, set):
+            set2 = set(set2)
+
+        return float(len(set1 & set2)) / float(len(set1 | set2))
+
+    def get_sim_score(self, set1, set2):
+        """Computes the normalized Jaccard similarity between two sets. Simply call get_raw_score.
+
+        Args:
+            set1,set2 (set or list): Input sets (or lists). Input lists are converted to sets.
+
+        Returns:
+            Normalized Jaccard similarity (float).
+
+        Raises:
+            TypeError : If the inputs are not sets (or lists) or if one of the inputs is None.
+
+        Examples:
+            >>> jac = Jaccard()
+            >>> jac.get_sim_score(['data', 'science'], ['data'])
+            0.5
+            >>> jac.get_sim_score({1, 1, 2, 3, 4}, {2, 3, 4, 5, 6, 7, 7, 8})
+            0.375
+            >>> jac.get_sim_score(['data', 'management'], ['data', 'data', 'science'])
+            0.3333333333333333
+        """
+        return self.get_raw_score(set1, set2)

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/jaro.py

@@ -0,0 +1,110 @@
+from py_stringmatching import utils
+from six.moves import xrange
+from py_stringmatching.similarity_measure.sequence_similarity_measure import \
+                                                    SequenceSimilarityMeasure
+
+
+class Jaro(SequenceSimilarityMeasure):
+    """Computes Jaro measure.
+
+    The Jaro measure is a type of edit distance, developed mainly to compare short strings,
+    such as first and last names.
+    """
+
+    def __init__(self):
+        super(Jaro, self).__init__()
+
+    def get_raw_score(self, string1, string2):
+        """Computes the raw Jaro score between two strings.
+
+        Args:
+            string1,string2 (str): Input strings.
+
+        Returns:
+            Jaro similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+
+        Examples:
+            >>> jaro = Jaro()
+            >>> jaro.get_raw_score('MARTHA', 'MARHTA')
+            0.9444444444444445
+            >>> jaro.get_raw_score('DWAYNE', 'DUANE')
+            0.8222222222222223
+            >>> jaro.get_raw_score('DIXON', 'DICKSONX')
+            0.7666666666666666
+
+        """
+        
+        # input validations
+        utils.sim_check_for_none(string1, string2)
+        utils.tok_check_for_string_input(string1, string2)
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(string1, string2):
+            return 0
+
+        len_s1 = len(string1)
+        len_s2 = len(string2)
+
+        max_len = max(len_s1, len_s2)
+        search_range = (max_len // 2) - 1
+        if search_range < 0:
+            search_range = 0
+
+        flags_s1 = [False] * len_s1
+        flags_s2 = [False] * len_s2
+
+        common_chars = 0
+        for i, ch_s1 in enumerate(string1):
+            low = i - search_range if i > search_range else 0
+            high = i + search_range if i + search_range < len_s2 else len_s2 - 1
+            for j in xrange(low, high + 1):
+                if not flags_s2[j] and string2[j] == ch_s1:
+                    flags_s1[i] = flags_s2[j] = True
+                    common_chars += 1
+                    break
+
+        if not common_chars:
+            return 0
+
+        k = trans_count = 0
+        for i, f_s1 in enumerate(flags_s1):
+            if f_s1:
+                for j in xrange(k, len_s2):
+                    if flags_s2[j]:
+                        k = j + 1
+                        break
+                if string1[i] != string2[j]:
+                    trans_count += 1
+
+        trans_count /= 2
+        common_chars = float(common_chars)
+        weight = ((common_chars / len_s1 + common_chars / len_s2 +
+                   (common_chars - trans_count) / common_chars)) / 3
+        return weight
+
+    def get_sim_score(self, string1, string2):
+        """Computes the normalized Jaro similarity score between two strings. Simply call get_raw_score.
+
+        Args:
+            string1,string2 (str): Input strings.
+
+        Returns:
+            Normalized Jaro similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+
+        Examples:
+            >>> jaro = Jaro()
+            >>> jaro.get_sim_score('MARTHA', 'MARHTA')
+            0.9444444444444445
+            >>> jaro.get_sim_score('DWAYNE', 'DUANE')
+            0.8222222222222223
+            >>> jaro.get_sim_score('DIXON', 'DICKSONX')
+            0.7666666666666666
+
+        """
+        return self.get_raw_score(string1, string2)

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/jaro_winkler.py

@@ -0,0 +1,106 @@
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.jaro import Jaro
+from py_stringmatching.similarity_measure.sequence_similarity_measure import \
+                                                    SequenceSimilarityMeasure
+
+
+class JaroWinkler(SequenceSimilarityMeasure):
+    """Computes Jaro-Winkler measure.
+
+    The Jaro-Winkler measure is designed to capture cases where two strings have a low Jaro score, but share a prefix and thus are likely to match.
+
+    Args:
+        prefix_weight (float): Weight to give to the prefix (defaults to 0.1).
+
+    Attributes:
+        prefix_weight (float): An attribute to store the prefix weight.
+    """
+
+    def __init__(self, prefix_weight=0.1):
+        self.prefix_weight = prefix_weight
+        super(JaroWinkler, self).__init__()
+
+    def get_raw_score(self, string1, string2):
+        """Computes the raw Jaro-Winkler score between two strings.
+
+        Args:
+            string1,string2 (str): Input strings.
+
+        Returns:
+            Jaro-Winkler similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+
+        Examples:
+            >>> jw = JaroWinkler()
+            >>> jw.get_raw_score('MARTHA', 'MARHTA')
+            0.9611111111111111
+            >>> jw.get_raw_score('DWAYNE', 'DUANE')
+            0.84
+            >>> jw.get_raw_score('DIXON', 'DICKSONX')
+            0.8133333333333332
+
+        """
+        
+        # input validations
+        utils.sim_check_for_none(string1, string2)
+        utils.tok_check_for_string_input(string1, string2)
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(string1, string2):
+            return 0
+
+        jw_score = Jaro().get_raw_score(string1, string2)
+        min_len = min(len(string1), len(string2))
+
+        # prefix length can be at max 4
+        j = min(min_len, 4)
+        i = 0
+        while i < j and string1[i] == string2[i] and string1[i]:
+            i += 1
+
+        if i:
+            jw_score += i * self.prefix_weight * (1 - jw_score)
+
+        return jw_score
+
+    def get_sim_score(self, string1, string2):
+        """Computes the normalized Jaro-Winkler similarity score between two strings. Simply call get_raw_score.
+
+        Args:
+            string1,string2 (str): Input strings.
+
+        Returns:
+            Normalized Jaro-Winkler similarity (float).
+
+        Raises:
+            TypeError : If the inputs are not strings or if one of the inputs is None.
+
+        Examples:
+            >>> jw = JaroWinkler()
+            >>> jw.get_sim_score('MARTHA', 'MARHTA')
+            0.9611111111111111
+            >>> jw.get_sim_score('DWAYNE', 'DUANE')
+            0.84
+            >>> jw.get_sim_score('DIXON', 'DICKSONX')
+            0.8133333333333332
+        """
+        return self.get_raw_score(string1, string2)
+
+    def get_prefix_weight(self):
+        """Get prefix weight.
+
+        Returns:
+            prefix weight (float).
+        """
+        return self.prefix_weight
+
+    def set_prefix_weight(self, prefix_weight):
+        """Set prefix weight.
+
+        Args:
+            prefix_weight (float): Weight to give to the prefix.
+        """
+        self.prefix_weight = prefix_weight
+        return True

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/levenshtein.py

@@ -0,0 +1,75 @@
+from __future__ import division
+
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.cython_levenshtein import levenshtein
+from py_stringmatching.similarity_measure.sequence_similarity_measure import \
+                                                    SequenceSimilarityMeasure
+
+
+class Levenshtein(SequenceSimilarityMeasure):
+    """Computes Levenshtein measure (also known as edit distance).
+
+    Levenshtein distance computes the minimum cost of transforming one string into the other. Transforming a string
+    is carried out using a sequence of the following operators: delete a character, insert a character, and
+    substitute one character for another.
+    """
+
+    def __init__(self):
+        super(Levenshtein, self).__init__()
+
+    def get_raw_score(self, string1, string2):
+        """Computes the raw Levenshtein distance between two strings.
+
+        Args:
+            string1,string2 (str): Input strings.
+
+        Returns:
+            Levenshtein distance (int).
+
+        Raises:
+            TypeError : If the inputs are not strings.
+
+        Examples:
+            >>> lev = Levenshtein()
+            >>> lev.get_raw_score('a', '')
+            1
+            >>> lev.get_raw_score('example', 'samples')
+            3
+            >>> lev.get_raw_score('levenshtein', 'frankenstein')
+            6
+        """
+        
+        # input validations
+        utils.sim_check_for_none(string1, string2)
+        utils.sim_check_for_string_inputs(string1, string2)
+        if utils.sim_check_for_exact_match(string1, string2):
+            return 0.0
+        return levenshtein(string1, string2)
+
+    def get_sim_score(self, string1, string2):
+        """Computes the normalized Levenshtein similarity score between two strings.
+
+        Args:
+            string1,string2 (str): Input strings.
+
+        Returns:
+            Normalized Levenshtein similarity (float).
+
+        Raises:
+            TypeError : If the inputs are not strings.
+
+        Examples:
+            >>> lev = Levenshtein()
+            >>> lev.get_sim_score('a', '')
+            0.0
+            >>> lev.get_sim_score('example', 'samples')
+            0.5714285714285714
+            >>> lev.get_sim_score('levenshtein', 'frankenstein')
+            0.5
+
+        """
+        raw_score = self.get_raw_score(string1, string2)
+        max_len = max(len(string1), len(string2))
+        if max_len == 0:
+            return 1.0
+        return 1 - (raw_score / max_len)

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/monge_elkan.py

@@ -0,0 +1,100 @@
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.jaro_winkler import JaroWinkler
+from py_stringmatching.similarity_measure.hybrid_similarity_measure import \
+                                                    HybridSimilarityMeasure
+
+
+class MongeElkan(HybridSimilarityMeasure):
+    """Computes Monge-Elkan measure.
+
+    The Monge-Elkan similarity measure is a type of hybrid similarity measure that combines the benefits of
+    sequence-based and set-based methods. This can be effective for domains in which more control is needed
+    over the similarity measure. It implicitly uses a secondary similarity measure, such as Levenshtein to compute
+    over all similarity score. See the string matching chapter in the DI book (Principles of Data Integration). 
+
+    Args:
+        sim_func (function): Secondary similarity function. This is expected to be a sequence-based
+                             similarity measure (defaults to Jaro-Winkler similarity measure).
+
+    Attributes:
+        sim_func (function): An attribute to store the secondary similarity function.
+    """
+
+    def __init__(self, sim_func=JaroWinkler().get_raw_score):
+        self.sim_func = sim_func
+        super(MongeElkan, self).__init__()
+
+    def get_raw_score(self, bag1, bag2):
+        """Computes the raw Monge-Elkan score between two bags (lists).
+
+        Args:
+            bag1,bag2 (list): Input lists.
+
+        Returns:
+            Monge-Elkan similarity score (float).
+
+        Raises:
+            TypeError : If the inputs are not lists or if one of the inputs is None.
+
+        Examples:
+            >>> me = MongeElkan()
+            >>> me.get_raw_score(['Niall'], ['Neal'])
+            0.8049999999999999
+            >>> me.get_raw_score(['Niall'], ['Nigel'])
+            0.7866666666666667
+            >>> me.get_raw_score(['Comput.', 'Sci.', 'and', 'Eng.', 'Dept.,', 'University', 'of', 'California,', 'San', 'Diego'], ['Department', 'of', 'Computer', 'Science,', 'Univ.', 'Calif.,', 'San', 'Diego'])
+            0.8677218614718616
+          >>> me.get_raw_score([''], ['a'])
+            0.0
+            >>> me = MongeElkan(sim_func=NeedlemanWunsch().get_raw_score)
+            >>> me.get_raw_score(['Comput.', 'Sci.', 'and', 'Eng.', 'Dept.,', 'University', 'of', 'California,', 'San', 'Diego'], ['Department', 'of', 'Computer', 'Science,', 'Univ.', 'Calif.,', 'San', 'Diego'])
+            2.0
+            >>> me = MongeElkan(sim_func=Affine().get_raw_score)
+            >>> me.get_raw_score(['Comput.', 'Sci.', 'and', 'Eng.', 'Dept.,', 'University', 'of', 'California,', 'San', 'Diego'], ['Department', 'of', 'Computer', 'Science,', 'Univ.', 'Calif.,', 'San', 'Diego'])
+            2.25
+
+        References:
+            * Principles of Data Integration book
+        """
+        
+        # input validations
+        utils.sim_check_for_none(bag1, bag2)
+        utils.sim_check_for_list_or_set_inputs(bag1, bag2)
+
+        # if exact match return 1.0
+        if utils.sim_check_for_exact_match(bag1, bag2):
+            return 1.0
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(bag1, bag2):
+            return 0
+
+        # aggregated sum of all the max sim score of all the elements in bag1
+        # with elements in bag2
+        sum_of_maxes = 0
+        for el1 in bag1:
+            max_sim = float('-inf')
+            for el2 in bag2:
+                max_sim = max(max_sim, self.sim_func(el1, el2))
+            sum_of_maxes += max_sim
+
+        sim = float(sum_of_maxes) / float(len(bag1))
+
+        return sim
+
+    def get_sim_func(self):
+        """Get the secondary similarity function.
+
+        Returns:
+            secondary similarity function (function).
+        """
+        return self.sim_func
+
+    def set_sim_func(self, sim_func):
+        """Set the secondary similarity function.
+
+        Args:
+            sim_func (function): Secondary similarity function.
+        """
+        self.sim_func = sim_func
+        return True