py-stringmatching 0.1.0__zip

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/tfidf.py

@@ -0,0 +1,193 @@
+from __future__ import division
+from math import log, sqrt
+import collections
+
+from py_stringmatching import utils
+from py_stringmatching.similarity_measure.token_similarity_measure import \
+                                                    TokenSimilarityMeasure
+
+
+class TfIdf(TokenSimilarityMeasure):
+    """Computes TF/IDF measure.
+
+    This measure employs the notion of TF/IDF score commonly used in information retrieval (IR) to
+    find documents that are relevant to keyword queries. The intuition underlying the TF/IDF measure
+    is that two strings are similar if they share distinguishing terms. See the string matching chapter in the book "Principles of Data Integration"
+    
+    Note:
+        Currently when you create a TF/IDF similarity measure object, the dampen flag is set to False by default. In most cases, you will want to set this flag to True, so that the TF and IDF formulas use logarithmic. So when creating this object, consider setting the flag to True. This will likely be fixed in the next release. 
+
+    Args:
+        corpus_list (list of lists): The corpus that will be used to compute TF and IDF values. This corpus is a list of strings, where each string has been tokenized into a list of tokens (that is, a bag of tokens). The default is set to None. In this case, when we call this TF/IDF measure on two input strings (using get_raw_score or get_sim_score), the corpus is taken to be the list of those two strings. 
+        dampen (boolean): Flag to indicate whether 'log' should be used in TF and IDF formulas. In general this flag should be set to True. 
+
+    Attributes:
+        dampen (boolean): An attribute to store the dampen flag.
+    """
+
+    def __init__(self, corpus_list=None, dampen=False):
+        self.__corpus_list = corpus_list
+        self.__document_frequency = {}
+        self.__compute_document_frequency()
+        self.__corpus_size = 0 if self.__corpus_list is None else (
+                                                         len(self.__corpus_list))
+        self.dampen = dampen        
+        super(TfIdf, self).__init__()
+
+    def get_raw_score(self, bag1, bag2):
+        """Computes the raw TF/IDF score between two lists.
+
+        Args:
+            bag1,bag2 (list): Input lists.
+
+        Returns:
+            TF/IDF score between the input lists (float).
+
+        Raises:
+            TypeError : If the inputs are not lists or if one of the inputs is None.
+
+        Examples:
+            
+            >>> # here the corpus is a list of three strings that 
+            >>> # have been tokenized into three lists of tokens
+            >>> tfidf = TfIdf([['a', 'b', 'a'], ['a', 'c'], ['a']])
+            >>> tfidf.get_raw_score(['a', 'b', 'a'], ['a', 'c'])
+            0.17541160386140586
+            >>> tfidf.get_raw_score(['a', 'b', 'a'], ['a'])
+            0.5547001962252291
+            >>> tfidf = TfIdf([['a', 'b', 'a'], ['a', 'c'], ['a'], ['b']], True)
+            >>> tfidf.get_raw_score(['a', 'b', 'a'], ['a', 'c'])
+            0.11166746710505392
+            >>> tfidf = TfIdf([['x', 'y'], ['w'], ['q']])
+            >>> tfidf.get_raw_score(['a', 'b', 'a'], ['a'])
+            0.0
+            >>> tfidf = TfIdf([['x', 'y'], ['w'], ['q']], True)
+            >>> tfidf.get_raw_score(['a', 'b', 'a'], ['a'])
+            0.0
+            >>> tfidf = TfIdf()
+            >>> tfidf.get_raw_score(['a', 'b', 'a'], ['a'])
+            0.7071067811865475
+        """
+        # input validations
+        utils.sim_check_for_none(bag1, bag2)
+        utils.sim_check_for_list_or_set_inputs(bag1, bag2)
+
+        # if the strings match exactly return 1.0
+        if utils.sim_check_for_exact_match(bag1, bag2):
+            return 1.0
+
+        # if one of the strings is empty return 0
+        if utils.sim_check_for_empty(bag1, bag2):
+            return 0
+
+        # term frequency for input strings
+        tf_x, tf_y = collections.Counter(bag1), collections.Counter(bag2)
+         
+        # find unique elements in the input lists and their document frequency 
+        local_df = {}
+        for element in tf_x:
+            local_df[element] = local_df.get(element, 0) + 1
+        for element in tf_y:
+            local_df[element] = local_df.get(element, 0) + 1
+
+        # if corpus is not provided treat input string as corpus
+        curr_df, corpus_size = (local_df, 2) if self.__corpus_list is None else (
+                                   (self.__document_frequency, self.__corpus_size))
+
+        idf_element, v_x, v_y, v_x_y, v_x_2, v_y_2 = (0.0, 0.0, 0.0, 
+                                                      0.0, 0.0, 0.0)
+
+        # tfidf calculation
+        for element in local_df.keys():
+            df_element = curr_df.get(element)
+            if df_element is None:
+                continue
+            idf_element = corpus_size * 1.0 / df_element
+            v_x = 0 if element not in tf_x else (log(idf_element) * log(tf_x[element] + 1)) if self.dampen else (
+                  idf_element * tf_x[element])
+            v_y = 0 if element not in tf_y else (log(idf_element) * log(tf_y[element] + 1)) if self.dampen else (
+                  idf_element * tf_y[element])
+            v_x_y += v_x * v_y
+            v_x_2 += v_x * v_x
+            v_y_2 += v_y * v_y
+
+        return 0.0 if v_x_y == 0 else v_x_y / (sqrt(v_x_2) * sqrt(v_y_2))
+
+    def get_sim_score(self, bag1, bag2):
+        """Computes the normalized TF/IDF similarity score between two lists. Simply call get_raw_score.
+
+        Args:
+            bag1,bag2 (list): Input lists.
+
+        Returns:
+            Normalized TF/IDF similarity score between the input lists (float).
+
+        Raises:
+            TypeError : If the inputs are not lists or if one of the inputs is None.
+
+        Examples:
+            
+            >>> tfidf = TfIdf([['a', 'b', 'a'], ['a', 'c'], ['a']])
+            >>> tfidf.get_sim_score(['a', 'b', 'a'], ['a', 'c'])
+            0.17541160386140586
+            >>> tfidf.get_sim_score(['a', 'b', 'a'], ['a'])
+            0.5547001962252291
+            >>> tfidf = TfIdf([['a', 'b', 'a'], ['a', 'c'], ['a'], ['b']], True)
+            >>> tfidf.get_sim_score(['a', 'b', 'a'], ['a', 'c'])
+            0.11166746710505392
+            >>> tfidf = TfIdf([['x', 'y'], ['w'], ['q']])
+            >>> tfidf.get_sim_score(['a', 'b', 'a'], ['a'])
+            0.0
+            >>> tfidf = TfIdf([['x', 'y'], ['w'], ['q']], True)
+            >>> tfidf.get_sim_score(['a', 'b', 'a'], ['a'])
+            0.0
+            >>> tfidf = TfIdf()
+            >>> tfidf.get_sim_score(['a', 'b', 'a'], ['a'])
+            0.7071067811865475
+        """
+        return self.get_raw_score(bag1, bag2)
+
+    def get_dampen(self):
+        """Get dampen flag.
+
+        Returns:
+            dampen flag (boolean).
+        """
+        return self.dampen
+
+    def get_corpus_list(self):
+        """Get corpus list.
+
+        Returns:
+            corpus list (list of lists).
+        """
+        return self.__corpus_list
+
+    def set_dampen(self, dampen):
+        """Set dampen flag.
+
+        Args:
+            dampen (boolean): Flag to indicate whether 'log' should be applied to TF and IDF formulas.
+        """
+        self.dampen = dampen
+        return True
+
+    def set_corpus_list(self, corpus_list):
+        """Set corpus list.
+
+        Args:
+            corpus_list (list of lists): Corpus list.
+        """
+        self.__corpus_list = corpus_list
+        self.__document_frequency = {}
+        self.__compute_document_frequency()
+        self.__corpus_size = 0 if self.__corpus_list is None else (
+                                                         len(self.__corpus_list))
+        return True
+
+    def __compute_document_frequency(self):
+        if self.__corpus_list != None:
+            for document in self.__corpus_list:
+                for element in set(document):
+                    self.__document_frequency[element] = ( 
+                        self.__document_frequency.get(element, 0) + 1)

py_stringmatching-0.1.0/py_stringmatching/similarity_measure/token_similarity_measure.py

@@ -0,0 +1,7 @@
+"""Token based similarity measure"""
+
+from py_stringmatching.similarity_measure.similarity_measure import \
+                                                        SimilarityMeasure
+
+class TokenSimilarityMeasure(SimilarityMeasure):
+    pass