py-stringmatching 0.1.0__zip

py_stringmatching-0.1.0/py_stringmatching/tests/test_tokenizers.py

@@ -0,0 +1,305 @@
+from __future__ import unicode_literals
+
+import unittest
+from nose.tools import *
+
+from py_stringmatching.tokenizer.alphabetic_tokenizer import AlphabeticTokenizer
+from py_stringmatching.tokenizer.alphanumeric_tokenizer import AlphanumericTokenizer
+from py_stringmatching.tokenizer.delimiter_tokenizer import DelimiterTokenizer
+from py_stringmatching.tokenizer.qgram_tokenizer import QgramTokenizer
+from py_stringmatching.tokenizer.whitespace_tokenizer import WhitespaceTokenizer
+
+
+class QgramTokenizerTestCases(unittest.TestCase):
+    def setUp(self):
+        self.qg1_tok = QgramTokenizer(1)
+        self.qg2_tok = QgramTokenizer()
+        self.qg2_tok_return_set = QgramTokenizer(return_set=True)
+        self.qg3_tok = QgramTokenizer(3)
+
+    def test_qgrams_valid(self):
+        self.assertEqual(self.qg2_tok.tokenize(''), [])
+        self.assertEqual(self.qg2_tok.tokenize('a'), [])
+        self.assertEqual(self.qg2_tok.tokenize('aa'), ['aa'])
+        self.assertEqual(self.qg2_tok.tokenize('database'),
+                         ['da', 'at', 'ta', 'ab', 'ba', 'as', 'se'])
+        self.assertEqual(self.qg2_tok.tokenize('aabaabcdba'),
+                         ['aa', 'ab', 'ba', 'aa', 'ab', 'bc', 'cd', 'db', 'ba'])
+        self.assertEqual(self.qg2_tok_return_set.tokenize('aabaabcdba'),
+                         ['aa', 'ab', 'ba', 'bc', 'cd', 'db'])
+        self.assertEqual(self.qg1_tok.tokenize('d'), ['d'])
+        self.assertEqual(self.qg3_tok.tokenize('database'),
+                         ['dat', 'ata', 'tab', 'aba', 'bas', 'ase'])
+
+    def test_get_return_set(self):
+        self.assertEqual(self.qg2_tok.get_return_set(), False)
+        self.assertEqual(self.qg2_tok_return_set.get_return_set(), True)
+
+    def test_get_qval(self):
+        self.assertEqual(self.qg2_tok.get_qval(), 2)
+        self.assertEqual(self.qg3_tok.get_qval(), 3)
+
+    def test_set_return_set(self):
+        tok = QgramTokenizer()
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(tok.tokenize('aabaabcdba'),
+                         ['aa', 'ab', 'ba', 'aa', 'ab', 'bc', 'cd', 'db', 'ba'])
+        self.assertEqual(tok.set_return_set(True), True)
+        self.assertEqual(tok.get_return_set(), True)
+        self.assertEqual(tok.tokenize('aabaabcdba'),
+                         ['aa', 'ab', 'ba', 'bc', 'cd', 'db'])
+        self.assertEqual(tok.set_return_set(False), True)
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(tok.tokenize('aabaabcdba'),
+                         ['aa', 'ab', 'ba', 'aa', 'ab', 'bc', 'cd', 'db', 'ba'])
+
+    def test_set_qval(self):
+        tok = QgramTokenizer()
+        self.assertEqual(tok.get_qval(), 2)
+        self.assertEqual(tok.tokenize('database'),
+                         ['da', 'at', 'ta', 'ab', 'ba', 'as', 'se'])
+        self.assertEqual(tok.set_qval(3), True)
+        self.assertEqual(tok.get_qval(), 3)
+        self.assertEqual(tok.tokenize('database'),
+                         ['dat', 'ata', 'tab', 'aba', 'bas', 'ase'])
+
+    @raises(TypeError)
+    def test_qgrams_none(self):
+        self.qg2_tok.tokenize(None)
+
+    @raises(AssertionError)
+    def test_qgrams_invalid1(self):
+        invalid_qg_tok = QgramTokenizer(0) 
+
+    @raises(TypeError)
+    def test_qgrams_invalid2(self):
+        self.qg2_tok.tokenize(99)
+
+    @raises(AssertionError)
+    def test_set_qval_invalid(self):
+        qg_tok = QgramTokenizer()
+        qg_tok.set_qval(0)
+
+
+class DelimiterTokenizerTestCases(unittest.TestCase):
+    def setUp(self):
+        self.delim_tok1 = DelimiterTokenizer()
+        self.delim_tok2 = DelimiterTokenizer(set([',']))
+        self.delim_tok3 = DelimiterTokenizer(set(['*', '.']))
+        self.delim_tok4 = DelimiterTokenizer(set(['..', 'ab']))    
+        self.delim_tok4_list = DelimiterTokenizer(['..', 'ab', '..'])
+        self.delim_tok4_return_set = DelimiterTokenizer(set(['..', 'ab']),
+                                                        return_set=True)
+
+    def test_delimiter_valid(self):
+        self.assertEqual(self.delim_tok1.tokenize('data science'),
+                         ['data', 'science'])
+        self.assertEqual(self.delim_tok2.tokenize('data,science'),
+                         ['data', 'science'])
+        self.assertEqual(self.delim_tok2.tokenize('data science'),
+                         ['data science'])
+        self.assertEqual(self.delim_tok3.tokenize('ab cd*ef.*bb. gg.'),
+                         ['ab cd', 'ef', 'bb', ' gg'])
+        self.assertEqual(
+            self.delim_tok4.tokenize('ab cd..efabbb....ggab cd..efabgh'),
+            [' cd', 'ef', 'bb', 'gg', ' cd', 'ef', 'gh'])
+        self.assertEqual(
+            self.delim_tok4_list.tokenize('ab cd..efabbb....ggab cd..efabgh'),
+            [' cd', 'ef', 'bb', 'gg', ' cd', 'ef', 'gh'])
+        self.assertEqual(
+            self.delim_tok4_return_set.tokenize(
+                'ab cd..efabbb....ggab cd..efabgh'),
+            [' cd', 'ef', 'bb', 'gg', 'gh'])
+
+    def test_get_return_set(self):
+        self.assertEqual(self.delim_tok4.get_return_set(), False)
+        self.assertEqual(self.delim_tok4_return_set.get_return_set(), True)
+
+    def test_get_delim_set(self):
+        self.assertSetEqual(self.delim_tok1.get_delim_set(), {' '})
+        self.assertSetEqual(self.delim_tok3.get_delim_set(), {'*', '.'})
+        self.assertSetEqual(self.delim_tok4_list.get_delim_set(), {'..', 'ab'})
+
+    def test_set_return_set(self):
+        tok = DelimiterTokenizer(set(['..', 'ab']))
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(
+            tok.tokenize('ab cd..efabbb....ggab cd..efabgh'),
+            [' cd', 'ef', 'bb', 'gg', ' cd', 'ef', 'gh'])
+        self.assertEqual(tok.set_return_set(True), True)
+        self.assertEqual(tok.get_return_set(), True)
+        self.assertEqual(
+            tok.tokenize('ab cd..efabbb....ggab cd..efabgh'),
+            [' cd', 'ef', 'bb', 'gg', 'gh'])
+        self.assertEqual(tok.set_return_set(False), True)
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(
+            tok.tokenize('ab cd..efabbb....ggab cd..efabgh'),
+            [' cd', 'ef', 'bb', 'gg', ' cd', 'ef', 'gh'])
+
+    def test_set_delim_set(self):
+        tok = DelimiterTokenizer(['*', '.'])
+        self.assertSetEqual(tok.get_delim_set(), {'*', '.'})
+        self.assertEqual(tok.tokenize('ab cd*ef.*bb. gg.'),
+                         ['ab cd', 'ef', 'bb', ' gg'])      
+        self.assertEqual(tok.set_delim_set({'..', 'ab'}), True)
+        self.assertSetEqual(tok.get_delim_set(), {'..', 'ab'})
+        self.assertEqual(
+            tok.tokenize('ab cd..efabbb....ggab cd..efabgh'),
+            [' cd', 'ef', 'bb', 'gg', ' cd', 'ef', 'gh'])
+
+    @raises(TypeError)
+    def test_delimiter_invalid1(self):
+        invalid_delim_tok = DelimiterTokenizer(set([',', 10]))
+
+    @raises(TypeError)
+    def test_delimiter_invalid2(self):
+        self.delim_tok1.tokenize(None)
+
+    @raises(TypeError)
+    def test_delimiter_invalid3(self):
+        self.delim_tok1.tokenize(99)
+
+
+class WhitespaceTokenizerTestCases(unittest.TestCase):
+    def setUp(self):
+        self.ws_tok = WhitespaceTokenizer()
+        self.ws_tok_return_set = WhitespaceTokenizer(return_set=True)
+
+    def test_whitespace_tok_valid(self):
+        self.assertEqual(self.ws_tok.tokenize('data science'),
+                         ['data', 'science'])
+        self.assertEqual(self.ws_tok.tokenize('data        science'),
+                         ['data', 'science'])
+        self.assertEqual(self.ws_tok.tokenize('data   science'),
+                         ['data', 'science'])
+        self.assertEqual(self.ws_tok.tokenize('data\tscience'),
+                         ['data', 'science'])
+        self.assertEqual(self.ws_tok.tokenize('data\nscience'),
+                         ['data', 'science'])
+        self.assertEqual(self.ws_tok.tokenize('ab cd ab bb cd db'),
+                         ['ab', 'cd', 'ab', 'bb', 'cd', 'db'])
+        self.assertEqual(self.ws_tok_return_set.tokenize('ab cd ab bb cd db'),
+                         ['ab', 'cd', 'bb', 'db'])
+
+    def test_get_return_set(self):
+        self.assertEqual(self.ws_tok.get_return_set(), False)
+        self.assertEqual(self.ws_tok_return_set.get_return_set(), True)
+
+    def test_set_return_set(self):
+        tok = WhitespaceTokenizer()
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(tok.tokenize('ab cd ab bb cd db'),
+                         ['ab', 'cd', 'ab', 'bb', 'cd', 'db'])
+        self.assertEqual(tok.set_return_set(True), True)
+        self.assertEqual(tok.get_return_set(), True)
+        self.assertEqual(tok.tokenize('ab cd ab bb cd db'),
+                         ['ab', 'cd', 'bb', 'db'])
+        self.assertEqual(tok.set_return_set(False), True)
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(tok.tokenize('ab cd ab bb cd db'),
+                         ['ab', 'cd', 'ab', 'bb', 'cd', 'db'])
+
+    def test_get_delim_set(self):
+        self.assertSetEqual(self.ws_tok.get_delim_set(), {' ', '\t', '\n'})
+
+    @raises(TypeError)
+    def test_whitespace_tok_invalid1(self):
+        self.ws_tok.tokenize(None)
+
+    @raises(TypeError)
+    def test_whitespace_tok_invalid2(self):
+        self.ws_tok.tokenize(99)
+
+    @raises(AttributeError)
+    def test_set_delim_set(self):
+        self.ws_tok.set_delim_set({'*', '.'})
+
+
+class AlphabeticTokenizerTestCases(unittest.TestCase):
+    def setUp(self):
+        self.al_tok = AlphabeticTokenizer()
+        self.al_tok_return_set = AlphabeticTokenizer(return_set=True)
+
+    def test_alphabetic_tok_valid(self):
+        self.assertEqual(self.al_tok.tokenize(''), [])
+        self.assertEqual(self.al_tok.tokenize('99'), [])
+        self.assertEqual(self.al_tok.tokenize('hello'), ['hello'])
+        self.assertEqual(self.al_tok.tokenize('ab bc. cd##de ef09 bc fg ab.'),
+                         ['ab', 'bc', 'cd', 'de', 'ef', 'bc', 'fg', 'ab'])
+        self.assertEqual(
+            self.al_tok_return_set.tokenize('ab bc. cd##de ef09 bc fg ab.'),
+            ['ab', 'bc', 'cd', 'de', 'ef', 'fg'])
+
+    def test_get_return_set(self):
+        self.assertEqual(self.al_tok.get_return_set(), False)
+        self.assertEqual(self.al_tok_return_set.get_return_set(), True)
+
+    def test_set_return_set(self):
+        tok = AlphabeticTokenizer()
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(tok.tokenize('ab bc. cd##de ef09 bc fg ab.'),
+                         ['ab', 'bc', 'cd', 'de', 'ef', 'bc', 'fg', 'ab'])
+        self.assertEqual(tok.set_return_set(True), True)
+        self.assertEqual(tok.get_return_set(), True)
+        self.assertEqual(
+            tok.tokenize('ab bc. cd##de ef09 bc fg ab.'),
+            ['ab', 'bc', 'cd', 'de', 'ef', 'fg'])
+        self.assertEqual(tok.set_return_set(False), True)
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(tok.tokenize('ab bc. cd##de ef09 bc fg ab.'),
+                         ['ab', 'bc', 'cd', 'de', 'ef', 'bc', 'fg', 'ab'])
+
+    @raises(TypeError)
+    def test_alphabetic_tok_invalid1(self):
+        self.al_tok.tokenize(None)
+
+    @raises(TypeError)
+    def test_alphabetic_tok_invalid2(self):
+        self.al_tok.tokenize(99)
+
+
+class AlphanumericTokenizerTestCases(unittest.TestCase):
+    def setUp(self):
+        self.alnum_tok = AlphanumericTokenizer()
+        self.alnum_tok_return_set = AlphanumericTokenizer(return_set=True)
+
+    def test_alphanumeric_tok_valid(self):
+        self.assertEqual(self.alnum_tok.tokenize(''), [])
+        self.assertEqual(self.alnum_tok.tokenize('#$'), [])
+        self.assertEqual(self.alnum_tok.tokenize('hello99'), ['hello99'])
+        self.assertEqual(
+            self.alnum_tok.tokenize(',data9,(science), data9#.(integration).88!'),
+            ['data9', 'science', 'data9', 'integration', '88'])
+        self.assertEqual(self.alnum_tok_return_set.tokenize(
+                             ',data9,(science), data9#.(integration).88!'),
+                         ['data9', 'science', 'integration', '88'])
+
+    def test_get_return_set(self):
+        self.assertEqual(self.alnum_tok.get_return_set(), False)
+        self.assertEqual(self.alnum_tok_return_set.get_return_set(), True)
+
+    def test_set_return_set(self):
+        tok = AlphanumericTokenizer()
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(
+            tok.tokenize(',data9,(science), data9#.(integration).88!'),
+            ['data9', 'science', 'data9', 'integration', '88'])
+        self.assertEqual(tok.set_return_set(True), True)
+        self.assertEqual(tok.get_return_set(), True)
+        self.assertEqual(
+            tok.tokenize(',data9,(science), data9#.(integration).88!'),
+            ['data9', 'science', 'integration', '88'])
+        self.assertEqual(tok.set_return_set(False), True)
+        self.assertEqual(tok.get_return_set(), False)
+        self.assertEqual(
+            tok.tokenize(',data9,(science), data9#.(integration).88!'),
+            ['data9', 'science', 'data9', 'integration', '88'])
+
+    @raises(TypeError)
+    def test_alphanumeric_tok_invalid1(self):
+        self.alnum_tok.tokenize(None)
+
+    @raises(TypeError)
+    def test_alphanumeric_tok_invalid2(self):
+        self.alnum_tok.tokenize(99)

py_stringmatching-0.1.0/py_stringmatching/tokenizer/alphabetic_tokenizer.py

@@ -0,0 +1,51 @@
+import re
+
+from py_stringmatching import utils
+from py_stringmatching.tokenizer.definition_tokenizer import DefinitionTokenizer
+
+
+class AlphabeticTokenizer(DefinitionTokenizer):
+    """Returns tokens that are maximal sequences of consecutive alphabetical characters.
+    
+    Args:
+        return_set (boolean): A flag to indicate whether to return a set of tokens instead of a bag of tokens (defaults to False).
+        
+    Attributes: 
+        return_set (boolean): An attribute that stores the value for the flag return_set. 
+    """
+    
+    def __init__(self, return_set=False):
+        self.__al_regex = re.compile('[a-zA-Z]+')
+        super(AlphabeticTokenizer, self).__init__(return_set)
+
+    def tokenize(self, input_string):
+        """Tokenizes input string into alphabetical tokens.
+        
+        Args:
+            input_string (str): The string to be tokenized.
+
+        Returns:
+            A Python list, which represents a set of tokens if the flag return_set is True, and a bag of tokens otherwise. 
+
+        Raises:
+            TypeError : If the input is not a string.
+
+        Examples:
+            >>> al_tok = AlphabeticTokenizer()
+            >>> al_tok.tokenize('data99science, data#integration.')
+            ['data', 'science', 'data', 'integration']
+            >>> al_tok.tokenize('99')
+            []
+            >>> al_tok = AlphabeticTokenizer(return_set=True) 
+            >>> al_tok.tokenize('data99science, data#integration.')
+            ['data', 'science', 'integration']
+        """
+        utils.tok_check_for_none(input_string)
+        utils.tok_check_for_string_input(input_string)
+
+        token_list = list(filter(None, self.__al_regex.findall(input_string)))
+
+        if self.return_set:
+            return utils.convert_bag_to_set(token_list)
+
+        return token_list

py_stringmatching-0.1.0/py_stringmatching/tokenizer/alphanumeric_tokenizer.py

@@ -0,0 +1,54 @@
+import re
+
+from py_stringmatching import utils
+from py_stringmatching.tokenizer.definition_tokenizer import DefinitionTokenizer
+
+
+class AlphanumericTokenizer(DefinitionTokenizer):
+    """Returns tokens that are maximal sequences of consecutive alphanumeric characters. 
+
+    Args:
+        return_set (boolean): A flag to indicate whether to return a set of
+                              tokens instead of a bag of tokens (defaults to False).
+                              
+    Attributes: 
+        return_set (boolean): An attribute to store the value of the flag return_set.
+    """
+    
+    def __init__(self, return_set=False):
+        self.__alnum_regex = re.compile('[a-zA-Z0-9]+')
+        super(AlphanumericTokenizer, self).__init__(return_set)
+
+    def tokenize(self, input_string):
+        """Tokenizes input string into alphanumeric tokens.
+
+        Args:
+            input_string (str): The string to be tokenized.
+
+        Returns:
+            A Python list, which represents a set of tokens if the flag return_set is true, and a bag of tokens otherwise. 
+
+        Raises:
+            TypeError : If the input is not a string.
+
+        Examples:
+            >>> alnum_tok = AlphanumericTokenizer()
+            >>> alnum_tok.tokenize('data9,(science), data9#.(integration).88')
+            ['data9', 'science', 'data9', 'integration', '88']
+            >>> alnum_tok.tokenize('#.&')
+            []
+            >>> alnum_tok = AlphanumericTokenizer(return_set=True) 
+            >>> alnum_tok.tokenize('data9,(science), data9#.(integration).88')
+            ['data9', 'science', 'integration', '88']
+                      
+        """
+        utils.tok_check_for_none(input_string)
+        utils.tok_check_for_string_input(input_string)
+
+        token_list = list(filter(None,
+                                 self.__alnum_regex.findall(input_string)))
+
+        if self.return_set:
+            return utils.convert_bag_to_set(token_list)
+
+        return token_list

py_stringmatching-0.1.0/py_stringmatching/tokenizer/definition_tokenizer.py

@@ -0,0 +1,18 @@
+from py_stringmatching.tokenizer.tokenizer import Tokenizer
+
+
+class DefinitionTokenizer(Tokenizer):
+    """A class of tokenizers that uses a definition to find tokens, as opposed to using delimiters.
+    
+    Examples of definitions include alphabetical tokens, qgram tokens. Examples of delimiters include white space, punctuations.
+
+    Args:
+        return_set (boolean): A flag to indicate whether to return a set of
+                              tokens instead of a bag of tokens (defaults to False).
+                              
+    Attributes: 
+        return_set (boolean): An attribute to store the flag return_set. 
+    """
+    
+    def __init__(self, return_set=False):
+        super(DefinitionTokenizer, self).__init__(return_set)

py_stringmatching-0.1.0/py_stringmatching/tokenizer/delimiter_tokenizer.py

@@ -0,0 +1,99 @@
+import re
+
+from py_stringmatching import utils
+from py_stringmatching.tokenizer.tokenizer import Tokenizer
+
+
+class DelimiterTokenizer(Tokenizer):
+    """Uses delimiters to find tokens, as apposed to using definitions. 
+    
+    Examples of delimiters include white space and punctuations. Examples of definitions include alphabetical and qgram tokens. 
+
+    Args:
+        delim_set (set): A set of delimiter strings (defaults to space delimiter).
+        return_set (boolean): A flag to indicate whether to return a set of
+                              tokens instead of a bag of tokens (defaults to False).
+                              
+    Attributes: 
+        return_set (boolean): An attribute to store the value of the flag return_set.
+    """
+
+    def __init__(self, delim_set=set([' ']), return_set=False):
+        self.__delim_set = None
+        self.__use_split = None
+        self.__delim_str = None
+        self.__delim_regex = None
+        self._update_delim_set(delim_set)
+        super(DelimiterTokenizer, self).__init__(return_set)
+
+    def tokenize(self, input_string):
+        """Tokenizes input string based on the set of delimiters.
+
+        Args:
+            input_string (str): The string to be tokenized. 
+
+        Returns:
+            A Python list which is a set or a bag of tokens, depending on whether return_set flag is set to True or False. 
+
+        Raises:
+            TypeError : If the input is not a string.
+
+        Examples:
+            >>> delim_tok = DelimiterTokenizer() 
+            >>> delim_tok.tokenize('data science')
+            ['data', 'science']
+            >>> delim_tok = DelimiterTokenizer(['$#$']) 
+            >>> delim_tok.tokenize('data$#$science')
+            ['data', 'science']
+            >>> delim_tok = DelimiterTokenizer([',', '.']) 
+            >>> delim_tok.tokenize('data,science.data,integration.')
+            ['data', 'science', 'data', 'integration']
+            >>> delim_tok = DelimiterTokenizer([',', '.'], return_set=True) 
+            >>> delim_tok.tokenize('data,science.data,integration.')
+            ['data', 'science', 'integration']
+
+        """
+        utils.tok_check_for_none(input_string)
+        utils.tok_check_for_string_input(input_string)
+    
+        if self.__use_split:
+            token_list = list(filter(None,
+                                     input_string.split(self.__delim_str)))
+        else:
+            token_list = list(filter(None,
+                                     self.__delim_regex.split(input_string)))
+
+        if self.return_set:
+            return utils.convert_bag_to_set(token_list)
+
+        return token_list
+
+    def get_delim_set(self):
+        """Gets the current set of delimiters.
+        
+        Returns:
+            A Python set which is the current set of delimiters. 
+        """
+        return self.__delim_set
+
+    def set_delim_set(self, delim_set):
+        """Sets the current set of delimiters.
+        
+        Args:
+            delim_set (set): A set of delimiter strings.
+        """
+        return self._update_delim_set(delim_set)
+
+    def _update_delim_set(self, delim_set):
+        if not isinstance(delim_set, set):
+            delim_set = set(delim_set)
+        self.__delim_set = delim_set
+        # if there is only one delimiter string, use split instead of regex
+        self.__use_split = False
+        if len(self.__delim_set) == 1:
+            self.__delim_str = list(self.__delim_set)[0]
+            self.__use_split = True
+        else:
+            self.__delim_regex = re.compile('|'.join(
+                                     map(re.escape, self.__delim_set)))
+        return True

py_stringmatching-0.1.0/py_stringmatching/tokenizer/qgram_tokenizer.py

@@ -0,0 +1,90 @@
+from py_stringmatching import utils
+from six.moves import xrange
+from py_stringmatching.tokenizer.definition_tokenizer import DefinitionTokenizer
+
+
+class QgramTokenizer(DefinitionTokenizer):
+    """Returns tokens that are sequences of q consecutive characters.
+    
+    A qgram of an input string s is a substring t (of s) which is a sequence of q consecutive characters. Qgrams are also known as
+    ngrams or kgrams. 
+
+    Args:
+        qval (int): A value for q, that is, the qgram's length (defaults to 2).
+        return_set (boolean): A flag to indicate whether to return a set of
+                              tokens or a bag of tokens (defaults to False).
+                              
+    Attributes:
+        qval (int): An attribute to store the q value.
+        return_set (boolean): An attribute to store the flag return_set.
+    """
+    
+    def __init__(self, qval=2, return_set=False):
+        if qval < 1:
+            raise AssertionError("qval cannot be less than 1") 
+        self.qval = qval
+        super(QgramTokenizer, self).__init__(return_set)
+
+    def tokenize(self, input_string):
+        """Tokenizes input string into qgrams.
+
+        Args:
+            input_string (str): The string to be tokenized. 
+
+        Returns:
+            A Python list, which is a set or a bag of qgrams, depending on whether return_set flag is True or False. 
+
+        Raises:
+            TypeError : If the input is not a string
+
+        Examples:
+            >>> qg2_tok = QgramTokenizer()
+            >>> qg2_tok.tokenize('database')
+            ['da','at','ta','ab','ba','as','se']
+            >>> qg2_tok.tokenize('a')
+            []
+            >>> qg3_tok = QgramTokenizer(3) 
+            >>> qg3_tok.tokenize('database')
+            ['dat', 'ata', 'tab', 'aba', 'bas', 'ase']
+                      
+            As these examples show, the current qgram tokenizer does not consider the case of appending #s at the 
+            start and the end of the input string. This is left for future work. 
+        """
+        utils.tok_check_for_none(input_string)
+        utils.tok_check_for_string_input(input_string)
+
+        qgram_list = []
+
+        if len(input_string) < self.qval:
+            return qgram_list
+
+        qgram_list = [input_string[i:i + self.qval] for i in 
+                          xrange(len(input_string) - (self.qval - 1))]
+        qgram_list = list(filter(None, qgram_list))
+
+        if self.return_set:
+            return utils.convert_bag_to_set(qgram_list)            
+
+        return qgram_list
+
+    def get_qval(self):
+        """Gets the value of the qval attribute, which is the length of qgrams. 
+
+        Returns:
+            The value of the qval attribute. 
+        """
+        return self.qval
+
+    def set_qval(self, qval):
+        """Sets the value of the qval attribute. 
+
+        Args:
+            qval (int): A value for q (the length of qgrams). 
+
+        Raises:
+            AssertionError : If qval is less than 1.
+        """
+        if qval < 1:
+            raise AssertionError("qval cannot be less than 1")
+        self.qval = qval
+        return True

py_stringmatching-0.1.0/py_stringmatching/tokenizer/tokenizer.py

@@ -0,0 +1,30 @@
+class Tokenizer(object):
+    """The root class for tokenizers.
+
+    Args:
+        return_set (boolean): A flag to indicate whether to return a set of
+                              tokens instead of a bag of tokens (defaults to False). 
+                              
+    Attributes: 
+        return_set (boolean): An attribute to store the flag return_set. 
+    """
+    
+    def __init__(self, return_set=False):
+        self.return_set = return_set
+
+    def get_return_set(self):
+        """Gets the value of the return_set flag.
+
+        Returns:
+            The boolean value of the return_set flag. 
+        """
+        return self.return_set
+
+    def set_return_set(self, return_set):
+        """Sets the value of the return_set flag.
+
+        Args:
+            return_set (boolean): a flag to indicate whether to return a set of tokens instead of a bag of tokens.
+        """
+        self.return_set = return_set
+        return True