nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/texttiling.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: TextTiling
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: George Boutsioukis
 #
 # URL: <http://nltk.org/>
@@ -55,12 +55,10 @@ class TextTilingTokenizer(TokenizerI):
 
     >>> from nltk.corpus import brown
     >>> tt = TextTilingTokenizer(demo_mode=True)
-    >>> text = brown.raw()[:10000]
+    >>> text = brown.raw()[:4000]
     >>> s, ss, d, b = tt.tokenize(text)
     >>> b
-    [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0,
-     0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0,
-     0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0]
+    [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0]
     """
 
     def __init__(self,
@@ -74,7 +72,6 @@ class TextTilingTokenizer(TokenizerI):
                  cutoff_policy=HC,
                  demo_mode=False):
 
-
         if stopwords is None:
             from nltk.corpus import stopwords
             stopwords = stopwords.words('english')
@@ -118,9 +115,13 @@ class TextTilingTokenizer(TokenizerI):
             gap_scores = self._block_comparison(tokseqs, token_table)
         elif self.similarity_method == VOCABULARY_INTRODUCTION:
             raise NotImplementedError("Vocabulary introduction not implemented")
+        else:
+            raise ValueError("Similarity method {} not recognized".format(self.similarity_method))
 
         if self.smoothing_method == DEFAULT_SMOOTHING:
             smooth_scores = self._smooth_scores(gap_scores)
+        else:
+            raise ValueError("Smoothing method {} not recognized".format(self.smoothing_method))
         # End of Lexical score Determination
 
         # Boundary identification
@@ -140,7 +141,7 @@ class TextTilingTokenizer(TokenizerI):
             segmented_text.append(text[prevb:b])
             prevb = b
 
-        if prevb < text_length: # append any text that may be remaining
+        if prevb < text_length:  # append any text that may be remaining
             segmented_text.append(text[prevb:])
 
         if not segmented_text:
@@ -151,7 +152,7 @@ class TextTilingTokenizer(TokenizerI):
         return segmented_text
 
     def _block_comparison(self, tokseqs, token_table):
-        "Implements the block comparison method"
+        """Implements the block comparison method"""
         def blk_frq(tok, block):
             ts_occs = filter(lambda o: o[0] in block,
                              token_table[tok].ts_occurences)
@@ -164,7 +165,7 @@ class TextTilingTokenizer(TokenizerI):
         for curr_gap in range(numgaps):
             score_dividend, score_divisor_b1, score_divisor_b2 = 0.0, 0.0, 0.0
             score = 0.0
-            #adjust window size for boundary conditions
+            # adjust window size for boundary conditions
             if curr_gap < self.k-1:
                 window_size = curr_gap + 1
             elif curr_gap > numgaps-self.k:
@@ -182,10 +183,10 @@ class TextTilingTokenizer(TokenizerI):
                 score_divisor_b1 += blk_frq(t, b1)**2
                 score_divisor_b2 += blk_frq(t, b2)**2
             try:
-                score = score_dividend/math.sqrt(score_divisor_b1*
-                                                 score_divisor_b2)
+                score = score_dividend / math.sqrt(score_divisor_b1 *
+                                                   score_divisor_b2)
             except ZeroDivisionError:
-                pass # score += 0.0
+                pass  # score += 0.0
 
             gap_scores.append(score)
 
@@ -194,12 +195,11 @@ class TextTilingTokenizer(TokenizerI):
     def _smooth_scores(self, gap_scores):
         "Wraps the smooth function from the SciPy Cookbook"
         return list(smooth(numpy.array(gap_scores[:]),
-                           window_len = self.smoothing_width+1))
+                           window_len=self.smoothing_width+1))
 
     def _mark_paragraph_breaks(self, text):
         """Identifies indented text or line breaks as the beginning of
         paragraphs"""
-
         MIN_PARAGRAPH = 100
         pattern = re.compile("[ \t\r\f\v]*\n[ \t\r\f\v]*\n[ \t\r\f\v]*")
         matches = pattern.finditer(text)
@@ -234,7 +234,7 @@ class TextTilingTokenizer(TokenizerI):
         current_par_break = next(pb_iter)
         if current_par_break == 0:
             try:
-                current_par_break = next(pb_iter) #skip break at 0
+                current_par_break = next(pb_iter)  # skip break at 0
             except StopIteration:
                 raise ValueError(
                     "No paragraph breaks were found(text too short perhaps?)"
@@ -246,7 +246,7 @@ class TextTilingTokenizer(TokenizerI):
                         current_par_break = next(pb_iter)
                         current_par += 1
                 except StopIteration:
-                    #hit bottom
+                    # hit bottom
                     pass
 
                 if word in token_table:
@@ -258,19 +258,16 @@ class TextTilingTokenizer(TokenizerI):
 
                     if token_table[word].last_tok_seq != current_tok_seq:
                         token_table[word].last_tok_seq = current_tok_seq
-                        token_table[word]\
-                                .ts_occurences.append([current_tok_seq,1])
+                        token_table[word].ts_occurences.append([current_tok_seq, 1])
                     else:
                         token_table[word].ts_occurences[-1][1] += 1
-                else: #new word
+                else:  # new word
                     token_table[word] = TokenTableField(first_pos=index,
-                                                        ts_occurences= \
-                                                          [[current_tok_seq,1]],
+                                                        ts_occurences=[[current_tok_seq, 1]],
                                                         total_count=1,
                                                         par_count=1,
                                                         last_par=current_par,
-                                                        last_tok_seq= \
-                                                          current_tok_seq)
+                                                        last_tok_seq=current_tok_seq)
 
             current_tok_seq += 1
 
@@ -285,7 +282,7 @@ class TextTilingTokenizer(TokenizerI):
         avg = sum(depth_scores)/len(depth_scores)
         stdev = numpy.std(depth_scores)
 
-        #SB: what is the purpose of this conditional?
+        # SB: what is the purpose of this conditional?
         if self.cutoff_policy == LC:
             cutoff = avg-stdev/2.0
         else:
@@ -293,11 +290,11 @@ class TextTilingTokenizer(TokenizerI):
 
         depth_tuples = sorted(zip(depth_scores, range(len(depth_scores))))
         depth_tuples.reverse()
-        hp = list(filter(lambda x:x[0]>cutoff, depth_tuples))
+        hp = list(filter(lambda x: x[0] > cutoff, depth_tuples))
 
         for dt in hp:
             boundaries[dt[1]] = 1
-            for dt2 in hp: #undo if there is a boundary close already
+            for dt2 in hp:  # undo if there is a boundary close already
                 if dt[1] != dt2[1] and abs(dt2[1]-dt[1]) < 4 \
                        and boundaries[dt2[1]] == 1:
                     boundaries[dt[1]] = 0
@@ -308,11 +305,11 @@ class TextTilingTokenizer(TokenizerI):
         between the left and right peaks and the gap's score"""
 
         depth_scores = [0 for x in scores]
-        #clip boundaries: this holds on the rule of thumb(my thumb)
-        #that a section shouldn't be smaller than at least 2
-        #pseudosentences for small texts and around 5 for larger ones.
+        # clip boundaries: this holds on the rule of thumb(my thumb)
+        # that a section shouldn't be smaller than at least 2
+        # pseudosentences for small texts and around 5 for larger ones.
 
-        clip = min(max(len(scores)/10, 2), 5)
+        clip = min(max(len(scores) // 10, 2), 5)
         index = clip
 
         for gapscore in scores[clip:-clip]:
@@ -347,11 +344,11 @@ class TextTilingTokenizer(TokenizerI):
                 seen_word = False
                 word_count += 1
             if char not in " \t\n" and not seen_word:
-                seen_word=True
+                seen_word = True
             if gaps_seen < len(boundaries) and word_count > \
-                                               (max(gaps_seen*self.w, self.w)):
+               (max(gaps_seen*self.w, self.w)):
                 if boundaries[gaps_seen] == 1:
-                    #find closest paragraph break
+                    # find closest paragraph break
                     best_fit = len(text)
                     for br in paragraph_breaks:
                         if best_fit > abs(br-char_count):
@@ -359,7 +356,7 @@ class TextTilingTokenizer(TokenizerI):
                             bestbr = br
                         else:
                             break
-                    if bestbr not in norm_boundaries: #avoid duplicates
+                    if bestbr not in norm_boundaries:  # avoid duplicates
                         norm_boundaries.append(bestbr)
                 gaps_seen += 1
 
@@ -379,18 +376,20 @@ class TokenTableField(object):
         self.__dict__.update(locals())
         del self.__dict__['self']
 
+
 class TokenSequence(object):
     "A token list with its original length and its index"
     def __init__(self,
                  index,
                  wrdindex_list,
                  original_length=None):
-        original_length=original_length or len(wrdindex_list)
+        original_length = original_length or len(wrdindex_list)
         self.__dict__.update(locals())
         del self.__dict__['self']
 
-#Pasted from the SciPy cookbook: http://www.scipy.org/Cookbook/SignalSmooth
-def smooth(x,window_len=11,window='flat'):
+
+# Pasted from the SciPy cookbook: http://www.scipy.org/Cookbook/SignalSmooth
+def smooth(x, window_len=11, window='flat'):
     """smooth the data using a window with requested size.
 
     This method is based on the convolution of a scaled window with the signal.
@@ -426,14 +425,14 @@ def smooth(x,window_len=11,window='flat'):
     if window_len < 3:
         return x
 
-    if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
+    if window not in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
         raise ValueError("Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")
 
-    s=numpy.r_[2*x[0]-x[window_len:1:-1],x,2*x[-1]-x[-1:-window_len:-1]]
+    s = numpy.r_[2*x[0]-x[window_len:1:-1], x, 2*x[-1]-x[-1:-window_len:-1]]
 
-    #print(len(s))
-    if window == 'flat': #moving average
-        w = numpy.ones(window_len,'d')
+    # print(len(s))
+    if window == 'flat':  # moving average
+        w = numpy.ones(window_len, 'd')
     else:
         w = eval('numpy.' + window + '(window_len)')
 
@@ -446,7 +445,8 @@ def demo(text=None):
     from nltk.corpus import brown
     from matplotlib import pylab
     tt = TextTilingTokenizer(demo_mode=True)
-    if text is None: text = brown.raw()[:10000]
+    if text is None:
+        text = brown.raw()[:10000]
     s, ss, d, b = tt.tokenize(text)
     pylab.xlabel("Sentence Gap index")
     pylab.ylabel("Gap Scores")
@@ -456,5 +456,3 @@ def demo(text=None):
     pylab.stem(range(len(b)), b)
     pylab.legend()
     pylab.show()
-
-

nltk-3.3/nltk/tokenize/toktok.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Python port of the tok-tok.pl tokenizer.
+#
+# Copyright (C) 2001-2015 NLTK Project
+# Author: Jon Dehdari
+# Contributors: Liling Tan, Selcuk Ayguney, ikegami, Martijn Pieters
+#
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+"""
+The tok-tok tokenizer is a simple, general tokenizer, where the input has one 
+sentence per line; thus only final period is tokenized.
+
+Tok-tok has been tested on, and gives reasonably good results for English, 
+Persian, Russian, Czech, French, German, Vietnamese, Tajik, and a few others. 
+The input should be in UTF-8 encoding.
+
+Reference:
+Jon Dehdari. 2014. A Neurophysiologically-Inspired Statistical Language 
+Model (Doctoral dissertation). Columbus, OH, USA: The Ohio State University. 
+"""
+
+import re
+from six import text_type
+
+from nltk.tokenize.api import TokenizerI
+
+class ToktokTokenizer(TokenizerI):
+    """
+    This is a Python port of the tok-tok.pl from
+    https://github.com/jonsafari/tok-tok/blob/master/tok-tok.pl
+    
+    >>> toktok = ToktokTokenizer()
+    >>> text = u'Is 9.5 or 525,600 my favorite number?'
+    >>> print (toktok.tokenize(text, return_str=True))
+    Is 9.5 or 525,600 my favorite number ?
+    >>> text = u'The https://github.com/jonsafari/tok-tok/blob/master/tok-tok.pl is a website with/and/or slashes and sort of weird : things'
+    >>> print (toktok.tokenize(text, return_str=True))
+    The https://github.com/jonsafari/tok-tok/blob/master/tok-tok.pl is a website with/and/or slashes and sort of weird : things
+    >>> text = u'\xa1This, is a sentence with weird\xbb symbols\u2026 appearing everywhere\xbf'
+    >>> expected = u'\xa1 This , is a sentence with weird \xbb symbols \u2026 appearing everywhere \xbf'
+    >>> assert toktok.tokenize(text, return_str=True) == expected
+    >>> toktok.tokenize(text) == [u'\xa1', u'This', u',', u'is', u'a', u'sentence', u'with', u'weird', u'\xbb', u'symbols', u'\u2026', u'appearing', u'everywhere', u'\xbf']
+    True
+    """
+    # Replace non-breaking spaces with normal spaces.
+    NON_BREAKING = re.compile(u"\u00A0"), " "
+    
+    # Pad some funky punctuation.
+    FUNKY_PUNCT_1 = re.compile(u'([،;؛¿!"\])}»›”؟¡%٪°±©®।॥…])'), r" \1 "
+    # Pad more funky punctuation.
+    FUNKY_PUNCT_2 = re.compile(u'([({\[“‘„‚«‹「『])'), r" \1 "
+    # Pad En dash and em dash
+    EN_EM_DASHES = re.compile(u'([–—])'), r" \1 "
+    
+    # Replace problematic character with numeric character reference.
+    AMPERCENT = re.compile('& '), '&amp; '
+    TAB = re.compile('\t'), ' &#9; '
+    PIPE = re.compile('\|'), ' &#124; '
+    
+    # Pad numbers with commas to keep them from further tokenization. 
+    COMMA_IN_NUM = re.compile(r'(?<!,)([,،])(?![,\d])'), r' \1 '
+    
+    # Just pad problematic (often neurotic) hyphen/single quote, etc.
+    PROB_SINGLE_QUOTES = re.compile(r"(['’`])"), r' \1 '
+    # Group ` ` stupid quotes ' ' into a single token.
+    STUPID_QUOTES_1 = re.compile(r" ` ` "), r" `` "
+    STUPID_QUOTES_2 = re.compile(r" ' ' "), r" '' "
+    
+    # Don't tokenize period unless it ends the line and that it isn't 
+    # preceded by another period, e.g.  
+    # "something ..." -> "something ..." 
+    # "something." -> "something ." 
+    FINAL_PERIOD_1 = re.compile(r"(?<!\.)\.$"), r" ."
+    # Don't tokenize period unless it ends the line eg. 
+    # " ... stuff." ->  "... stuff ."
+    FINAL_PERIOD_2 = re.compile(r"""(?<!\.)\.\s*(["'’»›”]) *$"""), r" . \1"
+
+    # Treat continuous commas as fake German,Czech, etc.: „
+    MULTI_COMMAS = re.compile(r'(,{2,})'), r' \1 '
+    # Treat continuous dashes as fake en-dash, etc.
+    MULTI_DASHES = re.compile(r'(-{2,})'), r' \1 '
+    # Treat multiple periods as a thing (eg. ellipsis)
+    MULTI_DOTS = re.compile(r'(\.{2,})'), r' \1 '
+
+    # This is the \p{Open_Punctuation} from Perl's perluniprops
+    # see http://perldoc.perl.org/perluniprops.html
+    OPEN_PUNCT = text_type(u'([{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d'
+                            u'\u208d\u2329\u2768\u276a\u276c\u276e\u2770\u2772'
+                            u'\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983'
+                            u'\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993'
+                            u'\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26'
+                            u'\u2e28\u3008\u300a\u300c\u300e\u3010\u3014\u3016'
+                            u'\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39'
+                            u'\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b'
+                            u'\ufe5d\uff08\uff3b\uff5b\uff5f\uff62')
+    # This is the \p{Close_Punctuation} from Perl's perluniprops
+    CLOSE_PUNCT = text_type(u')]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u232a'
+                            u'\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6'
+                            u'\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988'
+                            u'\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998'
+                            u'\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009'
+                            u'\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b'
+                            u'\u301e\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c'
+                            u'\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e'
+                            u'\uff09\uff3d\uff5d\uff60\uff63')
+    # This is the \p{Close_Punctuation} from Perl's perluniprops
+    CURRENCY_SYM = text_type(u'$\xa2\xa3\xa4\xa5\u058f\u060b\u09f2\u09f3\u09fb'
+                             u'\u0af1\u0bf9\u0e3f\u17db\u20a0\u20a1\u20a2\u20a3'
+                             u'\u20a4\u20a5\u20a6\u20a7\u20a8\u20a9\u20aa\u20ab'
+                             u'\u20ac\u20ad\u20ae\u20af\u20b0\u20b1\u20b2\u20b3'
+                             u'\u20b4\u20b5\u20b6\u20b7\u20b8\u20b9\u20ba\ua838'
+                             u'\ufdfc\ufe69\uff04\uffe0\uffe1\uffe5\uffe6')
+    
+    # Pad spaces after opening punctuations.
+    OPEN_PUNCT_RE = re.compile(u'([{}])'.format(OPEN_PUNCT)), r'\1 '
+    # Pad spaces before closing punctuations.
+    CLOSE_PUNCT_RE = re.compile(u'([{}])'.format(CLOSE_PUNCT)), r'\1 '
+    # Pad spaces after currency symbols.
+    CURRENCY_SYM_RE = re.compile(u'([{}])'.format(CURRENCY_SYM)), r'\1 '
+    
+    # Use for tokenizing URL-unfriendly characters: [:/?#]
+    URL_FOE_1 = re.compile(r':(?!//)'), r' : ' # in perl s{:(?!//)}{ : }g;
+    URL_FOE_2 = re.compile(r'\?(?!\S)'), r' ? ' # in perl s{\?(?!\S)}{ ? }g;
+    # in perl: m{://} or m{\S+\.\S+/\S+} or s{/}{ / }g;
+    URL_FOE_3 = re.compile(r'(:\/\/)[\S+\.\S+\/\S+][\/]'), ' / '
+    URL_FOE_4 = re.compile(r' /'), r' / ' # s{ /}{ / }g;
+    
+    # Left/Right strip, i.e. remove heading/trailing spaces.
+    # These strip regexes should NOT be used,
+    # instead use str.lstrip(), str.rstrip() or str.strip() 
+    # (They are kept for reference purposes to the original toktok.pl code)  
+    LSTRIP = re.compile(r'^ +'), ''
+    RSTRIP = re.compile(r'\s+$'),'\n' 
+    # Merge multiple spaces.
+    ONE_SPACE = re.compile(r' {2,}'), ' '
+    
+    TOKTOK_REGEXES = [NON_BREAKING, FUNKY_PUNCT_1, 
+                      URL_FOE_1, URL_FOE_2, URL_FOE_3, URL_FOE_4,
+                      AMPERCENT, TAB, PIPE,
+                      OPEN_PUNCT_RE, CLOSE_PUNCT_RE, 
+                      MULTI_COMMAS, COMMA_IN_NUM, FINAL_PERIOD_2,
+                      PROB_SINGLE_QUOTES, STUPID_QUOTES_1, STUPID_QUOTES_2,
+                      CURRENCY_SYM_RE, EN_EM_DASHES, MULTI_DASHES, MULTI_DOTS,
+                      FINAL_PERIOD_1, FINAL_PERIOD_2, ONE_SPACE]
+    
+    def tokenize(self, text, return_str=False):
+        text = text_type(text) # Converts input string into unicode.
+        for regexp, subsitution in self.TOKTOK_REGEXES:
+            text = regexp.sub(subsitution, text)
+        # Finally, strips heading and trailing spaces
+        # and converts output string into unicode.
+        text = text_type(text.strip()) 
+        return text if return_str else text.split()