nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/tgrep.py

@@ -3,7 +3,7 @@
 #
 # Natural Language Toolkit: TGrep search
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Will Roberts <wildwilhelm@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -111,15 +111,19 @@ macro definitions to ``m`` and initialises ``l`` to an empty dictionary.
 '''
 
 from __future__ import absolute_import, print_function, unicode_literals
-from nltk.compat import binary_type, text_type
+
 import functools
-import nltk.tree
+import re
+
+from six import binary_type, text_type
+
 try:
     import pyparsing
 except ImportError:
     print('Warning: nltk.tgrep will not work without the `pyparsing` package')
     print('installed.')
-import re
+
+import nltk.tree
 
 class TgrepException(Exception):
     '''Tgrep exception type.'''
@@ -932,6 +936,3 @@ def tgrep_nodes(pattern, trees, search_leaves=True):
                       if pattern(tree[position])]
         except AttributeError:
             yield []
-
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/__init__.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Tokenizers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# Contributors: matthewmc, clouds56
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
@@ -59,20 +60,23 @@ tokenization, see the other methods provided in this package.
 For further information, please see Chapter 3 of the NLTK book.
 """
 
+import re
+
 from nltk.data              import load
-from nltk.tokenize.simple   import (SpaceTokenizer, TabTokenizer, LineTokenizer,
-                                    line_tokenize)
+from nltk.tokenize.casual   import (TweetTokenizer, casual_tokenize)
+from nltk.tokenize.mwe      import MWETokenizer
+from nltk.tokenize.punkt    import PunktSentenceTokenizer
 from nltk.tokenize.regexp   import (RegexpTokenizer, WhitespaceTokenizer,
                                     BlanklineTokenizer, WordPunctTokenizer,
                                     wordpunct_tokenize, regexp_tokenize,
                                     blankline_tokenize)
-from nltk.tokenize.punkt    import PunktSentenceTokenizer
+from nltk.tokenize.repp     import ReppTokenizer
 from nltk.tokenize.sexpr    import SExprTokenizer, sexpr_tokenize
-from nltk.tokenize.treebank import TreebankWordTokenizer
-from nltk.tokenize.stanford import StanfordTokenizer
+from nltk.tokenize.simple   import (SpaceTokenizer, TabTokenizer, LineTokenizer,
+                                    line_tokenize)
 from nltk.tokenize.texttiling import TextTilingTokenizer
-from nltk.tokenize.casual   import (TweetTokenizer, casual_tokenize)
-from nltk.tokenize.mwe      import MWETokenizer
+from nltk.tokenize.toktok   import ToktokTokenizer
+from nltk.tokenize.treebank import TreebankWordTokenizer
 from nltk.tokenize.util     import string_span_tokenize, regexp_span_tokenize
 from nltk.tokenize.stanford_segmenter import StanfordSegmenter
 
@@ -91,18 +95,36 @@ def sent_tokenize(text, language='english'):
     return tokenizer.tokenize(text)
 
 # Standard word tokenizer.
-_treebank_word_tokenize = TreebankWordTokenizer().tokenize
-def word_tokenize(text, language='english'):
+_treebank_word_tokenizer = TreebankWordTokenizer()
+
+# See discussion on https://github.com/nltk/nltk/pull/1437
+# Adding to TreebankWordTokenizer, the splits on
+# - chervon quotes u'\xab' and u'\xbb' .
+# - unicode quotes u'\u2018', u'\u2019', u'\u201c' and u'\u201d'
+
+improved_open_quote_regex = re.compile(u'([«“‘„]|[`]+)', re.U)
+improved_close_quote_regex = re.compile(u'([»”’])', re.U)
+improved_punct_regex = re.compile(r'([^\.])(\.)([\]\)}>"\'' u'»”’ ' r']*)\s*$', re.U)
+_treebank_word_tokenizer.STARTING_QUOTES.insert(0, (improved_open_quote_regex, r' \1 '))
+_treebank_word_tokenizer.ENDING_QUOTES.insert(0, (improved_close_quote_regex, r' \1 '))
+_treebank_word_tokenizer.PUNCTUATION.insert(0, (improved_punct_regex, r'\1 \2 \3 '))
+
+
+def word_tokenize(text, language='english', preserve_line=False):
     """
     Return a tokenized copy of *text*,
     using NLTK's recommended word tokenizer
-    (currently :class:`.TreebankWordTokenizer`
+    (currently an improved :class:`.TreebankWordTokenizer`
     along with :class:`.PunktSentenceTokenizer`
     for the specified language).
 
-    :param text: text to split into sentences
+    :param text: text to split into words
+    :type text: str
     :param language: the model name in the Punkt corpus
+    :type language: str
+    :param preserve_line: An option to keep the preserve the sentence and not sentence tokenize it.
+    :type preserver_line: bool
     """
-    return [token for sent in sent_tokenize(text, language)
-            for token in _treebank_word_tokenize(sent)]
-
+    sentences = [text] if preserve_line else sent_tokenize(text, language)
+    return [token for sent in sentences
+            for token in _treebank_word_tokenizer.tokenize(sent)]

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/api.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Tokenizer Interface
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
@@ -11,11 +11,12 @@ Tokenizer Interface
 """
 
 from abc import ABCMeta, abstractmethod
-from nltk.six import add_metaclass
+from six import add_metaclass
 
 from nltk.internals import overridden
 from nltk.tokenize.util import string_span_tokenize
 
+
 @add_metaclass(ABCMeta)
 class TokenizerI(object):
     """
@@ -74,5 +75,3 @@ class StringTokenizer(TokenizerI):
     def span_tokenize(self, s):
         for span in string_span_tokenize(s, self._string):
             yield span
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/casual.py

@@ -2,7 +2,7 @@
 #
 # Natural Language Toolkit: Twitter Tokenizer
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Christopher Potts <cgpotts@stanford.edu>
 #         Ewan Klein <ewan@inf.ed.ac.uk> (modifications)
 #         Pierpaolo Pantone <> (modifications)
@@ -38,8 +38,9 @@ domains and tasks. The basic logic is this:
 
 from __future__ import unicode_literals
 import re
-from nltk.compat import htmlentitydefs, int2byte, unichr
 
+from six import int2byte, unichr
+from six.moves import html_entities
 
 ######################################################################
 # The following strings are components in the regular expression
@@ -126,15 +127,15 @@ REGEXPS = (
     (?:
       (?:            # (international)
         \+?[01]
-        [\-\s.]*
+        [ *\-.\)]*
       )?
       (?:            # (area code)
         [\(]?
         \d{3}
-        [\-\s.\)]*
+        [ *\-.\)]*
       )?
       \d{3}          # exchange
-      [\-\s.]*
+      [ *\-.\)]*
       \d{4}          # base
     )"""
     ,
@@ -246,7 +247,7 @@ def _replace_html_entities(text, keep=(), remove_illegal=True, encoding='utf-8')
             if entity_body in keep:
                 return match.group(0)
             else:
-                number = htmlentitydefs.name2codepoint.get(entity_body)
+                number = html_entities.name2codepoint.get(entity_body)
         if number is not None:
             try:
                 return unichr(number)
@@ -324,8 +325,9 @@ def remove_handles(text):
     """
     Remove Twitter username handles from text.
     """
-    pattern = re.compile(r"(^|(?<=[^\w.-]))@[A-Za-z_]+\w+")
-    return pattern.sub('', text)
+    pattern = re.compile(r"(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){20}(?!@))|(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){1,19})(?![A-Za-z0-9_]*@)")
+    # Substitute hadnles with ' ' to ensure that text on either side of removed handles are tokenized correctly
+    return pattern.sub(' ', text)
 
 ######################################################################
 # Tokenization Function
@@ -339,4 +341,3 @@ def casual_tokenize(text, preserve_case=True, reduce_len=False, strip_handles=Fa
                           strip_handles=strip_handles).tokenize(text)
 
 ###############################################################################
-

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/mwe.py

@@ -1,6 +1,6 @@
 # Multi-Word Expression tokenizer
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Rob Malouf <rmalouf@mail.sdsu.edu>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -71,7 +71,7 @@ class MWETokenizer(TokenizerI):
         >>> tokenizer.add_mwe(('a', 'b', 'c'))
         >>> tokenizer.add_mwe(('a', 'x'))
         >>> expected = {'a': {'x': {True: None}, 'b': {True: None, 'c': {True: None}}}}
-        >>> tokenizer._mwes.as_dict() == expected
+        >>> tokenizer._mwes == expected
         True
 
         """

nltk-3.3/nltk/tokenize/nist.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Python port of the mteval-v14.pl tokenizer.
+#
+# Copyright (C) 2001-2015 NLTK Project
+# Author: Liling Tan (ported from ftp://jaguar.ncsl.nist.gov/mt/resources/mteval-v14.pl)
+# Contributors: Ozan Caglayan, Wiktor Stribizew
+#
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+"""
+This is a NLTK port of the tokenizer used in the NIST BLEU evaluation script,
+https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v14.pl#L926
+which was also ported into Python in
+https://github.com/lium-lst/nmtpy/blob/master/nmtpy/metrics/mtevalbleu.py#L162
+"""
+
+from __future__ import unicode_literals
+
+import io
+import re
+from six import text_type
+
+from nltk.corpus import perluniprops
+from nltk.tokenize.api import TokenizerI
+from nltk.tokenize.util import xml_unescape
+
+
+class NISTTokenizer(TokenizerI):
+    """
+    This NIST tokenizer is sentence-based instead of the original
+    paragraph-based tokenization from mteval-14.pl; The sentence-based
+    tokenization is consistent with the other tokenizers available in NLTK.
+
+    >>> from six import text_type
+    >>> from nltk.tokenize.nist import NISTTokenizer
+    >>> nist = NISTTokenizer()
+    >>> s = "Good muffins cost $3.88 in New York."
+    >>> expected_lower = [u'good', u'muffins', u'cost', u'$', u'3.88', u'in', u'new', u'york', u'.']
+    >>> expected_cased = [u'Good', u'muffins', u'cost', u'$', u'3.88', u'in', u'New', u'York', u'.']
+    >>> nist.tokenize(s, lowercase=False) == expected_cased
+    True
+    >>> nist.tokenize(s, lowercase=True) == expected_lower  # Lowercased.
+    True
+
+    The international_tokenize() is the preferred function when tokenizing
+    non-european text, e.g.
+
+    >>> from nltk.tokenize.nist import NISTTokenizer
+    >>> nist = NISTTokenizer()
+
+    # Input strings.
+    >>> albb = u'Alibaba Group Holding Limited (Chinese: 阿里巴巴集团控股 有限公司) us a Chinese e-commerce company...'
+    >>> amz = u'Amazon.com, Inc. (/ˈæməzɒn/) is an American electronic commerce...'
+    >>> rkt = u'Rakuten, Inc. (楽天株式会社 Rakuten Kabushiki-gaisha) is a Japanese electronic commerce and Internet company based in Tokyo.'
+
+    # Expected tokens.
+    >>> expected_albb = [u'Alibaba', u'Group', u'Holding', u'Limited', u'(', u'Chinese', u':', u'\u963f\u91cc\u5df4\u5df4\u96c6\u56e2\u63a7\u80a1', u'\u6709\u9650\u516c\u53f8', u')']
+    >>> expected_amz = [u'Amazon', u'.', u'com', u',', u'Inc', u'.', u'(', u'/', u'\u02c8\xe6', u'm']
+    >>> expected_rkt = [u'Rakuten', u',', u'Inc', u'.', u'(', u'\u697d\u5929\u682a\u5f0f\u4f1a\u793e', u'Rakuten', u'Kabushiki', u'-', u'gaisha']
+
+    >>> nist.international_tokenize(albb)[:10] == expected_albb
+    True
+    >>> nist.international_tokenize(amz)[:10] == expected_amz
+    True
+    >>> nist.international_tokenize(rkt)[:10] == expected_rkt
+    True
+
+    # Doctest for patching issue #1926
+    >>> sent = u'this is a foo\u2604sentence.'
+    >>> expected_sent = [u'this', u'is', u'a', u'foo', u'\u2604', u'sentence', u'.']
+    >>> nist.international_tokenize(sent) == expected_sent
+    True
+    """
+    # Strip "skipped" tags
+    STRIP_SKIP = re.compile('<skipped>'), ''
+    #  Strip end-of-line hyphenation and join lines
+    STRIP_EOL_HYPHEN = re.compile(u'\u2028'), ' '
+    # Tokenize punctuation.
+    PUNCT = re.compile('([\{-\~\[-\` -\&\(-\+\:-\@\/])'), ' \\1 '
+    # Tokenize period and comma unless preceded by a digit.
+    PERIOD_COMMA_PRECEED = re.compile('([^0-9])([\.,])'), '\\1 \\2 '
+    # Tokenize period and comma unless followed by a digit.
+    PERIOD_COMMA_FOLLOW = re.compile('([\.,])([^0-9])'), ' \\1 \\2'
+    # Tokenize dash when preceded by a digit
+    DASH_PRECEED_DIGIT = re.compile('([0-9])(-)'), '\\1 \\2 '
+
+    LANG_DEPENDENT_REGEXES = [PUNCT, PERIOD_COMMA_PRECEED,
+                              PERIOD_COMMA_FOLLOW, DASH_PRECEED_DIGIT]
+
+    # Perluniprops characters used in NIST tokenizer.
+    pup_number = text_type(''.join(set(perluniprops.chars('Number')))) # i.e. \p{N}
+    pup_punct = text_type(''.join(set(perluniprops.chars('Punctuation')))) # i.e. \p{P}
+    pup_symbol = text_type(''.join(set(perluniprops.chars('Symbol')))) # i.e. \p{S}
+
+    # Python regexes needs to escape some special symbols, see
+    # see https://stackoverflow.com/q/45670950/610569
+    number_regex = re.sub(r'[]^\\-]', r'\\\g<0>', pup_number)
+    punct_regex = re.sub(r'[]^\\-]', r'\\\g<0>', pup_punct)
+    symbol_regex = re.sub(r'[]^\\-]', r'\\\g<0>', pup_symbol)
+
+    # Note: In the original perl implementation, \p{Z} and \p{Zl} were used to
+    #       (i) strip trailing and heading spaces  and
+    #       (ii) de-deuplicate spaces.
+    #       In Python, this would do: ' '.join(str.strip().split())
+    # Thus, the next two lines were commented out.
+    #Line_Separator = text_type(''.join(perluniprops.chars('Line_Separator'))) # i.e. \p{Zl}
+    #Separator = text_type(''.join(perluniprops.chars('Separator'))) # i.e. \p{Z}
+
+    # Pads non-ascii strings with space.
+    NONASCII = re.compile('([\x00-\x7f]+)'), r' \1 '
+    #  Tokenize any punctuation unless followed AND preceded by a digit.
+    PUNCT_1 = re.compile(u"([{n}])([{p}])".format(n=number_regex, p=punct_regex)), '\\1 \\2 '
+    PUNCT_2 = re.compile(u"([{p}])([{n}])".format(n=number_regex, p=punct_regex)), ' \\1 \\2'
+    # Tokenize symbols
+    SYMBOLS = re.compile(u"([{s}])".format(s=symbol_regex)), ' \\1 '
+
+    INTERNATIONAL_REGEXES = [NONASCII, PUNCT_1, PUNCT_2, SYMBOLS]
+
+    def lang_independent_sub(self, text):
+        """Performs the language independent string substituitions. """
+        # It's a strange order of regexes.
+        # It'll be better to unescape after STRIP_EOL_HYPHEN
+        # but let's keep it close to the original NIST implementation.
+        regexp, substitution = self.STRIP_SKIP
+        text = regexp.sub(substitution, text)
+        text = xml_unescape(text)
+        regexp, substitution = self.STRIP_EOL_HYPHEN
+        text = regexp.sub(substitution, text)
+        return text
+
+    def tokenize(self, text, lowercase=False,
+                 western_lang=True, return_str=False):
+        text = text_type(text)
+        # Language independent regex.
+        text = self.lang_independent_sub(text)
+        # Language dependent regex.
+        if western_lang:
+            # Pad string with whitespace.
+            text = ' ' + text + ' '
+            if lowercase:
+                text = text.lower()
+            for regexp, substitution in self.LANG_DEPENDENT_REGEXES:
+                text = regexp.sub(substitution, text)
+        # Remove contiguous whitespaces.
+        text = ' '.join(text.split())
+        # Finally, strips heading and trailing spaces
+        # and converts output string into unicode.
+        text = text_type(text.strip())
+        return text if return_str else text.split()
+
+    def international_tokenize(self, text, lowercase=False,
+                               split_non_ascii=True,
+                               return_str=False):
+        text = text_type(text)
+        # Different from the 'normal' tokenize(), STRIP_EOL_HYPHEN is applied
+        # first before unescaping.
+        regexp, substitution = self.STRIP_SKIP
+        text = regexp.sub(substitution, text)
+        regexp, substitution = self.STRIP_EOL_HYPHEN
+        text = regexp.sub(substitution, text)
+        text = xml_unescape(text)
+
+        if lowercase:
+            text = text.lower()
+
+        for regexp, substitution in self.INTERNATIONAL_REGEXES:
+            text = regexp.sub(substitution, text)
+
+        # Make sure that there's only one space only between words.
+        # Strip leading and trailing spaces.
+        text = ' '.join(text.strip().split())
+        return text if return_str else text.split()