nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/tokenize/mwe.py

@@ -1,120 +1,120 @@
-# Multi-Word Expression tokenizer
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Rob Malouf <rmalouf@mail.sdsu.edu>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Multi-Word Expression Tokenizer
-
-A ``MWETokenizer`` takes a string which has already been divided into tokens and
-retokenizes it, merging multi-word expressions into single tokens, using a lexicon
-of MWEs:
-
-
-    >>> from nltk.tokenize import MWETokenizer
-
-    >>> tokenizer = MWETokenizer([('a', 'little'), ('a', 'little', 'bit'), ('a', 'lot')])
-    >>> tokenizer.add_mwe(('in', 'spite', 'of'))
-
-    >>> tokenizer.tokenize('Testing testing testing one two three'.split())
-    ['Testing', 'testing', 'testing', 'one', 'two', 'three']
-
-    >>> tokenizer.tokenize('This is a test in spite'.split())
-    ['This', 'is', 'a', 'test', 'in', 'spite']
-
-    >>> tokenizer.tokenize('In a little or a little bit or a lot in spite of'.split())
-    ['In', 'a_little', 'or', 'a_little_bit', 'or', 'a_lot', 'in_spite_of']
-
-"""
-from nltk.util import Trie
-
-from nltk.tokenize.api import TokenizerI
-
-
-class MWETokenizer(TokenizerI):
-    """A tokenizer that processes tokenized text and merges multi-word expressions
-    into single tokens.
-    """
-
-    def __init__(self, mwes=None, separator='_'):
-        """Initialize the multi-word tokenizer with a list of expressions and a
-        separator
-
-        :type mwes: list(list(str))
-        :param mwes: A sequence of multi-word expressions to be merged, where
-            each MWE is a sequence of strings.
-        :type separator: str
-        :param separator: String that should be inserted between words in a multi-word
-            expression token. (Default is '_')
-
-        """
-        if not mwes:
-            mwes = []
-        self._mwes = Trie(mwes)
-        self._separator = separator
-
-    def add_mwe(self, mwe):
-        """Add a multi-word expression to the lexicon (stored as a word trie)
-
-        We use ``util.Trie`` to represent the trie. Its form is a dict of dicts. 
-        The key True marks the end of a valid MWE.
-
-        :param mwe: The multi-word expression we're adding into the word trie
-        :type mwe: tuple(str) or list(str)
-
-        :Example:
-
-        >>> tokenizer = MWETokenizer()
-        >>> tokenizer.add_mwe(('a', 'b'))
-        >>> tokenizer.add_mwe(('a', 'b', 'c'))
-        >>> tokenizer.add_mwe(('a', 'x'))
-        >>> expected = {'a': {'x': {True: None}, 'b': {True: None, 'c': {True: None}}}}
-        >>> tokenizer._mwes.as_dict() == expected
-        True
-
-        """
-        self._mwes.insert(mwe)
-
-    def tokenize(self, text):
-        """
-
-        :param text: A list containing tokenized text
-        :type text: list(str)
-        :return: A list of the tokenized text with multi-words merged together
-        :rtype: list(str)
-
-        :Example:
-
-        >>> tokenizer = MWETokenizer([('hors', "d'oeuvre")], separator='+')
-        >>> tokenizer.tokenize("An hors d'oeuvre tonight, sir?".split())
-        ['An', "hors+d'oeuvre", 'tonight,', 'sir?']
-        
-        """
-        i = 0
-        n = len(text)
-        result = []
-
-        while i < n:
-            if text[i] in self._mwes:
-                # possible MWE match
-                j = i
-                trie = self._mwes
-                while j < n and text[j] in trie:
-                    trie = trie[text[j]]
-                    j = j + 1
-                else:
-                    if Trie.LEAF in trie:
-                        # success!
-                        result.append(self._separator.join(text[i:j]))
-                        i = j
-                    else:
-                        # no match, so backtrack
-                        result.append(text[i])
-                        i += 1
-            else:
-                result.append(text[i])
-                i += 1
-
-        return result
+# Multi-Word Expression tokenizer
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Rob Malouf <rmalouf@mail.sdsu.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Multi-Word Expression Tokenizer
+
+A ``MWETokenizer`` takes a string which has already been divided into tokens and
+retokenizes it, merging multi-word expressions into single tokens, using a lexicon
+of MWEs:
+
+
+    >>> from nltk.tokenize import MWETokenizer
+
+    >>> tokenizer = MWETokenizer([('a', 'little'), ('a', 'little', 'bit'), ('a', 'lot')])
+    >>> tokenizer.add_mwe(('in', 'spite', 'of'))
+
+    >>> tokenizer.tokenize('Testing testing testing one two three'.split())
+    ['Testing', 'testing', 'testing', 'one', 'two', 'three']
+
+    >>> tokenizer.tokenize('This is a test in spite'.split())
+    ['This', 'is', 'a', 'test', 'in', 'spite']
+
+    >>> tokenizer.tokenize('In a little or a little bit or a lot in spite of'.split())
+    ['In', 'a_little', 'or', 'a_little_bit', 'or', 'a_lot', 'in_spite_of']
+
+"""
+from nltk.util import Trie
+
+from nltk.tokenize.api import TokenizerI
+
+
+class MWETokenizer(TokenizerI):
+    """A tokenizer that processes tokenized text and merges multi-word expressions
+    into single tokens.
+    """
+
+    def __init__(self, mwes=None, separator='_'):
+        """Initialize the multi-word tokenizer with a list of expressions and a
+        separator
+
+        :type mwes: list(list(str))
+        :param mwes: A sequence of multi-word expressions to be merged, where
+            each MWE is a sequence of strings.
+        :type separator: str
+        :param separator: String that should be inserted between words in a multi-word
+            expression token. (Default is '_')
+
+        """
+        if not mwes:
+            mwes = []
+        self._mwes = Trie(mwes)
+        self._separator = separator
+
+    def add_mwe(self, mwe):
+        """Add a multi-word expression to the lexicon (stored as a word trie)
+
+        We use ``util.Trie`` to represent the trie. Its form is a dict of dicts. 
+        The key True marks the end of a valid MWE.
+
+        :param mwe: The multi-word expression we're adding into the word trie
+        :type mwe: tuple(str) or list(str)
+
+        :Example:
+
+        >>> tokenizer = MWETokenizer()
+        >>> tokenizer.add_mwe(('a', 'b'))
+        >>> tokenizer.add_mwe(('a', 'b', 'c'))
+        >>> tokenizer.add_mwe(('a', 'x'))
+        >>> expected = {'a': {'x': {True: None}, 'b': {True: None, 'c': {True: None}}}}
+        >>> tokenizer._mwes == expected
+        True
+
+        """
+        self._mwes.insert(mwe)
+
+    def tokenize(self, text):
+        """
+
+        :param text: A list containing tokenized text
+        :type text: list(str)
+        :return: A list of the tokenized text with multi-words merged together
+        :rtype: list(str)
+
+        :Example:
+
+        >>> tokenizer = MWETokenizer([('hors', "d'oeuvre")], separator='+')
+        >>> tokenizer.tokenize("An hors d'oeuvre tonight, sir?".split())
+        ['An', "hors+d'oeuvre", 'tonight,', 'sir?']
+        
+        """
+        i = 0
+        n = len(text)
+        result = []
+
+        while i < n:
+            if text[i] in self._mwes:
+                # possible MWE match
+                j = i
+                trie = self._mwes
+                while j < n and text[j] in trie:
+                    trie = trie[text[j]]
+                    j = j + 1
+                else:
+                    if Trie.LEAF in trie:
+                        # success!
+                        result.append(self._separator.join(text[i:j]))
+                        i = j
+                    else:
+                        # no match, so backtrack
+                        result.append(text[i])
+                        i += 1
+            else:
+                result.append(text[i])
+                i += 1
+
+        return result

nltk-3.3/nltk/tokenize/nist.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Python port of the mteval-v14.pl tokenizer.
+#
+# Copyright (C) 2001-2015 NLTK Project
+# Author: Liling Tan (ported from ftp://jaguar.ncsl.nist.gov/mt/resources/mteval-v14.pl)
+# Contributors: Ozan Caglayan, Wiktor Stribizew
+#
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+"""
+This is a NLTK port of the tokenizer used in the NIST BLEU evaluation script,
+https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v14.pl#L926
+which was also ported into Python in
+https://github.com/lium-lst/nmtpy/blob/master/nmtpy/metrics/mtevalbleu.py#L162
+"""
+
+from __future__ import unicode_literals
+
+import io
+import re
+from six import text_type
+
+from nltk.corpus import perluniprops
+from nltk.tokenize.api import TokenizerI
+from nltk.tokenize.util import xml_unescape
+
+
+class NISTTokenizer(TokenizerI):
+    """
+    This NIST tokenizer is sentence-based instead of the original
+    paragraph-based tokenization from mteval-14.pl; The sentence-based
+    tokenization is consistent with the other tokenizers available in NLTK.
+
+    >>> from six import text_type
+    >>> from nltk.tokenize.nist import NISTTokenizer
+    >>> nist = NISTTokenizer()
+    >>> s = "Good muffins cost $3.88 in New York."
+    >>> expected_lower = [u'good', u'muffins', u'cost', u'$', u'3.88', u'in', u'new', u'york', u'.']
+    >>> expected_cased = [u'Good', u'muffins', u'cost', u'$', u'3.88', u'in', u'New', u'York', u'.']
+    >>> nist.tokenize(s, lowercase=False) == expected_cased
+    True
+    >>> nist.tokenize(s, lowercase=True) == expected_lower  # Lowercased.
+    True
+
+    The international_tokenize() is the preferred function when tokenizing
+    non-european text, e.g.
+
+    >>> from nltk.tokenize.nist import NISTTokenizer
+    >>> nist = NISTTokenizer()
+
+    # Input strings.
+    >>> albb = u'Alibaba Group Holding Limited (Chinese: 阿里巴巴集团控股 有限公司) us a Chinese e-commerce company...'
+    >>> amz = u'Amazon.com, Inc. (/ˈæməzɒn/) is an American electronic commerce...'
+    >>> rkt = u'Rakuten, Inc. (楽天株式会社 Rakuten Kabushiki-gaisha) is a Japanese electronic commerce and Internet company based in Tokyo.'
+
+    # Expected tokens.
+    >>> expected_albb = [u'Alibaba', u'Group', u'Holding', u'Limited', u'(', u'Chinese', u':', u'\u963f\u91cc\u5df4\u5df4\u96c6\u56e2\u63a7\u80a1', u'\u6709\u9650\u516c\u53f8', u')']
+    >>> expected_amz = [u'Amazon', u'.', u'com', u',', u'Inc', u'.', u'(', u'/', u'\u02c8\xe6', u'm']
+    >>> expected_rkt = [u'Rakuten', u',', u'Inc', u'.', u'(', u'\u697d\u5929\u682a\u5f0f\u4f1a\u793e', u'Rakuten', u'Kabushiki', u'-', u'gaisha']
+
+    >>> nist.international_tokenize(albb)[:10] == expected_albb
+    True
+    >>> nist.international_tokenize(amz)[:10] == expected_amz
+    True
+    >>> nist.international_tokenize(rkt)[:10] == expected_rkt
+    True
+
+    # Doctest for patching issue #1926
+    >>> sent = u'this is a foo\u2604sentence.'
+    >>> expected_sent = [u'this', u'is', u'a', u'foo', u'\u2604', u'sentence', u'.']
+    >>> nist.international_tokenize(sent) == expected_sent
+    True
+    """
+    # Strip "skipped" tags
+    STRIP_SKIP = re.compile('<skipped>'), ''
+    #  Strip end-of-line hyphenation and join lines
+    STRIP_EOL_HYPHEN = re.compile(u'\u2028'), ' '
+    # Tokenize punctuation.
+    PUNCT = re.compile('([\{-\~\[-\` -\&\(-\+\:-\@\/])'), ' \\1 '
+    # Tokenize period and comma unless preceded by a digit.
+    PERIOD_COMMA_PRECEED = re.compile('([^0-9])([\.,])'), '\\1 \\2 '
+    # Tokenize period and comma unless followed by a digit.
+    PERIOD_COMMA_FOLLOW = re.compile('([\.,])([^0-9])'), ' \\1 \\2'
+    # Tokenize dash when preceded by a digit
+    DASH_PRECEED_DIGIT = re.compile('([0-9])(-)'), '\\1 \\2 '
+
+    LANG_DEPENDENT_REGEXES = [PUNCT, PERIOD_COMMA_PRECEED,
+                              PERIOD_COMMA_FOLLOW, DASH_PRECEED_DIGIT]
+
+    # Perluniprops characters used in NIST tokenizer.
+    pup_number = text_type(''.join(set(perluniprops.chars('Number')))) # i.e. \p{N}
+    pup_punct = text_type(''.join(set(perluniprops.chars('Punctuation')))) # i.e. \p{P}
+    pup_symbol = text_type(''.join(set(perluniprops.chars('Symbol')))) # i.e. \p{S}
+
+    # Python regexes needs to escape some special symbols, see
+    # see https://stackoverflow.com/q/45670950/610569
+    number_regex = re.sub(r'[]^\\-]', r'\\\g<0>', pup_number)
+    punct_regex = re.sub(r'[]^\\-]', r'\\\g<0>', pup_punct)
+    symbol_regex = re.sub(r'[]^\\-]', r'\\\g<0>', pup_symbol)
+
+    # Note: In the original perl implementation, \p{Z} and \p{Zl} were used to
+    #       (i) strip trailing and heading spaces  and
+    #       (ii) de-deuplicate spaces.
+    #       In Python, this would do: ' '.join(str.strip().split())
+    # Thus, the next two lines were commented out.
+    #Line_Separator = text_type(''.join(perluniprops.chars('Line_Separator'))) # i.e. \p{Zl}
+    #Separator = text_type(''.join(perluniprops.chars('Separator'))) # i.e. \p{Z}
+
+    # Pads non-ascii strings with space.
+    NONASCII = re.compile('([\x00-\x7f]+)'), r' \1 '
+    #  Tokenize any punctuation unless followed AND preceded by a digit.
+    PUNCT_1 = re.compile(u"([{n}])([{p}])".format(n=number_regex, p=punct_regex)), '\\1 \\2 '
+    PUNCT_2 = re.compile(u"([{p}])([{n}])".format(n=number_regex, p=punct_regex)), ' \\1 \\2'
+    # Tokenize symbols
+    SYMBOLS = re.compile(u"([{s}])".format(s=symbol_regex)), ' \\1 '
+
+    INTERNATIONAL_REGEXES = [NONASCII, PUNCT_1, PUNCT_2, SYMBOLS]
+
+    def lang_independent_sub(self, text):
+        """Performs the language independent string substituitions. """
+        # It's a strange order of regexes.
+        # It'll be better to unescape after STRIP_EOL_HYPHEN
+        # but let's keep it close to the original NIST implementation.
+        regexp, substitution = self.STRIP_SKIP
+        text = regexp.sub(substitution, text)
+        text = xml_unescape(text)
+        regexp, substitution = self.STRIP_EOL_HYPHEN
+        text = regexp.sub(substitution, text)
+        return text
+
+    def tokenize(self, text, lowercase=False,
+                 western_lang=True, return_str=False):
+        text = text_type(text)
+        # Language independent regex.
+        text = self.lang_independent_sub(text)
+        # Language dependent regex.
+        if western_lang:
+            # Pad string with whitespace.
+            text = ' ' + text + ' '
+            if lowercase:
+                text = text.lower()
+            for regexp, substitution in self.LANG_DEPENDENT_REGEXES:
+                text = regexp.sub(substitution, text)
+        # Remove contiguous whitespaces.
+        text = ' '.join(text.split())
+        # Finally, strips heading and trailing spaces
+        # and converts output string into unicode.
+        text = text_type(text.strip())
+        return text if return_str else text.split()
+
+    def international_tokenize(self, text, lowercase=False,
+                               split_non_ascii=True,
+                               return_str=False):
+        text = text_type(text)
+        # Different from the 'normal' tokenize(), STRIP_EOL_HYPHEN is applied
+        # first before unescaping.
+        regexp, substitution = self.STRIP_SKIP
+        text = regexp.sub(substitution, text)
+        regexp, substitution = self.STRIP_EOL_HYPHEN
+        text = regexp.sub(substitution, text)
+        text = xml_unescape(text)
+
+        if lowercase:
+            text = text.lower()
+
+        for regexp, substitution in self.INTERNATIONAL_REGEXES:
+            text = regexp.sub(substitution, text)
+
+        # Make sure that there's only one space only between words.
+        # Strip leading and trailing spaces.
+        text = ' '.join(text.strip().split())
+        return text if return_str else text.split()