nltk 3.2__zip → 3.3__zip

nltk-3.2/nltk/test/unit/test_2x_compat.py

@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Unit tests for nltk.compat.
-See also nltk/test/compat.doctest.
-"""
-from __future__ import absolute_import, unicode_literals
-import unittest
-
-from nltk.text import Text
-from nltk.compat import PY3, python_2_unicode_compatible
-
-def setup_module(module):
-    from nose import SkipTest
-    if PY3:
-        raise SkipTest("test_2x_compat is for testing nltk.compat under Python 2.x")
-
-
-class TestTextTransliteration(unittest.TestCase):
-    txt = Text(["São", "Tomé", "and", "Príncipe"])
-
-    def test_repr(self):
-        self.assertEqual(repr(self.txt), br"<Text: S\xe3o Tom\xe9 and Pr\xedncipe...>")
-
-    def test_str(self):
-        self.assertEqual(str(self.txt), b"<Text: Sao Tome and Principe...>")
-

nltk-3.2/nltk/test/unit/test_stem.py

@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import print_function, unicode_literals
-import unittest
-from nltk.stem.snowball import SnowballStemmer
-
-
-class SnowballTest(unittest.TestCase):
-
-    def test_russian(self):
-        # Russian words both consisting of Cyrillic
-        # and Roman letters can be stemmed.
-        stemmer_russian = SnowballStemmer("russian")
-        assert stemmer_russian.stem("авантненькая") == "авантненьк"
-        assert stemmer_russian.stem("avenantnen'kai^a") == "avenantnen'k"
-
-    def test_german(self):
-        stemmer_german = SnowballStemmer("german")
-        stemmer_german2 = SnowballStemmer("german", ignore_stopwords=True)
-
-        assert stemmer_german.stem("Schr\xe4nke") == 'schrank'
-        assert stemmer_german2.stem("Schr\xe4nke") == 'schrank'
-
-        assert stemmer_german.stem("keinen") == 'kein'
-        assert stemmer_german2.stem("keinen") == 'keinen'
-
-    def test_spanish(self):
-        stemmer = SnowballStemmer('spanish')
-
-        assert stemmer.stem("Visionado") == 'vision'
-
-        # The word 'algue' was raising an IndexError
-        assert stemmer.stem("algue") == 'algu'
-
-    def test_short_strings_bug(self):
-        stemmer = SnowballStemmer('english')
-        assert stemmer.stem("y's") == 'y'

nltk-3.2/nltk/test/unit/test_tokenize.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Unit tests for nltk.tokenize.
-See also nltk/test/tokenize.doctest
-"""
-
-from __future__ import unicode_literals
-from nltk.tokenize import TweetTokenizer
-import unittest
-
-class TestTokenize(unittest.TestCase):
-
-    def test_tweet_tokenizer(self):
-        """
-        Test TweetTokenizer using words with special and accented characters.
-        """
-
-        tokenizer = TweetTokenizer(strip_handles=True, reduce_len=True)
-        s9 = "@myke: Let's test these words: resumé España München français"
-        tokens = tokenizer.tokenize(s9)
-        expected = [':', "Let's", 'test', 'these', 'words', ':', 'resumé',
-                    'España', 'München', 'français']
-        self.assertEqual(tokens, expected)

nltk-3.2/nltk/test/unit/translate/test_bleu.py

@@ -1,84 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Tests for BLEU translation evaluation metric
-"""
-
-import unittest
-from nltk.translate.bleu_score import _modified_precision
-
-class TestBLEU(unittest.TestCase):
-    def test__modified_precision(self):
-        """
-        Examples from the original BLEU paper 
-        http://www.aclweb.org/anthology/P02-1040.pdf
-        """
-        # Example 1: the "the*" example.
-        # Reference sentences.
-        ref1 = 'the cat is on the mat'.split()
-        ref2 = 'there is a cat on the mat'.split()
-        # Hypothesis sentence(s).
-        hyp1 = 'the the the the the the the'.split()
-        
-        references = [ref1, ref2] 
-        
-        # Testing modified unigram precision.
-        hyp1_unigram_precision =  float(_modified_precision(references, hyp1, n=1))
-        assert (round(hyp1_unigram_precision, 4) == 0.2857)
-        # With assertAlmostEqual at 4 place precision.
-        self.assertAlmostEqual(hyp1_unigram_precision, 0.28571428, places=4)
-        
-        # Testing modified bigram precision.
-        assert(_modified_precision(references, hyp1, n=2) == 0.0)
-        
-        
-        # Example 2: the "of the" example.
-        # Reference sentences
-        ref1 = str('It is a guide to action that ensures that the military '
-                   'will forever heed Party commands').split()
-        ref2 = str('It is the guiding principle which guarantees the military '
-                   'forces always being under the command of the Party').split()
-        ref3 = str('It is the practical guide for the army always to heed '
-                   'the directions of the party').split()
-        # Hypothesis sentence(s).
-        hyp1 = 'of the'.split()
-        
-        references = [ref1, ref2, ref3] 
-        # Testing modified unigram precision.
-        assert (_modified_precision(references, hyp1, n=1) == 1.0)
-        
-        # Testing modified bigram precision.
-        assert(_modified_precision(references, hyp1, n=2) == 1.0)
-        
-
-        # Example 3: Proper MT outputs.
-        hyp1 = str('It is a guide to action which ensures that the military '
-                   'always obeys the commands of the party').split()
-        hyp2 = str('It is to insure the troops forever hearing the activity '
-                   'guidebook that party direct').split()
-        
-        references = [ref1, ref2, ref3]
-        
-        # Unigram precision.
-        hyp1_unigram_precision = float(_modified_precision(references, hyp1, n=1))
-        hyp2_unigram_precision = float(_modified_precision(references, hyp2, n=1))
-        # Test unigram precision with assertAlmostEqual at 4 place precision.
-        self.assertAlmostEqual(hyp1_unigram_precision, 0.94444444, places=4)
-        self.assertAlmostEqual(hyp2_unigram_precision, 0.57142857, places=4)
-        # Test unigram precision with rounding.
-        assert (round(hyp1_unigram_precision, 4) == 0.9444)
-        assert (round(hyp2_unigram_precision, 4) == 0.5714)
-        
-        
-        # Bigram precision
-        hyp1_bigram_precision = float(_modified_precision(references, hyp1, n=2))
-        hyp2_bigram_precision = float(_modified_precision(references, hyp2, n=2))
-        # Test bigram precision with assertAlmostEqual at 4 place precision.
-        self.assertAlmostEqual(hyp1_bigram_precision, 0.58823529, places=4)
-        self.assertAlmostEqual(hyp2_bigram_precision, 0.07692307, places=4)
-        # Test bigram precision with rounding.
-        assert (round(hyp1_bigram_precision, 4) == 0.5882)
-        assert (round(hyp2_bigram_precision, 4) == 0.0769)
-        
-    def test_brevity_penalty(self):
-        pass
-    

nltk-3.2/nltk/tokenize/treebank.py

@@ -1,124 +0,0 @@
-# Natural Language Toolkit: Tokenizers
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Edward Loper <edloper@gmail.com>
-#         Michael Heilman <mheilman@cmu.edu> (re-port from http://www.cis.upenn.edu/~treebank/tokenizer.sed)
-#
-# URL: <http://nltk.sourceforge.net>
-# For license information, see LICENSE.TXT
-
-r"""
-
-Penn Treebank Tokenizer
-
-The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
-This implementation is a port of the tokenizer sed script written by Robert McIntyre
-and available at http://www.cis.upenn.edu/~treebank/tokenizer.sed.
-"""
-
-import re
-from nltk.tokenize.api import TokenizerI
-
-
-class TreebankWordTokenizer(TokenizerI):
-    """
-    The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
-    This is the method that is invoked by ``word_tokenize()``.  It assumes that the
-    text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
-
-    This tokenizer performs the following steps:
-
-    - split standard contractions, e.g. ``don't`` -> ``do n't`` and ``they'll`` -> ``they 'll``
-    - treat most punctuation characters as separate tokens
-    - split off commas and single quotes, when followed by whitespace
-    - separate periods that appear at the end of line
-
-        >>> from nltk.tokenize import TreebankWordTokenizer
-        >>> s = '''Good muffins cost $3.88\\nin New York.  Please buy me\\ntwo of them.\\nThanks.'''
-        >>> TreebankWordTokenizer().tokenize(s)
-        ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks', '.']
-        >>> s = "They'll save and invest more."
-        >>> TreebankWordTokenizer().tokenize(s)
-        ['They', "'ll", 'save', 'and', 'invest', 'more', '.']
-        >>> s = "hi, my name can't hello,"
-        >>> TreebankWordTokenizer().tokenize(s)
-        ['hi', ',', 'my', 'name', 'ca', "n't", 'hello', ',']
-    """
-
-    #starting quotes
-    STARTING_QUOTES = [
-        (re.compile(r'^\"'), r'``'),
-        (re.compile(r'(``)'), r' \1 '),
-        (re.compile(r'([ (\[{<])"'), r'\1 `` '),
-    ]
-
-    #punctuation
-    PUNCTUATION = [
-        (re.compile(r'([:,])([^\d])'), r' \1 \2'),
-        (re.compile(r'([:,])$'), r' \1 '),
-        (re.compile(r'\.\.\.'), r' ... '),
-        (re.compile(r'[;@#$%&]'), r' \g<0> '),
-        (re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'), r'\1 \2\3 '),
-        (re.compile(r'[?!]'), r' \g<0> '),
-
-        (re.compile(r"([^'])' "), r"\1 ' "),
-    ]
-
-    #parens, brackets, etc.
-    PARENS_BRACKETS = [
-        (re.compile(r'[\]\[\(\)\{\}\<\>]'), r' \g<0> '),
-        (re.compile(r'--'), r' -- '),
-    ]
-
-    #ending quotes
-    ENDING_QUOTES = [
-        (re.compile(r'"'), " '' "),
-        (re.compile(r'(\S)(\'\')'), r'\1 \2 '),
-
-        (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
-        (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
-    ]
-
-    # List of contractions adapted from Robert MacIntyre's tokenizer.
-    CONTRACTIONS2 = [re.compile(r"(?i)\b(can)(not)\b"),
-                     re.compile(r"(?i)\b(d)('ye)\b"),
-                     re.compile(r"(?i)\b(gim)(me)\b"),
-                     re.compile(r"(?i)\b(gon)(na)\b"),
-                     re.compile(r"(?i)\b(got)(ta)\b"),
-                     re.compile(r"(?i)\b(lem)(me)\b"),
-                     re.compile(r"(?i)\b(mor)('n)\b"),
-                     re.compile(r"(?i)\b(wan)(na) ")]
-    CONTRACTIONS3 = [re.compile(r"(?i) ('t)(is)\b"),
-                     re.compile(r"(?i) ('t)(was)\b")]
-    CONTRACTIONS4 = [re.compile(r"(?i)\b(whad)(dd)(ya)\b"),
-                     re.compile(r"(?i)\b(wha)(t)(cha)\b")]
-
-    def tokenize(self, text):
-        for regexp, substitution in self.STARTING_QUOTES:
-            text = regexp.sub(substitution, text)
-
-        for regexp, substitution in self.PUNCTUATION:
-            text = regexp.sub(substitution, text)
-
-        for regexp, substitution in self.PARENS_BRACKETS:
-            text = regexp.sub(substitution, text)
-
-        #add extra space to make things easier
-        text = " " + text + " "
-
-        for regexp, substitution in self.ENDING_QUOTES:
-            text = regexp.sub(substitution, text)
-
-        for regexp in self.CONTRACTIONS2:
-            text = regexp.sub(r' \1 \2 ', text)
-        for regexp in self.CONTRACTIONS3:
-            text = regexp.sub(r' \1 \2 ', text)
-
-        # We are not using CONTRACTIONS4 since
-        # they are also commented out in the SED scripts
-        # for regexp in self.CONTRACTIONS4:
-        #     text = regexp.sub(r' \1 \2 \3 ', text)
-
-        return text.split()
-
-

nltk-3.2/nltk/tokenize/util.py

@@ -1,90 +0,0 @@
-# Natural Language Toolkit: Tokenizer Utilities
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Steven Bird <stevenbird1@gmail.com>
-# URL: <http://nltk.sourceforge.net>
-# For license information, see LICENSE.TXT
-
-from re import finditer
-
-def string_span_tokenize(s, sep):
-    r"""
-    Return the offsets of the tokens in *s*, as a sequence of ``(start, end)``
-    tuples, by splitting the string at each occurrence of *sep*.
-
-        >>> from nltk.tokenize.util import string_span_tokenize
-        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
-        ... two of them.\n\nThanks.'''
-        >>> list(string_span_tokenize(s, " "))
-        [(0, 4), (5, 12), (13, 17), (18, 26), (27, 30), (31, 36), (37, 37),
-        (38, 44), (45, 48), (49, 55), (56, 58), (59, 73)]
-
-    :param s: the string to be tokenized
-    :type s: str
-    :param sep: the token separator
-    :type sep: str
-    :rtype: iter(tuple(int, int))
-    """
-    if len(sep) == 0:
-        raise ValueError("Token delimiter must not be empty")
-    left = 0
-    while True:
-        try:
-            right = s.index(sep, left)
-            if right != 0:
-                yield left, right
-        except ValueError:
-            if left != len(s):
-                yield left, len(s)
-            break
-
-        left = right + len(sep)
-
-def regexp_span_tokenize(s, regexp):
-    r"""
-    Return the offsets of the tokens in *s*, as a sequence of ``(start, end)``
-    tuples, by splitting the string at each successive match of *regexp*.
-
-        >>> from nltk.tokenize.util import regexp_span_tokenize
-        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
-        ... two of them.\n\nThanks.'''
-        >>> list(regexp_span_tokenize(s, r'\s'))
-        [(0, 4), (5, 12), (13, 17), (18, 23), (24, 26), (27, 30), (31, 36),
-        (38, 44), (45, 48), (49, 51), (52, 55), (56, 58), (59, 64), (66, 73)]
-
-    :param s: the string to be tokenized
-    :type s: str
-    :param regexp: regular expression that matches token separators (must not be empty)
-    :type regexp: str
-    :rtype: iter(tuple(int, int))
-    """
-    left = 0
-    for m in finditer(regexp, s):
-        right, next = m.span()
-        if right != left:
-            yield left, right
-        left = next
-    yield left, len(s)
-
-def spans_to_relative(spans):
-    r"""
-    Return a sequence of relative spans, given a sequence of spans.
-
-        >>> from nltk.tokenize import WhitespaceTokenizer
-        >>> from nltk.tokenize.util import spans_to_relative
-        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
-        ... two of them.\n\nThanks.'''
-        >>> list(spans_to_relative(WhitespaceTokenizer().span_tokenize(s)))
-        [(0, 4), (1, 7), (1, 4), (1, 5), (1, 2), (1, 3), (1, 5), (2, 6),
-        (1, 3), (1, 2), (1, 3), (1, 2), (1, 5), (2, 7)]
-
-    :param spans: a sequence of (start, end) offsets of the tokens
-    :type spans: iter(tuple(int, int))
-    :rtype: iter(tuple(int, int))
-    """
-    prev = 0
-    for left, right in spans:
-        yield left - prev, right - left
-        prev = right
-
-