nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/test/grammartestsuites.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ==========================

{nltk-3.2.1 → nltk-3.3}/nltk/test/index.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 .. _align howto: align.html

{nltk-3.2.1 → nltk-3.3}/nltk/test/inference.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ====================================

{nltk-3.2.1 → nltk-3.3}/nltk/test/inference_fixt.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+
 def setup_module(module):
     from nose import SkipTest
     from nltk.inference.mace import Mace

{nltk-3.2.1 → nltk-3.3}/nltk/test/internals.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ==========================================

{nltk-3.2.1 → nltk-3.3}/nltk/test/japanese.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ============================

{nltk-3.2.1 → nltk-3.3}/nltk/test/logic.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 =======================

{nltk-3.2.1 → nltk-3.3}/nltk/test/metrics.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 =======
@@ -62,7 +62,7 @@ Other distance measures:
     >>> print(jaccard_distance(s1, s2))
     0.6
     >>> print(masi_distance(s1, s2))
-    0.868...
+    0.868
 
 ----------------------
 Miscellaneous Measures
@@ -239,7 +239,7 @@ For other associations, we ensure the ordering of the measures:
     True
     >>> bam.dice(20, (42, 20), N) > bam.dice(20, (41, 27), N)
     True
-    >>> bam.fisher(20, (42, 20), N) > bam.fisher(20, (41, 27), N)
+    >>> bam.fisher(20, (42, 20), N) > bam.fisher(20, (41, 27), N) # doctest: +SKIP
     False
 
 For trigrams, we have to provide more count information:

{nltk-3.2.1 → nltk-3.3}/nltk/test/misc.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 --------------------------------------------------------------------------------

{nltk-3.2.1 → nltk-3.3}/nltk/test/nonmonotonic.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ======================

{nltk-3.2.1 → nltk-3.3}/nltk/test/nonmonotonic_fixt.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+
 def setup_module(module):
     from nose import SkipTest
     from nltk.inference.mace import Mace

{nltk-3.2.1 → nltk-3.3}/nltk/test/parse.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 =========

{nltk-3.2.1 → nltk-3.3}/nltk/test/portuguese_en.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ==================================

{nltk-3.2.1 → nltk-3.3}/nltk/test/probability.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ===========

{nltk-3.2.1 → nltk-3.3}/nltk/test/probability_fixt.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+
 # probability.doctest uses HMM which requires numpy;
 # skip probability.doctest if numpy is not available
 
@@ -9,4 +10,4 @@ def setup_module(module):
     try:
         import numpy
     except ImportError:
-        raise SkipTest("probability.doctest requires numpy")
+        raise SkipTest("probability.doctest requires numpy")

{nltk-3.2.1 → nltk-3.3}/nltk/test/propbank.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ========

{nltk-3.2.1 → nltk-3.3}/nltk/test/relextract.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ======================

{nltk-3.2.1 → nltk-3.3}/nltk/test/resolution.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 =========================

{nltk-3.2.1 → nltk-3.3}/nltk/test/runtests.py

@@ -17,18 +17,32 @@ if __name__ == '__main__':
     # there shouldn't be import from NLTK for coverage to work properly
     from doctest_nose_plugin import DoctestFix
 
+    try:
+        # Import RedNose plugin for colored test output
+        from rednose import RedNose
+
+        rednose_available = True
+    except ImportError:
+        rednose_available = False
+
+
     class NltkPluginManager(PluginManager):
         """
         Nose plugin manager that replaces standard doctest plugin
-        with a patched version.
+        with a patched version and adds RedNose plugin for colored test output.
         """
+
         def loadPlugins(self):
             for plug in builtin.plugins:
                 if plug != Doctest:
                     self.addPlugin(plug())
             self.addPlugin(DoctestFix())
+            if rednose_available:
+                self.addPlugin(RedNose())
+
             super(NltkPluginManager, self).loadPlugins()
 
+
     manager = NltkPluginManager()
     manager.loadPlugins()
 
@@ -47,18 +61,26 @@ if __name__ == '__main__':
         # only extra options were passed
         args += [NLTK_TEST_DIR]
 
+    # Activate RedNose and hide skipped test messages from output
+    if rednose_available:
+        args += [
+            '--rednose',
+            '--hide-skips'
+        ]
+
     arguments = [
-        '--exclude=', # why is this needed?
-        #'--with-xunit',
-        #'--xunit-file=$WORKSPACE/nosetests.xml',
-        #'--nocapture',
-        '--with-doctest',
-        #'--doctest-tests',
-        #'--debug=nose,nose.importer,nose.inspector,nose.plugins,nose.result,nose.selector',
-        '--doctest-extension=.doctest',
-        '--doctest-fixtures=_fixt',
-        '--doctest-options=+ELLIPSIS,+NORMALIZE_WHITESPACE,+IGNORE_EXCEPTION_DETAIL,+ALLOW_UNICODE,doctestencoding=utf-8',
-        #'--verbosity=3',
-    ] + args
+                    '--exclude=',  # why is this needed?
+                    # '--with-xunit',
+                    # '--xunit-file=$WORKSPACE/nosetests.xml',
+                    # '--nocapture',
+                    '--with-doctest',
+                    # '--doctest-tests',
+                    # '--debug=nose,nose.importer,nose.inspector,nose.plugins,nose.result,nose.selector',
+                    '--doctest-extension=.doctest',
+                    '--doctest-fixtures=_fixt',
+                    '--doctest-options=+ELLIPSIS,+NORMALIZE_WHITESPACE,+IGNORE_EXCEPTION_DETAIL,+ALLOW_UNICODE,'
+                    'doctestencoding=utf-8',
+                    # '--verbosity=3',
+                ] + args
 
     nose.main(argv=arguments, plugins=manager.plugins)

{nltk-3.2.1 → nltk-3.3}/nltk/test/semantics.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 =========

{nltk-3.2.1 → nltk-3.3}/nltk/test/sentiment.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ===================
@@ -128,8 +128,8 @@ Vader
     ...    under orders and in the ''least offensive way possible.''"
     ... ]
     >>> sentences.extend(tricky_sentences)
-    >>> sid = SentimentIntensityAnalyzer()
     >>> for sentence in sentences:
+    ...     sid = SentimentIntensityAnalyzer()
     ...     print(sentence)
     ...     ss = sid.polarity_scores(sentence)
     ...     for k in sorted(ss):
@@ -230,4 +230,4 @@ Vader
     the twin towers collapsed today
     compound: -0.2732, neg: 0.344, neu: 0.656, pos: 0.0,
     However, Mr. Carter solemnly argues, his client carried out the kidnapping under orders and in the ''least offensive way possible.''
-    compound: -0.5859, neg: 0.23, neu: 0.697, pos: 0.074,
+    compound: -0.5859, neg: 0.23, neu: 0.697, pos: 0.074,

{nltk-3.2.1 → nltk-3.3}/nltk/test/sentiwordnet.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ======================

{nltk-3.2.1 → nltk-3.3}/nltk/test/simple.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 =================

{nltk-3.2.1 → nltk-3.3}/nltk/test/stem.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ==========
@@ -46,7 +46,7 @@ Unit tests for Snowball stemmer
 See which languages are supported.
 
     >>> print(" ".join(SnowballStemmer.languages))
-    danish dutch english finnish french german hungarian italian
+    arabic danish dutch english finnish french german hungarian italian
     norwegian porter portuguese romanian russian spanish swedish
 
 Create a new instance of a language specific subclass.

{nltk-3.2.1 → nltk-3.3}/nltk/test/tag.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 Regression Tests

{nltk-3.2.1 → nltk-3.3}/nltk/test/tokenize.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
     >>> from __future__ import print_function
@@ -40,6 +40,19 @@ Some test strings.
     >>> word_tokenize(s10)
     ['There', 'were', '300,000', ',', 'but', 'that', 'was', "n't", 'enough', '.']
 
+
+Testing improvement made to the TreebankWordTokenizer
+
+    >>> sx1 = u'\xabNow that I can do.\xbb'
+    >>> expected = [u'\xab', u'Now', u'that', u'I', u'can', u'do', u'.', u'\xbb']
+    >>> word_tokenize(sx1) == expected
+    True
+    >>> sx2 = u'The unicode 201C and 201D \u201cLEFT(RIGHT) DOUBLE QUOTATION MARK\u201d is also OPEN_PUNCT and CLOSE_PUNCT.'
+    >>> expected = [u'The', u'unicode', u'201C', u'and', u'201D', u'\u201c', u'LEFT', u'(', u'RIGHT', u')', u'DOUBLE', u'QUOTATION', u'MARK', u'\u201d', u'is', u'also', u'OPEN_PUNCT', u'and', u'CLOSE_PUNCT', u'.']
+    >>> word_tokenize(sx2) == expected
+    True
+
+
 Sentence tokenization in word_tokenize:
 
     >>> s11 = "I called Dr. Jones. I called Dr. Jones."
@@ -163,4 +176,98 @@ It should not hang on long sequences of the same punctuation character.
     >>> tknzr = TweetTokenizer()
     >>> s10 = "Photo: Aujourd'hui sur http://t.co/0gebOFDUzn Projet... http://t.co/bKfIUbydz2.............................. http://fb.me/3b6uXpz0L"
     >>> tknzr.tokenize(s10)
-    [u'Photo', u':', u"Aujourd'hui", u'sur', u'http://t.co/0gebOFDUzn', u'Projet', u'...', u'http://t.co/bKfIUbydz2', u'...', u'http://fb.me/3b6uXpz0L']
+    [u'Photo', u':', u"Aujourd'hui", u'sur', u'http://t.co/0gebOFDUzn', u'Projet', u'...', u'http://t.co/bKfIUbydz2', u'...', u'http://fb.me/3b6uXpz0L']
+
+
+Regression Tests: PunktSentenceTokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The sentence splitter should remove whitespace following the sentence boundary.
+
+    >>> pst = PunktSentenceTokenizer()
+    >>> pst.tokenize('See Section 3).  Or Section 2).  ')
+    ['See Section 3).', 'Or Section 2).']
+    >>> pst.tokenize('See Section 3.)  Or Section 2.)  ')
+    ['See Section 3.)', 'Or Section 2.)']
+    >>> pst.tokenize('See Section 3.)  Or Section 2.)  ', realign_boundaries=False)
+    ['See Section 3.', ')  Or Section 2.', ')']
+
+
+Two instances of PunktSentenceTokenizer should not share PunktParameters.
+
+    >>> pst = PunktSentenceTokenizer()
+    >>> pst2 = PunktSentenceTokenizer()
+    >>> pst._params is pst2._params
+    False
+
+Regression Tests: align_tokens
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Post-hoc alignment of tokens with a source string
+
+    >>> from nltk.tokenize.util import align_tokens
+    >>> list(align_tokens([''], ""))
+    [(0, 0)]
+    >>> list(align_tokens([''], " "))
+    [(0, 0)]
+    >>> list(align_tokens([], ""))
+    []
+    >>> list(align_tokens([], " "))
+    []
+    >>> list(align_tokens(['a'], "a"))
+    [(0, 1)]
+    >>> list(align_tokens(['abc', 'def'], "abcdef"))
+    [(0, 3), (3, 6)]
+    >>> list(align_tokens(['abc', 'def'], "abc def"))
+    [(0, 3), (4, 7)]
+    >>> list(align_tokens(['ab', 'cd'], "ab cd ef"))
+    [(0, 2), (3, 5)]
+    >>> list(align_tokens(['ab', 'cd', 'ef'], "ab cd ef"))
+    [(0, 2), (3, 5), (6, 8)]
+    >>> list(align_tokens(['ab', 'cd', 'efg'], "ab cd ef"))
+    Traceback (most recent call last):
+    ....
+    ValueError: substring "efg" not found in "ab cd ef"
+    >>> list(align_tokens(['ab', 'cd', 'ef', 'gh'], "ab cd ef"))
+    Traceback (most recent call last):
+    ....
+    ValueError: substring "gh" not found in "ab cd ef"
+    >>> list(align_tokens(['The', 'plane', ',', 'bound', 'for', 'St', 'Petersburg', ',', 'crashed', 'in', 'Egypt', "'s", 'Sinai', 'desert', 'just', '23', 'minutes', 'after', 'take-off', 'from', 'Sharm', 'el-Sheikh', 'on', 'Saturday', '.'], "The plane, bound for St Petersburg, crashed in Egypt's Sinai desert just 23 minutes after take-off from Sharm el-Sheikh on Saturday."))
+    [(0, 3), (4, 9), (9, 10), (11, 16), (17, 20), (21, 23), (24, 34), (34, 35), (36, 43), (44, 46), (47, 52), (52, 54), (55, 60), (61, 67), (68, 72), (73, 75), (76, 83), (84, 89), (90, 98), (99, 103), (104, 109), (110, 119), (120, 122), (123, 131), (131, 132)]
+
+
+Regression Tests: MWETokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Pickle an MWETokenizer
+
+    >>> from nltk.tokenize import MWETokenizer
+    >>> import pickle
+
+    >>> tokenizer = MWETokenizer([('hors', "d'oeuvre")], separator='+')
+    >>> p = pickle.dumps(tokenizer)
+    >>> unpickeled = pickle.loads(p)
+    >>> unpickeled.tokenize("An hors d'oeuvre tonight, sir?".split())
+    ['An', "hors+d'oeuvre", 'tonight,', 'sir?']
+
+
+Regression Tests: TextTilingTokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+TextTilingTokneizer tokenizes text into coherent subtopic chunks based upon Hearst's TextTiling algorithm.
+
+    >>> from nltk.tokenize import TextTilingTokenizer
+    >>> from nltk.corpus import brown
+    >>> tt = TextTilingTokenizer()
+    >>> tt.tokenize(brown.raw()[0:1000])
+    ["\n\n\tThe/at Fulton/np-tl County/nn-tl Grand/jj-tl Jury/nn-tl said/vbd Friday/nr an/at investigation/nn of/in Atlanta's/np$ recent/jj primary/nn election/nn produced/vbd ``/`` no/at evidence/nn ''/'' that/cs any/dti irregularities/nns took/vbd place/nn ./.\n\n\n\tThe/at jury/nn further/rbr said/vbd in/in term-end/nn presentments/nns that/cs the/at City/nn-tl Executive/jj-tl Committee/nn-tl ,/, which/wdt had/hvd over-all/jj charge/nn of/in the/at election/nn ,/, ``/`` deserves/vbz the/at praise/nn and/cc thanks/nns of/in the/at City/nn-tl of/in-tl Atlanta/np-tl ''/'' for/in the/at manner/nn in/in which/wdt the/at election/nn was/bedz conducted/vbn ./.\n\n\n\tThe/at September-October/np term/nn jury/nn had/hvd been/ben charged/vbn by/in Fulton/np-tl Superior/jj-tl Court/nn-tl Judge/nn-tl Durwood/np Pye/np to/to investigate/vb reports/nns of/in possible/jj ``/`` irregularities/nns ''/'' in/in the/at hard-fought/jj primary/nn which/wdt was/bedz won/vbn by/in Mayor-nominate/nn-tl Ivan/np Allen/np Jr./"]
+
+Test that `ValueError` exceptions are raised when illegal arguments are used.
+
+    >>> TextTilingTokenizer(similarity_method='foo').tokenize(brown.raw()[0:1000])
+    Traceback (most recent call last):
+      ...
+    ValueError: Similarity method foo not recognized
+    >>> TextTilingTokenizer(smoothing_method='bar').tokenize(brown.raw()[0:1000])
+    Traceback (most recent call last):
+      ...
+    ValueError: Smoothing method bar not recognized
+

{nltk-3.2.1 → nltk-3.3}/nltk/test/toolbox.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ===============================

{nltk-3.2.1 → nltk-3.3}/nltk/test/translate.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 .. -*- coding: utf-8 -*-

{nltk-3.2.1 → nltk-3.3}/nltk/test/tree.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ===============================

{nltk-3.2.1 → nltk-3.3}/nltk/test/treeprettyprinter.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ========================================================

{nltk-3.2.1 → nltk-3.3}/nltk/test/treetransforms.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 -------------------------------------------

nltk-3.3/nltk/test/unit/test_aline.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for nltk.metrics.aline
+"""
+
+from __future__ import unicode_literals
+
+import unittest
+
+from nltk.metrics import aline
+
+
+class TestAline(unittest.TestCase):
+    """
+    Test Aline algorithm for aligning phonetic sequences
+    """
+
+    def test_aline(self):
+        result = aline.align('θin', 'tenwis')
+        expected = [[('θ', 't'), ('i', 'e'), ('n', 'n'), ('-', 'w'), ('-', 'i'), ('-', 's')]]
+
+        self.assertEqual(result, expected)
+
+        result = aline.align('jo', 'ʒə')
+        expected = [[('j', 'ʒ'), ('o', 'ə')]]
+
+        self.assertEqual(result, expected)
+
+        result = aline.align('pematesiweni', 'pematesewen')
+        expected = [[('p', 'p'), ('e', 'e'), ('m', 'm'), ('a', 'a'), ('t', 't'), ('e', 'e'),
+                     ('s', 's'), ('i', 'e'), ('w', 'w'), ('e', 'e'), ('n', 'n'), ('i', '-')]]
+
+        self.assertEqual(result, expected)
+
+        result = aline.align('tuwθ', 'dentis')
+        expected = [[('t', 'd'), ('u', 'e'), ('w', '-'), ('-', 'n'), ('-', 't'), ('-', 'i'), ('θ', 's')]]
+
+        self.assertEqual(result, expected)
+
+    def test_aline_delta(self):
+        """
+        Test aline for computing the difference between two segments
+        """
+        result = aline.delta('p', 'q')
+        expected = 20.0
+
+        self.assertEqual(result, expected)
+
+        result = aline.delta('a', 'A')
+        expected = 0.0
+
+        self.assertEqual(result, expected)

nltk-3.3/nltk/test/unit/test_chunk.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+import unittest
+
+from nltk import RegexpParser
+
+
+class TestChunkRule(unittest.TestCase):
+
+    def test_tag_pattern2re_pattern_quantifier(self):
+        """Test for bug https://github.com/nltk/nltk/issues/1597
+    
+        Ensures that curly bracket quantifiers can be used inside a chunk rule.
+        This type of quantifier has been used for the supplementary example
+        in http://www.nltk.org/book/ch07.html#exploring-text-corpora.
+        """
+        sent = [('The', 'AT'), ('September-October', 'NP'), ('term', 'NN'), ('jury', 'NN'), ('had', 'HVD'),
+                ('been', 'BEN'), ('charged', 'VBN'), ('by', 'IN'), ('Fulton', 'NP-TL'), ('Superior', 'JJ-TL'),
+                ('Court', 'NN-TL'), ('Judge', 'NN-TL'), ('Durwood', 'NP'), ('Pye', 'NP'), ('to', 'TO'),
+                ('investigate', 'VB'), ('reports', 'NNS'), ('of', 'IN'), ('possible', 'JJ'), ('``', '``'),
+                ('irregularities', 'NNS'), ("''", "''"), ('in', 'IN'), ('the', 'AT'), ('hard-fought', 'JJ'),
+                ('primary', 'NN'), ('which', 'WDT'), ('was', 'BEDZ'), ('won', 'VBN'), ('by', 'IN'),
+                ('Mayor-nominate', 'NN-TL'), ('Ivan', 'NP'), ('Allen', 'NP'), ('Jr.', 'NP'),
+                ('.', '.')]  # source: brown corpus
+        cp = RegexpParser('CHUNK: {<N.*>{4,}}')
+        tree = cp.parse(sent)
+        assert tree.pformat() == """(S
+  The/AT
+  September-October/NP
+  term/NN
+  jury/NN
+  had/HVD
+  been/BEN
+  charged/VBN
+  by/IN
+  Fulton/NP-TL
+  Superior/JJ-TL
+  (CHUNK Court/NN-TL Judge/NN-TL Durwood/NP Pye/NP)
+  to/TO
+  investigate/VB
+  reports/NNS
+  of/IN
+  possible/JJ
+  ``/``
+  irregularities/NNS
+  ''/''
+  in/IN
+  the/AT
+  hard-fought/JJ
+  primary/NN
+  which/WDT
+  was/BEDZ
+  won/VBN
+  by/IN
+  (CHUNK Mayor-nominate/NN-TL Ivan/NP Allen/NP Jr./NP)
+  ./.)"""

{nltk-3.2.1 → nltk-3.3}/nltk/test/unit/test_classify.py

@@ -7,31 +7,32 @@ from nose import SkipTest
 from nltk import classify
 
 TRAIN = [
-     (dict(a=1,b=1,c=1), 'y'),
-     (dict(a=1,b=1,c=1), 'x'),
-     (dict(a=1,b=1,c=0), 'y'),
-     (dict(a=0,b=1,c=1), 'x'),
-     (dict(a=0,b=1,c=1), 'y'),
-     (dict(a=0,b=0,c=1), 'y'),
-     (dict(a=0,b=1,c=0), 'x'),
-     (dict(a=0,b=0,c=0), 'x'),
-     (dict(a=0,b=1,c=1), 'y'),
- ]
+    (dict(a=1, b=1, c=1), 'y'),
+    (dict(a=1, b=1, c=1), 'x'),
+    (dict(a=1, b=1, c=0), 'y'),
+    (dict(a=0, b=1, c=1), 'x'),
+    (dict(a=0, b=1, c=1), 'y'),
+    (dict(a=0, b=0, c=1), 'y'),
+    (dict(a=0, b=1, c=0), 'x'),
+    (dict(a=0, b=0, c=0), 'x'),
+    (dict(a=0, b=1, c=1), 'y'),
+]
 
 TEST = [
-     (dict(a=1,b=0,c=1)), # unseen
-     (dict(a=1,b=0,c=0)), # unseen
-     (dict(a=0,b=1,c=1)), # seen 3 times, labels=y,y,x
-     (dict(a=0,b=1,c=0)), # seen 1 time, label=x
+    (dict(a=1, b=0, c=1)),  # unseen
+    (dict(a=1, b=0, c=0)),  # unseen
+    (dict(a=0, b=1, c=1)),  # seen 3 times, labels=y,y,x
+    (dict(a=0, b=1, c=0)),  # seen 1 time, label=x
 ]
 
 RESULTS = [
-    (0.16,  0.84),
-    (0.46,  0.54),
-    (0.41,  0.59),
-    (0.76,  0.24),
+    (0.16, 0.84),
+    (0.46, 0.54),
+    (0.41, 0.59),
+    (0.76, 0.24),
 ]
 
+
 def assert_classifier_correct(algorithm):
     try:
         classifier = classify.MaxentClassifier.train(
@@ -49,5 +50,6 @@ def assert_classifier_correct(algorithm):
 def test_megam():
     assert_classifier_correct('MEGAM')
 
+
 def test_tadm():
     assert_classifier_correct('TADM')