nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/test/unit/test_collocations.py

@@ -27,10 +27,11 @@ class TestBigram(unittest.TestCase):
 
         b = BigramCollocationFinder.from_words(sent)
 
-        #python 2.6 does not have assertItemsEqual or assertListEqual
+        # python 2.6 does not have assertItemsEqual or assertListEqual
         self.assertEqual(
             sorted(b.ngram_fd.items()),
-            sorted([(('a', 'a'), 1), (('a', 'test'), 1), (('is', 'a'), 1), (('is', 'is'), 1), (('test', 'test'), 1), (('this', 'is'), 1), (('this', 'this'), 1)])
+            sorted([(('a', 'a'), 1), (('a', 'test'), 1), (('is', 'a'), 1), (('is', 'is'), 1), (('test', 'test'), 1),
+                    (('this', 'is'), 1), (('this', 'this'), 1)])
         )
         self.assertEqual(
             sorted(b.word_fd.items()),
@@ -39,7 +40,8 @@ class TestBigram(unittest.TestCase):
         self.assertTrue(len(sent) == sum(b.word_fd.values()) == sum(b.ngram_fd.values()) + 1)
         self.assertTrue(close_enough(
             sorted(b.score_ngrams(BigramAssocMeasures.pmi)),
-            sorted([(('a', 'a'), 1.0), (('a', 'test'), 1.0), (('is', 'a'), 1.0), (('is', 'is'), 1.0), (('test', 'test'), 1.0), (('this', 'is'), 1.0), (('this', 'this'), 1.0)])
+            sorted([(('a', 'a'), 1.0), (('a', 'test'), 1.0), (('is', 'a'), 1.0), (('is', 'is'), 1.0),
+                    (('test', 'test'), 1.0), (('this', 'is'), 1.0), (('this', 'this'), 1.0)])
         ))
 
     def test_bigram3(self):
@@ -48,7 +50,8 @@ class TestBigram(unittest.TestCase):
         b = BigramCollocationFinder.from_words(sent, window_size=3)
         self.assertEqual(
             sorted(b.ngram_fd.items()),
-            sorted([(('a', 'test'), 3), (('is', 'a'), 3), (('this', 'is'), 3), (('a', 'a'), 1), (('is', 'is'), 1), (('test', 'test'), 1), (('this', 'this'), 1)])
+            sorted([(('a', 'test'), 3), (('is', 'a'), 3), (('this', 'is'), 3), (('a', 'a'), 1), (('is', 'is'), 1),
+                    (('test', 'test'), 1), (('this', 'this'), 1)])
         )
         self.assertEqual(
             sorted(b.word_fd.items()),
@@ -57,7 +60,9 @@ class TestBigram(unittest.TestCase):
         self.assertTrue(len(sent) == sum(b.word_fd.values()) == (sum(b.ngram_fd.values()) + 2 + 1) / 2.0)
         self.assertTrue(close_enough(
             sorted(b.score_ngrams(BigramAssocMeasures.pmi)),
-            sorted([(('a', 'test'), 1.584962500721156), (('is', 'a'), 1.584962500721156), (('this', 'is'), 1.584962500721156), (('a', 'a'), 0.0), (('is', 'is'), 0.0), (('test', 'test'), 0.0), (('this', 'this'), 0.0)])
+            sorted([(('a', 'test'), 1.584962500721156), (('is', 'a'), 1.584962500721156),
+                    (('this', 'is'), 1.584962500721156), (('a', 'a'), 0.0), (('is', 'is'), 0.0),
+                    (('test', 'test'), 0.0), (('this', 'this'), 0.0)])
         ))
 
     def test_bigram5(self):
@@ -66,7 +71,8 @@ class TestBigram(unittest.TestCase):
         b = BigramCollocationFinder.from_words(sent, window_size=5)
         self.assertEqual(
             sorted(b.ngram_fd.items()),
-            sorted([(('a', 'test'), 4), (('is', 'a'), 4), (('this', 'is'), 4), (('is', 'test'), 3), (('this', 'a'), 3), (('a', 'a'), 1), (('is', 'is'), 1), (('test', 'test'), 1), (('this', 'this'), 1)])
+            sorted([(('a', 'test'), 4), (('is', 'a'), 4), (('this', 'is'), 4), (('is', 'test'), 3), (('this', 'a'), 3),
+                    (('a', 'a'), 1), (('is', 'is'), 1), (('test', 'test'), 1), (('this', 'this'), 1)])
         )
         self.assertEqual(
             sorted(b.word_fd.items()),
@@ -75,5 +81,8 @@ class TestBigram(unittest.TestCase):
         self.assertTrue(len(sent) == sum(b.word_fd.values()) == (sum(b.ngram_fd.values()) + 4 + 3 + 2 + 1) / 4.0)
         self.assertTrue(close_enough(
             sorted(b.score_ngrams(BigramAssocMeasures.pmi)),
-            sorted([(('a', 'test'), 1.0), (('is', 'a'), 1.0), (('this', 'is'), 1.0), (('is', 'test'), 0.5849625007211562), (('this', 'a'), 0.5849625007211562), (('a', 'a'), -1.0), (('is', 'is'), -1.0), (('test', 'test'), -1.0), (('this', 'this'), -1.0)])
+            sorted(
+                [(('a', 'test'), 1.0), (('is', 'a'), 1.0), (('this', 'is'), 1.0), (('is', 'test'), 0.5849625007211562),
+                 (('this', 'a'), 0.5849625007211562), (('a', 'a'), -1.0), (('is', 'is'), -1.0),
+                 (('test', 'test'), -1.0), (('this', 'this'), -1.0)])
         ))

nltk-3.3/nltk/test/unit/test_concordance.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+
+import unittest
+from nose import with_setup
+
+from nltk.corpus import gutenberg
+from nltk.text import Text
+
+import contextlib
+import sys
+
+try:
+    from StringIO import StringIO
+except ImportError as e:
+    from io import StringIO
+
+
+@contextlib.contextmanager
+def stdout_redirect(where):
+    sys.stdout = where
+    try:
+        yield where
+    finally:
+        sys.stdout = sys.__stdout__
+
+
+class TestConcordance(unittest.TestCase):
+    """Text constructed using: http://www.nltk.org/book/ch01.html"""
+
+    @classmethod
+    def setup_class(cls):
+        cls.corpus = gutenberg.words('melville-moby_dick.txt')
+
+    @classmethod
+    def teardown_class(cls):
+        pass
+
+    def setUp(self):
+        self.text = Text(TestConcordance.corpus)
+        self.query = "monstrous"
+        self.maxDiff = None
+        self.list_out = [
+            'ong the former , one was of a most monstrous size . ... This came towards us , ',
+            'ON OF THE PSALMS . " Touching that monstrous bulk of the whale or ork we have r',
+            'll over with a heathenish array of monstrous clubs and spears . Some were thick',
+            'd as you gazed , and wondered what monstrous cannibal and savage could ever hav',
+            'that has survived the flood ; most monstrous and most mountainous ! That Himmal',
+            'they might scout at Moby Dick as a monstrous fable , or still worse and more de',
+            'th of Radney .\'" CHAPTER 55 Of the Monstrous Pictures of Whales . I shall ere l',
+            'ing Scenes . In connexion with the monstrous pictures of whales , I am strongly',
+            'ere to enter upon those still more monstrous stories of them which are to be fo',
+            'ght have been rummaged out of this monstrous cabinet there is no telling . But ',
+            'of Whale - Bones ; for Whales of a monstrous size are oftentimes cast up dead u']
+
+    def tearDown(self):
+        pass
+
+    def test_concordance_list(self):
+        concordance_out = self.text.concordance_list(self.query)
+        self.assertEqual(self.list_out, [c.line for c in concordance_out])
+
+    def test_concordance_width(self):
+        list_out = ["monstrous", "monstrous", "monstrous",
+                    "monstrous", "monstrous", "monstrous",
+                    "Monstrous", "monstrous", "monstrous",
+                    "monstrous", "monstrous"]
+
+        concordance_out = self.text.concordance_list(self.query, width=0)
+        self.assertEqual(list_out, [c.query for c in concordance_out])
+
+    def test_concordance_lines(self):
+        concordance_out = self.text.concordance_list(self.query, lines=3)
+        self.assertEqual(self.list_out[:3], [c.line for c in concordance_out])
+
+    def test_concordance_print(self):
+        print_out = """Displaying 11 of 11 matches:
+        ong the former , one was of a most monstrous size . ... This came towards us ,
+        ON OF THE PSALMS . " Touching that monstrous bulk of the whale or ork we have r
+        ll over with a heathenish array of monstrous clubs and spears . Some were thick
+        d as you gazed , and wondered what monstrous cannibal and savage could ever hav
+        that has survived the flood ; most monstrous and most mountainous ! That Himmal
+        they might scout at Moby Dick as a monstrous fable , or still worse and more de
+        th of Radney .'" CHAPTER 55 Of the Monstrous Pictures of Whales . I shall ere l
+        ing Scenes . In connexion with the monstrous pictures of whales , I am strongly
+        ere to enter upon those still more monstrous stories of them which are to be fo
+        ght have been rummaged out of this monstrous cabinet there is no telling . But
+        of Whale - Bones ; for Whales of a monstrous size are oftentimes cast up dead u
+        """
+
+        with stdout_redirect(StringIO()) as stdout:
+            self.text.concordance(self.query)
+
+        def strip_space(raw_str):
+            return raw_str.replace(" ", "")
+
+        self.assertEqual(strip_space(print_out),
+                         strip_space(stdout.getvalue()))