nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpora.py

@@ -1,183 +1,200 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import, unicode_literals
-import unittest
-from nltk.corpus import (sinica_treebank, conll2007, indian, cess_cat, cess_esp,
-                         floresta, ptb, udhr)
-from nltk.tree import Tree
-from nltk.test.unit.utils import skipIf
-
-
-class TestUdhr(unittest.TestCase):
-
-    def test_words(self):
-        for name in udhr.fileids():
-            try:
-                words = list(udhr.words(name))
-            except AssertionError:
-                print(name)
-                raise
-            self.assertTrue(words)
-
-    def test_raw_unicode(self):
-        for name in udhr.fileids():
-            txt = udhr.raw(name)
-            assert not isinstance(txt, bytes), name
-
-
-class TestIndian(unittest.TestCase):
-
-    def test_words(self):
-        words = indian.words()[:3]
-        self.assertEqual(words, ['মহিষের', 'সন্তান', ':'])
-
-    def test_tagged_words(self):
-        tagged_words = indian.tagged_words()[:3]
-        self.assertEqual(tagged_words, [('মহিষের', 'NN'), ('সন্তান', 'NN'), (':', 'SYM')])
-
-
-class TestCess(unittest.TestCase):
-    def test_catalan(self):
-        words = cess_cat.words()[:15]
-        txt = "El Tribunal_Suprem -Fpa- TS -Fpt- ha confirmat la condemna a quatre anys d' inhabilitació especial"
-        self.assertEqual(words, txt.split())
-        self.assertEqual(cess_cat.tagged_sents()[0][34][0], "càrrecs")
-
-    def test_esp(self):
-        words = cess_esp.words()[:15]
-        txt = "El grupo estatal Electricité_de_France -Fpa- EDF -Fpt- anunció hoy , jueves , la compra del"
-        self.assertEqual(words, txt.split())
-        self.assertEqual(cess_esp.words()[115], "años")
-
-
-class TestFloresta(unittest.TestCase):
-    def test_words(self):
-        words = floresta.words()[:10]
-        txt = "Um revivalismo refrescante O 7_e_Meio é um ex-libris de a"
-        self.assertEqual(words, txt.split())
-
-class TestSinicaTreebank(unittest.TestCase):
-
-    def test_sents(self):
-        first_3_sents = sinica_treebank.sents()[:3]
-        self.assertEqual(
-            first_3_sents,
-            [['一'], ['友情'], ['嘉珍', '和', '我', '住在', '同一條', '巷子']]
-        )
-
-    def test_parsed_sents(self):
-        parsed_sents = sinica_treebank.parsed_sents()[25]
-        self.assertEqual(parsed_sents,
-            Tree('S', [
-                Tree('NP', [
-                    Tree('Nba', ['嘉珍'])
-                ]),
-                Tree('V‧地', [
-                    Tree('VA11', ['不停']),
-                    Tree('DE', ['的'])
-                ]),
-                Tree('VA4', ['哭泣'])
-            ]))
-
-
-class TestCoNLL2007(unittest.TestCase):
-    # Reading the CoNLL 2007 Dependency Treebanks
-
-    def test_sents(self):
-        sents = conll2007.sents('esp.train')[0]
-        self.assertEqual(
-            sents[:6],
-            ['El', 'aumento', 'del', 'índice', 'de', 'desempleo']
-        )
-
-    def test_parsed_sents(self):
-
-        parsed_sents = conll2007.parsed_sents('esp.train')[0]
-
-        self.assertEqual(parsed_sents.tree(),
-            Tree('fortaleció', [
-                Tree('aumento', [
-                    'El',
-                    Tree('del', [
-                        Tree('índice', [
-                            Tree('de', [
-                                Tree('desempleo', ['estadounidense'])
-                            ])
-                        ])
-                    ])
-                ]),
-                'hoy',
-                'considerablemente',
-                Tree('al', [
-                    Tree('euro', [
-                        Tree('cotizaba', [
-                            ',',
-                            'que',
-                            Tree('a', [
-                                Tree('15.35', ['las', 'GMT'])
-                            ]),
-                            'se',
-                            Tree('en', [
-                                Tree('mercado', [
-                                    'el',
-                                    Tree('de', ['divisas']),
-                                    Tree('de', ['Fráncfort'])
-                                ])
-                            ]),
-                            Tree('a', ['0,9452_dólares']),
-                            Tree('frente_a', [
-                                ',',
-                                Tree('0,9349_dólares', [
-                                    'los',
-                                    Tree('de', [
-                                        Tree('mañana', ['esta'])
-                                    ])
-                                ])
-                            ])
-                        ])
-                    ])
-                ]),
-                '.'
-            ])
-        )
-
-
-@skipIf(not ptb.fileids(), "A full installation of the Penn Treebank is not available")
-class TestPTB(unittest.TestCase):
-    def test_fileids(self):
-        self.assertEqual(
-            ptb.fileids()[:4],
-            ['BROWN/CF/CF01.MRG', 'BROWN/CF/CF02.MRG', 'BROWN/CF/CF03.MRG', 'BROWN/CF/CF04.MRG']
-        )
-
-    def test_words(self):
-        self.assertEqual(
-            ptb.words('WSJ/00/WSJ_0003.MRG')[:7],
-            ['A', 'form', 'of', 'asbestos', 'once', 'used', '*']
-        )
-
-    def test_tagged_words(self):
-        self.assertEqual(
-            ptb.tagged_words('WSJ/00/WSJ_0003.MRG')[:3],
-            [('A', 'DT'), ('form', 'NN'), ('of', 'IN')]
-        )
-
-    def test_categories(self):
-        self.assertEqual(
-            ptb.categories(),
-            ['adventure', 'belles_lettres', 'fiction', 'humor', 'lore', 'mystery', 'news', 'romance', 'science_fiction']
-        )
-
-    def test_news_fileids(self):
-        self.assertEqual(
-            ptb.fileids('news')[:3],
-            ['WSJ/00/WSJ_0001.MRG', 'WSJ/00/WSJ_0002.MRG', 'WSJ/00/WSJ_0003.MRG']
-        )
-
-    def test_category_words(self):
-        self.assertEqual(
-            ptb.words(categories=['humor','fiction'])[:6],
-            ['Thirty-three', 'Scotty', 'did', 'not', 'go', 'back']
-        )
-
-# unload corpora
-from nltk.corpus import teardown_module
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+import unittest
+
+from nltk.corpus import (sinica_treebank, conll2007, indian, cess_cat, cess_esp,
+                         floresta, ptb, udhr) # mwa_ppdb
+
+from nltk.compat import python_2_unicode_compatible
+from nltk.tree import Tree
+from nltk.test.unit.utils import skipIf
+
+
+class TestUdhr(unittest.TestCase):
+
+    def test_words(self):
+        for name in udhr.fileids():
+            try:
+                words = list(udhr.words(name))
+            except AssertionError:
+                print(name)
+                raise
+            self.assertTrue(words)
+
+    def test_raw_unicode(self):
+        for name in udhr.fileids():
+            txt = udhr.raw(name)
+            assert not isinstance(txt, bytes), name
+
+
+class TestIndian(unittest.TestCase):
+
+    def test_words(self):
+        words = indian.words()[:3]
+        self.assertEqual(words, ['মহিষের', 'সন্তান', ':'])
+
+    def test_tagged_words(self):
+        tagged_words = indian.tagged_words()[:3]
+        self.assertEqual(tagged_words, [('মহিষের', 'NN'), ('সন্তান', 'NN'), (':', 'SYM')])
+
+
+class TestCess(unittest.TestCase):
+    def test_catalan(self):
+        words = cess_cat.words()[:15]
+        txt = "El Tribunal_Suprem -Fpa- TS -Fpt- ha confirmat la condemna a quatre anys d' inhabilitació especial"
+        self.assertEqual(words, txt.split())
+        self.assertEqual(cess_cat.tagged_sents()[0][34][0], "càrrecs")
+
+    def test_esp(self):
+        words = cess_esp.words()[:15]
+        txt = "El grupo estatal Electricité_de_France -Fpa- EDF -Fpt- anunció hoy , jueves , la compra del"
+        self.assertEqual(words, txt.split())
+        self.assertEqual(cess_esp.words()[115], "años")
+
+
+class TestFloresta(unittest.TestCase):
+    def test_words(self):
+        words = floresta.words()[:10]
+        txt = "Um revivalismo refrescante O 7_e_Meio é um ex-libris de a"
+        self.assertEqual(words, txt.split())
+
+class TestSinicaTreebank(unittest.TestCase):
+
+    def test_sents(self):
+        first_3_sents = sinica_treebank.sents()[:3]
+        self.assertEqual(
+            first_3_sents,
+            [['一'], ['友情'], ['嘉珍', '和', '我', '住在', '同一條', '巷子']]
+        )
+
+    def test_parsed_sents(self):
+        parsed_sents = sinica_treebank.parsed_sents()[25]
+        self.assertEqual(parsed_sents,
+            Tree('S', [
+                Tree('NP', [
+                    Tree('Nba', ['嘉珍'])
+                ]),
+                Tree('V‧地', [
+                    Tree('VA11', ['不停']),
+                    Tree('DE', ['的'])
+                ]),
+                Tree('VA4', ['哭泣'])
+            ]))
+
+
+class TestCoNLL2007(unittest.TestCase):
+    # Reading the CoNLL 2007 Dependency Treebanks
+
+    def test_sents(self):
+        sents = conll2007.sents('esp.train')[0]
+        self.assertEqual(
+            sents[:6],
+            ['El', 'aumento', 'del', 'índice', 'de', 'desempleo']
+        )
+
+    def test_parsed_sents(self):
+
+        parsed_sents = conll2007.parsed_sents('esp.train')[0]
+
+        self.assertEqual(parsed_sents.tree(),
+            Tree('fortaleció', [
+                Tree('aumento', [
+                    'El',
+                    Tree('del', [
+                        Tree('índice', [
+                            Tree('de', [
+                                Tree('desempleo', ['estadounidense'])
+                            ])
+                        ])
+                    ])
+                ]),
+                'hoy',
+                'considerablemente',
+                Tree('al', [
+                    Tree('euro', [
+                        Tree('cotizaba', [
+                            ',',
+                            'que',
+                            Tree('a', [
+                                Tree('15.35', ['las', 'GMT'])
+                            ]),
+                            'se',
+                            Tree('en', [
+                                Tree('mercado', [
+                                    'el',
+                                    Tree('de', ['divisas']),
+                                    Tree('de', ['Fráncfort'])
+                                ])
+                            ]),
+                            Tree('a', ['0,9452_dólares']),
+                            Tree('frente_a', [
+                                ',',
+                                Tree('0,9349_dólares', [
+                                    'los',
+                                    Tree('de', [
+                                        Tree('mañana', ['esta'])
+                                    ])
+                                ])
+                            ])
+                        ])
+                    ])
+                ]),
+                '.'
+            ])
+        )
+
+
+@skipIf(not ptb.fileids(), "A full installation of the Penn Treebank is not available")
+class TestPTB(unittest.TestCase):
+    def test_fileids(self):
+        self.assertEqual(
+            ptb.fileids()[:4],
+            ['BROWN/CF/CF01.MRG', 'BROWN/CF/CF02.MRG', 'BROWN/CF/CF03.MRG', 'BROWN/CF/CF04.MRG']
+        )
+
+    def test_words(self):
+        self.assertEqual(
+            ptb.words('WSJ/00/WSJ_0003.MRG')[:7],
+            ['A', 'form', 'of', 'asbestos', 'once', 'used', '*']
+        )
+
+    def test_tagged_words(self):
+        self.assertEqual(
+            ptb.tagged_words('WSJ/00/WSJ_0003.MRG')[:3],
+            [('A', 'DT'), ('form', 'NN'), ('of', 'IN')]
+        )
+
+    def test_categories(self):
+        self.assertEqual(
+            ptb.categories(),
+            ['adventure', 'belles_lettres', 'fiction', 'humor', 'lore', 'mystery', 'news', 'romance', 'science_fiction']
+        )
+
+    def test_news_fileids(self):
+        self.assertEqual(
+            ptb.fileids('news')[:3],
+            ['WSJ/00/WSJ_0001.MRG', 'WSJ/00/WSJ_0002.MRG', 'WSJ/00/WSJ_0003.MRG']
+        )
+
+    def test_category_words(self):
+        self.assertEqual(
+            ptb.words(categories=['humor','fiction'])[:6],
+            ['Thirty-three', 'Scotty', 'did', 'not', 'go', 'back']
+        )
+
+@unittest.skip("Skipping test for mwa_ppdb.")
+class TestMWAPPDB(unittest.TestCase):
+    def test_fileids(self):
+        self.assertEqual(mwa_ppdb.fileids(),
+            ['ppdb-1.0-xxxl-lexical.extended.synonyms.uniquepairs'])
+
+    def test_entries(self):
+        self.assertEqual(mwa_ppdb.entries()[:10],
+            [('10/17/01', '17/10/2001'), ('102,70', '102.70'),
+            ('13,53', '13.53'), ('3.2.5.3.2.1', '3.2.5.3.2.1.'),
+            ('53,76', '53.76'), ('6.9.5', '6.9.5.'),
+            ('7.7.6.3', '7.7.6.3.'), ('76,20', '76.20'),
+            ('79,85', '79.85'), ('93,65', '93.65')] )
+
+# unload corpora
+from nltk.corpus import teardown_module

{nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpus_views.py

@@ -1,45 +1,45 @@
-# -*- coding: utf-8 -*-
-"""
-Corpus View Regression Tests
-"""
-from __future__ import absolute_import, unicode_literals
-import unittest
-import nltk.data
-from nltk.corpus.reader.util import (StreamBackedCorpusView,
-                                     read_whitespace_block, read_line_block)
-
-class TestCorpusViews(unittest.TestCase):
-
-    linetok = nltk.LineTokenizer(blanklines='keep')
-    names = [
-        'corpora/inaugural/README', # A very short file (160 chars)
-        'corpora/inaugural/1793-Washington.txt', # A relatively short file (791 chars)
-        'corpora/inaugural/1909-Taft.txt', # A longer file (32k chars)
-    ]
-
-    def data(self):
-        for name in self.names:
-            f = nltk.data.find(name)
-            with f.open() as fp:
-                file_data = fp.read().decode('utf8')
-            yield f, file_data
-
-    def test_correct_values(self):
-        # Check that corpus views produce the correct sequence of values.
-
-        for f, file_data in self.data():
-            v = StreamBackedCorpusView(f, read_whitespace_block)
-            self.assertEqual(list(v), file_data.split())
-
-            v = StreamBackedCorpusView(f, read_line_block)
-            self.assertEqual(list(v), self.linetok.tokenize(file_data))
-
-    def test_correct_length(self):
-        # Check that the corpus views report the correct lengths:
-
-        for f, file_data in self.data():
-            v = StreamBackedCorpusView(f, read_whitespace_block)
-            self.assertEqual(len(v), len(file_data.split()))
-
-            v = StreamBackedCorpusView(f, read_line_block)
-            self.assertEqual(len(v), len(self.linetok.tokenize(file_data)))
+# -*- coding: utf-8 -*-
+"""
+Corpus View Regression Tests
+"""
+from __future__ import absolute_import, unicode_literals
+import unittest
+import nltk.data
+from nltk.corpus.reader.util import (StreamBackedCorpusView,
+                                     read_whitespace_block, read_line_block)
+
+class TestCorpusViews(unittest.TestCase):
+
+    linetok = nltk.LineTokenizer(blanklines='keep')
+    names = [
+        'corpora/inaugural/README', # A very short file (160 chars)
+        'corpora/inaugural/1793-Washington.txt', # A relatively short file (791 chars)
+        'corpora/inaugural/1909-Taft.txt', # A longer file (32k chars)
+    ]
+
+    def data(self):
+        for name in self.names:
+            f = nltk.data.find(name)
+            with f.open() as fp:
+                file_data = fp.read().decode('utf8')
+            yield f, file_data
+
+    def test_correct_values(self):
+        # Check that corpus views produce the correct sequence of values.
+
+        for f, file_data in self.data():
+            v = StreamBackedCorpusView(f, read_whitespace_block)
+            self.assertEqual(list(v), file_data.split())
+
+            v = StreamBackedCorpusView(f, read_line_block)
+            self.assertEqual(list(v), self.linetok.tokenize(file_data))
+
+    def test_correct_length(self):
+        # Check that the corpus views report the correct lengths:
+
+        for f, file_data in self.data():
+            v = StreamBackedCorpusView(f, read_whitespace_block)
+            self.assertEqual(len(v), len(file_data.split()))
+
+            v = StreamBackedCorpusView(f, read_line_block)
+            self.assertEqual(len(v), len(self.linetok.tokenize(file_data)))

{nltk-3.2 → nltk-3.3}/nltk/test/unit/test_hmm.py

@@ -1,88 +1,88 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import, unicode_literals
-from nltk.tag import hmm
-
-def _wikipedia_example_hmm():
-    # Example from wikipedia
-    # (http://en.wikipedia.org/wiki/Forward%E2%80%93backward_algorithm)
-
-    states = ['rain', 'no rain']
-    symbols = ['umbrella', 'no umbrella']
-
-    A = [[0.7, 0.3], [0.3, 0.7]]  # transition probabilities
-    B = [[0.9, 0.1], [0.2, 0.8]]  # emission probabilities
-    pi = [0.5, 0.5]  # initial probabilities
-
-    seq = ['umbrella', 'umbrella', 'no umbrella', 'umbrella', 'umbrella']
-    seq = list(zip(seq, [None]*len(seq)))
-
-    model = hmm._create_hmm_tagger(states, symbols, A, B, pi)
-    return model, states, symbols, seq
-
-
-def test_forward_probability():
-    from numpy.testing import assert_array_almost_equal
-
-    # example from p. 385, Huang et al
-    model, states, symbols = hmm._market_hmm_example()
-    seq = [('up', None), ('up', None)]
-    expected = [
-        [0.35, 0.02, 0.09],
-        [0.1792, 0.0085, 0.0357]
-    ]
-
-    fp = 2**model._forward_probability(seq)
-
-    assert_array_almost_equal(fp, expected)
-
-
-def test_forward_probability2():
-    from numpy.testing import assert_array_almost_equal
-
-    model, states, symbols, seq = _wikipedia_example_hmm()
-    fp = 2**model._forward_probability(seq)
-
-    # examples in wikipedia are normalized
-    fp = (fp.T / fp.sum(axis=1)).T
-
-    wikipedia_results = [
-        [0.8182, 0.1818],
-        [0.8834, 0.1166],
-        [0.1907, 0.8093],
-        [0.7308, 0.2692],
-        [0.8673, 0.1327],
-    ]
-
-    assert_array_almost_equal(wikipedia_results, fp, 4)
-
-
-def test_backward_probability():
-    from numpy.testing import assert_array_almost_equal
-
-    model, states, symbols, seq = _wikipedia_example_hmm()
-
-    bp = 2**model._backward_probability(seq)
-    # examples in wikipedia are normalized
-
-    bp = (bp.T / bp.sum(axis=1)).T
-
-    wikipedia_results = [
-        # Forward-backward algorithm doesn't need b0_5,
-        # so .backward_probability doesn't compute it.
-        # [0.6469, 0.3531],
-        [0.5923, 0.4077],
-        [0.3763, 0.6237],
-        [0.6533, 0.3467],
-        [0.6273, 0.3727],
-        [0.5, 0.5],
-    ]
-
-    assert_array_almost_equal(wikipedia_results, bp, 4)
-
-
-def setup_module(module):
-    from nose import SkipTest
-    try:
-        import numpy
-    except ImportError:
-        raise SkipTest("numpy is required for nltk.test.test_hmm")
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+from nltk.tag import hmm
+
+def _wikipedia_example_hmm():
+    # Example from wikipedia
+    # (http://en.wikipedia.org/wiki/Forward%E2%80%93backward_algorithm)
+
+    states = ['rain', 'no rain']
+    symbols = ['umbrella', 'no umbrella']
+
+    A = [[0.7, 0.3], [0.3, 0.7]]  # transition probabilities
+    B = [[0.9, 0.1], [0.2, 0.8]]  # emission probabilities
+    pi = [0.5, 0.5]  # initial probabilities
+
+    seq = ['umbrella', 'umbrella', 'no umbrella', 'umbrella', 'umbrella']
+    seq = list(zip(seq, [None]*len(seq)))
+
+    model = hmm._create_hmm_tagger(states, symbols, A, B, pi)
+    return model, states, symbols, seq
+
+
+def test_forward_probability():
+    from numpy.testing import assert_array_almost_equal
+
+    # example from p. 385, Huang et al
+    model, states, symbols = hmm._market_hmm_example()
+    seq = [('up', None), ('up', None)]
+    expected = [
+        [0.35, 0.02, 0.09],
+        [0.1792, 0.0085, 0.0357]
+    ]
+
+    fp = 2**model._forward_probability(seq)
+
+    assert_array_almost_equal(fp, expected)
+
+
+def test_forward_probability2():
+    from numpy.testing import assert_array_almost_equal
+
+    model, states, symbols, seq = _wikipedia_example_hmm()
+    fp = 2**model._forward_probability(seq)
+
+    # examples in wikipedia are normalized
+    fp = (fp.T / fp.sum(axis=1)).T
+
+    wikipedia_results = [
+        [0.8182, 0.1818],
+        [0.8834, 0.1166],
+        [0.1907, 0.8093],
+        [0.7308, 0.2692],
+        [0.8673, 0.1327],
+    ]
+
+    assert_array_almost_equal(wikipedia_results, fp, 4)
+
+
+def test_backward_probability():
+    from numpy.testing import assert_array_almost_equal
+
+    model, states, symbols, seq = _wikipedia_example_hmm()
+
+    bp = 2**model._backward_probability(seq)
+    # examples in wikipedia are normalized
+
+    bp = (bp.T / bp.sum(axis=1)).T
+
+    wikipedia_results = [
+        # Forward-backward algorithm doesn't need b0_5,
+        # so .backward_probability doesn't compute it.
+        # [0.6469, 0.3531],
+        [0.5923, 0.4077],
+        [0.3763, 0.6237],
+        [0.6533, 0.3467],
+        [0.6273, 0.3727],
+        [0.5, 0.5],
+    ]
+
+    assert_array_almost_equal(wikipedia_results, bp, 4)
+
+
+def setup_module(module):
+    from nose import SkipTest
+    try:
+        import numpy
+    except ImportError:
+        raise SkipTest("numpy is required for nltk.test.test_hmm")