nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/test/unit/test_seekable_unicode_stream_reader.py

@@ -1,124 +1,124 @@
-# -*- coding: utf-8 -*-
-"""
-The following test performs a random series of reads, seeks, and
-tells, and checks that the results are consistent.
-"""
-from __future__ import absolute_import, unicode_literals
-import random
-import functools
-from io import BytesIO
-from nltk.corpus.reader import SeekableUnicodeStreamReader
-
-def check_reader(unicode_string, encoding, n=1000):
-    bytestr = unicode_string.encode(encoding)
-    strlen = len(unicode_string)
-    stream = BytesIO(bytestr)
-    reader = SeekableUnicodeStreamReader(stream, encoding)
-    # Find all character positions
-    chars = []
-    while True:
-        pos = reader.tell()
-        chars.append( (pos, reader.read(1)) )
-        if chars[-1][1] == '': break
-    # Find all strings
-    strings = dict( (pos,'') for (pos,c) in chars )
-    for pos1, char in chars:
-        for pos2, _ in chars:
-            if pos2 <= pos1:
-                strings[pos2] += char
-    while True:
-        op = random.choice('tsrr')
-        # Check our position?
-        if op == 't': # tell
-            reader.tell()
-        # Perform a seek?
-        if op == 's': # seek
-            new_pos = random.choice([p for (p,c) in chars])
-            reader.seek(new_pos)
-        # Perform a read?
-        if op == 'r': # read
-            if random.random() < .3: pos = reader.tell()
-            else: pos = None
-            if random.random() < .2: size = None
-            elif random.random() < .8:
-                size = random.randint(0, int(strlen/6))
-            else: size = random.randint(0, strlen+20)
-            if random.random() < .8:
-                s = reader.read(size)
-            else:
-                s = reader.readline(size)
-            # check that everything's consistent
-            if pos is not None:
-                assert pos in strings
-                assert strings[pos].startswith(s)
-                n -= 1
-                if n == 0:
-                    return 'passed'
-
-
-#Call the randomized test function `check_reader` with a variety of
-#input strings and encodings.
-
-ENCODINGS = ['ascii', 'latin1', 'greek', 'hebrew', 'utf-16', 'utf-8']
-
-STRINGS = [
-    """
-    This is a test file.
-    It is fairly short.
-    """,
-
-    "This file can be encoded with latin1. \x83",
-
-    """\
-    This is a test file.
-    Here's a blank line:
-
-    And here's some unicode: \xee \u0123 \uffe3
-    """,
-
-    """\
-    This is a test file.
-    Unicode characters: \xf3 \u2222 \u3333\u4444 \u5555
-    """,
-
-]
-
-def test_reader():
-    for string in STRINGS:
-        for encoding in ENCODINGS:
-            try:
-                # skip strings that can't be encoded with the current encoding
-                string.encode(encoding)
-                yield check_reader, string, encoding
-            except UnicodeEncodeError:
-                pass
-
-
-
-# nose shows the whole string arguments in a verbose mode; this is annoying,
-# so large string test is separated.
-
-LARGE_STRING = """\
-This is a larger file.  It has some lines that are longer \
-than 72 characters.  It's got lots of repetition.  Here's \
-some unicode chars: \xee \u0123 \uffe3 \ueeee \u2345
-
-How fun!  Let's repeat it twenty times.
-"""*10
-
-def test_reader_on_large_string():
-    for encoding in ENCODINGS:
-        try:
-            # skip strings that can't be encoded with the current encoding
-            LARGE_STRING.encode(encoding)
-            def _check(encoding, n=1000):
-                check_reader(LARGE_STRING, encoding, n)
-
-            yield _check, encoding
-
-        except UnicodeEncodeError:
-            pass
-
-def teardown_module(module=None):
-    import gc
-    gc.collect()
+# -*- coding: utf-8 -*-
+"""
+The following test performs a random series of reads, seeks, and
+tells, and checks that the results are consistent.
+"""
+from __future__ import absolute_import, unicode_literals
+import random
+import functools
+from io import BytesIO
+from nltk.corpus.reader import SeekableUnicodeStreamReader
+
+def check_reader(unicode_string, encoding, n=1000):
+    bytestr = unicode_string.encode(encoding)
+    strlen = len(unicode_string)
+    stream = BytesIO(bytestr)
+    reader = SeekableUnicodeStreamReader(stream, encoding)
+    # Find all character positions
+    chars = []
+    while True:
+        pos = reader.tell()
+        chars.append( (pos, reader.read(1)) )
+        if chars[-1][1] == '': break
+    # Find all strings
+    strings = dict( (pos,'') for (pos,c) in chars )
+    for pos1, char in chars:
+        for pos2, _ in chars:
+            if pos2 <= pos1:
+                strings[pos2] += char
+    while True:
+        op = random.choice('tsrr')
+        # Check our position?
+        if op == 't': # tell
+            reader.tell()
+        # Perform a seek?
+        if op == 's': # seek
+            new_pos = random.choice([p for (p,c) in chars])
+            reader.seek(new_pos)
+        # Perform a read?
+        if op == 'r': # read
+            if random.random() < .3: pos = reader.tell()
+            else: pos = None
+            if random.random() < .2: size = None
+            elif random.random() < .8:
+                size = random.randint(0, int(strlen/6))
+            else: size = random.randint(0, strlen+20)
+            if random.random() < .8:
+                s = reader.read(size)
+            else:
+                s = reader.readline(size)
+            # check that everything's consistent
+            if pos is not None:
+                assert pos in strings
+                assert strings[pos].startswith(s)
+                n -= 1
+                if n == 0:
+                    return 'passed'
+
+
+#Call the randomized test function `check_reader` with a variety of
+#input strings and encodings.
+
+ENCODINGS = ['ascii', 'latin1', 'greek', 'hebrew', 'utf-16', 'utf-8']
+
+STRINGS = [
+    """
+    This is a test file.
+    It is fairly short.
+    """,
+
+    "This file can be encoded with latin1. \x83",
+
+    """\
+    This is a test file.
+    Here's a blank line:
+
+    And here's some unicode: \xee \u0123 \uffe3
+    """,
+
+    """\
+    This is a test file.
+    Unicode characters: \xf3 \u2222 \u3333\u4444 \u5555
+    """,
+
+]
+
+def test_reader():
+    for string in STRINGS:
+        for encoding in ENCODINGS:
+            try:
+                # skip strings that can't be encoded with the current encoding
+                string.encode(encoding)
+                yield check_reader, string, encoding
+            except UnicodeEncodeError:
+                pass
+
+
+
+# nose shows the whole string arguments in a verbose mode; this is annoying,
+# so large string test is separated.
+
+LARGE_STRING = """\
+This is a larger file.  It has some lines that are longer \
+than 72 characters.  It's got lots of repetition.  Here's \
+some unicode chars: \xee \u0123 \uffe3 \ueeee \u2345
+
+How fun!  Let's repeat it twenty times.
+"""*10
+
+def test_reader_on_large_string():
+    for encoding in ENCODINGS:
+        try:
+            # skip strings that can't be encoded with the current encoding
+            LARGE_STRING.encode(encoding)
+            def _check(encoding, n=1000):
+                check_reader(LARGE_STRING, encoding, n)
+
+            yield _check, encoding
+
+        except UnicodeEncodeError:
+            pass
+
+def teardown_module(module=None):
+    import gc
+    gc.collect()

nltk-3.3/nltk/test/unit/test_senna.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for Senna
+"""
+
+from __future__ import unicode_literals
+from os import environ, path, sep
+
+import logging
+import unittest
+
+from nltk.classify import Senna
+from nltk.tag import SennaTagger, SennaChunkTagger, SennaNERTagger
+
+# Set Senna executable path for tests if it is not specified as an environment variable
+if 'SENNA' in environ:
+    SENNA_EXECUTABLE_PATH = path.normpath(environ['SENNA']) + sep
+else:
+    SENNA_EXECUTABLE_PATH = '/usr/share/senna-v3.0'
+
+senna_is_installed = path.exists(SENNA_EXECUTABLE_PATH)
+
+@unittest.skipUnless(senna_is_installed, "Requires Senna executable")
+class TestSennaPipeline(unittest.TestCase):
+    """Unittest for nltk.classify.senna"""
+
+    def test_senna_pipeline(self):
+        """Senna pipeline interface"""
+
+        pipeline = Senna(SENNA_EXECUTABLE_PATH, ['pos', 'chk', 'ner'])
+        sent = 'Dusseldorf is an international business center'.split()
+        result = [(token['word'], token['chk'], token['ner'], token['pos']) for token in pipeline.tag(sent)]
+        expected = [('Dusseldorf', 'B-NP', 'B-LOC', 'NNP'), ('is', 'B-VP',
+            'O', 'VBZ'), ('an', 'B-NP', 'O', 'DT'), ('international', 'I-NP',
+            'O', 'JJ'), ('business', 'I-NP', 'O', 'NN'), ('center', 'I-NP',
+            'O', 'NN')]
+        self.assertEqual(result, expected)
+
+@unittest.skipUnless(senna_is_installed, "Requires Senna executable")
+class TestSennaTagger(unittest.TestCase):
+    """Unittest for nltk.tag.senna"""
+
+    def test_senna_tagger(self):
+        tagger = SennaTagger(SENNA_EXECUTABLE_PATH)
+        result = tagger.tag('What is the airspeed of an unladen swallow ?'.split())
+        expected = [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed',
+            'NN'),('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow',
+            'NN'), ('?', '.')]
+        self.assertEqual(result, expected)
+
+    def test_senna_chunk_tagger(self):
+        chktagger = SennaChunkTagger(SENNA_EXECUTABLE_PATH)
+        result_1 = chktagger.tag('What is the airspeed of an unladen swallow ?'.split())
+        expected_1 = [('What', 'B-NP'), ('is', 'B-VP'), ('the', 'B-NP'), ('airspeed',
+            'I-NP'), ('of', 'B-PP'), ('an', 'B-NP'), ('unladen', 'I-NP'), ('swallow',
+            'I-NP'), ('?', 'O')]
+
+        result_2 = list(chktagger.bio_to_chunks(result_1, chunk_type='NP'))
+        expected_2 = [('What', '0'), ('the airspeed', '2-3'), ('an unladen swallow',
+            '5-6-7')]
+        self.assertEqual(result_1, expected_1)
+        self.assertEqual(result_2, expected_2)
+
+    def test_senna_ner_tagger(self):
+        nertagger = SennaNERTagger(SENNA_EXECUTABLE_PATH)
+        result_1 = nertagger.tag('Shakespeare theatre was in London .'.split())
+        expected_1 = [('Shakespeare', 'B-PER'), ('theatre', 'O'), ('was', 'O'),
+                ('in', 'O'), ('London', 'B-LOC'), ('.', 'O')]
+
+        result_2 = nertagger.tag('UN headquarters are in NY , USA .'.split())
+        expected_2 = [('UN', 'B-ORG'), ('headquarters', 'O'), ('are', 'O'),
+        ('in', 'O'), ('NY', 'B-LOC'), (',', 'O'), ('USA', 'B-LOC'), ('.', 'O')]
+        self.assertEqual(result_1, expected_1)
+        self.assertEqual(result_2, expected_2)

nltk-3.3/nltk/test/unit/test_stem.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+import unittest
+from contextlib import closing
+from nltk import data
+from nltk.stem.snowball import SnowballStemmer
+from nltk.stem.porter import PorterStemmer
+import os
+
+
+class SnowballTest(unittest.TestCase):
+
+    def test_arabic(self):
+        """
+        this unit testing for test the snowball arabic light stemmer
+        this stemmer deals with prefixes and suffixes
+        """
+        ar_stemmer = SnowballStemmer("arabic")
+        assert ar_stemmer.stem('الْعَرَبِــــــيَّة') == "عرب"
+        assert ar_stemmer.stem("العربية") == "عرب"
+        assert ar_stemmer.stem("فقالوا") == "قال"
+        assert ar_stemmer.stem("الطالبات") == "طالب"
+        assert ar_stemmer.stem("فالطالبات") == "طالب"
+        assert ar_stemmer.stem("والطالبات") == "طالب"
+        assert ar_stemmer.stem("الطالبون") == "طالب"
+
+    def test_russian(self):
+        # Russian words both consisting of Cyrillic
+        # and Roman letters can be stemmed.
+        stemmer_russian = SnowballStemmer("russian")
+        assert stemmer_russian.stem("авантненькая") == "авантненьк"
+        assert stemmer_russian.stem("avenantnen'kai^a") == "avenantnen'k"
+
+    def test_german(self):
+        stemmer_german = SnowballStemmer("german")
+        stemmer_german2 = SnowballStemmer("german", ignore_stopwords=True)
+
+        assert stemmer_german.stem("Schr\xe4nke") == 'schrank'
+        assert stemmer_german2.stem("Schr\xe4nke") == 'schrank'
+
+        assert stemmer_german.stem("keinen") == 'kein'
+        assert stemmer_german2.stem("keinen") == 'keinen'
+
+    def test_spanish(self):
+        stemmer = SnowballStemmer('spanish')
+
+        assert stemmer.stem("Visionado") == 'vision'
+
+        # The word 'algue' was raising an IndexError
+        assert stemmer.stem("algue") == 'algu'
+
+    def test_short_strings_bug(self):
+        stemmer = SnowballStemmer('english')
+        assert stemmer.stem("y's") == 'y'
+
+class PorterTest(unittest.TestCase):
+    
+    def _vocabulary(self):
+        with closing(data.find('stemmers/porter_test/porter_vocabulary.txt').open(encoding='utf-8')) as fp:
+            return fp.read().splitlines()
+        
+    def _test_against_expected_output(self, stemmer_mode, expected_stems):
+        stemmer = PorterStemmer(mode=stemmer_mode)
+        for word, true_stem in zip(self._vocabulary(), expected_stems):
+            our_stem = stemmer.stem(word)
+            assert our_stem == true_stem, (
+                "%s should stem to %s in %s mode but got %s" % (
+                    word, true_stem, stemmer_mode, our_stem
+                )
+            )
+    
+    def test_vocabulary_martin_mode(self):
+        """Tests all words from the test vocabulary provided by M Porter
+        
+        The sample vocabulary and output were sourced from:
+            http://tartarus.org/martin/PorterStemmer/voc.txt
+            http://tartarus.org/martin/PorterStemmer/output.txt
+        and are linked to from the Porter Stemmer algorithm's homepage
+        at
+            http://tartarus.org/martin/PorterStemmer/
+        """
+        with closing(data.find('stemmers/porter_test/porter_martin_output.txt').open(encoding='utf-8')) as fp:
+            self._test_against_expected_output(
+                PorterStemmer.MARTIN_EXTENSIONS,
+                fp.read().splitlines()
+            )
+        
+    def test_vocabulary_nltk_mode(self):
+        with closing(data.find('stemmers/porter_test/porter_nltk_output.txt').open(encoding='utf-8')) as fp:
+            self._test_against_expected_output(
+                PorterStemmer.NLTK_EXTENSIONS,
+                fp.read().splitlines()
+            )
+        
+    def test_vocabulary_original_mode(self):
+        # The list of stems for this test was generated by taking the
+        # Martin-blessed stemmer from
+        # http://tartarus.org/martin/PorterStemmer/c.txt
+        # and removing all the --DEPARTURE-- sections from it and
+        # running it against Martin's test vocabulary.
+
+        with closing(data.find('stemmers/porter_test/porter_original_output.txt').open(encoding='utf-8')) as fp:
+            self._test_against_expected_output(
+                PorterStemmer.ORIGINAL_ALGORITHM,
+                fp.read().splitlines()
+            )
+
+        self._test_against_expected_output(
+            PorterStemmer.ORIGINAL_ALGORITHM,
+            data.find('stemmers/porter_test/porter_original_output.txt')
+                .open(encoding='utf-8')
+                .read()
+                .splitlines()
+        )
+
+    def test_oed_bug(self):
+        """Test for bug https://github.com/nltk/nltk/issues/1581
+
+        Ensures that 'oed' can be stemmed without throwing an error.
+        """
+        assert PorterStemmer().stem('oed') == 'o'

{nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tag.py

@@ -1,20 +1,20 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import, unicode_literals
-
-def test_basic():
-    from nltk.tag import pos_tag
-    from nltk.tokenize import word_tokenize
-
-    result = pos_tag(word_tokenize("John's big idea isn't all that bad."))
-    assert result == [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'),
-                      ('idea', 'NN'), ('is', 'VBZ'), ("n't", 'RB'),
-                      ('all', 'PDT'), ('that', 'DT'), ('bad', 'JJ'),
-                      ('.', '.')]
-
-
-def setup_module(module):
-    from nose import SkipTest
-    try:
-        import numpy
-    except ImportError:
-        raise SkipTest("numpy is required for nltk.test.test_tag")
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+
+def test_basic():
+    from nltk.tag import pos_tag
+    from nltk.tokenize import word_tokenize
+
+    result = pos_tag(word_tokenize("John's big idea isn't all that bad."))
+    assert result == [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'),
+                      ('idea', 'NN'), ('is', 'VBZ'), ("n't", 'RB'),
+                      ('all', 'PDT'), ('that', 'DT'), ('bad', 'JJ'),
+                      ('.', '.')]
+
+
+def setup_module(module):
+    from nose import SkipTest
+    try:
+        import numpy
+    except ImportError:
+        raise SkipTest("numpy is required for nltk.test.test_tag")