nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/test/simple.doctest

@@ -1,85 +1,85 @@
-.. Copyright (C) 2001-2015 NLTK Project
-.. For license information, see LICENSE.TXT
-
-=================
-EasyInstall Tests
-=================
-
-This file contains some simple tests that will be run by EasyInstall in
-order to test the installation when NLTK-Data is absent.
-
-    >>> from __future__ import print_function
-
-------------
-Tokenization
-------------
-
-    >>> from nltk.tokenize import wordpunct_tokenize
-    >>> s = ("Good muffins cost $3.88\nin New York.  Please buy me\n"
-    ...      "two of them.\n\nThanks.")
-    >>> wordpunct_tokenize(s) # doctest: +NORMALIZE_WHITESPACE
-    ['Good', 'muffins', 'cost', '$', '3', '.', '88', 'in', 'New', 'York', '.',
-    'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
-
--------
-Metrics
--------
-
-    >>> from nltk.metrics import precision, recall, f_measure
-    >>> reference = 'DET NN VB DET JJ NN NN IN DET NN'.split()
-    >>> test    = 'DET VB VB DET NN NN NN IN DET NN'.split()
-    >>> reference_set = set(reference)
-    >>> test_set = set(test)
-    >>> precision(reference_set, test_set)
-    1.0
-    >>> print(recall(reference_set, test_set))
-    0.8
-    >>> print(f_measure(reference_set, test_set))
-    0.88888888888...
-
-------------------
-Feature Structures
-------------------
-
-    >>> from nltk import FeatStruct
-    >>> fs1 = FeatStruct(PER=3, NUM='pl', GND='fem')
-    >>> fs2 = FeatStruct(POS='N', AGR=fs1)
-    >>> print(fs2)
-    [       [ GND = 'fem' ] ]
-    [ AGR = [ NUM = 'pl'  ] ]
-    [       [ PER = 3     ] ]
-    [                       ]
-    [ POS = 'N'             ]
-    >>> print(fs2['AGR'])
-    [ GND = 'fem' ]
-    [ NUM = 'pl'  ]
-    [ PER = 3     ]
-    >>> print(fs2['AGR']['PER'])
-    3
-
--------
-Parsing
--------
-
-    >>> from nltk.parse.recursivedescent import RecursiveDescentParser
-    >>> from nltk.grammar import CFG
-    >>> grammar = CFG.fromstring("""
-    ... S -> NP VP
-    ... PP -> P NP
-    ... NP -> 'the' N | N PP | 'the' N PP
-    ... VP -> V NP | V PP | V NP PP
-    ... N -> 'cat' | 'dog' | 'rug'
-    ... V -> 'chased'
-    ... P -> 'on'
-    ... """)
-    >>> rd = RecursiveDescentParser(grammar)
-    >>> sent = 'the cat chased the dog on the rug'.split()
-    >>> for t in rd.parse(sent):
-    ...     print(t)
-    (S
-      (NP the (N cat))
-      (VP (V chased) (NP the (N dog) (PP (P on) (NP the (N rug))))))
-    (S
-      (NP the (N cat))
-      (VP (V chased) (NP the (N dog)) (PP (P on) (NP the (N rug)))))
-
+.. Copyright (C) 2001-2018 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=================
+EasyInstall Tests
+=================
+
+This file contains some simple tests that will be run by EasyInstall in
+order to test the installation when NLTK-Data is absent.
+
+    >>> from __future__ import print_function
+
+------------
+Tokenization
+------------
+
+    >>> from nltk.tokenize import wordpunct_tokenize
+    >>> s = ("Good muffins cost $3.88\nin New York.  Please buy me\n"
+    ...      "two of them.\n\nThanks.")
+    >>> wordpunct_tokenize(s) # doctest: +NORMALIZE_WHITESPACE
+    ['Good', 'muffins', 'cost', '$', '3', '.', '88', 'in', 'New', 'York', '.',
+    'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+
+-------
+Metrics
+-------
+
+    >>> from nltk.metrics import precision, recall, f_measure
+    >>> reference = 'DET NN VB DET JJ NN NN IN DET NN'.split()
+    >>> test    = 'DET VB VB DET NN NN NN IN DET NN'.split()
+    >>> reference_set = set(reference)
+    >>> test_set = set(test)
+    >>> precision(reference_set, test_set)
+    1.0
+    >>> print(recall(reference_set, test_set))
+    0.8
+    >>> print(f_measure(reference_set, test_set))
+    0.88888888888...
+
+------------------
+Feature Structures
+------------------
+
+    >>> from nltk import FeatStruct
+    >>> fs1 = FeatStruct(PER=3, NUM='pl', GND='fem')
+    >>> fs2 = FeatStruct(POS='N', AGR=fs1)
+    >>> print(fs2)
+    [       [ GND = 'fem' ] ]
+    [ AGR = [ NUM = 'pl'  ] ]
+    [       [ PER = 3     ] ]
+    [                       ]
+    [ POS = 'N'             ]
+    >>> print(fs2['AGR'])
+    [ GND = 'fem' ]
+    [ NUM = 'pl'  ]
+    [ PER = 3     ]
+    >>> print(fs2['AGR']['PER'])
+    3
+
+-------
+Parsing
+-------
+
+    >>> from nltk.parse.recursivedescent import RecursiveDescentParser
+    >>> from nltk.grammar import CFG
+    >>> grammar = CFG.fromstring("""
+    ... S -> NP VP
+    ... PP -> P NP
+    ... NP -> 'the' N | N PP | 'the' N PP
+    ... VP -> V NP | V PP | V NP PP
+    ... N -> 'cat' | 'dog' | 'rug'
+    ... V -> 'chased'
+    ... P -> 'on'
+    ... """)
+    >>> rd = RecursiveDescentParser(grammar)
+    >>> sent = 'the cat chased the dog on the rug'.split()
+    >>> for t in rd.parse(sent):
+    ...     print(t)
+    (S
+      (NP the (N cat))
+      (VP (V chased) (NP the (N dog) (PP (P on) (NP the (N rug))))))
+    (S
+      (NP the (N cat))
+      (VP (V chased) (NP the (N dog)) (PP (P on) (NP the (N rug)))))
+

{nltk-3.2 → nltk-3.3}/nltk/test/stem.doctest

@@ -1,78 +1,78 @@
-.. Copyright (C) 2001-2015 NLTK Project
-.. For license information, see LICENSE.TXT
-
-==========
- Stemmers
-==========
-
-Overview
-~~~~~~~~
-
-Stemmers remove morphological affixes from words, leaving only the
-word stem.
-
-    >>> from __future__ import print_function
-    >>> from nltk.stem import *
-
-Unit tests for the Porter stemmer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    >>> from nltk.stem.porter import *
-
-Create a new Porter stemmer.
-
-    >>> stemmer = PorterStemmer()
-
-Test the stemmer on various pluralised words.
-
-    >>> plurals = ['caresses', 'flies', 'dies', 'mules', 'denied',
-    ...            'died', 'agreed', 'owned', 'humbled', 'sized',
-    ...            'meeting', 'stating', 'siezing', 'itemization',
-    ...            'sensational', 'traditional', 'reference', 'colonizer',
-    ...            'plotted']
-
-    >>> singles = [stemmer.stem(plural) for plural in plurals]
-
-    >>> print(' '.join(singles))  # doctest: +NORMALIZE_WHITESPACE
-    caress fli die mule deni die agre own humbl size meet
-    state siez item sensat tradit refer colon plot
-
-
-Unit tests for Snowball stemmer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    >>> from nltk.stem.snowball import SnowballStemmer
-
-See which languages are supported.
-
-    >>> print(" ".join(SnowballStemmer.languages))
-    danish dutch english finnish french german hungarian italian
-    norwegian porter portuguese romanian russian spanish swedish
-
-Create a new instance of a language specific subclass.
-
-    >>> stemmer = SnowballStemmer("english")
-
-Stem a word.
-
-    >>> print(stemmer.stem("running"))
-    run
-
-Decide not to stem stopwords.
-
-    >>> stemmer2 = SnowballStemmer("english", ignore_stopwords=True)
-    >>> print(stemmer.stem("having"))
-    have
-    >>> print(stemmer2.stem("having"))
-    having
-
-The 'english' stemmer is better than the original 'porter' stemmer.
-
-    >>> print(SnowballStemmer("english").stem("generously"))
-    generous
-    >>> print(SnowballStemmer("porter").stem("generously"))
-    gener
-
-.. note::
-
-    Extra stemmer tests can be found in `nltk.test.unit.test_stem`.
+.. Copyright (C) 2001-2018 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==========
+ Stemmers
+==========
+
+Overview
+~~~~~~~~
+
+Stemmers remove morphological affixes from words, leaving only the
+word stem.
+
+    >>> from __future__ import print_function
+    >>> from nltk.stem import *
+
+Unit tests for the Porter stemmer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    >>> from nltk.stem.porter import *
+
+Create a new Porter stemmer.
+
+    >>> stemmer = PorterStemmer()
+
+Test the stemmer on various pluralised words.
+
+    >>> plurals = ['caresses', 'flies', 'dies', 'mules', 'denied',
+    ...            'died', 'agreed', 'owned', 'humbled', 'sized',
+    ...            'meeting', 'stating', 'siezing', 'itemization',
+    ...            'sensational', 'traditional', 'reference', 'colonizer',
+    ...            'plotted']
+
+    >>> singles = [stemmer.stem(plural) for plural in plurals]
+
+    >>> print(' '.join(singles))  # doctest: +NORMALIZE_WHITESPACE
+    caress fli die mule deni die agre own humbl size meet
+    state siez item sensat tradit refer colon plot
+
+
+Unit tests for Snowball stemmer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    >>> from nltk.stem.snowball import SnowballStemmer
+
+See which languages are supported.
+
+    >>> print(" ".join(SnowballStemmer.languages))
+    arabic danish dutch english finnish french german hungarian italian
+    norwegian porter portuguese romanian russian spanish swedish
+
+Create a new instance of a language specific subclass.
+
+    >>> stemmer = SnowballStemmer("english")
+
+Stem a word.
+
+    >>> print(stemmer.stem("running"))
+    run
+
+Decide not to stem stopwords.
+
+    >>> stemmer2 = SnowballStemmer("english", ignore_stopwords=True)
+    >>> print(stemmer.stem("having"))
+    have
+    >>> print(stemmer2.stem("having"))
+    having
+
+The 'english' stemmer is better than the original 'porter' stemmer.
+
+    >>> print(SnowballStemmer("english").stem("generously"))
+    generous
+    >>> print(SnowballStemmer("porter").stem("generously"))
+    gener
+
+.. note::
+
+    Extra stemmer tests can be found in `nltk.test.unit.test_stem`.

{nltk-3.2 → nltk-3.3}/nltk/test/tag.doctest

@@ -1,33 +1,33 @@
-.. Copyright (C) 2001-2015 NLTK Project
-.. For license information, see LICENSE.TXT
-
-Regression Tests
-~~~~~~~~~~~~~~~~
-
-Sequential Taggers
-------------------
-
-Add tests for:
-  - make sure backoff is being done correctly.
-  - make sure ngram taggers don't use previous sentences for context.
-  - make sure ngram taggers see 'beginning of the sentence' as a
-    unique context
-  - make sure regexp tagger's regexps are tried in order
-  - train on some simple examples, & make sure that the size & the
-    generated models are correct.
-  - make sure cutoff works as intended
-  - make sure that ngram models only exclude contexts covered by the
-    backoff tagger if the backoff tagger gets that context correct at
-    *all* locations.
-
-
-Regression Testing for issue #1025
-==================================
-
-We want to ensure that a RegexpTagger can be created with more than 100 patterns
-and does not fail with:
- "AssertionError: sorry, but this version only supports 100 named groups"
-
-    >>> from nltk.tag import RegexpTagger
-    >>> patterns = [(str(i), 'NNP',) for i in range(200)]
-    >>> tagger = RegexpTagger(patterns)
+.. Copyright (C) 2001-2018 NLTK Project
+.. For license information, see LICENSE.TXT
+
+Regression Tests
+~~~~~~~~~~~~~~~~
+
+Sequential Taggers
+------------------
+
+Add tests for:
+  - make sure backoff is being done correctly.
+  - make sure ngram taggers don't use previous sentences for context.
+  - make sure ngram taggers see 'beginning of the sentence' as a
+    unique context
+  - make sure regexp tagger's regexps are tried in order
+  - train on some simple examples, & make sure that the size & the
+    generated models are correct.
+  - make sure cutoff works as intended
+  - make sure that ngram models only exclude contexts covered by the
+    backoff tagger if the backoff tagger gets that context correct at
+    *all* locations.
+
+
+Regression Testing for issue #1025
+==================================
+
+We want to ensure that a RegexpTagger can be created with more than 100 patterns
+and does not fail with:
+ "AssertionError: sorry, but this version only supports 100 named groups"
+
+    >>> from nltk.tag import RegexpTagger
+    >>> patterns = [(str(i), 'NNP',) for i in range(200)]
+    >>> tagger = RegexpTagger(patterns)