nltk 3.2.1__zip → 3.3__zip

nltk-3.3/nltk/test/unit/translate/test_bleu.py

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for BLEU translation evaluation metric
+"""
+
+import functools
+import io
+import unittest
+
+from nltk.data import find
+from nltk.translate.bleu_score import modified_precision, brevity_penalty, closest_ref_length
+from nltk.translate.bleu_score import sentence_bleu, corpus_bleu, SmoothingFunction
+
+
+class TestBLEU(unittest.TestCase):
+    def test_modified_precision(self):
+        """
+        Examples from the original BLEU paper
+        http://www.aclweb.org/anthology/P02-1040.pdf
+        """
+        # Example 1: the "the*" example.
+        # Reference sentences.
+        ref1 = 'the cat is on the mat'.split()
+        ref2 = 'there is a cat on the mat'.split()
+        # Hypothesis sentence(s).
+        hyp1 = 'the the the the the the the'.split()
+
+        references = [ref1, ref2]
+
+        # Testing modified unigram precision.
+        hyp1_unigram_precision =  float(modified_precision(references, hyp1, n=1))
+        assert (round(hyp1_unigram_precision, 4) == 0.2857)
+        # With assertAlmostEqual at 4 place precision.
+        self.assertAlmostEqual(hyp1_unigram_precision, 0.28571428, places=4)
+
+        # Testing modified bigram precision.
+        assert(float(modified_precision(references, hyp1, n=2)) == 0.0)
+
+
+        # Example 2: the "of the" example.
+        # Reference sentences
+        ref1 = str('It is a guide to action that ensures that the military '
+                   'will forever heed Party commands').split()
+        ref2 = str('It is the guiding principle which guarantees the military '
+                   'forces always being under the command of the Party').split()
+        ref3 = str('It is the practical guide for the army always to heed '
+                   'the directions of the party').split()
+        # Hypothesis sentence(s).
+        hyp1 = 'of the'.split()
+
+        references = [ref1, ref2, ref3]
+        # Testing modified unigram precision.
+        assert (float(modified_precision(references, hyp1, n=1)) == 1.0)
+
+        # Testing modified bigram precision.
+        assert(float(modified_precision(references, hyp1, n=2)) == 1.0)
+
+
+        # Example 3: Proper MT outputs.
+        hyp1 = str('It is a guide to action which ensures that the military '
+                   'always obeys the commands of the party').split()
+        hyp2 = str('It is to insure the troops forever hearing the activity '
+                   'guidebook that party direct').split()
+
+        references = [ref1, ref2, ref3]
+
+        # Unigram precision.
+        hyp1_unigram_precision = float(modified_precision(references, hyp1, n=1))
+        hyp2_unigram_precision = float(modified_precision(references, hyp2, n=1))
+        # Test unigram precision with assertAlmostEqual at 4 place precision.
+        self.assertAlmostEqual(hyp1_unigram_precision, 0.94444444, places=4)
+        self.assertAlmostEqual(hyp2_unigram_precision, 0.57142857, places=4)
+        # Test unigram precision with rounding.
+        assert (round(hyp1_unigram_precision, 4) == 0.9444)
+        assert (round(hyp2_unigram_precision, 4) == 0.5714)
+
+        # Bigram precision
+        hyp1_bigram_precision = float(modified_precision(references, hyp1, n=2))
+        hyp2_bigram_precision = float(modified_precision(references, hyp2, n=2))
+        # Test bigram precision with assertAlmostEqual at 4 place precision.
+        self.assertAlmostEqual(hyp1_bigram_precision, 0.58823529, places=4)
+        self.assertAlmostEqual(hyp2_bigram_precision, 0.07692307, places=4)
+        # Test bigram precision with rounding.
+        assert (round(hyp1_bigram_precision, 4) == 0.5882)
+        assert (round(hyp2_bigram_precision, 4) == 0.0769)
+
+    def test_brevity_penalty(self):
+        # Test case from brevity_penalty_closest function in mteval-v13a.pl.
+        # Same test cases as in the doctest in nltk.translate.bleu_score.py
+        references = [['a'] * 11, ['a'] * 8]
+        hypothesis = ['a'] * 7
+        hyp_len = len(hypothesis)
+        closest_ref_len =  closest_ref_length(references, hyp_len)
+        self.assertAlmostEqual(brevity_penalty(closest_ref_len, hyp_len), 0.8669, places=4)
+
+        references = [['a'] * 11, ['a'] * 8, ['a'] * 6, ['a'] * 7]
+        hypothesis = ['a'] * 7
+        hyp_len = len(hypothesis)
+        closest_ref_len =  closest_ref_length(references, hyp_len)
+        assert brevity_penalty(closest_ref_len, hyp_len) == 1.0
+
+    def test_zero_matches(self):
+        # Test case where there's 0 matches
+        references = ['The candidate has no alignment to any of the references'.split()]
+        hypothesis = 'John loves Mary'.split()
+
+        # Test BLEU to nth order of n-grams, where n is len(hypothesis).
+        for n in range(1,len(hypothesis)):
+            weights = [1.0/n] * n # Uniform weights.
+            assert(sentence_bleu(references, hypothesis, weights) == 0)
+
+    def test_full_matches(self):
+        # Test case where there's 100% matches
+        references = ['John loves Mary'.split()]
+        hypothesis = 'John loves Mary'.split()
+
+        # Test BLEU to nth order of n-grams, where n is len(hypothesis).
+        for n in range(1,len(hypothesis)):
+            weights = [1.0/n] * n # Uniform weights.
+            assert(sentence_bleu(references, hypothesis, weights) == 1.0)
+
+    def test_partial_matches_hypothesis_longer_than_reference(self):
+        references = ['John loves Mary'.split()]
+        hypothesis = 'John loves Mary who loves Mike'.split()
+        # Since no 4-grams matches were found the result should be zero
+        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
+        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.0, places=4)
+        # Checks that the warning has been raised because len(reference) < 4.
+        try:
+            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
+        except AttributeError:
+            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
+
+
+#@unittest.skip("Skipping fringe cases for BLEU.")
+class TestBLEUFringeCases(unittest.TestCase):
+
+    def test_case_where_n_is_bigger_than_hypothesis_length(self):
+        # Test BLEU to nth order of n-grams, where n > len(hypothesis).
+        references = ['John loves Mary ?'.split()]
+        hypothesis = 'John loves Mary'.split()
+        n = len(hypothesis) + 1 #
+        weights = [1.0/n] * n # Uniform weights.
+        # Since no n-grams matches were found the result should be zero
+        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
+        self.assertAlmostEqual(sentence_bleu(references, hypothesis, weights), 0.0, places=4)
+        # Checks that the warning has been raised because len(hypothesis) < 4.
+        try:
+            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
+        except AttributeError:
+            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
+
+        # Test case where n > len(hypothesis) but so is n > len(reference), and
+        # it's a special case where reference == hypothesis.
+        references = ['John loves Mary'.split()]
+        hypothesis = 'John loves Mary'.split()
+        # Since no 4-grams matches were found the result should be zero
+        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
+        self.assertAlmostEqual(sentence_bleu(references, hypothesis, weights), 0.0, places=4)
+
+    def test_empty_hypothesis(self):
+        # Test case where there's hypothesis is empty.
+        references = ['The candidate has no alignment to any of the references'.split()]
+        hypothesis = []
+        assert(sentence_bleu(references, hypothesis) == 0)
+
+    def test_empty_references(self):
+        # Test case where there's reference is empty.
+        references = [[]]
+        hypothesis = 'John loves Mary'.split()
+        assert(sentence_bleu(references, hypothesis) == 0)
+
+    def test_empty_references_and_hypothesis(self):
+        # Test case where both references and hypothesis is empty.
+        references = [[]]
+        hypothesis = []
+        assert(sentence_bleu(references, hypothesis) == 0)
+
+    def test_reference_or_hypothesis_shorter_than_fourgrams(self):
+        # Tese case where the length of reference or hypothesis
+        # is shorter than 4.
+        references = ['let it go'.split()]
+        hypothesis = 'let go it'.split()
+        # Checks that the value the hypothesis and reference returns is 0.0
+        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
+        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.0, places=4)
+        # Checks that the warning has been raised.
+        try:
+            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
+        except AttributeError:
+            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
+
+class TestBLEUvsMteval13a(unittest.TestCase):
+
+    def test_corpus_bleu(self):
+        ref_file = find('models/wmt15_eval/ref.ru')
+        hyp_file = find('models/wmt15_eval/google.ru')
+        mteval_output_file = find('models/wmt15_eval/mteval-13a.output')
+
+        # Reads the BLEU scores from the `mteval-13a.output` file.
+        # The order of the list corresponds to the order of the ngrams.
+        with open(mteval_output_file, 'r') as mteval_fin:
+            # The numbers are located in the last 2nd line of the file.
+            # The first and 2nd item in the list are the score and system names.
+            mteval_bleu_scores = map(float, mteval_fin.readlines()[-2].split()[1:-1])
+
+        with io.open(ref_file, 'r', encoding='utf8') as ref_fin:
+            with io.open(hyp_file, 'r', encoding='utf8') as hyp_fin:
+                # Whitespace tokenize the file.
+                # Note: split() automatically strip().
+                hypothesis = list(map(lambda x: x.split(), hyp_fin))
+                # Note that the corpus_bleu input is list of list of references.
+                references = list(map(lambda x: [x.split()], ref_fin))
+                # Without smoothing.
+                for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
+                    nltk_bleu = corpus_bleu(references, hypothesis, weights=(1.0/i,)*i)
+                    # Check that the BLEU scores difference is less than 0.005 .
+                    # Note: This is an approximate comparison; as much as
+                    #       +/- 0.01 BLEU might be "statistically significant",
+                    #       the actual translation quality might not be.
+                    assert abs(mteval_bleu - nltk_bleu) < 0.005
+
+                # With the same smoothing method used in mteval-v13a.pl
+                chencherry = SmoothingFunction()
+                for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
+                    nltk_bleu = corpus_bleu(references, hypothesis,
+                                            weights=(1.0/i,)*i,
+                                            smoothing_function=chencherry.method3)
+                    assert abs(mteval_bleu - nltk_bleu) < 0.005
+
+class TestBLEUWithBadSentence(unittest.TestCase):
+    def test_corpus_bleu_with_bad_sentence(self):
+        hyp = "Teo S yb , oe uNb , R , T t , , t Tue Ar saln S , , 5istsi l , 5oe R ulO sae oR R"
+        ref = str("Their tasks include changing a pump on the faulty stokehold ."
+                  "Likewise , two species that are very similar in morphology "
+                  "were distinguished using genetics .")
+        references = [[ref.split()]]
+        hypotheses = [hyp.split()]
+        try: # Check that the warning is raised since no. of 2-grams < 0.
+            with self.assertWarns(UserWarning):
+                # Verify that the BLEU output is undesired since no. of 2-grams < 0.
+                self.assertAlmostEqual(corpus_bleu(references, hypotheses), 0.0, places=4)
+        except AttributeError: # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
+            self.assertAlmostEqual(corpus_bleu(references, hypotheses), 0.0, places=4)

nltk-3.3/nltk/test/unit/translate/test_gdfa.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+Tests GDFA alignments
+"""
+
+import functools
+import io
+import unittest
+
+from nltk.translate.gdfa import grow_diag_final_and
+
+class TestGDFA(unittest.TestCase):
+    def test_from_eflomal_outputs(self):
+        """
+        Testing GDFA with first 10 eflomal outputs from issue #1829
+        https://github.com/nltk/nltk/issues/1829
+        """
+        # Input.
+        forwards = ['0-0 1-2',
+                    '0-0 1-1',
+                    '0-0 2-1 3-2 4-3 5-4 6-5 7-6 8-7 7-8 9-9 10-10 9-11 11-12 12-13 13-14',
+                    '0-0 1-1 1-2 2-3 3-4 4-5 4-6 5-7 6-8 8-9 9-10',
+                    '0-0 14-1 15-2 16-3 20-5 21-6 22-7 5-8 6-9 7-10 8-11 9-12 10-13 11-14 12-15 13-16 14-17 17-18 18-19 19-20 20-21 23-22 24-23 25-24 26-25 27-27 28-28 29-29 30-30 31-31',
+                    '0-0 1-1 0-2 2-3',
+                    '0-0 2-2 4-4',
+                    '0-0 1-1 2-3 3-4 5-5 7-6 8-7 9-8 10-9 11-10 12-11 13-12 14-13 15-14 16-16 17-17 18-18 19-19 20-20',
+                    '3-0 4-1 6-2 5-3 6-4 7-5 8-6 9-7 10-8 11-9 16-10 9-12 10-13 12-14',
+                    '1-0']
+        backwards = ['0-0 1-2',
+                     '0-0 1-1',
+                     '0-0 2-1 3-2 4-3 5-4 6-5 7-6 8-7 9-8 10-10 11-12 12-11 13-13',
+                     '0-0 1-2 2-3 3-4 4-6 6-8 7-5 8-7 9-8',
+                     '0-0 1-8 2-9 3-10 4-11 5-12 6-11 8-13 9-14 10-15 11-16 12-17 13-18 14-19 15-20 16-21 17-22 18-23 19-24 20-29 21-30 22-31 23-2 24-3 25-4 26-5 27-5 28-6 29-7 30-28 31-31',
+                     '0-0 1-1 2-3',
+                     '0-0 1-1 2-3 4-4',
+                     '0-0 1-1 2-3 3-4 5-5 7-6 8-7 9-8 10-9 11-10 12-11 13-12 14-13 15-14 16-16 17-17 18-18 19-19 20-16 21-18',
+                     '0-0 1-1 3-2 4-1 5-3 6-4 7-5 8-6 9-7 10-8 11-9 12-8 13-9 14-8 15-9 16-10',
+                     '1-0']
+        source_lens = [2, 3, 3, 15, 11, 33, 4, 6, 23, 18]
+        target_lens = [2, 4, 3, 16, 12, 33, 5, 6, 22, 16]
+        # Expected Output.
+        expected = [ [(0, 0), (1, 2)],
+                     [(0, 0), (1, 1)],
+                     [(0, 0), (2, 1), (3, 2), (4, 3), (5, 4), (6, 5), (7, 6), (8, 7), (10, 10), (11, 12)],
+                     [(0, 0), (1, 1), (1, 2), (2, 3), (3, 4), (4, 5), (4, 6), (5, 7), (6, 8), (7, 5), (8, 7), (8, 9), (9, 8), (9, 10)],
+                     [(0, 0), (1, 8), (2, 9), (3, 10), (4, 11), (5, 8), (6, 9), (6, 11), (7, 10), (8, 11), (31, 31)],
+                     [(0, 0), (0, 2), (1, 1), (2, 3)],
+                     [(0, 0), (1, 1), (2, 2), (2, 3), (4, 4)],
+                     [(0, 0), (1, 1), (2, 3), (3, 4), (5, 5), (7, 6), (8, 7), (9, 8), (10, 9), (11, 10), (12, 11), (13, 12), (14, 13), (15, 14), (16, 16), (17, 17), (18, 18), (19, 19)],
+                     [(0, 0), (1, 1), (3, 0), (3, 2), (4, 1), (5, 3), (6, 2), (6, 4), (7, 5), (8, 6), (9, 7), (9, 12), (10, 8), (10, 13), (11, 9), (12, 8), (12, 14), (13, 9), (14, 8), (15, 9), (16, 10)],
+                     [(1, 0)],
+                     [(0, 0), (1, 1), (3, 2), (4, 3), (5, 4), (6, 5), (7, 6), (9, 10), (10, 12), (11, 13), (12, 14), (13, 15)],
+                   ]
+
+        # Iterate through all 10 examples and check for expected outputs.
+        for fw, bw, src_len, trg_len, expect in zip(forwards, backwards, source_lens, target_lens, expected):
+            self.assertListEqual(expect, grow_diag_final_and(src_len, trg_len, fw, bw))

nltk-3.3/nltk/test/unit/translate/test_nist.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for NIST translation evaluation metric
+"""
+
+import io
+import unittest
+
+from nltk.data import find
+from nltk.translate.nist_score import sentence_nist, corpus_nist
+
+
+class TestNIST(unittest.TestCase):
+    def test_sentence_nist(self):
+        ref_file = find('models/wmt15_eval/ref.ru')
+        hyp_file = find('models/wmt15_eval/google.ru')
+        mteval_output_file = find('models/wmt15_eval/mteval-13a.output')
+
+        # Reads the NIST scores from the `mteval-13a.output` file.
+        # The order of the list corresponds to the order of the ngrams.
+        with open(mteval_output_file, 'r') as mteval_fin:
+            # The numbers are located in the last 4th line of the file.
+            # The first and 2nd item in the list are the score and system names.
+            mteval_nist_scores = map(float, mteval_fin.readlines()[-4].split()[1:-1])
+
+        with io.open(ref_file, 'r', encoding='utf8') as ref_fin:
+            with io.open(hyp_file, 'r', encoding='utf8') as hyp_fin:
+                # Whitespace tokenize the file.
+                # Note: split() automatically strip().
+                hypotheses = list(map(lambda x: x.split(), hyp_fin))
+                # Note that the corpus_bleu input is list of list of references.
+                references = list(map(lambda x: [x.split()], ref_fin))
+                # Without smoothing.
+                for i, mteval_nist in zip(range(1,10), mteval_nist_scores):
+                    nltk_nist = corpus_nist(references, hypotheses, i)
+                    # Check that the NIST scores difference is less than 0.5
+                    assert abs(mteval_nist - nltk_nist) < 0.05

{nltk-3.2.1 → nltk-3.3}/nltk/test/unit/translate/test_stack_decoder.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Stack decoder
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Tah Wei Hoon <hoon.tw@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/test/util.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 =================

{nltk-3.2.1 → nltk-3.3}/nltk/test/wordnet.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 =================
@@ -48,8 +48,9 @@ The WordNet corpus reader gives access to the Open Multilingual
 WordNet, using ISO-639 language codes.
 
     >>> sorted(wn.langs()) # doctest: +NORMALIZE_WHITESPACE
-    ['als', 'arb', 'bul', 'cat', 'cmn', 'dan', 'ell', 'eng', 'eus', 'fas', 'fin', 'fra', 'glg', 'heb', 'hrv', 'ind', 
-    'ita', 'jpn', 'nno', 'nob', 'pol', 'por', 'qcn', 'slv', 'spa', 'swe', 'tha', 'zsm']
+    ['als', 'arb', 'bul', 'cat', 'cmn', 'dan', 'ell', 'eng', 'eus', 'fas',
+    'fin', 'fra', 'glg', 'heb', 'hrv', 'ind', 'ita', 'jpn', 'nld', 'nno',
+    'nob', 'pol', 'por', 'qcn', 'slv', 'spa', 'swe', 'tha', 'zsm']
     >>> wn.synsets(b'\xe7\x8a\xac'.decode('utf-8'), lang='jpn')
     [Synset('dog.n.01'), Synset('spy.n.01')]
     
@@ -113,6 +114,12 @@ Note that some relations are defined by WordNet only over Lemmas:
 The relations that are currently defined in this way are `antonyms`,
 `derivationally_related_forms` and `pertainyms`.
 
+If you know the byte offset used to identify a synset in the original
+Princeton WordNet data file, you can use that to instantiate the synset
+in NLTK:
+
+    >>> wn.synset_from_pos_and_offset('n', 4543158)
+    Synset('wagon.n.01')
 
 ------
 Lemmas
@@ -579,3 +586,10 @@ Issue 395: wordnet returns incorrect result for lowest_common_hypernyms of chef
 
     >>> wn.synset('policeman.n.01').lowest_common_hypernyms(wn.synset('chef.n.01'))
     [Synset('person.n.01')]
+
+Bug https://github.com/nltk/nltk/issues/1641: Non-English lemmas containing capital letters cannot be looked up using wordnet.lemmas() or wordnet.synsets()
+
+    >>> wn.lemmas('Londres', lang='fra')
+    [Lemma('united_kingdom.n.01.Londres'), Lemma('london.n.01.Londres'), Lemma('london.n.02.Londres')]
+    >>> wn.lemmas('londres', lang='fra')
+    [Lemma('united_kingdom.n.01.Londres'), Lemma('london.n.01.Londres'), Lemma('london.n.02.Londres')]

{nltk-3.2.1 → nltk-3.3}/nltk/test/wordnet_fixt.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+
 def teardown_module(module=None):
     from nltk.corpus import wordnet
     wordnet._unload()

{nltk-3.2.1 → nltk-3.3}/nltk/test/wordnet_lch.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 ===============================

{nltk-3.2.1 → nltk-3.3}/nltk/test/wsd.doctest

@@ -1,4 +1,4 @@
-.. Copyright (C) 2001-2016 NLTK Project
+.. Copyright (C) 2001-2018 NLTK Project
 .. For license information, see LICENSE.TXT
 
 .. -*- coding: utf-8 -*-

{nltk-3.2.1 → nltk-3.3}/nltk/text.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Texts
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -13,21 +13,26 @@ Functionality includes: concordancing, collocation discovery,
 regular expression search over tokenized strings, and
 distributional similarity.
 """
-from __future__ import print_function, division, unicode_literals
+from __future__ import print_function, division, unicode_literals, absolute_import
 
 from math import log
-from collections import defaultdict
+from collections import defaultdict, Counter, namedtuple
 from functools import reduce
 from itertools import islice
 import re
 
+from six import text_type
+
 from nltk.probability import FreqDist, LidstoneProbDist
 from nltk.probability import ConditionalFreqDist as CFD
 from nltk.util import tokenwrap, LazyConcatenation
 from nltk.metrics import f_measure, BigramAssocMeasures
 from nltk.collocations import BigramCollocationFinder
-from nltk.compat import python_2_unicode_compatible, text_type, Counter
+from nltk.compat import python_2_unicode_compatible
 
+ConcordanceLine = namedtuple('ConcordanceLine',
+                             ['left', 'query', 'right', 'offset',
+                              'left_print', 'right_print', 'line'])
 
 class ContextIndex(object):
     """
@@ -115,6 +120,7 @@ class ContextIndex(object):
                           if c in common)
             return fd
 
+
 @python_2_unicode_compatible
 class ConcordanceIndex(object):
     """
@@ -141,9 +147,7 @@ class ConcordanceIndex(object):
         """Function mapping each token to an index key (or None)."""
 
         self._offsets = defaultdict(list)
-        """Dictionary mapping words (or keys) to lists of offset
-           indices."""
-
+        """Dictionary mapping words (or keys) to lists of offset indices."""
         # Initialize the index (self._offsets)
         for index, word in enumerate(tokens):
             word = self._key(word)
@@ -171,36 +175,56 @@ class ConcordanceIndex(object):
         return '<ConcordanceIndex for %d tokens (%d types)>' % (
             len(self._tokens), len(self._offsets))
 
-    def print_concordance(self, word, width=75, lines=25):
+    def find_concordance(self, word, width=80, lines=25):
         """
-        Print a concordance for ``word`` with the specified context window.
+        Find the concordance lines given the query word.
+        """
+        half_width = (width - len(word) - 2) // 2
+        context = width // 4  # approx number of words of context
 
+        # Find the instances of the word to create the ConcordanceLine
+        concordance_list = []
+        offsets = self.offsets(word)
+        if offsets:
+            for i in offsets:
+                query_word = self._tokens[i]
+                # Find the context of query word.
+                left_context = self._tokens[i-context:i]
+                right_context = self._tokens[i+1:i+context]
+                # Create the pretty lines with the query_word in the middle.
+                left_print= ' '.join(left_context)[-half_width:]
+                right_print = ' '.join(right_context)[:half_width]
+                # The WYSIWYG line of the concordance.
+                line_print = ' '.join([left_print, query_word, right_print])
+                # Create the ConcordanceLine
+                concordance_line = ConcordanceLine(left_context, query_word,
+                                                    right_context, i,
+                                                    left_print, right_print, line_print)
+                concordance_list.append(concordance_line)
+        return concordance_list[:lines]
+
+    def print_concordance(self, word, width=80, lines=25):
+        """
+        Print concordance lines given the query word.
         :param word: The target word
         :type word: str
-        :param width: The width of each line, in characters (default=80)
-        :type width: int
         :param lines: The number of lines to display (default=25)
         :type lines: int
+        :param width: The width of each line, in characters (default=80)
+        :type width: int
+        :param save: The option to save the concordance.
+        :type save: bool
         """
-        half_width = (width - len(word) - 2) // 2
-        context = width // 4 # approx number of words of context
+        concordance_list = self.find_concordance(word, width=80, lines=25)
 
-        offsets = self.offsets(word)
-        if offsets:
-            lines = min(lines, len(offsets))
-            print("Displaying %s of %s matches:" % (lines, len(offsets)))
-            for i in offsets:
-                if lines <= 0:
-                    break
-                left = (' ' * half_width +
-                        ' '.join(self._tokens[i-context:i]))
-                right = ' '.join(self._tokens[i+1:i+context])
-                left = left[-half_width:]
-                right = right[:half_width]
-                print(left, self._tokens[i], right)
-                lines -= 1
+        if not concordance_list:
+            print("no matches")
         else:
-            print("No matches")
+            lines = min(lines, len(concordance_list))
+            print("Displaying {} of {} matches:".format(lines,len(concordance_list)))
+            for i, concordance_line in enumerate(concordance_list[:lines]):
+                print(concordance_line.line)
+
 
 class TokenSearcher(object):
     """
@@ -308,10 +332,7 @@ class Text(object):
     #////////////////////////////////////////////////////////////
 
     def __getitem__(self, i):
-        if isinstance(i, slice):
-            return self.tokens[i.start:i.stop]
-        else:
-            return self.tokens[i]
+        return self.tokens[i]
 
     def __len__(self):
         return len(self.tokens)
@@ -322,16 +343,40 @@ class Text(object):
 
     def concordance(self, word, width=79, lines=25):
         """
-        Print a concordance for ``word`` with the specified context window.
+        Prints a concordance for ``word`` with the specified context window.
         Word matching is not case-sensitive.
+
+        :param word: The target word
+        :type word: str
+        :param width: The width of each line, in characters (default=80)
+        :type width: int
+        :param lines: The number of lines to display (default=25)
+        :type lines: int
+
         :seealso: ``ConcordanceIndex``
         """
         if '_concordance_index' not in self.__dict__:
-            #print("Building index...")
-            self._concordance_index = ConcordanceIndex(self.tokens,
-                                                       key=lambda s:s.lower())
+            self._concordance_index = ConcordanceIndex(self.tokens, key=lambda s:s.lower())
+
+        return self._concordance_index.print_concordance(word, width, lines)
+
+    def concordance_list(self,  word, width=79, lines=25):
+        """
+        Generate a concordance for ``word`` with the specified context window.
+        Word matching is not case-sensitive.
+
+        :param word: The target word
+        :type word: str
+        :param width: The width of each line, in characters (default=80)
+        :type width: int
+        :param lines: The number of lines to display (default=25)
+        :type lines: int
 
-        self._concordance_index.print_concordance(word, width, lines)
+        :seealso: ``ConcordanceIndex``
+        """
+        if '_concordance_index' not in self.__dict__:
+            self._concordance_index = ConcordanceIndex(self.tokens, key=lambda s:s.lower())
+        return self._concordance_index.find_concordance(word, width, lines)
 
     def collocations(self, num=20, window_size=2):
         """
@@ -391,7 +436,7 @@ class Text(object):
                                                     filter=lambda x:x.isalpha(),
                                                     key=lambda s:s.lower())
 
-#        words = self._word_context_index.similar_words(word, num)
+        # words = self._word_context_index.similar_words(word, num)
 
         word = word.lower()
         wci = self._word_context_index._word_to_contexts
@@ -444,6 +489,13 @@ class Text(object):
         from nltk.draw import dispersion_plot
         dispersion_plot(self, words)
 
+    def generate(self, words):
+        """
+        Issues a reminder to users following the book online
+        """
+        import warnings
+        warnings.warn('The generate() method is no longer available.', DeprecationWarning)
+
     def plot(self, *args):
         """
         See documentation for FreqDist.plot()
@@ -564,7 +616,8 @@ class TextCollection(Text):
         idf = self._idf_cache.get(term)
         if idf is None:
             matches = len([True for text in self._texts if term in text])
-            # FIXME Should this raise some kind of error instead?
+            if len(self._texts) == 0:
+                raise ValueError('IDF undefined for empty document collection')
             idf = (log(len(self._texts) / matches) if matches else 0.0)
             self._idf_cache[term] = idf
         return idf