nltk 3.2__zip → 3.3__zip

nltk-3.3/nltk/test/unit/translate/test_bleu.py

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for BLEU translation evaluation metric
+"""
+
+import functools
+import io
+import unittest
+
+from nltk.data import find
+from nltk.translate.bleu_score import modified_precision, brevity_penalty, closest_ref_length
+from nltk.translate.bleu_score import sentence_bleu, corpus_bleu, SmoothingFunction
+
+
+class TestBLEU(unittest.TestCase):
+    def test_modified_precision(self):
+        """
+        Examples from the original BLEU paper
+        http://www.aclweb.org/anthology/P02-1040.pdf
+        """
+        # Example 1: the "the*" example.
+        # Reference sentences.
+        ref1 = 'the cat is on the mat'.split()
+        ref2 = 'there is a cat on the mat'.split()
+        # Hypothesis sentence(s).
+        hyp1 = 'the the the the the the the'.split()
+
+        references = [ref1, ref2]
+
+        # Testing modified unigram precision.
+        hyp1_unigram_precision =  float(modified_precision(references, hyp1, n=1))
+        assert (round(hyp1_unigram_precision, 4) == 0.2857)
+        # With assertAlmostEqual at 4 place precision.
+        self.assertAlmostEqual(hyp1_unigram_precision, 0.28571428, places=4)
+
+        # Testing modified bigram precision.
+        assert(float(modified_precision(references, hyp1, n=2)) == 0.0)
+
+
+        # Example 2: the "of the" example.
+        # Reference sentences
+        ref1 = str('It is a guide to action that ensures that the military '
+                   'will forever heed Party commands').split()
+        ref2 = str('It is the guiding principle which guarantees the military '
+                   'forces always being under the command of the Party').split()
+        ref3 = str('It is the practical guide for the army always to heed '
+                   'the directions of the party').split()
+        # Hypothesis sentence(s).
+        hyp1 = 'of the'.split()
+
+        references = [ref1, ref2, ref3]
+        # Testing modified unigram precision.
+        assert (float(modified_precision(references, hyp1, n=1)) == 1.0)
+
+        # Testing modified bigram precision.
+        assert(float(modified_precision(references, hyp1, n=2)) == 1.0)
+
+
+        # Example 3: Proper MT outputs.
+        hyp1 = str('It is a guide to action which ensures that the military '
+                   'always obeys the commands of the party').split()
+        hyp2 = str('It is to insure the troops forever hearing the activity '
+                   'guidebook that party direct').split()
+
+        references = [ref1, ref2, ref3]
+
+        # Unigram precision.
+        hyp1_unigram_precision = float(modified_precision(references, hyp1, n=1))
+        hyp2_unigram_precision = float(modified_precision(references, hyp2, n=1))
+        # Test unigram precision with assertAlmostEqual at 4 place precision.
+        self.assertAlmostEqual(hyp1_unigram_precision, 0.94444444, places=4)
+        self.assertAlmostEqual(hyp2_unigram_precision, 0.57142857, places=4)
+        # Test unigram precision with rounding.
+        assert (round(hyp1_unigram_precision, 4) == 0.9444)
+        assert (round(hyp2_unigram_precision, 4) == 0.5714)
+
+        # Bigram precision
+        hyp1_bigram_precision = float(modified_precision(references, hyp1, n=2))
+        hyp2_bigram_precision = float(modified_precision(references, hyp2, n=2))
+        # Test bigram precision with assertAlmostEqual at 4 place precision.
+        self.assertAlmostEqual(hyp1_bigram_precision, 0.58823529, places=4)
+        self.assertAlmostEqual(hyp2_bigram_precision, 0.07692307, places=4)
+        # Test bigram precision with rounding.
+        assert (round(hyp1_bigram_precision, 4) == 0.5882)
+        assert (round(hyp2_bigram_precision, 4) == 0.0769)
+
+    def test_brevity_penalty(self):
+        # Test case from brevity_penalty_closest function in mteval-v13a.pl.
+        # Same test cases as in the doctest in nltk.translate.bleu_score.py
+        references = [['a'] * 11, ['a'] * 8]
+        hypothesis = ['a'] * 7
+        hyp_len = len(hypothesis)
+        closest_ref_len =  closest_ref_length(references, hyp_len)
+        self.assertAlmostEqual(brevity_penalty(closest_ref_len, hyp_len), 0.8669, places=4)
+
+        references = [['a'] * 11, ['a'] * 8, ['a'] * 6, ['a'] * 7]
+        hypothesis = ['a'] * 7
+        hyp_len = len(hypothesis)
+        closest_ref_len =  closest_ref_length(references, hyp_len)
+        assert brevity_penalty(closest_ref_len, hyp_len) == 1.0
+
+    def test_zero_matches(self):
+        # Test case where there's 0 matches
+        references = ['The candidate has no alignment to any of the references'.split()]
+        hypothesis = 'John loves Mary'.split()
+
+        # Test BLEU to nth order of n-grams, where n is len(hypothesis).
+        for n in range(1,len(hypothesis)):
+            weights = [1.0/n] * n # Uniform weights.
+            assert(sentence_bleu(references, hypothesis, weights) == 0)
+
+    def test_full_matches(self):
+        # Test case where there's 100% matches
+        references = ['John loves Mary'.split()]
+        hypothesis = 'John loves Mary'.split()
+
+        # Test BLEU to nth order of n-grams, where n is len(hypothesis).
+        for n in range(1,len(hypothesis)):
+            weights = [1.0/n] * n # Uniform weights.
+            assert(sentence_bleu(references, hypothesis, weights) == 1.0)
+
+    def test_partial_matches_hypothesis_longer_than_reference(self):
+        references = ['John loves Mary'.split()]
+        hypothesis = 'John loves Mary who loves Mike'.split()
+        # Since no 4-grams matches were found the result should be zero
+        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
+        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.0, places=4)
+        # Checks that the warning has been raised because len(reference) < 4.
+        try:
+            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
+        except AttributeError:
+            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
+
+
+#@unittest.skip("Skipping fringe cases for BLEU.")
+class TestBLEUFringeCases(unittest.TestCase):
+
+    def test_case_where_n_is_bigger_than_hypothesis_length(self):
+        # Test BLEU to nth order of n-grams, where n > len(hypothesis).
+        references = ['John loves Mary ?'.split()]
+        hypothesis = 'John loves Mary'.split()
+        n = len(hypothesis) + 1 #
+        weights = [1.0/n] * n # Uniform weights.
+        # Since no n-grams matches were found the result should be zero
+        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
+        self.assertAlmostEqual(sentence_bleu(references, hypothesis, weights), 0.0, places=4)
+        # Checks that the warning has been raised because len(hypothesis) < 4.
+        try:
+            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
+        except AttributeError:
+            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
+
+        # Test case where n > len(hypothesis) but so is n > len(reference), and
+        # it's a special case where reference == hypothesis.
+        references = ['John loves Mary'.split()]
+        hypothesis = 'John loves Mary'.split()
+        # Since no 4-grams matches were found the result should be zero
+        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
+        self.assertAlmostEqual(sentence_bleu(references, hypothesis, weights), 0.0, places=4)
+
+    def test_empty_hypothesis(self):
+        # Test case where there's hypothesis is empty.
+        references = ['The candidate has no alignment to any of the references'.split()]
+        hypothesis = []
+        assert(sentence_bleu(references, hypothesis) == 0)
+
+    def test_empty_references(self):
+        # Test case where there's reference is empty.
+        references = [[]]
+        hypothesis = 'John loves Mary'.split()
+        assert(sentence_bleu(references, hypothesis) == 0)
+
+    def test_empty_references_and_hypothesis(self):
+        # Test case where both references and hypothesis is empty.
+        references = [[]]
+        hypothesis = []
+        assert(sentence_bleu(references, hypothesis) == 0)
+
+    def test_reference_or_hypothesis_shorter_than_fourgrams(self):
+        # Tese case where the length of reference or hypothesis
+        # is shorter than 4.
+        references = ['let it go'.split()]
+        hypothesis = 'let go it'.split()
+        # Checks that the value the hypothesis and reference returns is 0.0
+        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
+        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.0, places=4)
+        # Checks that the warning has been raised.
+        try:
+            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
+        except AttributeError:
+            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
+
+class TestBLEUvsMteval13a(unittest.TestCase):
+
+    def test_corpus_bleu(self):
+        ref_file = find('models/wmt15_eval/ref.ru')
+        hyp_file = find('models/wmt15_eval/google.ru')
+        mteval_output_file = find('models/wmt15_eval/mteval-13a.output')
+
+        # Reads the BLEU scores from the `mteval-13a.output` file.
+        # The order of the list corresponds to the order of the ngrams.
+        with open(mteval_output_file, 'r') as mteval_fin:
+            # The numbers are located in the last 2nd line of the file.
+            # The first and 2nd item in the list are the score and system names.
+            mteval_bleu_scores = map(float, mteval_fin.readlines()[-2].split()[1:-1])
+
+        with io.open(ref_file, 'r', encoding='utf8') as ref_fin:
+            with io.open(hyp_file, 'r', encoding='utf8') as hyp_fin:
+                # Whitespace tokenize the file.
+                # Note: split() automatically strip().
+                hypothesis = list(map(lambda x: x.split(), hyp_fin))
+                # Note that the corpus_bleu input is list of list of references.
+                references = list(map(lambda x: [x.split()], ref_fin))
+                # Without smoothing.
+                for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
+                    nltk_bleu = corpus_bleu(references, hypothesis, weights=(1.0/i,)*i)
+                    # Check that the BLEU scores difference is less than 0.005 .
+                    # Note: This is an approximate comparison; as much as
+                    #       +/- 0.01 BLEU might be "statistically significant",
+                    #       the actual translation quality might not be.
+                    assert abs(mteval_bleu - nltk_bleu) < 0.005
+
+                # With the same smoothing method used in mteval-v13a.pl
+                chencherry = SmoothingFunction()
+                for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
+                    nltk_bleu = corpus_bleu(references, hypothesis,
+                                            weights=(1.0/i,)*i,
+                                            smoothing_function=chencherry.method3)
+                    assert abs(mteval_bleu - nltk_bleu) < 0.005
+
+class TestBLEUWithBadSentence(unittest.TestCase):
+    def test_corpus_bleu_with_bad_sentence(self):
+        hyp = "Teo S yb , oe uNb , R , T t , , t Tue Ar saln S , , 5istsi l , 5oe R ulO sae oR R"
+        ref = str("Their tasks include changing a pump on the faulty stokehold ."
+                  "Likewise , two species that are very similar in morphology "
+                  "were distinguished using genetics .")
+        references = [[ref.split()]]
+        hypotheses = [hyp.split()]
+        try: # Check that the warning is raised since no. of 2-grams < 0.
+            with self.assertWarns(UserWarning):
+                # Verify that the BLEU output is undesired since no. of 2-grams < 0.
+                self.assertAlmostEqual(corpus_bleu(references, hypotheses), 0.0, places=4)
+        except AttributeError: # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
+            self.assertAlmostEqual(corpus_bleu(references, hypotheses), 0.0, places=4)

nltk-3.3/nltk/test/unit/translate/test_gdfa.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+Tests GDFA alignments
+"""
+
+import functools
+import io
+import unittest
+
+from nltk.translate.gdfa import grow_diag_final_and
+
+class TestGDFA(unittest.TestCase):
+    def test_from_eflomal_outputs(self):
+        """
+        Testing GDFA with first 10 eflomal outputs from issue #1829
+        https://github.com/nltk/nltk/issues/1829
+        """
+        # Input.
+        forwards = ['0-0 1-2',
+                    '0-0 1-1',
+                    '0-0 2-1 3-2 4-3 5-4 6-5 7-6 8-7 7-8 9-9 10-10 9-11 11-12 12-13 13-14',
+                    '0-0 1-1 1-2 2-3 3-4 4-5 4-6 5-7 6-8 8-9 9-10',
+                    '0-0 14-1 15-2 16-3 20-5 21-6 22-7 5-8 6-9 7-10 8-11 9-12 10-13 11-14 12-15 13-16 14-17 17-18 18-19 19-20 20-21 23-22 24-23 25-24 26-25 27-27 28-28 29-29 30-30 31-31',
+                    '0-0 1-1 0-2 2-3',
+                    '0-0 2-2 4-4',
+                    '0-0 1-1 2-3 3-4 5-5 7-6 8-7 9-8 10-9 11-10 12-11 13-12 14-13 15-14 16-16 17-17 18-18 19-19 20-20',
+                    '3-0 4-1 6-2 5-3 6-4 7-5 8-6 9-7 10-8 11-9 16-10 9-12 10-13 12-14',
+                    '1-0']
+        backwards = ['0-0 1-2',
+                     '0-0 1-1',
+                     '0-0 2-1 3-2 4-3 5-4 6-5 7-6 8-7 9-8 10-10 11-12 12-11 13-13',
+                     '0-0 1-2 2-3 3-4 4-6 6-8 7-5 8-7 9-8',
+                     '0-0 1-8 2-9 3-10 4-11 5-12 6-11 8-13 9-14 10-15 11-16 12-17 13-18 14-19 15-20 16-21 17-22 18-23 19-24 20-29 21-30 22-31 23-2 24-3 25-4 26-5 27-5 28-6 29-7 30-28 31-31',
+                     '0-0 1-1 2-3',
+                     '0-0 1-1 2-3 4-4',
+                     '0-0 1-1 2-3 3-4 5-5 7-6 8-7 9-8 10-9 11-10 12-11 13-12 14-13 15-14 16-16 17-17 18-18 19-19 20-16 21-18',
+                     '0-0 1-1 3-2 4-1 5-3 6-4 7-5 8-6 9-7 10-8 11-9 12-8 13-9 14-8 15-9 16-10',
+                     '1-0']
+        source_lens = [2, 3, 3, 15, 11, 33, 4, 6, 23, 18]
+        target_lens = [2, 4, 3, 16, 12, 33, 5, 6, 22, 16]
+        # Expected Output.
+        expected = [ [(0, 0), (1, 2)],
+                     [(0, 0), (1, 1)],
+                     [(0, 0), (2, 1), (3, 2), (4, 3), (5, 4), (6, 5), (7, 6), (8, 7), (10, 10), (11, 12)],
+                     [(0, 0), (1, 1), (1, 2), (2, 3), (3, 4), (4, 5), (4, 6), (5, 7), (6, 8), (7, 5), (8, 7), (8, 9), (9, 8), (9, 10)],
+                     [(0, 0), (1, 8), (2, 9), (3, 10), (4, 11), (5, 8), (6, 9), (6, 11), (7, 10), (8, 11), (31, 31)],
+                     [(0, 0), (0, 2), (1, 1), (2, 3)],
+                     [(0, 0), (1, 1), (2, 2), (2, 3), (4, 4)],
+                     [(0, 0), (1, 1), (2, 3), (3, 4), (5, 5), (7, 6), (8, 7), (9, 8), (10, 9), (11, 10), (12, 11), (13, 12), (14, 13), (15, 14), (16, 16), (17, 17), (18, 18), (19, 19)],
+                     [(0, 0), (1, 1), (3, 0), (3, 2), (4, 1), (5, 3), (6, 2), (6, 4), (7, 5), (8, 6), (9, 7), (9, 12), (10, 8), (10, 13), (11, 9), (12, 8), (12, 14), (13, 9), (14, 8), (15, 9), (16, 10)],
+                     [(1, 0)],
+                     [(0, 0), (1, 1), (3, 2), (4, 3), (5, 4), (6, 5), (7, 6), (9, 10), (10, 12), (11, 13), (12, 14), (13, 15)],
+                   ]
+
+        # Iterate through all 10 examples and check for expected outputs.
+        for fw, bw, src_len, trg_len, expect in zip(forwards, backwards, source_lens, target_lens, expected):
+            self.assertListEqual(expect, grow_diag_final_and(src_len, trg_len, fw, bw))

{nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm1.py

@@ -1,75 +1,75 @@
-# -*- coding: utf-8 -*-
-"""
-Tests for IBM Model 1 training methods
-"""
-
-import unittest
-
-from collections import defaultdict
-from nltk.translate import AlignedSent
-from nltk.translate import IBMModel
-from nltk.translate import IBMModel1
-from nltk.translate.ibm_model import AlignmentInfo
-
-
-class TestIBMModel1(unittest.TestCase):
-    def test_set_uniform_translation_probabilities(self):
-        # arrange
-        corpus = [
-            AlignedSent(['ham', 'eggs'], ['schinken', 'schinken', 'eier']),
-            AlignedSent(['spam', 'spam', 'spam', 'spam'], ['spam', 'spam']),
-        ]
-        model1 = IBMModel1(corpus, 0)
-
-        # act
-        model1.set_uniform_probabilities(corpus)
-
-        # assert
-        # expected_prob = 1.0 / (target vocab size + 1)
-        self.assertEqual(model1.translation_table['ham']['eier'], 1.0 / 3)
-        self.assertEqual(model1.translation_table['eggs'][None], 1.0 / 3)
-
-    def test_set_uniform_translation_probabilities_of_non_domain_values(self):
-        # arrange
-        corpus = [
-            AlignedSent(['ham', 'eggs'], ['schinken', 'schinken', 'eier']),
-            AlignedSent(['spam', 'spam', 'spam', 'spam'], ['spam', 'spam']),
-        ]
-        model1 = IBMModel1(corpus, 0)
-
-        # act
-        model1.set_uniform_probabilities(corpus)
-
-        # assert
-        # examine target words that are not in the training data domain
-        self.assertEqual(model1.translation_table['parrot']['eier'],
-                         IBMModel.MIN_PROB)
-
-    def test_prob_t_a_given_s(self):
-        # arrange
-        src_sentence = ["ich", 'esse', 'ja', 'gern', 'räucherschinken']
-        trg_sentence = ['i', 'love', 'to', 'eat', 'smoked', 'ham']
-        corpus = [AlignedSent(trg_sentence, src_sentence)]
-        alignment_info = AlignmentInfo((0, 1, 4, 0, 2, 5, 5),
-                                       [None] + src_sentence,
-                                       ['UNUSED'] + trg_sentence,
-                                       None)
-
-        translation_table = defaultdict(lambda: defaultdict(float))
-        translation_table['i']['ich'] = 0.98
-        translation_table['love']['gern'] = 0.98
-        translation_table['to'][None] = 0.98
-        translation_table['eat']['esse'] = 0.98
-        translation_table['smoked']['räucherschinken'] = 0.98
-        translation_table['ham']['räucherschinken'] = 0.98
-
-        model1 = IBMModel1(corpus, 0)
-        model1.translation_table = translation_table
-
-        # act
-        probability = model1.prob_t_a_given_s(alignment_info)
-
-        # assert
-        lexical_translation = 0.98 * 0.98 * 0.98 * 0.98 * 0.98 * 0.98
-        expected_probability = lexical_translation
-        self.assertEqual(round(probability, 4), round(expected_probability, 4))
+# -*- coding: utf-8 -*-
+"""
+Tests for IBM Model 1 training methods
+"""
+
+import unittest
+
+from collections import defaultdict
+from nltk.translate import AlignedSent
+from nltk.translate import IBMModel
+from nltk.translate import IBMModel1
+from nltk.translate.ibm_model import AlignmentInfo
+
+
+class TestIBMModel1(unittest.TestCase):
+    def test_set_uniform_translation_probabilities(self):
+        # arrange
+        corpus = [
+            AlignedSent(['ham', 'eggs'], ['schinken', 'schinken', 'eier']),
+            AlignedSent(['spam', 'spam', 'spam', 'spam'], ['spam', 'spam']),
+        ]
+        model1 = IBMModel1(corpus, 0)
+
+        # act
+        model1.set_uniform_probabilities(corpus)
+
+        # assert
+        # expected_prob = 1.0 / (target vocab size + 1)
+        self.assertEqual(model1.translation_table['ham']['eier'], 1.0 / 3)
+        self.assertEqual(model1.translation_table['eggs'][None], 1.0 / 3)
+
+    def test_set_uniform_translation_probabilities_of_non_domain_values(self):
+        # arrange
+        corpus = [
+            AlignedSent(['ham', 'eggs'], ['schinken', 'schinken', 'eier']),
+            AlignedSent(['spam', 'spam', 'spam', 'spam'], ['spam', 'spam']),
+        ]
+        model1 = IBMModel1(corpus, 0)
+
+        # act
+        model1.set_uniform_probabilities(corpus)
+
+        # assert
+        # examine target words that are not in the training data domain
+        self.assertEqual(model1.translation_table['parrot']['eier'],
+                         IBMModel.MIN_PROB)
+
+    def test_prob_t_a_given_s(self):
+        # arrange
+        src_sentence = ["ich", 'esse', 'ja', 'gern', 'räucherschinken']
+        trg_sentence = ['i', 'love', 'to', 'eat', 'smoked', 'ham']
+        corpus = [AlignedSent(trg_sentence, src_sentence)]
+        alignment_info = AlignmentInfo((0, 1, 4, 0, 2, 5, 5),
+                                       [None] + src_sentence,
+                                       ['UNUSED'] + trg_sentence,
+                                       None)
+
+        translation_table = defaultdict(lambda: defaultdict(float))
+        translation_table['i']['ich'] = 0.98
+        translation_table['love']['gern'] = 0.98
+        translation_table['to'][None] = 0.98
+        translation_table['eat']['esse'] = 0.98
+        translation_table['smoked']['räucherschinken'] = 0.98
+        translation_table['ham']['räucherschinken'] = 0.98
+
+        model1 = IBMModel1(corpus, 0)
+        model1.translation_table = translation_table
+
+        # act
+        probability = model1.prob_t_a_given_s(alignment_info)
+
+        # assert
+        lexical_translation = 0.98 * 0.98 * 0.98 * 0.98 * 0.98 * 0.98
+        expected_probability = lexical_translation
+        self.assertEqual(round(probability, 4), round(expected_probability, 4))

{nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm2.py

@@ -1,87 +1,87 @@
-# -*- coding: utf-8 -*-
-"""
-Tests for IBM Model 2 training methods
-"""
-
-import unittest
-
-from collections import defaultdict
-from nltk.translate import AlignedSent
-from nltk.translate import IBMModel
-from nltk.translate import IBMModel2
-from nltk.translate.ibm_model import AlignmentInfo
-
-
-class TestIBMModel2(unittest.TestCase):
-    def test_set_uniform_alignment_probabilities(self):
-        # arrange
-        corpus = [
-            AlignedSent(['ham', 'eggs'], ['schinken', 'schinken', 'eier']),
-            AlignedSent(['spam', 'spam', 'spam', 'spam'], ['spam', 'spam']),
-        ]
-        model2 = IBMModel2(corpus, 0)
-
-        # act
-        model2.set_uniform_probabilities(corpus)
-
-        # assert
-        # expected_prob = 1.0 / (length of source sentence + 1)
-        self.assertEqual(model2.alignment_table[0][1][3][2], 1.0 / 4)
-        self.assertEqual(model2.alignment_table[2][4][2][4], 1.0 / 3)
-
-    def test_set_uniform_alignment_probabilities_of_non_domain_values(self):
-        # arrange
-        corpus = [
-            AlignedSent(['ham', 'eggs'], ['schinken', 'schinken', 'eier']),
-            AlignedSent(['spam', 'spam', 'spam', 'spam'], ['spam', 'spam']),
-        ]
-        model2 = IBMModel2(corpus, 0)
-
-        # act
-        model2.set_uniform_probabilities(corpus)
-
-        # assert
-        # examine i and j values that are not in the training data domain
-        self.assertEqual(model2.alignment_table[99][1][3][2], IBMModel.MIN_PROB)
-        self.assertEqual(model2.alignment_table[2][99][2][4], IBMModel.MIN_PROB)
-
-    def test_prob_t_a_given_s(self):
-        # arrange
-        src_sentence = ["ich", 'esse', 'ja', 'gern', 'räucherschinken']
-        trg_sentence = ['i', 'love', 'to', 'eat', 'smoked', 'ham']
-        corpus = [AlignedSent(trg_sentence, src_sentence)]
-        alignment_info = AlignmentInfo((0, 1, 4, 0, 2, 5, 5),
-                                       [None] + src_sentence,
-                                       ['UNUSED'] + trg_sentence,
-                                       None)
-
-        translation_table = defaultdict(lambda: defaultdict(float))
-        translation_table['i']['ich'] = 0.98
-        translation_table['love']['gern'] = 0.98
-        translation_table['to'][None] = 0.98
-        translation_table['eat']['esse'] = 0.98
-        translation_table['smoked']['räucherschinken'] = 0.98
-        translation_table['ham']['räucherschinken'] = 0.98
-
-        alignment_table = defaultdict(
-            lambda: defaultdict(lambda: defaultdict(
-                lambda: defaultdict(float))))
-        alignment_table[0][3][5][6] = 0.97  # None -> to
-        alignment_table[1][1][5][6] = 0.97  # ich -> i
-        alignment_table[2][4][5][6] = 0.97  # esse -> eat
-        alignment_table[4][2][5][6] = 0.97  # gern -> love
-        alignment_table[5][5][5][6] = 0.96  # räucherschinken -> smoked
-        alignment_table[5][6][5][6] = 0.96  # räucherschinken -> ham
-
-        model2 = IBMModel2(corpus, 0)
-        model2.translation_table = translation_table
-        model2.alignment_table = alignment_table
-
-        # act
-        probability = model2.prob_t_a_given_s(alignment_info)
-
-        # assert
-        lexical_translation = 0.98 * 0.98 * 0.98 * 0.98 * 0.98 * 0.98
-        alignment = 0.97 * 0.97 * 0.97 * 0.97 * 0.96 * 0.96
-        expected_probability = lexical_translation * alignment
-        self.assertEqual(round(probability, 4), round(expected_probability, 4))
+# -*- coding: utf-8 -*-
+"""
+Tests for IBM Model 2 training methods
+"""
+
+import unittest
+
+from collections import defaultdict
+from nltk.translate import AlignedSent
+from nltk.translate import IBMModel
+from nltk.translate import IBMModel2
+from nltk.translate.ibm_model import AlignmentInfo
+
+
+class TestIBMModel2(unittest.TestCase):
+    def test_set_uniform_alignment_probabilities(self):
+        # arrange
+        corpus = [
+            AlignedSent(['ham', 'eggs'], ['schinken', 'schinken', 'eier']),
+            AlignedSent(['spam', 'spam', 'spam', 'spam'], ['spam', 'spam']),
+        ]
+        model2 = IBMModel2(corpus, 0)
+
+        # act
+        model2.set_uniform_probabilities(corpus)
+
+        # assert
+        # expected_prob = 1.0 / (length of source sentence + 1)
+        self.assertEqual(model2.alignment_table[0][1][3][2], 1.0 / 4)
+        self.assertEqual(model2.alignment_table[2][4][2][4], 1.0 / 3)
+
+    def test_set_uniform_alignment_probabilities_of_non_domain_values(self):
+        # arrange
+        corpus = [
+            AlignedSent(['ham', 'eggs'], ['schinken', 'schinken', 'eier']),
+            AlignedSent(['spam', 'spam', 'spam', 'spam'], ['spam', 'spam']),
+        ]
+        model2 = IBMModel2(corpus, 0)
+
+        # act
+        model2.set_uniform_probabilities(corpus)
+
+        # assert
+        # examine i and j values that are not in the training data domain
+        self.assertEqual(model2.alignment_table[99][1][3][2], IBMModel.MIN_PROB)
+        self.assertEqual(model2.alignment_table[2][99][2][4], IBMModel.MIN_PROB)
+
+    def test_prob_t_a_given_s(self):
+        # arrange
+        src_sentence = ["ich", 'esse', 'ja', 'gern', 'räucherschinken']
+        trg_sentence = ['i', 'love', 'to', 'eat', 'smoked', 'ham']
+        corpus = [AlignedSent(trg_sentence, src_sentence)]
+        alignment_info = AlignmentInfo((0, 1, 4, 0, 2, 5, 5),
+                                       [None] + src_sentence,
+                                       ['UNUSED'] + trg_sentence,
+                                       None)
+
+        translation_table = defaultdict(lambda: defaultdict(float))
+        translation_table['i']['ich'] = 0.98
+        translation_table['love']['gern'] = 0.98
+        translation_table['to'][None] = 0.98
+        translation_table['eat']['esse'] = 0.98
+        translation_table['smoked']['räucherschinken'] = 0.98
+        translation_table['ham']['räucherschinken'] = 0.98
+
+        alignment_table = defaultdict(
+            lambda: defaultdict(lambda: defaultdict(
+                lambda: defaultdict(float))))
+        alignment_table[0][3][5][6] = 0.97  # None -> to
+        alignment_table[1][1][5][6] = 0.97  # ich -> i
+        alignment_table[2][4][5][6] = 0.97  # esse -> eat
+        alignment_table[4][2][5][6] = 0.97  # gern -> love
+        alignment_table[5][5][5][6] = 0.96  # räucherschinken -> smoked
+        alignment_table[5][6][5][6] = 0.96  # räucherschinken -> ham
+
+        model2 = IBMModel2(corpus, 0)
+        model2.translation_table = translation_table
+        model2.alignment_table = alignment_table
+
+        # act
+        probability = model2.prob_t_a_given_s(alignment_info)
+
+        # assert
+        lexical_translation = 0.98 * 0.98 * 0.98 * 0.98 * 0.98 * 0.98
+        alignment = 0.97 * 0.97 * 0.97 * 0.97 * 0.96 * 0.96
+        expected_probability = lexical_translation * alignment
+        self.assertEqual(round(probability, 4), round(expected_probability, 4))