nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/translate/ibm3.py

@@ -1,337 +1,337 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: IBM Model 3
-#
-# Copyright (C) 2001-2013 NLTK Project
-# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Translation model that considers how a word can be aligned to
-multiple words in another language.
-
-IBM Model 3 improves on Model 2 by directly modeling the phenomenon
-where a word in one language may be translated into zero or more words
-in another. This is expressed by the fertility probability,
-n(phi | source word).
-
-If a source word translates into more than one word, it is possible to
-generate sentences that have the same alignment in multiple ways. This
-is modeled by a distortion step. The distortion probability, d(j|i,l,m),
-predicts a target word position, given its aligned source word's
-position. The distortion probability replaces the alignment probability
-of Model 2.
-
-The fertility probability is not applicable for NULL. Target words that
-align to NULL are assumed to be distributed uniformly in the target
-sentence. The existence of these words is modeled by p1, the probability
-that a target word produced by a real source word requires another
-target word that is produced by NULL.
-
-The EM algorithm used in Model 3 is:
-E step - In the training data, collect counts, weighted by prior
-         probabilities.
-         (a) count how many times a source language word is translated
-             into a target language word
-         (b) count how many times a particular position in the target
-             sentence is aligned to a particular position in the source
-             sentence
-         (c) count how many times a source word is aligned to phi number
-             of target words
-         (d) count how many times NULL is aligned to a target word
-
-M step - Estimate new probabilities based on the counts from the E step
-
-Because there are too many possible alignments, only the most probable
-ones are considered. First, the best alignment is determined using prior
-probabilities. Then, a hill climbing approach is used to find other good
-candidates.
-
-
-Notations:
-i: Position in the source sentence
-    Valid values are 0 (for NULL), 1, 2, ..., length of source sentence
-j: Position in the target sentence
-    Valid values are 1, 2, ..., length of target sentence
-l: Number of words in the source sentence, excluding NULL
-m: Number of words in the target sentence
-s: A word in the source language
-t: A word in the target language
-phi: Fertility, the number of target words produced by a source word
-p1: Probability that a target word produced by a source word is
-    accompanied by another target word that is aligned to NULL
-p0: 1 - p1
-
-
-References:
-Philipp Koehn. 2010. Statistical Machine Translation.
-Cambridge University Press, New York.
-
-Peter E Brown, Stephen A. Della Pietra, Vincent J. Della Pietra, and
-Robert L. Mercer. 1993. The Mathematics of Statistical Machine
-Translation: Parameter Estimation. Computational Linguistics, 19 (2),
-263-311.
-"""
-
-from __future__ import division
-from collections import defaultdict
-from math import factorial
-from nltk.translate import AlignedSent
-from nltk.translate import Alignment
-from nltk.translate import IBMModel
-from nltk.translate import IBMModel2
-from nltk.translate.ibm_model import Counts
-import warnings
-
-
-class IBMModel3(IBMModel):
-    """
-    Translation model that considers how a word can be aligned to
-    multiple words in another language
-
-    >>> bitext = []
-    >>> bitext.append(AlignedSent(['klein', 'ist', 'das', 'haus'], ['the', 'house', 'is', 'small']))
-    >>> bitext.append(AlignedSent(['das', 'haus', 'war', 'ja', 'groß'], ['the', 'house', 'was', 'big']))
-    >>> bitext.append(AlignedSent(['das', 'buch', 'ist', 'ja', 'klein'], ['the', 'book', 'is', 'small']))
-    >>> bitext.append(AlignedSent(['ein', 'haus', 'ist', 'klein'], ['a', 'house', 'is', 'small']))
-    >>> bitext.append(AlignedSent(['das', 'haus'], ['the', 'house']))
-    >>> bitext.append(AlignedSent(['das', 'buch'], ['the', 'book']))
-    >>> bitext.append(AlignedSent(['ein', 'buch'], ['a', 'book']))
-    >>> bitext.append(AlignedSent(['ich', 'fasse', 'das', 'buch', 'zusammen'], ['i', 'summarize', 'the', 'book']))
-    >>> bitext.append(AlignedSent(['fasse', 'zusammen'], ['summarize']))
-
-    >>> ibm3 = IBMModel3(bitext, 5)
-
-    >>> print(round(ibm3.translation_table['buch']['book'], 3))
-    1.0
-    >>> print(round(ibm3.translation_table['das']['book'], 3))
-    0.0
-    >>> print(round(ibm3.translation_table['ja'][None], 3))
-    1.0
-
-    >>> print(round(ibm3.distortion_table[1][1][2][2], 3))
-    1.0
-    >>> print(round(ibm3.distortion_table[1][2][2][2], 3))
-    0.0
-    >>> print(round(ibm3.distortion_table[2][2][4][5], 3))
-    0.75
-
-    >>> print(round(ibm3.fertility_table[2]['summarize'], 3))
-    1.0
-    >>> print(round(ibm3.fertility_table[1]['book'], 3))
-    1.0
-
-    >>> print(ibm3.p1)
-    0.054...
-
-    >>> test_sentence = bitext[2]
-    >>> test_sentence.words
-    ['das', 'buch', 'ist', 'ja', 'klein']
-    >>> test_sentence.mots
-    ['the', 'book', 'is', 'small']
-    >>> test_sentence.alignment
-    Alignment([(0, 0), (1, 1), (2, 2), (3, None), (4, 3)])
-
-    """
-
-    def __init__(self, sentence_aligned_corpus, iterations,
-                 probability_tables=None):
-        """
-        Train on ``sentence_aligned_corpus`` and create a lexical
-        translation model, a distortion model, a fertility model, and a
-        model for generating NULL-aligned words.
-
-        Translation direction is from ``AlignedSent.mots`` to
-        ``AlignedSent.words``.
-
-        :param sentence_aligned_corpus: Sentence-aligned parallel corpus
-        :type sentence_aligned_corpus: list(AlignedSent)
-
-        :param iterations: Number of iterations to run training algorithm
-        :type iterations: int
-
-        :param probability_tables: Optional. Use this to pass in custom
-            probability values. If not specified, probabilities will be
-            set to a uniform distribution, or some other sensible value.
-            If specified, all the following entries must be present:
-            ``translation_table``, ``alignment_table``,
-            ``fertility_table``, ``p1``, ``distortion_table``.
-            See ``IBMModel`` for the type and purpose of these tables.
-        :type probability_tables: dict[str]: object
-        """
-        super(IBMModel3, self).__init__(sentence_aligned_corpus)
-        self.reset_probabilities()
-
-        if probability_tables is None:
-            # Get translation and alignment probabilities from IBM Model 2
-            ibm2 = IBMModel2(sentence_aligned_corpus, iterations)
-            self.translation_table = ibm2.translation_table
-            self.alignment_table = ibm2.alignment_table
-            self.set_uniform_probabilities(sentence_aligned_corpus)
-        else:
-            # Set user-defined probabilities
-            self.translation_table = probability_tables['translation_table']
-            self.alignment_table = probability_tables['alignment_table']
-            self.fertility_table = probability_tables['fertility_table']
-            self.p1 = probability_tables['p1']
-            self.distortion_table = probability_tables['distortion_table']
-
-        for n in range(0, iterations):
-            self.train(sentence_aligned_corpus)
-
-    def reset_probabilities(self):
-        super(IBMModel3, self).reset_probabilities()
-        self.distortion_table = defaultdict(
-            lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(
-                lambda: self.MIN_PROB))))
-        """
-        dict[int][int][int][int]: float. Probability(j | i,l,m).
-        Values accessed as ``distortion_table[j][i][l][m]``.
-        """
-
-    def set_uniform_probabilities(self, sentence_aligned_corpus):
-        # d(j | i,l,m) = 1 / m for all i, j, l, m
-        l_m_combinations = set()
-        for aligned_sentence in sentence_aligned_corpus:
-            l = len(aligned_sentence.mots)
-            m = len(aligned_sentence.words)
-            if (l, m) not in l_m_combinations:
-                l_m_combinations.add((l, m))
-                initial_prob = 1 / m
-                if initial_prob < IBMModel.MIN_PROB:
-                    warnings.warn("A target sentence is too long (" + str(m) +
-                                  " words). Results may be less accurate.")
-                for j in range(1, m + 1):
-                    for i in range(0, l + 1):
-                        self.distortion_table[j][i][l][m] = initial_prob
-
-        # simple initialization, taken from GIZA++
-        self.fertility_table[0] = defaultdict(lambda: 0.2)
-        self.fertility_table[1] = defaultdict(lambda: 0.65)
-        self.fertility_table[2] = defaultdict(lambda: 0.1)
-        self.fertility_table[3] = defaultdict(lambda: 0.04)
-        MAX_FERTILITY = 10
-        initial_fert_prob = 0.01 / (MAX_FERTILITY - 4)
-        for phi in range(4, MAX_FERTILITY):
-            self.fertility_table[phi] = defaultdict(lambda: initial_fert_prob)
-
-        self.p1 = 0.5
-
-    def train(self, parallel_corpus):
-        counts = Model3Counts()
-        for aligned_sentence in parallel_corpus:
-            l = len(aligned_sentence.mots)
-            m = len(aligned_sentence.words)
-
-            # Sample the alignment space
-            sampled_alignments, best_alignment = self.sample(aligned_sentence)
-            # Record the most probable alignment
-            aligned_sentence.alignment = Alignment(
-                best_alignment.zero_indexed_alignment())
-
-            # E step (a): Compute normalization factors to weigh counts
-            total_count = self.prob_of_alignments(sampled_alignments)
-
-            # E step (b): Collect counts
-            for alignment_info in sampled_alignments:
-                count = self.prob_t_a_given_s(alignment_info)
-                normalized_count = count / total_count
-
-                for j in range(1, m + 1):
-                    counts.update_lexical_translation(
-                        normalized_count, alignment_info, j)
-                    counts.update_distortion(
-                        normalized_count, alignment_info, j, l, m)
-
-                counts.update_null_generation(normalized_count, alignment_info)
-                counts.update_fertility(normalized_count, alignment_info)
-
-        # M step: Update probabilities with maximum likelihood estimates
-        # If any probability is less than MIN_PROB, clamp it to MIN_PROB
-        existing_alignment_table = self.alignment_table
-        self.reset_probabilities()
-        self.alignment_table = existing_alignment_table  # don't retrain
-
-        self.maximize_lexical_translation_probabilities(counts)
-        self.maximize_distortion_probabilities(counts)
-        self.maximize_fertility_probabilities(counts)
-        self.maximize_null_generation_probabilities(counts)
-
-    def maximize_distortion_probabilities(self, counts):
-        MIN_PROB = IBMModel.MIN_PROB
-        for j, i_s in counts.distortion.items():
-            for i, src_sentence_lengths in i_s.items():
-                for l, trg_sentence_lengths in src_sentence_lengths.items():
-                    for m in trg_sentence_lengths:
-                        estimate = (counts.distortion[j][i][l][m] /
-                                    counts.distortion_for_any_j[i][l][m])
-                        self.distortion_table[j][i][l][m] = max(estimate,
-                                                                MIN_PROB)
-
-    def prob_t_a_given_s(self, alignment_info):
-        """
-        Probability of target sentence and an alignment given the
-        source sentence
-        """
-        src_sentence = alignment_info.src_sentence
-        trg_sentence = alignment_info.trg_sentence
-        l = len(src_sentence) - 1  # exclude NULL
-        m = len(trg_sentence) - 1
-        p1 = self.p1
-        p0 = 1 - p1
-
-        probability = 1.0
-        MIN_PROB = IBMModel.MIN_PROB
-
-        # Combine NULL insertion probability
-        null_fertility = alignment_info.fertility_of_i(0)
-        probability *= (pow(p1, null_fertility) *
-                        pow(p0, m - 2 * null_fertility))
-        if probability < MIN_PROB:
-            return MIN_PROB
-
-        # Compute combination (m - null_fertility) choose null_fertility
-        for i in range(1, null_fertility + 1):
-            probability *= (m - null_fertility - i + 1) / i
-            if probability < MIN_PROB:
-                return MIN_PROB
-
-        # Combine fertility probabilities
-        for i in range(1, l + 1):
-            fertility = alignment_info.fertility_of_i(i)
-            probability *= (factorial(fertility) *
-                self.fertility_table[fertility][src_sentence[i]])
-            if probability < MIN_PROB:
-                return MIN_PROB
-
-        # Combine lexical and distortion probabilities
-        for j in range(1, m + 1):
-            t = trg_sentence[j]
-            i = alignment_info.alignment[j]
-            s = src_sentence[i]
-
-            probability *= (self.translation_table[t][s] *
-                self.distortion_table[j][i][l][m])
-            if probability < MIN_PROB:
-                return MIN_PROB
-
-        return probability
-
-
-class Model3Counts(Counts):
-    """
-    Data object to store counts of various parameters during training.
-    Includes counts for distortion.
-    """
-    def __init__(self):
-        super(Model3Counts, self).__init__()
-        self.distortion = defaultdict(
-            lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(
-                lambda: 0.0))))
-        self.distortion_for_any_j = defaultdict(
-            lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
-
-    def update_distortion(self, count, alignment_info, j, l, m):
-        i = alignment_info.alignment[j]
-        self.distortion[j][i][l][m] += count
-        self.distortion_for_any_j[i][l][m] += count
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: IBM Model 3
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Translation model that considers how a word can be aligned to
+multiple words in another language.
+
+IBM Model 3 improves on Model 2 by directly modeling the phenomenon
+where a word in one language may be translated into zero or more words
+in another. This is expressed by the fertility probability,
+n(phi | source word).
+
+If a source word translates into more than one word, it is possible to
+generate sentences that have the same alignment in multiple ways. This
+is modeled by a distortion step. The distortion probability, d(j|i,l,m),
+predicts a target word position, given its aligned source word's
+position. The distortion probability replaces the alignment probability
+of Model 2.
+
+The fertility probability is not applicable for NULL. Target words that
+align to NULL are assumed to be distributed uniformly in the target
+sentence. The existence of these words is modeled by p1, the probability
+that a target word produced by a real source word requires another
+target word that is produced by NULL.
+
+The EM algorithm used in Model 3 is:
+E step - In the training data, collect counts, weighted by prior
+         probabilities.
+         (a) count how many times a source language word is translated
+             into a target language word
+         (b) count how many times a particular position in the target
+             sentence is aligned to a particular position in the source
+             sentence
+         (c) count how many times a source word is aligned to phi number
+             of target words
+         (d) count how many times NULL is aligned to a target word
+
+M step - Estimate new probabilities based on the counts from the E step
+
+Because there are too many possible alignments, only the most probable
+ones are considered. First, the best alignment is determined using prior
+probabilities. Then, a hill climbing approach is used to find other good
+candidates.
+
+
+Notations:
+i: Position in the source sentence
+    Valid values are 0 (for NULL), 1, 2, ..., length of source sentence
+j: Position in the target sentence
+    Valid values are 1, 2, ..., length of target sentence
+l: Number of words in the source sentence, excluding NULL
+m: Number of words in the target sentence
+s: A word in the source language
+t: A word in the target language
+phi: Fertility, the number of target words produced by a source word
+p1: Probability that a target word produced by a source word is
+    accompanied by another target word that is aligned to NULL
+p0: 1 - p1
+
+
+References:
+Philipp Koehn. 2010. Statistical Machine Translation.
+Cambridge University Press, New York.
+
+Peter E Brown, Stephen A. Della Pietra, Vincent J. Della Pietra, and
+Robert L. Mercer. 1993. The Mathematics of Statistical Machine
+Translation: Parameter Estimation. Computational Linguistics, 19 (2),
+263-311.
+"""
+
+from __future__ import division
+from collections import defaultdict
+from math import factorial
+from nltk.translate import AlignedSent
+from nltk.translate import Alignment
+from nltk.translate import IBMModel
+from nltk.translate import IBMModel2
+from nltk.translate.ibm_model import Counts
+import warnings
+
+
+class IBMModel3(IBMModel):
+    """
+    Translation model that considers how a word can be aligned to
+    multiple words in another language
+
+    >>> bitext = []
+    >>> bitext.append(AlignedSent(['klein', 'ist', 'das', 'haus'], ['the', 'house', 'is', 'small']))
+    >>> bitext.append(AlignedSent(['das', 'haus', 'war', 'ja', 'groß'], ['the', 'house', 'was', 'big']))
+    >>> bitext.append(AlignedSent(['das', 'buch', 'ist', 'ja', 'klein'], ['the', 'book', 'is', 'small']))
+    >>> bitext.append(AlignedSent(['ein', 'haus', 'ist', 'klein'], ['a', 'house', 'is', 'small']))
+    >>> bitext.append(AlignedSent(['das', 'haus'], ['the', 'house']))
+    >>> bitext.append(AlignedSent(['das', 'buch'], ['the', 'book']))
+    >>> bitext.append(AlignedSent(['ein', 'buch'], ['a', 'book']))
+    >>> bitext.append(AlignedSent(['ich', 'fasse', 'das', 'buch', 'zusammen'], ['i', 'summarize', 'the', 'book']))
+    >>> bitext.append(AlignedSent(['fasse', 'zusammen'], ['summarize']))
+
+    >>> ibm3 = IBMModel3(bitext, 5)
+
+    >>> print(round(ibm3.translation_table['buch']['book'], 3))
+    1.0
+    >>> print(round(ibm3.translation_table['das']['book'], 3))
+    0.0
+    >>> print(round(ibm3.translation_table['ja'][None], 3))
+    1.0
+
+    >>> print(round(ibm3.distortion_table[1][1][2][2], 3))
+    1.0
+    >>> print(round(ibm3.distortion_table[1][2][2][2], 3))
+    0.0
+    >>> print(round(ibm3.distortion_table[2][2][4][5], 3))
+    0.75
+
+    >>> print(round(ibm3.fertility_table[2]['summarize'], 3))
+    1.0
+    >>> print(round(ibm3.fertility_table[1]['book'], 3))
+    1.0
+
+    >>> print(ibm3.p1)
+    0.054...
+
+    >>> test_sentence = bitext[2]
+    >>> test_sentence.words
+    ['das', 'buch', 'ist', 'ja', 'klein']
+    >>> test_sentence.mots
+    ['the', 'book', 'is', 'small']
+    >>> test_sentence.alignment
+    Alignment([(0, 0), (1, 1), (2, 2), (3, None), (4, 3)])
+
+    """
+
+    def __init__(self, sentence_aligned_corpus, iterations,
+                 probability_tables=None):
+        """
+        Train on ``sentence_aligned_corpus`` and create a lexical
+        translation model, a distortion model, a fertility model, and a
+        model for generating NULL-aligned words.
+
+        Translation direction is from ``AlignedSent.mots`` to
+        ``AlignedSent.words``.
+
+        :param sentence_aligned_corpus: Sentence-aligned parallel corpus
+        :type sentence_aligned_corpus: list(AlignedSent)
+
+        :param iterations: Number of iterations to run training algorithm
+        :type iterations: int
+
+        :param probability_tables: Optional. Use this to pass in custom
+            probability values. If not specified, probabilities will be
+            set to a uniform distribution, or some other sensible value.
+            If specified, all the following entries must be present:
+            ``translation_table``, ``alignment_table``,
+            ``fertility_table``, ``p1``, ``distortion_table``.
+            See ``IBMModel`` for the type and purpose of these tables.
+        :type probability_tables: dict[str]: object
+        """
+        super(IBMModel3, self).__init__(sentence_aligned_corpus)
+        self.reset_probabilities()
+
+        if probability_tables is None:
+            # Get translation and alignment probabilities from IBM Model 2
+            ibm2 = IBMModel2(sentence_aligned_corpus, iterations)
+            self.translation_table = ibm2.translation_table
+            self.alignment_table = ibm2.alignment_table
+            self.set_uniform_probabilities(sentence_aligned_corpus)
+        else:
+            # Set user-defined probabilities
+            self.translation_table = probability_tables['translation_table']
+            self.alignment_table = probability_tables['alignment_table']
+            self.fertility_table = probability_tables['fertility_table']
+            self.p1 = probability_tables['p1']
+            self.distortion_table = probability_tables['distortion_table']
+
+        for n in range(0, iterations):
+            self.train(sentence_aligned_corpus)
+
+    def reset_probabilities(self):
+        super(IBMModel3, self).reset_probabilities()
+        self.distortion_table = defaultdict(
+            lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(
+                lambda: self.MIN_PROB))))
+        """
+        dict[int][int][int][int]: float. Probability(j | i,l,m).
+        Values accessed as ``distortion_table[j][i][l][m]``.
+        """
+
+    def set_uniform_probabilities(self, sentence_aligned_corpus):
+        # d(j | i,l,m) = 1 / m for all i, j, l, m
+        l_m_combinations = set()
+        for aligned_sentence in sentence_aligned_corpus:
+            l = len(aligned_sentence.mots)
+            m = len(aligned_sentence.words)
+            if (l, m) not in l_m_combinations:
+                l_m_combinations.add((l, m))
+                initial_prob = 1 / m
+                if initial_prob < IBMModel.MIN_PROB:
+                    warnings.warn("A target sentence is too long (" + str(m) +
+                                  " words). Results may be less accurate.")
+                for j in range(1, m + 1):
+                    for i in range(0, l + 1):
+                        self.distortion_table[j][i][l][m] = initial_prob
+
+        # simple initialization, taken from GIZA++
+        self.fertility_table[0] = defaultdict(lambda: 0.2)
+        self.fertility_table[1] = defaultdict(lambda: 0.65)
+        self.fertility_table[2] = defaultdict(lambda: 0.1)
+        self.fertility_table[3] = defaultdict(lambda: 0.04)
+        MAX_FERTILITY = 10
+        initial_fert_prob = 0.01 / (MAX_FERTILITY - 4)
+        for phi in range(4, MAX_FERTILITY):
+            self.fertility_table[phi] = defaultdict(lambda: initial_fert_prob)
+
+        self.p1 = 0.5
+
+    def train(self, parallel_corpus):
+        counts = Model3Counts()
+        for aligned_sentence in parallel_corpus:
+            l = len(aligned_sentence.mots)
+            m = len(aligned_sentence.words)
+
+            # Sample the alignment space
+            sampled_alignments, best_alignment = self.sample(aligned_sentence)
+            # Record the most probable alignment
+            aligned_sentence.alignment = Alignment(
+                best_alignment.zero_indexed_alignment())
+
+            # E step (a): Compute normalization factors to weigh counts
+            total_count = self.prob_of_alignments(sampled_alignments)
+
+            # E step (b): Collect counts
+            for alignment_info in sampled_alignments:
+                count = self.prob_t_a_given_s(alignment_info)
+                normalized_count = count / total_count
+
+                for j in range(1, m + 1):
+                    counts.update_lexical_translation(
+                        normalized_count, alignment_info, j)
+                    counts.update_distortion(
+                        normalized_count, alignment_info, j, l, m)
+
+                counts.update_null_generation(normalized_count, alignment_info)
+                counts.update_fertility(normalized_count, alignment_info)
+
+        # M step: Update probabilities with maximum likelihood estimates
+        # If any probability is less than MIN_PROB, clamp it to MIN_PROB
+        existing_alignment_table = self.alignment_table
+        self.reset_probabilities()
+        self.alignment_table = existing_alignment_table  # don't retrain
+
+        self.maximize_lexical_translation_probabilities(counts)
+        self.maximize_distortion_probabilities(counts)
+        self.maximize_fertility_probabilities(counts)
+        self.maximize_null_generation_probabilities(counts)
+
+    def maximize_distortion_probabilities(self, counts):
+        MIN_PROB = IBMModel.MIN_PROB
+        for j, i_s in counts.distortion.items():
+            for i, src_sentence_lengths in i_s.items():
+                for l, trg_sentence_lengths in src_sentence_lengths.items():
+                    for m in trg_sentence_lengths:
+                        estimate = (counts.distortion[j][i][l][m] /
+                                    counts.distortion_for_any_j[i][l][m])
+                        self.distortion_table[j][i][l][m] = max(estimate,
+                                                                MIN_PROB)
+
+    def prob_t_a_given_s(self, alignment_info):
+        """
+        Probability of target sentence and an alignment given the
+        source sentence
+        """
+        src_sentence = alignment_info.src_sentence
+        trg_sentence = alignment_info.trg_sentence
+        l = len(src_sentence) - 1  # exclude NULL
+        m = len(trg_sentence) - 1
+        p1 = self.p1
+        p0 = 1 - p1
+
+        probability = 1.0
+        MIN_PROB = IBMModel.MIN_PROB
+
+        # Combine NULL insertion probability
+        null_fertility = alignment_info.fertility_of_i(0)
+        probability *= (pow(p1, null_fertility) *
+                        pow(p0, m - 2 * null_fertility))
+        if probability < MIN_PROB:
+            return MIN_PROB
+
+        # Compute combination (m - null_fertility) choose null_fertility
+        for i in range(1, null_fertility + 1):
+            probability *= (m - null_fertility - i + 1) / i
+            if probability < MIN_PROB:
+                return MIN_PROB
+
+        # Combine fertility probabilities
+        for i in range(1, l + 1):
+            fertility = alignment_info.fertility_of_i(i)
+            probability *= (factorial(fertility) *
+                self.fertility_table[fertility][src_sentence[i]])
+            if probability < MIN_PROB:
+                return MIN_PROB
+
+        # Combine lexical and distortion probabilities
+        for j in range(1, m + 1):
+            t = trg_sentence[j]
+            i = alignment_info.alignment[j]
+            s = src_sentence[i]
+
+            probability *= (self.translation_table[t][s] *
+                self.distortion_table[j][i][l][m])
+            if probability < MIN_PROB:
+                return MIN_PROB
+
+        return probability
+
+
+class Model3Counts(Counts):
+    """
+    Data object to store counts of various parameters during training.
+    Includes counts for distortion.
+    """
+    def __init__(self):
+        super(Model3Counts, self).__init__()
+        self.distortion = defaultdict(
+            lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(
+                lambda: 0.0))))
+        self.distortion_for_any_j = defaultdict(
+            lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+
+    def update_distortion(self, count, alignment_info, j, l, m):
+        i = alignment_info.alignment[j]
+        self.distortion[j][i][l][m] += count
+        self.distortion_for_any_j[i][l][m] += count