nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/translate/ibm2.py

@@ -1,308 +1,308 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: IBM Model 2
-#
-# Copyright (C) 2001-2013 NLTK Project
-# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Lexical translation model that considers word order.
-
-IBM Model 2 improves on Model 1 by accounting for word order.
-An alignment probability is introduced, a(i | j,l,m), which predicts
-a source word position, given its aligned target word's position.
-
-The EM algorithm used in Model 2 is:
-E step - In the training data, collect counts, weighted by prior
-         probabilities.
-         (a) count how many times a source language word is translated
-             into a target language word
-         (b) count how many times a particular position in the source
-             sentence is aligned to a particular position in the target
-             sentence
-
-M step - Estimate new probabilities based on the counts from the E step
-
-
-Notations:
-i: Position in the source sentence
-    Valid values are 0 (for NULL), 1, 2, ..., length of source sentence
-j: Position in the target sentence
-    Valid values are 1, 2, ..., length of target sentence
-l: Number of words in the source sentence, excluding NULL
-m: Number of words in the target sentence
-s: A word in the source language
-t: A word in the target language
-
-
-References:
-Philipp Koehn. 2010. Statistical Machine Translation.
-Cambridge University Press, New York.
-
-Peter E Brown, Stephen A. Della Pietra, Vincent J. Della Pietra, and
-Robert L. Mercer. 1993. The Mathematics of Statistical Machine
-Translation: Parameter Estimation. Computational Linguistics, 19 (2),
-263-311.
-"""
-
-from __future__ import division
-from collections import defaultdict
-from nltk.translate import AlignedSent
-from nltk.translate import Alignment
-from nltk.translate import IBMModel
-from nltk.translate import IBMModel1
-from nltk.translate.ibm_model import Counts
-import warnings
-
-
-class IBMModel2(IBMModel):
-    """
-    Lexical translation model that considers word order
-
-    >>> bitext = []
-    >>> bitext.append(AlignedSent(['klein', 'ist', 'das', 'haus'], ['the', 'house', 'is', 'small']))
-    >>> bitext.append(AlignedSent(['das', 'haus', 'ist', 'ja', 'groß'], ['the', 'house', 'is', 'big']))
-    >>> bitext.append(AlignedSent(['das', 'buch', 'ist', 'ja', 'klein'], ['the', 'book', 'is', 'small']))
-    >>> bitext.append(AlignedSent(['das', 'haus'], ['the', 'house']))
-    >>> bitext.append(AlignedSent(['das', 'buch'], ['the', 'book']))
-    >>> bitext.append(AlignedSent(['ein', 'buch'], ['a', 'book']))
-
-    >>> ibm2 = IBMModel2(bitext, 5)
-
-    >>> print(round(ibm2.translation_table['buch']['book'], 3))
-    1.0
-    >>> print(round(ibm2.translation_table['das']['book'], 3))
-    0.0
-    >>> print(round(ibm2.translation_table['buch'][None], 3))
-    0.0
-    >>> print(round(ibm2.translation_table['ja'][None], 3))
-    0.0
-
-    >>> print(ibm2.alignment_table[1][1][2][2])
-    0.938...
-    >>> print(round(ibm2.alignment_table[1][2][2][2], 3))
-    0.0
-    >>> print(round(ibm2.alignment_table[2][2][4][5], 3))
-    1.0
-
-    >>> test_sentence = bitext[2]
-    >>> test_sentence.words
-    ['das', 'buch', 'ist', 'ja', 'klein']
-    >>> test_sentence.mots
-    ['the', 'book', 'is', 'small']
-    >>> test_sentence.alignment
-    Alignment([(0, 0), (1, 1), (2, 2), (3, 2), (4, 3)])
-
-    """
-
-    def __init__(self, sentence_aligned_corpus, iterations,
-                 probability_tables=None):
-        """
-        Train on ``sentence_aligned_corpus`` and create a lexical
-        translation model and an alignment model.
-
-        Translation direction is from ``AlignedSent.mots`` to
-        ``AlignedSent.words``.
-
-        :param sentence_aligned_corpus: Sentence-aligned parallel corpus
-        :type sentence_aligned_corpus: list(AlignedSent)
-
-        :param iterations: Number of iterations to run training algorithm
-        :type iterations: int
-
-        :param probability_tables: Optional. Use this to pass in custom
-            probability values. If not specified, probabilities will be
-            set to a uniform distribution, or some other sensible value.
-            If specified, all the following entries must be present:
-            ``translation_table``, ``alignment_table``.
-            See ``IBMModel`` for the type and purpose of these tables.
-        :type probability_tables: dict[str]: object
-        """
-        super(IBMModel2, self).__init__(sentence_aligned_corpus)
-
-        if probability_tables is None:
-            # Get translation probabilities from IBM Model 1
-            # Run more iterations of training for Model 1, since it is
-            # faster than Model 2
-            ibm1 = IBMModel1(sentence_aligned_corpus, 2 * iterations)
-            self.translation_table = ibm1.translation_table
-            self.set_uniform_probabilities(sentence_aligned_corpus)
-        else:
-            # Set user-defined probabilities
-            self.translation_table = probability_tables['translation_table']
-            self.alignment_table = probability_tables['alignment_table']
-
-        for n in range(0, iterations):
-            self.train(sentence_aligned_corpus)
-
-        self.__align_all(sentence_aligned_corpus)
-
-    def set_uniform_probabilities(self, sentence_aligned_corpus):
-        # a(i | j,l,m) = 1 / (l+1) for all i, j, l, m
-        l_m_combinations = set()
-        for aligned_sentence in sentence_aligned_corpus:
-            l = len(aligned_sentence.mots)
-            m = len(aligned_sentence.words)
-            if (l, m) not in l_m_combinations:
-                l_m_combinations.add((l, m))
-                initial_prob = 1 / (l + 1)
-                if initial_prob < IBMModel.MIN_PROB:
-                    warnings.warn("A source sentence is too long (" + str(l) +
-                                  " words). Results may be less accurate.")
-
-                for i in range(0, l + 1):
-                    for j in range(1, m + 1):
-                        self.alignment_table[i][j][l][m] = initial_prob
-
-    def train(self, parallel_corpus):
-        counts = Model2Counts()
-        for aligned_sentence in parallel_corpus:
-            src_sentence = [None] + aligned_sentence.mots
-            trg_sentence = ['UNUSED'] + aligned_sentence.words  # 1-indexed
-            l = len(aligned_sentence.mots)
-            m = len(aligned_sentence.words)
-
-            # E step (a): Compute normalization factors to weigh counts
-            total_count = self.prob_all_alignments(src_sentence, trg_sentence)
-
-            # E step (b): Collect counts
-            for j in range(1, m + 1):
-                t = trg_sentence[j]
-                for i in range(0, l + 1):
-                    s = src_sentence[i]
-                    count = self.prob_alignment_point(
-                        i, j, src_sentence, trg_sentence)
-                    normalized_count = count / total_count[t]
-
-                    counts.update_lexical_translation(normalized_count, s, t)
-                    counts.update_alignment(normalized_count, i, j, l, m)
-
-        # M step: Update probabilities with maximum likelihood estimates
-        self.maximize_lexical_translation_probabilities(counts)
-        self.maximize_alignment_probabilities(counts)
-
-    def maximize_alignment_probabilities(self, counts):
-        MIN_PROB = IBMModel.MIN_PROB
-        for i, j_s in counts.alignment.items():
-            for j, src_sentence_lengths in j_s.items():
-                for l, trg_sentence_lengths in src_sentence_lengths.items():
-                    for m in trg_sentence_lengths:
-                        estimate = (counts.alignment[i][j][l][m] /
-                                    counts.alignment_for_any_i[j][l][m])
-                        self.alignment_table[i][j][l][m] = max(estimate,
-                                                               MIN_PROB)
-
-    def prob_all_alignments(self, src_sentence, trg_sentence):
-        """
-        Computes the probability of all possible word alignments,
-        expressed as a marginal distribution over target words t
-
-        Each entry in the return value represents the contribution to
-        the total alignment probability by the target word t.
-
-        To obtain probability(alignment | src_sentence, trg_sentence),
-        simply sum the entries in the return value.
-
-        :return: Probability of t for all s in ``src_sentence``
-        :rtype: dict(str): float
-        """
-        alignment_prob_for_t = defaultdict(lambda: 0.0)
-        for j in range(1, len(trg_sentence)):
-            t = trg_sentence[j]
-            for i in range(0, len(src_sentence)):
-                alignment_prob_for_t[t] += self.prob_alignment_point(
-                    i, j, src_sentence, trg_sentence)
-        return alignment_prob_for_t
-
-    def prob_alignment_point(self, i, j, src_sentence, trg_sentence):
-        """
-        Probability that position j in ``trg_sentence`` is aligned to
-        position i in the ``src_sentence``
-        """
-        l = len(src_sentence) - 1
-        m = len(trg_sentence) - 1
-        s = src_sentence[i]
-        t = trg_sentence[j]
-        return self.translation_table[t][s] * self.alignment_table[i][j][l][m]
-
-    def prob_t_a_given_s(self, alignment_info):
-        """
-        Probability of target sentence and an alignment given the
-        source sentence
-        """
-        prob = 1.0
-        l = len(alignment_info.src_sentence) - 1
-        m = len(alignment_info.trg_sentence) - 1
-
-        for j, i in enumerate(alignment_info.alignment):
-            if j == 0:
-                continue  # skip the dummy zeroeth element
-            trg_word = alignment_info.trg_sentence[j]
-            src_word = alignment_info.src_sentence[i]
-            prob *= (self.translation_table[trg_word][src_word] *
-                     self.alignment_table[i][j][l][m])
-
-        return max(prob, IBMModel.MIN_PROB)
-
-    def __align_all(self, parallel_corpus):
-        for sentence_pair in parallel_corpus:
-            self.__align(sentence_pair)
-
-    def __align(self, sentence_pair):
-        """
-        Determines the best word alignment for one sentence pair from
-        the corpus that the model was trained on.
-
-        The best alignment will be set in ``sentence_pair`` when the
-        method returns. In contrast with the internal implementation of
-        IBM models, the word indices in the ``Alignment`` are zero-
-        indexed, not one-indexed.
-
-        :param sentence_pair: A sentence in the source language and its
-            counterpart sentence in the target language
-        :type sentence_pair: AlignedSent
-        """
-        best_alignment = []
-
-        l = len(sentence_pair.mots)
-        m = len(sentence_pair.words)
-
-        for j, trg_word in enumerate(sentence_pair.words):
-            # Initialize trg_word to align with the NULL token
-            best_prob = (self.translation_table[trg_word][None] *
-                         self.alignment_table[0][j + 1][l][m])
-            best_prob = max(best_prob, IBMModel.MIN_PROB)
-            best_alignment_point = None
-            for i, src_word in enumerate(sentence_pair.mots):
-                align_prob = (self.translation_table[trg_word][src_word] *
-                              self.alignment_table[i + 1][j + 1][l][m])
-                if align_prob >= best_prob:
-                    best_prob = align_prob
-                    best_alignment_point = i
-
-            best_alignment.append((j, best_alignment_point))
-
-        sentence_pair.alignment = Alignment(best_alignment)
-
-
-class Model2Counts(Counts):
-    """
-    Data object to store counts of various parameters during training.
-    Includes counts for alignment.
-    """
-    def __init__(self):
-        super(Model2Counts, self).__init__()
-        self.alignment = defaultdict(
-            lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(
-                lambda: 0.0))))
-        self.alignment_for_any_i = defaultdict(
-            lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
-
-    def update_lexical_translation(self, count, s, t):
-        self.t_given_s[t][s] += count
-        self.any_t_given_s[s] += count
-
-    def update_alignment(self, count, i, j, l, m):
-        self.alignment[i][j][l][m] += count
-        self.alignment_for_any_i[j][l][m] += count
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: IBM Model 2
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Lexical translation model that considers word order.
+
+IBM Model 2 improves on Model 1 by accounting for word order.
+An alignment probability is introduced, a(i | j,l,m), which predicts
+a source word position, given its aligned target word's position.
+
+The EM algorithm used in Model 2 is:
+E step - In the training data, collect counts, weighted by prior
+         probabilities.
+         (a) count how many times a source language word is translated
+             into a target language word
+         (b) count how many times a particular position in the source
+             sentence is aligned to a particular position in the target
+             sentence
+
+M step - Estimate new probabilities based on the counts from the E step
+
+
+Notations:
+i: Position in the source sentence
+    Valid values are 0 (for NULL), 1, 2, ..., length of source sentence
+j: Position in the target sentence
+    Valid values are 1, 2, ..., length of target sentence
+l: Number of words in the source sentence, excluding NULL
+m: Number of words in the target sentence
+s: A word in the source language
+t: A word in the target language
+
+
+References:
+Philipp Koehn. 2010. Statistical Machine Translation.
+Cambridge University Press, New York.
+
+Peter E Brown, Stephen A. Della Pietra, Vincent J. Della Pietra, and
+Robert L. Mercer. 1993. The Mathematics of Statistical Machine
+Translation: Parameter Estimation. Computational Linguistics, 19 (2),
+263-311.
+"""
+
+from __future__ import division
+from collections import defaultdict
+from nltk.translate import AlignedSent
+from nltk.translate import Alignment
+from nltk.translate import IBMModel
+from nltk.translate import IBMModel1
+from nltk.translate.ibm_model import Counts
+import warnings
+
+
+class IBMModel2(IBMModel):
+    """
+    Lexical translation model that considers word order
+
+    >>> bitext = []
+    >>> bitext.append(AlignedSent(['klein', 'ist', 'das', 'haus'], ['the', 'house', 'is', 'small']))
+    >>> bitext.append(AlignedSent(['das', 'haus', 'ist', 'ja', 'groß'], ['the', 'house', 'is', 'big']))
+    >>> bitext.append(AlignedSent(['das', 'buch', 'ist', 'ja', 'klein'], ['the', 'book', 'is', 'small']))
+    >>> bitext.append(AlignedSent(['das', 'haus'], ['the', 'house']))
+    >>> bitext.append(AlignedSent(['das', 'buch'], ['the', 'book']))
+    >>> bitext.append(AlignedSent(['ein', 'buch'], ['a', 'book']))
+
+    >>> ibm2 = IBMModel2(bitext, 5)
+
+    >>> print(round(ibm2.translation_table['buch']['book'], 3))
+    1.0
+    >>> print(round(ibm2.translation_table['das']['book'], 3))
+    0.0
+    >>> print(round(ibm2.translation_table['buch'][None], 3))
+    0.0
+    >>> print(round(ibm2.translation_table['ja'][None], 3))
+    0.0
+
+    >>> print(ibm2.alignment_table[1][1][2][2])
+    0.938...
+    >>> print(round(ibm2.alignment_table[1][2][2][2], 3))
+    0.0
+    >>> print(round(ibm2.alignment_table[2][2][4][5], 3))
+    1.0
+
+    >>> test_sentence = bitext[2]
+    >>> test_sentence.words
+    ['das', 'buch', 'ist', 'ja', 'klein']
+    >>> test_sentence.mots
+    ['the', 'book', 'is', 'small']
+    >>> test_sentence.alignment
+    Alignment([(0, 0), (1, 1), (2, 2), (3, 2), (4, 3)])
+
+    """
+
+    def __init__(self, sentence_aligned_corpus, iterations,
+                 probability_tables=None):
+        """
+        Train on ``sentence_aligned_corpus`` and create a lexical
+        translation model and an alignment model.
+
+        Translation direction is from ``AlignedSent.mots`` to
+        ``AlignedSent.words``.
+
+        :param sentence_aligned_corpus: Sentence-aligned parallel corpus
+        :type sentence_aligned_corpus: list(AlignedSent)
+
+        :param iterations: Number of iterations to run training algorithm
+        :type iterations: int
+
+        :param probability_tables: Optional. Use this to pass in custom
+            probability values. If not specified, probabilities will be
+            set to a uniform distribution, or some other sensible value.
+            If specified, all the following entries must be present:
+            ``translation_table``, ``alignment_table``.
+            See ``IBMModel`` for the type and purpose of these tables.
+        :type probability_tables: dict[str]: object
+        """
+        super(IBMModel2, self).__init__(sentence_aligned_corpus)
+
+        if probability_tables is None:
+            # Get translation probabilities from IBM Model 1
+            # Run more iterations of training for Model 1, since it is
+            # faster than Model 2
+            ibm1 = IBMModel1(sentence_aligned_corpus, 2 * iterations)
+            self.translation_table = ibm1.translation_table
+            self.set_uniform_probabilities(sentence_aligned_corpus)
+        else:
+            # Set user-defined probabilities
+            self.translation_table = probability_tables['translation_table']
+            self.alignment_table = probability_tables['alignment_table']
+
+        for n in range(0, iterations):
+            self.train(sentence_aligned_corpus)
+
+        self.align_all(sentence_aligned_corpus)
+
+    def set_uniform_probabilities(self, sentence_aligned_corpus):
+        # a(i | j,l,m) = 1 / (l+1) for all i, j, l, m
+        l_m_combinations = set()
+        for aligned_sentence in sentence_aligned_corpus:
+            l = len(aligned_sentence.mots)
+            m = len(aligned_sentence.words)
+            if (l, m) not in l_m_combinations:
+                l_m_combinations.add((l, m))
+                initial_prob = 1 / (l + 1)
+                if initial_prob < IBMModel.MIN_PROB:
+                    warnings.warn("A source sentence is too long (" + str(l) +
+                                  " words). Results may be less accurate.")
+
+                for i in range(0, l + 1):
+                    for j in range(1, m + 1):
+                        self.alignment_table[i][j][l][m] = initial_prob
+
+    def train(self, parallel_corpus):
+        counts = Model2Counts()
+        for aligned_sentence in parallel_corpus:
+            src_sentence = [None] + aligned_sentence.mots
+            trg_sentence = ['UNUSED'] + aligned_sentence.words  # 1-indexed
+            l = len(aligned_sentence.mots)
+            m = len(aligned_sentence.words)
+
+            # E step (a): Compute normalization factors to weigh counts
+            total_count = self.prob_all_alignments(src_sentence, trg_sentence)
+
+            # E step (b): Collect counts
+            for j in range(1, m + 1):
+                t = trg_sentence[j]
+                for i in range(0, l + 1):
+                    s = src_sentence[i]
+                    count = self.prob_alignment_point(
+                        i, j, src_sentence, trg_sentence)
+                    normalized_count = count / total_count[t]
+
+                    counts.update_lexical_translation(normalized_count, s, t)
+                    counts.update_alignment(normalized_count, i, j, l, m)
+
+        # M step: Update probabilities with maximum likelihood estimates
+        self.maximize_lexical_translation_probabilities(counts)
+        self.maximize_alignment_probabilities(counts)
+
+    def maximize_alignment_probabilities(self, counts):
+        MIN_PROB = IBMModel.MIN_PROB
+        for i, j_s in counts.alignment.items():
+            for j, src_sentence_lengths in j_s.items():
+                for l, trg_sentence_lengths in src_sentence_lengths.items():
+                    for m in trg_sentence_lengths:
+                        estimate = (counts.alignment[i][j][l][m] /
+                                    counts.alignment_for_any_i[j][l][m])
+                        self.alignment_table[i][j][l][m] = max(estimate,
+                                                               MIN_PROB)
+
+    def prob_all_alignments(self, src_sentence, trg_sentence):
+        """
+        Computes the probability of all possible word alignments,
+        expressed as a marginal distribution over target words t
+
+        Each entry in the return value represents the contribution to
+        the total alignment probability by the target word t.
+
+        To obtain probability(alignment | src_sentence, trg_sentence),
+        simply sum the entries in the return value.
+
+        :return: Probability of t for all s in ``src_sentence``
+        :rtype: dict(str): float
+        """
+        alignment_prob_for_t = defaultdict(lambda: 0.0)
+        for j in range(1, len(trg_sentence)):
+            t = trg_sentence[j]
+            for i in range(0, len(src_sentence)):
+                alignment_prob_for_t[t] += self.prob_alignment_point(
+                    i, j, src_sentence, trg_sentence)
+        return alignment_prob_for_t
+
+    def prob_alignment_point(self, i, j, src_sentence, trg_sentence):
+        """
+        Probability that position j in ``trg_sentence`` is aligned to
+        position i in the ``src_sentence``
+        """
+        l = len(src_sentence) - 1
+        m = len(trg_sentence) - 1
+        s = src_sentence[i]
+        t = trg_sentence[j]
+        return self.translation_table[t][s] * self.alignment_table[i][j][l][m]
+
+    def prob_t_a_given_s(self, alignment_info):
+        """
+        Probability of target sentence and an alignment given the
+        source sentence
+        """
+        prob = 1.0
+        l = len(alignment_info.src_sentence) - 1
+        m = len(alignment_info.trg_sentence) - 1
+
+        for j, i in enumerate(alignment_info.alignment):
+            if j == 0:
+                continue  # skip the dummy zeroeth element
+            trg_word = alignment_info.trg_sentence[j]
+            src_word = alignment_info.src_sentence[i]
+            prob *= (self.translation_table[trg_word][src_word] *
+                     self.alignment_table[i][j][l][m])
+
+        return max(prob, IBMModel.MIN_PROB)
+
+    def align_all(self, parallel_corpus):
+        for sentence_pair in parallel_corpus:
+            self.align(sentence_pair)
+
+    def align(self, sentence_pair):
+        """
+        Determines the best word alignment for one sentence pair from
+        the corpus that the model was trained on.
+
+        The best alignment will be set in ``sentence_pair`` when the
+        method returns. In contrast with the internal implementation of
+        IBM models, the word indices in the ``Alignment`` are zero-
+        indexed, not one-indexed.
+
+        :param sentence_pair: A sentence in the source language and its
+            counterpart sentence in the target language
+        :type sentence_pair: AlignedSent
+        """
+        best_alignment = []
+
+        l = len(sentence_pair.mots)
+        m = len(sentence_pair.words)
+
+        for j, trg_word in enumerate(sentence_pair.words):
+            # Initialize trg_word to align with the NULL token
+            best_prob = (self.translation_table[trg_word][None] *
+                         self.alignment_table[0][j + 1][l][m])
+            best_prob = max(best_prob, IBMModel.MIN_PROB)
+            best_alignment_point = None
+            for i, src_word in enumerate(sentence_pair.mots):
+                align_prob = (self.translation_table[trg_word][src_word] *
+                              self.alignment_table[i + 1][j + 1][l][m])
+                if align_prob >= best_prob:
+                    best_prob = align_prob
+                    best_alignment_point = i
+
+            best_alignment.append((j, best_alignment_point))
+
+        sentence_pair.alignment = Alignment(best_alignment)
+
+
+class Model2Counts(Counts):
+    """
+    Data object to store counts of various parameters during training.
+    Includes counts for alignment.
+    """
+    def __init__(self):
+        super(Model2Counts, self).__init__()
+        self.alignment = defaultdict(
+            lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(
+                lambda: 0.0))))
+        self.alignment_for_any_i = defaultdict(
+            lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+
+    def update_lexical_translation(self, count, s, t):
+        self.t_given_s[t][s] += count
+        self.any_t_given_s[s] += count
+
+    def update_alignment(self, count, i, j, l, m):
+        self.alignment[i][j][l][m] += count
+        self.alignment_for_any_i[j][l][m] += count