nltk 3.2.1__zip → 3.3__zip

nltk-3.3/nltk/translate/chrf_score.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: ChrF score
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Authors: Maja Popovic
+# Contributors: Liling Tan
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+""" ChrF score implementation """
+from __future__ import division
+from collections import Counter
+
+from nltk.util import ngrams, everygrams
+
+def sentence_chrf(reference, hypothesis, min_len=1, max_len=6, beta=3.0):
+    """
+    Calculates the sentence level CHRF (Character n-gram F-score) described in
+     - Maja Popovic. 2015. CHRF: Character n-gram F-score for Automatic MT Evaluation.
+       In Proceedings of the 10th Workshop on Machine Translation.
+       http://www.statmt.org/wmt15/pdf/WMT49.pdf
+     - Maja Popovic. 2016. CHRF Deconstructed: β Parameters and n-gram Weights.
+       In Proceedings of the 1st Conference on Machine Translation.
+       http://www.statmt.org/wmt16/pdf/W16-2341.pdf
+
+    Unlike multi-reference BLEU, CHRF only supports a single reference.
+
+    An example from the original BLEU paper
+    http://www.aclweb.org/anthology/P02-1040.pdf
+
+        >>> ref1 = str('It is a guide to action that ensures that the military '
+        ...            'will forever heed Party commands').split()
+        >>> hyp1 = str('It is a guide to action which ensures that the military '
+        ...            'always obeys the commands of the party').split()
+        >>> hyp2 = str('It is to insure the troops forever hearing the activity '
+        ...            'guidebook that party direct').split()
+        >>> sentence_chrf(ref1, hyp1) # doctest: +ELLIPSIS
+        0.6768...
+        >>> sentence_chrf(ref1, hyp2) # doctest: +ELLIPSIS
+        0.4201...
+
+    The infamous "the the the ... " example
+
+        >>> ref = 'the cat is on the mat'.split()
+        >>> hyp = 'the the the the the the the'.split()
+        >>> sentence_chrf(ref, hyp)  # doctest: +ELLIPSIS
+        0.2530...
+
+    An example to show that this function allows users to use strings instead of
+    tokens, i.e. list(str) as inputs.
+
+        >>> ref1 = str('It is a guide to action that ensures that the military '
+        ...            'will forever heed Party commands')
+        >>> hyp1 = str('It is a guide to action which ensures that the military '
+        ...            'always obeys the commands of the party')
+        >>> sentence_chrf(ref1, hyp1) # doctest: +ELLIPSIS
+        0.6768...
+        >>> type(ref1) == type(hyp1) == str
+        True
+        >>> sentence_chrf(ref1.split(), hyp1.split()) # doctest: +ELLIPSIS
+        0.6768...
+
+    To skip the unigrams and only use 2- to 3-grams:
+
+        >>> sentence_chrf(ref1, hyp1, min_len=2, max_len=3) # doctest: +ELLIPSIS
+        0.7018...
+
+    :param references: reference sentence
+    :type references: list(str) / str
+    :param hypothesis: a hypothesis sentence
+    :type hypothesis: list(str) / str
+    :param min_len: The minimum order of n-gram this function should extract.
+    :type min_len: int
+    :param max_len: The maximum order of n-gram this function should extract.
+    :type max_len: int
+    :param beta: the parameter to assign more importance to recall over precision
+    :type beta: float
+    :return: the sentence level CHRF score.
+    :rtype: float
+    """
+    return corpus_chrf([reference], [hypothesis], min_len, max_len, beta=beta)
+
+
+def corpus_chrf(list_of_references, hypotheses, min_len=1, max_len=6, beta=3.0):
+    """
+    Calculates the corpus level CHRF (Character n-gram F-score), it is the
+    micro-averaged value of the sentence/segment level CHRF score.
+
+    CHRF only supports a single reference.
+
+        >>> ref1 = str('It is a guide to action that ensures that the military '
+        ...            'will forever heed Party commands').split()
+        >>> ref2 = str('It is the guiding principle which guarantees the military '
+        ...            'forces always being under the command of the Party').split()
+        >>>
+        >>> hyp1 = str('It is a guide to action which ensures that the military '
+        ...            'always obeys the commands of the party').split()
+        >>> hyp2 = str('It is to insure the troops forever hearing the activity '
+        ...            'guidebook that party direct')
+        >>> corpus_chrf([ref1, ref2, ref1, ref2], [hyp1, hyp2, hyp2, hyp1]) # doctest: +ELLIPSIS
+        0.4915...
+
+    :param references: a corpus of list of reference sentences, w.r.t. hypotheses
+    :type references: list(list(str)) / list(str)
+    :param hypotheses: a list of hypothesis sentences
+    :type hypotheses: list(list(str)) / list(str)
+    :param min_len: The minimum order of n-gram this function should extract.
+    :type min_len: int
+    :param max_len: The maximum order of n-gram this function should extract.
+    :type max_len: int
+    :param beta: the parameter to assign more importance to recall over precision
+    :type beta: float
+    :return: the sentence level CHRF score.
+    :rtype: float
+    """
+
+    assert len(list_of_references) == len(hypotheses), "The number of hypotheses and their references should be the same"
+
+    # Iterate through each hypothesis and their corresponding references.
+    for reference, hypothesis in zip(list_of_references, hypotheses):
+        # Cheating condition to allow users to input strings instead of tokens.
+        if type(reference) and type(hypothesis) != str:
+            reference, hypothesis = ' '.join(reference), ' '.join(hypothesis)
+        # For each order of ngram, calculate the no. of ngram matches and
+        # keep track of no. of ngram in references.
+        ref_ngrams = Counter(everygrams(reference, min_len, max_len))
+        hyp_ngrams = Counter(everygrams(hypothesis, min_len, max_len))
+        overlap_ngrams = ref_ngrams & hyp_ngrams
+        tp = sum(overlap_ngrams.values()) # True positives.
+        tpfp = sum(hyp_ngrams.values()) # True positives + False positives.
+        tffn = sum(ref_ngrams.values()) # True posities + False negatives.
+
+    precision = tp / tpfp
+    recall = tp / tffn
+    factor = beta**2
+    score = (1+ factor ) * (precision * recall) / ( factor * precision + recall)
+    return score

{nltk-3.2.1 → nltk-3.3}/nltk/translate/gale_church.py

@@ -2,8 +2,9 @@
 
 # Natural Language Toolkit: Gale-Church Aligner
 #
-# Copyright (C) 2001-2013 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Torsten Marek <marek@ifi.uzh.ch>
+# Contributor: Cassidy Laidlaw, Liling Tan
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
@@ -76,16 +77,30 @@ class LanguageIndependent(object):
     VARIANCE_CHARACTERS = 6.8
 
 
-def trace(backlinks, source, target):
+def trace(backlinks, source_sents_lens, target_sents_lens):
+    """
+    Traverse the alignment cost from the tracebacks and retrieves
+    appropriate sentence pairs. 
+    
+    :param backlinks: A dictionary where the key is the alignment points and value is the cost (referencing the LanguageIndependent.PRIORS)
+    :type backlinks: dict
+    :param source_sents_lens: A list of target sentences' lengths
+    :type source_sents_lens: list(int)
+    :param target_sents_lens: A list of target sentences' lengths
+    :type target_sents_lens: list(int)
+    """
     links = []
-    pos = (len(source), len(target))
-
-    while pos != (0, 0):
-        s, t = backlinks[pos]
+    position = (len(source_sents_lens), len(target_sents_lens))
+    while position != (0, 0) and all(p >=0 for p in position):
+        try:
+            s, t = backlinks[position]
+        except TypeError:
+            position = (position[0]-1 , position[1]-1)
+            continue
         for i in range(s):
             for j in range(t):
-                links.append((pos[0] - i - 1, pos[1] - j - 1))
-        pos = (pos[0] - s, pos[1] - t)
+                links.append((position[0] - i - 1, position[1] - j - 1))
+        position = (position[0] - s, position[1] - t)
 
     return links[::-1]
 
@@ -116,7 +131,7 @@ def align_log_prob(i, j, source_sents, target_sents, alignment, params):
     return - (LOG2 + norm_logsf(abs(delta)) + math.log(params.PRIORS[alignment]))
 
 
-def align_blocks(source_sents, target_sents, params = LanguageIndependent):
+def align_blocks(source_sents_lens, target_sents_lens, params = LanguageIndependent):
     """Return the sentence alignment of two text blocks (usually paragraphs).
 
         >>> align_blocks([5,5,5], [7,7,7])
@@ -128,8 +143,8 @@ def align_blocks(source_sents, target_sents, params = LanguageIndependent):
         >>> align_blocks([10,2,10,10,2,10], [12,3,20,3,12])
         [(0, 0), (1, 1), (2, 2), (3, 2), (4, 3), (5, 4)]
 
-    @param source_sents: The list of source sentence lengths.
-    @param target_sents: The list of target sentence lengths.
+    @param source_sents_lens: The list of source sentence lengths.
+    @param target_sents_lens: The list of target sentence lengths.
     @param params: the sentence alignment parameters.
     @return: The sentence alignments, a list of index pairs.
     """
@@ -141,8 +156,8 @@ def align_blocks(source_sents, target_sents, params = LanguageIndependent):
 
     backlinks = {}
 
-    for i in range(len(source_sents) + 1):
-        for j in range(len(target_sents) + 1):
+    for i in range(len(source_sents_lens) + 1): 
+        for j in range(len(target_sents_lens) + 1):
             min_dist = float('inf')
             min_align = None
             for a in alignment_types:
@@ -150,7 +165,8 @@ def align_blocks(source_sents, target_sents, params = LanguageIndependent):
                 prev_j = j - a[1]
                 if prev_i < -len(D) or prev_j < 0:
                     continue
-                p = D[prev_i][prev_j] + align_log_prob(i, j, source_sents, target_sents, a, params)
+                p = D[prev_i][prev_j] + align_log_prob(i, j, source_sents_lens, 
+                                                       target_sents_lens, a, params)
                 if p < min_dist:
                     min_dist = p
                     min_align = a
@@ -164,8 +180,8 @@ def align_blocks(source_sents, target_sents, params = LanguageIndependent):
         if len(D) > 2:
             D.pop(0)
         D.append([])
-
-    return trace(backlinks, source_sents, target_sents)
+    
+    return trace(backlinks, source_sents_lens, target_sents_lens)
 
 
 def align_texts(source_blocks, target_blocks, params = LanguageIndependent):
@@ -230,6 +246,3 @@ def parse_token_stream(stream, soft_delimiter, hard_delimiter):
 #        print align_texts(source, target)
 
 
-
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/translate/gdfa.py

@@ -1,12 +1,11 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: GDFA word alignment symmetrization
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Authors: Liling Tan
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
-import codecs
 from collections import defaultdict
 
 def grow_diag_final_and(srclen, trglen, e2f, f2e):
@@ -39,12 +38,12 @@ def grow_diag_final_and(srclen, trglen, e2f, f2e):
         >>> trglen = len(trgtext.split())
         >>>
         >>> gdfa = grow_diag_final_and(srclen, trglen, forw, back)
-        >>> gdfa == set([(28, 18), (6, 6), (24, 17), (2, 1), (15, 12), (13, 12),
+        >>> gdfa == sorted(set([(28, 18), (6, 6), (24, 17), (2, 1), (15, 12), (13, 12),
         ...         (2, 9), (3, 10), (26, 17), (25, 15), (8, 6), (9, 7), (20,
         ...         13), (18, 13), (0, 0), (10, 4), (13, 15), (23, 14), (7, 5),
         ...         (25, 14), (1, 9), (17, 13), (4, 11), (11, 17), (9, 2), (22,
         ...         12), (27, 18), (24, 16), (21, 3), (19, 12), (17, 12), (5,
-        ...         12), (11, 6), (12, 8)])
+        ...         12), (11, 6), (12, 8)]))
         True
     
     References:
@@ -73,12 +72,12 @@ def grow_diag_final_and(srclen, trglen, e2f, f2e):
     neighbors = [(-1,0),(0,-1),(1,0),(0,1),(-1,-1),(-1,1),(1,-1),(1,1)]
     alignment = set(e2f).intersection(set(f2e)) # Find the intersection.
     union = set(e2f).union(set(f2e))
-
+    
     # *aligned* is used to check if neighbors are aligned in grow_diag()
     aligned = defaultdict(set)
     for i,j in alignment:
         aligned['e'].add(i)
-        aligned['j'].add(j)
+        aligned['f'].add(j)
     
     def grow_diag():
         """
@@ -88,6 +87,7 @@ def grow_diag_final_and(srclen, trglen, e2f, f2e):
         prev_len = len(alignment) - 1
         # iterate until no new points added
         while prev_len < len(alignment):
+            no_new_points = True
             # for english word e = 0 ... en
             for e in range(srclen):
                 # for foreign word f = 0 ... fn
@@ -105,6 +105,11 @@ def grow_diag_final_and(srclen, trglen, e2f, f2e):
                                 alignment.add(neighbor)
                                 aligned['e'].add(e_new); aligned['f'].add(f_new)
                                 prev_len+=1
+                                no_new_points = False
+            # iterate until no new points added
+            if no_new_points:
+                break
+
                                                                     
     def final_and(a):
         """
@@ -119,13 +124,12 @@ def grow_diag_final_and(srclen, trglen, e2f, f2e):
                 # and (e-new, f-new in union(e2f, f2e) )
                 if (e_new not in aligned
                     and f_new not in aligned
-                    and (e_new, f_new) in a):
-
+                    and (e_new, f_new) in union):
                     alignment.add((e_new, f_new))
                     aligned['e'].add(e_new); aligned['f'].add(f_new)
 
+    
     grow_diag()
     final_and(e2f)
     final_and(f2e)
-    return alignment
-
+    return sorted(alignment)

nltk-3.3/nltk/translate/gleu_score.py

@@ -0,0 +1,193 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: GLEU Score
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Authors:
+# Contributors: Mike Schuster, Michael Wayne Goodman, Liling Tan
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+""" GLEU score implementation. """
+from __future__ import division
+from collections import Counter
+
+from nltk.util import ngrams, everygrams
+
+
+def sentence_gleu(references, hypothesis, min_len=1, max_len=4):
+    """
+    Calculates the sentence level GLEU (Google-BLEU) score described in
+
+        Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc V. Le, Mohammad Norouzi,
+        Wolfgang Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey,
+        Jeff Klingner, Apurva Shah, Melvin Johnson, Xiaobing Liu, Lukasz Kaiser,
+        Stephan Gouws, Yoshikiyo Kato, Taku Kudo, Hideto Kazawa, Keith Stevens,
+        George Kurian, Nishant Patil, Wei Wang, Cliff Young, Jason Smith,
+        Jason Riesa, Alex Rudnick, Oriol Vinyals, Greg Corrado, Macduff Hughes,
+        Jeffrey Dean. (2016) Google’s Neural Machine Translation System:
+        Bridging the Gap between Human and Machine Translation.
+        eprint arXiv:1609.08144. https://arxiv.org/pdf/1609.08144v2.pdf
+        Retrieved on 27 Oct 2016.
+
+    From Wu et al. (2016):
+        "The BLEU score has some undesirable properties when used for single
+         sentences, as it was designed to be a corpus measure. We therefore
+         use a slightly different score for our RL experiments which we call
+         the 'GLEU score'. For the GLEU score, we record all sub-sequences of
+         1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then
+         compute a recall, which is the ratio of the number of matching n-grams
+         to the number of total n-grams in the target (ground truth) sequence,
+         and a precision, which is the ratio of the number of matching n-grams
+         to the number of total n-grams in the generated output sequence. Then
+         GLEU score is simply the minimum of recall and precision. This GLEU
+         score's range is always between 0 (no matches) and 1 (all match) and
+         it is symmetrical when switching output and target. According to
+         our experiments, GLEU score correlates quite well with the BLEU
+         metric on a corpus level but does not have its drawbacks for our per
+         sentence reward objective."
+
+    Note: The initial implementation only allowed a single reference, but now
+          a list of references is required (which is consistent with
+          bleu_score.sentence_bleu()).
+
+    The infamous "the the the ... " example
+
+        >>> ref = 'the cat is on the mat'.split()
+        >>> hyp = 'the the the the the the the'.split()
+        >>> sentence_gleu([ref], hyp)  # doctest: +ELLIPSIS
+        0.0909...
+
+    An example to evaluate normal machine translation outputs
+
+        >>> ref1 = str('It is a guide to action that ensures that the military '
+        ...            'will forever heed Party commands').split()
+        >>> hyp1 = str('It is a guide to action which ensures that the military '
+        ...            'always obeys the commands of the party').split()
+        >>> hyp2 = str('It is to insure the troops forever hearing the activity '
+        ...            'guidebook that party direct').split()
+        >>> sentence_gleu([ref1], hyp1) # doctest: +ELLIPSIS
+        0.4393...
+        >>> sentence_gleu([ref1], hyp2) # doctest: +ELLIPSIS
+        0.1206...
+
+    :param references: a list of reference sentences
+    :type references: list(list(str))
+    :param hypothesis: a hypothesis sentence
+    :type hypothesis: list(str)
+    :param min_len: The minimum order of n-gram this function should extract.
+    :type min_len: int
+    :param max_len: The maximum order of n-gram this function should extract.
+    :type max_len: int
+    :return: the sentence level GLEU score.
+    :rtype: float
+    """
+    return corpus_gleu(
+        [references],
+        [hypothesis],
+        min_len=min_len,
+        max_len=max_len
+    )
+
+def corpus_gleu(list_of_references, hypotheses, min_len=1, max_len=4):
+    """
+    Calculate a single corpus-level GLEU score (aka. system-level GLEU) for all
+    the hypotheses and their respective references.
+
+    Instead of averaging the sentence level GLEU scores (i.e. macro-average
+    precision), Wu et al. (2016) sum up the matching tokens and the max of
+    hypothesis and reference tokens for each sentence, then compute using the
+    aggregate values.
+
+    From Mike Schuster (via email):
+        "For the corpus, we just add up the two statistics n_match and
+         n_all = max(n_all_output, n_all_target) for all sentences, then
+         calculate gleu_score = n_match / n_all, so it is not just a mean of
+         the sentence gleu scores (in our case, longer sentences count more,
+         which I think makes sense as they are more difficult to translate)."
+
+    >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
+    ...         'ensures', 'that', 'the', 'military', 'always',
+    ...         'obeys', 'the', 'commands', 'of', 'the', 'party']
+    >>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
+    ...          'ensures', 'that', 'the', 'military', 'will', 'forever',
+    ...          'heed', 'Party', 'commands']
+    >>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which',
+    ...          'guarantees', 'the', 'military', 'forces', 'always',
+    ...          'being', 'under', 'the', 'command', 'of', 'the', 'Party']
+    >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
+    ...          'army', 'always', 'to', 'heed', 'the', 'directions',
+    ...          'of', 'the', 'party']
+
+    >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was',
+    ...         'interested', 'in', 'world', 'history']
+    >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history',
+    ...          'because', 'he', 'read', 'the', 'book']
+
+    >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
+    >>> hypotheses = [hyp1, hyp2]
+    >>> corpus_gleu(list_of_references, hypotheses) # doctest: +ELLIPSIS
+    0.5673...
+
+    The example below show that corpus_gleu() is different from averaging
+    sentence_gleu() for hypotheses
+
+    >>> score1 = sentence_gleu([ref1a], hyp1)
+    >>> score2 = sentence_gleu([ref2a], hyp2)
+    >>> (score1 + score2) / 2 # doctest: +ELLIPSIS
+    0.6144...
+
+    :param list_of_references: a list of reference sentences, w.r.t. hypotheses
+    :type list_of_references: list(list(list(str)))
+    :param hypotheses: a list of hypothesis sentences
+    :type hypotheses: list(list(str))
+    :param min_len: The minimum order of n-gram this function should extract.
+    :type min_len: int
+    :param max_len: The maximum order of n-gram this function should extract.
+    :type max_len: int
+    :return: The corpus-level GLEU score.
+    :rtype: float
+    """
+    # sanity check
+    assert len(list_of_references) == len(hypotheses), "The number of hypotheses and their reference(s) should be the same"
+
+    # sum matches and max-token-lengths over all sentences
+    corpus_n_match = 0
+    corpus_n_all = 0
+
+    for references, hypothesis in zip(list_of_references, hypotheses):
+        hyp_ngrams = Counter(everygrams(hypothesis, min_len, max_len))
+        tpfp = sum(hyp_ngrams.values())  # True positives + False positives.
+        
+        hyp_counts = []
+        for reference in references:
+            ref_ngrams = Counter(everygrams(reference, min_len, max_len))
+            tpfn = sum(ref_ngrams.values())  # True positives + False negatives.
+
+            overlap_ngrams = ref_ngrams & hyp_ngrams
+            tp = sum(overlap_ngrams.values())  # True positives.
+
+            # While GLEU is defined as the minimum of precision and
+            # recall, we can reduce the number of division operations by one by
+            # instead finding the maximum of the denominators for the precision
+            # and recall formulae, since the numerators are the same:
+            #     precision = tp / tpfp
+            #     recall = tp / tpfn
+            #     gleu_score = min(precision, recall) == tp / max(tpfp, tpfn)
+            n_all = max(tpfp, tpfn)
+
+            if n_all > 0:
+                hyp_counts.append((tp, n_all))
+
+        # use the reference yielding the highest score
+        if hyp_counts:
+            n_match, n_all = max(hyp_counts, key=lambda hc: hc[0]/hc[1])
+            corpus_n_match += n_match
+            corpus_n_all += n_all
+
+    # corner case: empty corpus or empty references---don't divide by zero!
+    if corpus_n_all == 0:
+        gleu_score = 0.0
+    else:
+        gleu_score = corpus_n_match / corpus_n_all
+
+    return gleu_score

{nltk-3.2.1 → nltk-3.3}/nltk/translate/ibm1.py

@@ -15,16 +15,25 @@
 """
 Lexical translation model that ignores word order.
 
-In IBM Model 1, word order is ignored for simplicity. Thus, the
-following two alignments are equally likely.
+In IBM Model 1, word order is ignored for simplicity. As long as the
+word alignments are equivalent, it doesn't matter where the word occurs
+in the source or target sentence. Thus, the following three alignments
+are equally likely.
 
 Source: je mange du jambon
 Target: i eat some ham
-Alignment: (1,1) (2,2) (3,3) (4,4)
+Alignment: (0,0) (1,1) (2,2) (3,3)
 
 Source: je mange du jambon
 Target: some ham eat i
-Alignment: (1,4) (2,3) (3,2) (4,1)
+Alignment: (0,2) (1,3) (2,1) (3,1)
+
+Source: du jambon je mange
+Target: eat i some ham
+Alignment: (0,3) (1,2) (2,0) (3,1)
+
+Note that an alignment is represented here as
+(word_index_in_target, word_index_in_source).
 
 The EM algorithm used in Model 1 is:
 E step - In the training data, count how many times a source language
@@ -130,7 +139,7 @@ class IBMModel1(IBMModel):
         for n in range(0, iterations):
             self.train(sentence_aligned_corpus)
 
-        self.__align_all(sentence_aligned_corpus)
+        self.align_all(sentence_aligned_corpus)
 
     def set_uniform_probabilities(self, sentence_aligned_corpus):
         initial_prob = 1 / len(self.trg_vocab)
@@ -205,11 +214,11 @@ class IBMModel1(IBMModel):
 
         return max(prob, IBMModel.MIN_PROB)
 
-    def __align_all(self, parallel_corpus):
+    def align_all(self, parallel_corpus):
         for sentence_pair in parallel_corpus:
-            self.__align(sentence_pair)
+            self.align(sentence_pair)
 
-    def __align(self, sentence_pair):
+    def align(self, sentence_pair):
         """
         Determines the best word alignment for one sentence pair from
         the corpus that the model was trained on.

{nltk-3.2.1 → nltk-3.3}/nltk/translate/ibm2.py

@@ -136,7 +136,7 @@ class IBMModel2(IBMModel):
         for n in range(0, iterations):
             self.train(sentence_aligned_corpus)
 
-        self.__align_all(sentence_aligned_corpus)
+        self.align_all(sentence_aligned_corpus)
 
     def set_uniform_probabilities(self, sentence_aligned_corpus):
         # a(i | j,l,m) = 1 / (l+1) for all i, j, l, m
@@ -245,11 +245,11 @@ class IBMModel2(IBMModel):
 
         return max(prob, IBMModel.MIN_PROB)
 
-    def __align_all(self, parallel_corpus):
+    def align_all(self, parallel_corpus):
         for sentence_pair in parallel_corpus:
-            self.__align(sentence_pair)
+            self.align(sentence_pair)
 
-    def __align(self, sentence_pair):
+    def align(self, sentence_pair):
         """
         Determines the best word alignment for one sentence pair from
         the corpus that the model was trained on.

{nltk-3.2.1 → nltk-3.3}/nltk/translate/ibm4.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: IBM Model 4
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Tah Wei Hoon <hoon.tw@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/translate/ibm5.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: IBM Model 5
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Tah Wei Hoon <hoon.tw@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/translate/ibm_model.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: IBM Model Core
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Tah Wei Hoon <hoon.tw@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -496,6 +496,9 @@ class AlignmentInfo(object):
     def __eq__(self, other):
         return self.alignment == other.alignment
 
+    def __ne__(self, other):
+        return not self == other
+
     def __hash__(self):
         return hash(self.alignment)
 

{nltk-3.2.1 → nltk-3.3}/nltk/translate/metrics.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Translation metrics
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Will Zhang <wilzzha@gmail.com>
 #         Guan Gui <ggui@student.unimelb.edu.au>
 #         Steven Bird <stevenbird1@gmail.com>