nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/translate/gdfa.py

@@ -1,131 +1,135 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: GDFA word alignment symmetrization
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Authors: Liling Tan
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-import codecs
-from collections import defaultdict
-
-def grow_diag_final_and(srclen, trglen, e2f, f2e):
-    """
-    This module symmetrisatizes the source-to-target and target-to-source
-    word alignment output and produces, aka. GDFA algorithm (Koehn, 2005).
-    
-    Step 1: Find the intersection of the bidirectional alignment.
-    
-    Step 2: Search for additional neighbor alignment points to be added, given
-            these criteria: (i) neighbor alignments points are not in the
-            intersection and (ii) neighbor alignments are in the union.
-            
-    Step 3: Add all other alignment points thats not in the intersection, not in
-            the neighboring alignments that met the criteria but in the original
-            foward/backward alignment outputs.
-    
-        >>> forw = ('0-0 2-1 9-2 21-3 10-4 7-5 11-6 9-7 12-8 1-9 3-10 '
-        ...         '4-11 17-12 17-13 25-14 13-15 24-16 11-17 28-18')
-        >>> back = ('0-0 1-9 2-9 3-10 4-11 5-12 6-6 7-5 8-6 9-7 10-4 '
-        ...         '11-6 12-8 13-12 15-12 17-13 18-13 19-12 20-13 '
-        ...         '21-3 22-12 23-14 24-17 25-15 26-17 27-18 28-18')
-        >>> srctext = ("この よう な ハロー 白色 わい 星 の Ｌ 関数 "
-        ...            "は Ｌ と 共 に 不連続 に 増加 する こと が "
-        ...            "期待 さ れる こと を 示し た 。")
-        >>> trgtext = ("Therefore , we expect that the luminosity function "
-        ...            "of such halo white dwarfs increases discontinuously "
-        ...            "with the luminosity .")
-        >>> srclen = len(srctext.split())
-        >>> trglen = len(trgtext.split())
-        >>>
-        >>> gdfa = grow_diag_final_and(srclen, trglen, forw, back)
-        >>> gdfa == set([(28, 18), (6, 6), (24, 17), (2, 1), (15, 12), (13, 12),
-        ...         (2, 9), (3, 10), (26, 17), (25, 15), (8, 6), (9, 7), (20,
-        ...         13), (18, 13), (0, 0), (10, 4), (13, 15), (23, 14), (7, 5),
-        ...         (25, 14), (1, 9), (17, 13), (4, 11), (11, 17), (9, 2), (22,
-        ...         12), (27, 18), (24, 16), (21, 3), (19, 12), (17, 12), (5,
-        ...         12), (11, 6), (12, 8)])
-        True
-    
-    References:
-    Koehn, P., A. Axelrod, A. Birch, C. Callison, M. Osborne, and D. Talbot. 
-    2005. Edinburgh System Description for the 2005 IWSLT Speech 
-    Translation Evaluation. In MT Eval Workshop.
-
-    :type srclen: int
-    :param srclen: the number of tokens in the source language
-    :type trglen: int
-    :param trglen: the number of tokens in the target language
-    :type e2f: str
-    :param e2f: the forward word alignment outputs from source-to-target
-                language (in pharaoh output format)
-    :type f2e: str
-    :param f2e: the backward word alignment outputs from target-to-source
-                language (in pharaoh output format)
-    :rtype: set(tuple(int))
-    :return: the symmetrized alignment points from the GDFA algorithm
-    """
-
-    # Converts pharaoh text format into list of tuples.
-    e2f = [tuple(map(int,a.split('-'))) for a in e2f.split()]
-    f2e = [tuple(map(int,a.split('-'))) for a in f2e.split()]
-    
-    neighbors = [(-1,0),(0,-1),(1,0),(0,1),(-1,-1),(-1,1),(1,-1),(1,1)]
-    alignment = set(e2f).intersection(set(f2e)) # Find the intersection.
-    union = set(e2f).union(set(f2e))
-
-    # *aligned* is used to check if neighbors are aligned in grow_diag()
-    aligned = defaultdict(set)
-    for i,j in alignment:
-        aligned['e'].add(i)
-        aligned['j'].add(j)
-    
-    def grow_diag():
-        """
-        Search for the neighbor points and them to the intersected alignment
-        points if criteria are met.
-        """
-        prev_len = len(alignment) - 1
-        # iterate until no new points added
-        while prev_len < len(alignment):
-            # for english word e = 0 ... en
-            for e in range(srclen):
-                # for foreign word f = 0 ... fn
-                for f in range(trglen): 
-                    # if ( e aligned with f)
-                    if (e,f) in alignment:
-                        # for each neighboring point (e-new, f-new)
-                        for neighbor in neighbors:
-                            neighbor = tuple(i+j for i,j in zip((e,f),neighbor))
-                            e_new, f_new = neighbor
-                            # if ( ( e-new not aligned and f-new not aligned) 
-                            # and (e-new, f-new in union(e2f, f2e) )
-                            if (e_new not in aligned and f_new not in aligned)\
-                            and neighbor in union:
-                                alignment.add(neighbor)
-                                aligned['e'].add(e_new); aligned['f'].add(f_new)
-                                prev_len+=1
-                                                                    
-    def final_and(a):
-        """
-        Adds remaining points that are not in the intersection, not in the 
-        neighboring alignments but in the original *e2f* and *f2e* alignments
-        """
-        # for english word e = 0 ... en
-        for e_new in range(srclen):
-            # for foreign word f = 0 ... fn
-            for f_new in range(trglen):
-                # if ( ( e-new not aligned and f-new not aligned) 
-                # and (e-new, f-new in union(e2f, f2e) )
-                if (e_new not in aligned
-                    and f_new not in aligned
-                    and (e_new, f_new) in a):
-
-                    alignment.add((e_new, f_new))
-                    aligned['e'].add(e_new); aligned['f'].add(f_new)
-
-    grow_diag()
-    final_and(e2f)
-    final_and(f2e)
-    return alignment
-
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: GDFA word alignment symmetrization
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Authors: Liling Tan
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from collections import defaultdict
+
+def grow_diag_final_and(srclen, trglen, e2f, f2e):
+    """
+    This module symmetrisatizes the source-to-target and target-to-source
+    word alignment output and produces, aka. GDFA algorithm (Koehn, 2005).
+    
+    Step 1: Find the intersection of the bidirectional alignment.
+    
+    Step 2: Search for additional neighbor alignment points to be added, given
+            these criteria: (i) neighbor alignments points are not in the
+            intersection and (ii) neighbor alignments are in the union.
+            
+    Step 3: Add all other alignment points thats not in the intersection, not in
+            the neighboring alignments that met the criteria but in the original
+            foward/backward alignment outputs.
+    
+        >>> forw = ('0-0 2-1 9-2 21-3 10-4 7-5 11-6 9-7 12-8 1-9 3-10 '
+        ...         '4-11 17-12 17-13 25-14 13-15 24-16 11-17 28-18')
+        >>> back = ('0-0 1-9 2-9 3-10 4-11 5-12 6-6 7-5 8-6 9-7 10-4 '
+        ...         '11-6 12-8 13-12 15-12 17-13 18-13 19-12 20-13 '
+        ...         '21-3 22-12 23-14 24-17 25-15 26-17 27-18 28-18')
+        >>> srctext = ("この よう な ハロー 白色 わい 星 の Ｌ 関数 "
+        ...            "は Ｌ と 共 に 不連続 に 増加 する こと が "
+        ...            "期待 さ れる こと を 示し た 。")
+        >>> trgtext = ("Therefore , we expect that the luminosity function "
+        ...            "of such halo white dwarfs increases discontinuously "
+        ...            "with the luminosity .")
+        >>> srclen = len(srctext.split())
+        >>> trglen = len(trgtext.split())
+        >>>
+        >>> gdfa = grow_diag_final_and(srclen, trglen, forw, back)
+        >>> gdfa == sorted(set([(28, 18), (6, 6), (24, 17), (2, 1), (15, 12), (13, 12),
+        ...         (2, 9), (3, 10), (26, 17), (25, 15), (8, 6), (9, 7), (20,
+        ...         13), (18, 13), (0, 0), (10, 4), (13, 15), (23, 14), (7, 5),
+        ...         (25, 14), (1, 9), (17, 13), (4, 11), (11, 17), (9, 2), (22,
+        ...         12), (27, 18), (24, 16), (21, 3), (19, 12), (17, 12), (5,
+        ...         12), (11, 6), (12, 8)]))
+        True
+    
+    References:
+    Koehn, P., A. Axelrod, A. Birch, C. Callison, M. Osborne, and D. Talbot. 
+    2005. Edinburgh System Description for the 2005 IWSLT Speech 
+    Translation Evaluation. In MT Eval Workshop.
+
+    :type srclen: int
+    :param srclen: the number of tokens in the source language
+    :type trglen: int
+    :param trglen: the number of tokens in the target language
+    :type e2f: str
+    :param e2f: the forward word alignment outputs from source-to-target
+                language (in pharaoh output format)
+    :type f2e: str
+    :param f2e: the backward word alignment outputs from target-to-source
+                language (in pharaoh output format)
+    :rtype: set(tuple(int))
+    :return: the symmetrized alignment points from the GDFA algorithm
+    """
+
+    # Converts pharaoh text format into list of tuples.
+    e2f = [tuple(map(int,a.split('-'))) for a in e2f.split()]
+    f2e = [tuple(map(int,a.split('-'))) for a in f2e.split()]
+    
+    neighbors = [(-1,0),(0,-1),(1,0),(0,1),(-1,-1),(-1,1),(1,-1),(1,1)]
+    alignment = set(e2f).intersection(set(f2e)) # Find the intersection.
+    union = set(e2f).union(set(f2e))
+    
+    # *aligned* is used to check if neighbors are aligned in grow_diag()
+    aligned = defaultdict(set)
+    for i,j in alignment:
+        aligned['e'].add(i)
+        aligned['f'].add(j)
+    
+    def grow_diag():
+        """
+        Search for the neighbor points and them to the intersected alignment
+        points if criteria are met.
+        """
+        prev_len = len(alignment) - 1
+        # iterate until no new points added
+        while prev_len < len(alignment):
+            no_new_points = True
+            # for english word e = 0 ... en
+            for e in range(srclen):
+                # for foreign word f = 0 ... fn
+                for f in range(trglen): 
+                    # if ( e aligned with f)
+                    if (e,f) in alignment:
+                        # for each neighboring point (e-new, f-new)
+                        for neighbor in neighbors:
+                            neighbor = tuple(i+j for i,j in zip((e,f),neighbor))
+                            e_new, f_new = neighbor
+                            # if ( ( e-new not aligned and f-new not aligned) 
+                            # and (e-new, f-new in union(e2f, f2e) )
+                            if (e_new not in aligned and f_new not in aligned)\
+                            and neighbor in union:
+                                alignment.add(neighbor)
+                                aligned['e'].add(e_new); aligned['f'].add(f_new)
+                                prev_len+=1
+                                no_new_points = False
+            # iterate until no new points added
+            if no_new_points:
+                break
+
+                                                                    
+    def final_and(a):
+        """
+        Adds remaining points that are not in the intersection, not in the 
+        neighboring alignments but in the original *e2f* and *f2e* alignments
+        """
+        # for english word e = 0 ... en
+        for e_new in range(srclen):
+            # for foreign word f = 0 ... fn
+            for f_new in range(trglen):
+                # if ( ( e-new not aligned and f-new not aligned) 
+                # and (e-new, f-new in union(e2f, f2e) )
+                if (e_new not in aligned
+                    and f_new not in aligned
+                    and (e_new, f_new) in union):
+                    alignment.add((e_new, f_new))
+                    aligned['e'].add(e_new); aligned['f'].add(f_new)
+
+    
+    grow_diag()
+    final_and(e2f)
+    final_and(f2e)
+    return sorted(alignment)

nltk-3.3/nltk/translate/gleu_score.py

@@ -0,0 +1,193 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: GLEU Score
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Authors:
+# Contributors: Mike Schuster, Michael Wayne Goodman, Liling Tan
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+""" GLEU score implementation. """
+from __future__ import division
+from collections import Counter
+
+from nltk.util import ngrams, everygrams
+
+
+def sentence_gleu(references, hypothesis, min_len=1, max_len=4):
+    """
+    Calculates the sentence level GLEU (Google-BLEU) score described in
+
+        Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc V. Le, Mohammad Norouzi,
+        Wolfgang Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey,
+        Jeff Klingner, Apurva Shah, Melvin Johnson, Xiaobing Liu, Lukasz Kaiser,
+        Stephan Gouws, Yoshikiyo Kato, Taku Kudo, Hideto Kazawa, Keith Stevens,
+        George Kurian, Nishant Patil, Wei Wang, Cliff Young, Jason Smith,
+        Jason Riesa, Alex Rudnick, Oriol Vinyals, Greg Corrado, Macduff Hughes,
+        Jeffrey Dean. (2016) Google’s Neural Machine Translation System:
+        Bridging the Gap between Human and Machine Translation.
+        eprint arXiv:1609.08144. https://arxiv.org/pdf/1609.08144v2.pdf
+        Retrieved on 27 Oct 2016.
+
+    From Wu et al. (2016):
+        "The BLEU score has some undesirable properties when used for single
+         sentences, as it was designed to be a corpus measure. We therefore
+         use a slightly different score for our RL experiments which we call
+         the 'GLEU score'. For the GLEU score, we record all sub-sequences of
+         1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then
+         compute a recall, which is the ratio of the number of matching n-grams
+         to the number of total n-grams in the target (ground truth) sequence,
+         and a precision, which is the ratio of the number of matching n-grams
+         to the number of total n-grams in the generated output sequence. Then
+         GLEU score is simply the minimum of recall and precision. This GLEU
+         score's range is always between 0 (no matches) and 1 (all match) and
+         it is symmetrical when switching output and target. According to
+         our experiments, GLEU score correlates quite well with the BLEU
+         metric on a corpus level but does not have its drawbacks for our per
+         sentence reward objective."
+
+    Note: The initial implementation only allowed a single reference, but now
+          a list of references is required (which is consistent with
+          bleu_score.sentence_bleu()).
+
+    The infamous "the the the ... " example
+
+        >>> ref = 'the cat is on the mat'.split()
+        >>> hyp = 'the the the the the the the'.split()
+        >>> sentence_gleu([ref], hyp)  # doctest: +ELLIPSIS
+        0.0909...
+
+    An example to evaluate normal machine translation outputs
+
+        >>> ref1 = str('It is a guide to action that ensures that the military '
+        ...            'will forever heed Party commands').split()
+        >>> hyp1 = str('It is a guide to action which ensures that the military '
+        ...            'always obeys the commands of the party').split()
+        >>> hyp2 = str('It is to insure the troops forever hearing the activity '
+        ...            'guidebook that party direct').split()
+        >>> sentence_gleu([ref1], hyp1) # doctest: +ELLIPSIS
+        0.4393...
+        >>> sentence_gleu([ref1], hyp2) # doctest: +ELLIPSIS
+        0.1206...
+
+    :param references: a list of reference sentences
+    :type references: list(list(str))
+    :param hypothesis: a hypothesis sentence
+    :type hypothesis: list(str)
+    :param min_len: The minimum order of n-gram this function should extract.
+    :type min_len: int
+    :param max_len: The maximum order of n-gram this function should extract.
+    :type max_len: int
+    :return: the sentence level GLEU score.
+    :rtype: float
+    """
+    return corpus_gleu(
+        [references],
+        [hypothesis],
+        min_len=min_len,
+        max_len=max_len
+    )
+
+def corpus_gleu(list_of_references, hypotheses, min_len=1, max_len=4):
+    """
+    Calculate a single corpus-level GLEU score (aka. system-level GLEU) for all
+    the hypotheses and their respective references.
+
+    Instead of averaging the sentence level GLEU scores (i.e. macro-average
+    precision), Wu et al. (2016) sum up the matching tokens and the max of
+    hypothesis and reference tokens for each sentence, then compute using the
+    aggregate values.
+
+    From Mike Schuster (via email):
+        "For the corpus, we just add up the two statistics n_match and
+         n_all = max(n_all_output, n_all_target) for all sentences, then
+         calculate gleu_score = n_match / n_all, so it is not just a mean of
+         the sentence gleu scores (in our case, longer sentences count more,
+         which I think makes sense as they are more difficult to translate)."
+
+    >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
+    ...         'ensures', 'that', 'the', 'military', 'always',
+    ...         'obeys', 'the', 'commands', 'of', 'the', 'party']
+    >>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
+    ...          'ensures', 'that', 'the', 'military', 'will', 'forever',
+    ...          'heed', 'Party', 'commands']
+    >>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which',
+    ...          'guarantees', 'the', 'military', 'forces', 'always',
+    ...          'being', 'under', 'the', 'command', 'of', 'the', 'Party']
+    >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
+    ...          'army', 'always', 'to', 'heed', 'the', 'directions',
+    ...          'of', 'the', 'party']
+
+    >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was',
+    ...         'interested', 'in', 'world', 'history']
+    >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history',
+    ...          'because', 'he', 'read', 'the', 'book']
+
+    >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
+    >>> hypotheses = [hyp1, hyp2]
+    >>> corpus_gleu(list_of_references, hypotheses) # doctest: +ELLIPSIS
+    0.5673...
+
+    The example below show that corpus_gleu() is different from averaging
+    sentence_gleu() for hypotheses
+
+    >>> score1 = sentence_gleu([ref1a], hyp1)
+    >>> score2 = sentence_gleu([ref2a], hyp2)
+    >>> (score1 + score2) / 2 # doctest: +ELLIPSIS
+    0.6144...
+
+    :param list_of_references: a list of reference sentences, w.r.t. hypotheses
+    :type list_of_references: list(list(list(str)))
+    :param hypotheses: a list of hypothesis sentences
+    :type hypotheses: list(list(str))
+    :param min_len: The minimum order of n-gram this function should extract.
+    :type min_len: int
+    :param max_len: The maximum order of n-gram this function should extract.
+    :type max_len: int
+    :return: The corpus-level GLEU score.
+    :rtype: float
+    """
+    # sanity check
+    assert len(list_of_references) == len(hypotheses), "The number of hypotheses and their reference(s) should be the same"
+
+    # sum matches and max-token-lengths over all sentences
+    corpus_n_match = 0
+    corpus_n_all = 0
+
+    for references, hypothesis in zip(list_of_references, hypotheses):
+        hyp_ngrams = Counter(everygrams(hypothesis, min_len, max_len))
+        tpfp = sum(hyp_ngrams.values())  # True positives + False positives.
+        
+        hyp_counts = []
+        for reference in references:
+            ref_ngrams = Counter(everygrams(reference, min_len, max_len))
+            tpfn = sum(ref_ngrams.values())  # True positives + False negatives.
+
+            overlap_ngrams = ref_ngrams & hyp_ngrams
+            tp = sum(overlap_ngrams.values())  # True positives.
+
+            # While GLEU is defined as the minimum of precision and
+            # recall, we can reduce the number of division operations by one by
+            # instead finding the maximum of the denominators for the precision
+            # and recall formulae, since the numerators are the same:
+            #     precision = tp / tpfp
+            #     recall = tp / tpfn
+            #     gleu_score = min(precision, recall) == tp / max(tpfp, tpfn)
+            n_all = max(tpfp, tpfn)
+
+            if n_all > 0:
+                hyp_counts.append((tp, n_all))
+
+        # use the reference yielding the highest score
+        if hyp_counts:
+            n_match, n_all = max(hyp_counts, key=lambda hc: hc[0]/hc[1])
+            corpus_n_match += n_match
+            corpus_n_all += n_all
+
+    # corner case: empty corpus or empty references---don't divide by zero!
+    if corpus_n_all == 0:
+        gleu_score = 0.0
+    else:
+        gleu_score = corpus_n_match / corpus_n_all
+
+    return gleu_score