nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/translate/metrics.py

@@ -1,40 +1,40 @@
-# Natural Language Toolkit: Translation metrics
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Will Zhang <wilzzha@gmail.com>
-#         Guan Gui <ggui@student.unimelb.edu.au>
-#         Steven Bird <stevenbird1@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-from __future__ import division
-
-def alignment_error_rate(reference, hypothesis, possible=None):
-    """
-    Return the Alignment Error Rate (AER) of an alignment
-    with respect to a "gold standard" reference alignment.
-    Return an error rate between 0.0 (perfect alignment) and 1.0 (no
-    alignment).
-
-        >>> from nltk.translate import Alignment
-        >>> ref = Alignment([(0, 0), (1, 1), (2, 2)])
-        >>> test = Alignment([(0, 0), (1, 2), (2, 1)])
-        >>> alignment_error_rate(ref, test) # doctest: +ELLIPSIS
-        0.6666666666666667
-
-    :type reference: Alignment
-    :param reference: A gold standard alignment (sure alignments)
-    :type hypothesis: Alignment
-    :param hypothesis: A hypothesis alignment (aka. candidate alignments)
-    :type possible: Alignment or None
-    :param possible: A gold standard reference of possible alignments
-        (defaults to *reference* if None)
-    :rtype: float or None
-    """
-
-    if possible is None:
-        possible = reference
-    else:
-        assert(reference.issubset(possible)) # sanity check
-
-    return (1.0 - (len(hypothesis & reference) + len(hypothesis & possible)) /
-            float(len(hypothesis) + len(reference)))
+# Natural Language Toolkit: Translation metrics
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Will Zhang <wilzzha@gmail.com>
+#         Guan Gui <ggui@student.unimelb.edu.au>
+#         Steven Bird <stevenbird1@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import division
+
+def alignment_error_rate(reference, hypothesis, possible=None):
+    """
+    Return the Alignment Error Rate (AER) of an alignment
+    with respect to a "gold standard" reference alignment.
+    Return an error rate between 0.0 (perfect alignment) and 1.0 (no
+    alignment).
+
+        >>> from nltk.translate import Alignment
+        >>> ref = Alignment([(0, 0), (1, 1), (2, 2)])
+        >>> test = Alignment([(0, 0), (1, 2), (2, 1)])
+        >>> alignment_error_rate(ref, test) # doctest: +ELLIPSIS
+        0.6666666666666667
+
+    :type reference: Alignment
+    :param reference: A gold standard alignment (sure alignments)
+    :type hypothesis: Alignment
+    :param hypothesis: A hypothesis alignment (aka. candidate alignments)
+    :type possible: Alignment or None
+    :param possible: A gold standard reference of possible alignments
+        (defaults to *reference* if None)
+    :rtype: float or None
+    """
+
+    if possible is None:
+        possible = reference
+    else:
+        assert(reference.issubset(possible)) # sanity check
+
+    return (1.0 - (len(hypothesis & reference) + len(hypothesis & possible)) /
+            float(len(hypothesis) + len(reference)))

nltk-3.3/nltk/translate/nist_score.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: NIST Score
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Authors:
+# Contributors:
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""NIST score implementation."""
+from __future__ import division
+
+import math
+import fractions
+from collections import Counter
+
+from nltk.util import ngrams
+
+def sentence_nist(references, hypothesis, n=5):
+    """
+    Calculate NIST score from
+    George Doddington. 2002. "Automatic evaluation of machine translation quality
+    using n-gram co-occurrence statistics." Proceedings of HLT.
+    Morgan Kaufmann Publishers Inc. http://dl.acm.org/citation.cfm?id=1289189.1289273
+
+    DARPA commissioned NIST to develop an MT evaluation facility based on the BLEU
+    score. The official script used by NIST to compute BLEU and NIST score is
+    mteval-14.pl. The main differences are:
+
+     - BLEU uses geometric mean of the ngram overlaps, NIST uses arithmetic mean.
+     - NIST has a different brevity penalty
+     - NIST score from mteval-14.pl has a self-contained tokenizer
+
+    Note: The mteval-14.pl includes a smoothing function for BLEU score that is NOT
+          used in the NIST score computation.
+
+    >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
+    ...               'ensures', 'that', 'the', 'military', 'always',
+    ...               'obeys', 'the', 'commands', 'of', 'the', 'party']
+
+    >>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops',
+    ...               'forever', 'hearing', 'the', 'activity', 'guidebook',
+    ...               'that', 'party', 'direct']
+
+    >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
+    ...               'ensures', 'that', 'the', 'military', 'will', 'forever',
+    ...               'heed', 'Party', 'commands']
+
+    >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which',
+    ...               'guarantees', 'the', 'military', 'forces', 'always',
+    ...               'being', 'under', 'the', 'command', 'of', 'the',
+    ...               'Party']
+
+    >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
+    ...               'army', 'always', 'to', 'heed', 'the', 'directions',
+    ...               'of', 'the', 'party']
+
+    >>> sentence_nist([reference1, reference2, reference3], hypothesis1) # doctest: +ELLIPSIS
+    3.3709...
+
+    >>> sentence_nist([reference1, reference2, reference3], hypothesis2) # doctest: +ELLIPSIS
+    1.4619...
+
+    :param references: reference sentences
+    :type references: list(list(str))
+    :param hypothesis: a hypothesis sentence
+    :type hypothesis: list(str)
+    :param n: highest n-gram order
+    :type n: int
+    """
+    return corpus_nist([references], [hypothesis], n)
+
+def corpus_nist(list_of_references, hypotheses, n=5):
+    """
+    Calculate a single corpus-level NIST score (aka. system-level BLEU) for all
+    the hypotheses and their respective references.
+
+    :param references: a corpus of lists of reference sentences, w.r.t. hypotheses
+    :type references: list(list(list(str)))
+    :param hypotheses: a list of hypothesis sentences
+    :type hypotheses: list(list(str))
+    :param n: highest n-gram order
+    :type n: int
+    """
+    # Before proceeding to compute NIST, perform sanity checks.
+    assert len(list_of_references) == len(hypotheses), "The number of hypotheses and their reference(s) should be the same"
+
+    # Collect the ngram coounts from the reference sentences.
+    ngram_freq = Counter()
+    total_reference_words = 0
+    for references in list_of_references: # For each source sent, there's a list of reference sents.
+        for reference in references:
+            # For each order of ngram, count the ngram occurrences.
+            for i in range(1,n+1):
+                ngram_freq.update(ngrams(reference, i))
+            total_reference_words += len(reference)
+
+    # Compute the information weights based on the reference sentences.
+    # Eqn 2 in Doddington (2002):
+    # Info(w_1 ... w_n) = log_2 [ (# of occurrences of w_1 ... w_n-1) / (# of occurrences of w_1 ... w_n) ]
+    information_weights = {}
+    for _ngram in ngram_freq: # w_1 ... w_n
+        _mgram = _ngram[:-1] #  w_1 ... w_n-1
+        # From https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v13a.pl#L546
+        # it's computed as such:
+        #     denominator = ngram_freq[_mgram] if _mgram and _mgram in ngram_freq else denominator = total_reference_words
+        #     information_weights[_ngram] = -1 * math.log(ngram_freq[_ngram]/denominator) / math.log(2)
+        #
+        # Mathematically, it's equivalent to the our implementation:
+        if _mgram and _mgram in ngram_freq:
+            numerator = ngram_freq[_mgram]
+        else:
+            numerator = total_reference_words
+        information_weights[_ngram] = math.log(numerator/ngram_freq[_ngram], 2)
+
+    # Micro-average.
+    nist_precision_numerator_per_ngram = Counter()
+    nist_precision_denominator_per_ngram = Counter()
+    l_ref, l_sys = 0, 0
+    # For each order of ngram.
+    for i in range(1,n+1):
+        # Iterate through each hypothesis and their corresponding references.
+        for references, hypothesis in zip(list_of_references, hypotheses):
+            hyp_len = len(hypothesis)
+
+            # Find reference with the best NIST score.
+            nist_score_per_ref = []
+            for reference in references:
+                _ref_len = len(reference)
+                # Counter of ngrams in hypothesis.
+                hyp_ngrams = Counter(ngrams(hypothesis, i)) if len(hypothesis) >= i else Counter()
+                ref_ngrams = Counter(ngrams(reference, i)) if len(reference) >=i else Counter()
+                ngram_overlaps = hyp_ngrams & ref_ngrams
+                # Precision part of the score in Eqn 3
+                _numerator = sum(information_weights[_ngram] * count
+                                for _ngram, count in ngram_overlaps.items())
+                _denominator = sum(hyp_ngrams.values())
+                _precision = 0 if _denominator == 0 else _numerator/_denominator
+                nist_score_per_ref.append((_precision, _numerator, _denominator, _ref_len))
+            # Best reference.
+            precision, numerator, denominator, ref_len = max(nist_score_per_ref)
+            nist_precision_numerator_per_ngram[i] += numerator
+            nist_precision_denominator_per_ngram[i] += denominator
+            l_ref += ref_len
+            l_sys += hyp_len
+
+    # Final NIST micro-average mean aggregation.
+    nist_precision = 0
+    for i in nist_precision_numerator_per_ngram:
+        precision = nist_precision_numerator_per_ngram[i] / nist_precision_denominator_per_ngram[i]
+        nist_precision+= precision
+    # Eqn 3 in Doddington(2002)
+    return nist_precision * nist_length_penalty(l_ref, l_sys)
+
+
+def nist_length_penalty(ref_len, hyp_len):
+    """
+    Calculates the NIST length penalty, from Eq. 3 in Doddington (2002)
+
+        penalty = exp( beta * log( min( len(hyp)/len(ref) , 1.0 )))
+
+    where,
+
+        `beta` is chosen to make the brevity penalty factor = 0.5 when the
+        no. of words in the system output (hyp) is 2/3 of the average
+        no. of words in the reference translation (ref)
+
+    The NIST penalty is different from BLEU's such that it minimize the impact
+    of the score of small variations in the length of a translation.
+    See Fig. 4 in  Doddington (2002)
+    """
+    ratio = hyp_len / ref_len
+    if 0 < ratio < 1:
+        ratio_x, score_x = 1.5, 0.5
+        beta = math.log(score_x) / math.log(ratio_x)**2
+        return math.exp(beta * math.log(ratio)**2)
+    else: # ratio <= 0 or ratio >= 1
+        return max(min(ratio, 1.0), 0.0)

{nltk-3.2 → nltk-3.3}/nltk/translate/phrase_based.py

@@ -1,175 +1,175 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: Phrase Extraction Algorithm
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Authors: Liling Tan, Fredrik Hedman, Petra Barancikova
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-def extract(f_start, f_end, e_start, e_end, 
-            alignment, f_aligned,
-            srctext, trgtext, srclen, trglen, max_phrase_length):
-    """
-    This function checks for alignment point consistency and extracts 
-    phrases using the chunk of consistent phrases.
-    
-    A phrase pair (e, f ) is consistent with an alignment A if and only if:
-
-    (i) No English words in the phrase pair are aligned to words outside it.
-    
-           ∀e i ∈ e, (e i , f j ) ∈ A ⇒ f j ∈ f
-    
-    (ii) No Foreign words in the phrase pair are aligned to words outside it. 
-            
-            ∀f j ∈ f , (e i , f j ) ∈ A ⇒ e i ∈ e
-    
-    (iii) The phrase pair contains at least one alignment point. 
-            
-            ∃e i ∈ e  ̄ , f j ∈ f  ̄ s.t. (e i , f j ) ∈ A
-    
-    :type f_start: int
-    :param f_start: Starting index of the possible foreign language phrases
-    :type f_end: int
-    :param f_end: Starting index of the possible foreign language phrases
-    :type e_start: int
-    :param e_start: Starting index of the possible source language phrases
-    :type e_end: int
-    :param e_end: Starting index of the possible source language phrases
-    :type srctext: list
-    :param srctext: The source language tokens, a list of string.
-    :type trgtext: list
-    :param trgtext: The target language tokens, a list of string.
-    :type srclen: int
-    :param srclen: The number of tokens in the source language tokens.
-    :type trglen: int
-    :param trglen: The number of tokens in the target language tokens.
-    """
-
-    if f_end < 0:  # 0-based indexing.
-        return {}
-    # Check if alignment points are consistent.
-    for e,f in alignment:
-        if ((f_start <= f <= f_end) and (e < e_start or e > e_end)):
-            return {}
-
-    # Add phrase pairs (incl. additional unaligned f)
-    phrases = set()
-    fs = f_start
-    while True:
-        fe = min(f_end, f_start + max_phrase_length - 1)
-        while True:
-            # add phrase pair ([e_start, e_end], [fs, fe]) to set E
-            # Need to +1 in range  to include the end-point.
-            src_phrase = " ".join(srctext[e_start:e_end+1])
-            trg_phrase = " ".join(trgtext[fs:fe+1])
-            # Include more data for later ordering.
-            phrases.add(((e_start, e_end+1), (f_start, f_end+1), 
-                         src_phrase, trg_phrase))
-            fe += 1
-            if fe in f_aligned or fe == trglen:
-                break
-        fs -=1 
-        if fs in f_aligned or fs < 0:
-            break
-    return phrases
-
-def phrase_extraction(srctext, trgtext, alignment, max_phrase_length=0):
-    """
-    Phrase extraction algorithm extracts all consistent phrase pairs from 
-    a word-aligned sentence pair.
-
-    The idea is to loop over all possible source language (e) phrases and find 
-    the minimal foreign phrase (f) that matches each of them. Matching is done 
-    by identifying all alignment points for the source phrase and finding the 
-    shortest foreign phrase that includes all the foreign counterparts for the 
-    source words.
-
-    In short, a phrase alignment has to 
-    (a) contain all alignment points for all covered words
-    (b) contain at least one alignment point
-            
-    >>> srctext = "michael assumes that he will stay in the house"
-    >>> trgtext = "michael geht davon aus , dass er im haus bleibt"
-    >>> alignment = [(0,0), (1,1), (1,2), (1,3), (2,5), (3,6), (4,9), 
-    ... (5,9), (6,7), (7,7), (8,8)]
-    >>> phrases = phrase_extraction(srctext, trgtext, alignment)
-    >>> for i in sorted(phrases):
-    ...    print(i)
-    ...
-    ((0, 1), (0, 1), 'michael', 'michael')
-    ((0, 2), (0, 4), 'michael assumes', 'michael geht davon aus')
-    ((0, 2), (0, 4), 'michael assumes', 'michael geht davon aus ,')
-    ((0, 3), (0, 6), 'michael assumes that', 'michael geht davon aus , dass')
-    ((0, 4), (0, 7), 'michael assumes that he', 'michael geht davon aus , dass er')
-    ((0, 9), (0, 10), 'michael assumes that he will stay in the house', 'michael geht davon aus , dass er im haus bleibt')
-    ((1, 2), (1, 4), 'assumes', 'geht davon aus')
-    ((1, 2), (1, 4), 'assumes', 'geht davon aus ,')
-    ((1, 3), (1, 6), 'assumes that', 'geht davon aus , dass')
-    ((1, 4), (1, 7), 'assumes that he', 'geht davon aus , dass er')
-    ((1, 9), (1, 10), 'assumes that he will stay in the house', 'geht davon aus , dass er im haus bleibt')
-    ((2, 3), (5, 6), 'that', ', dass')
-    ((2, 3), (5, 6), 'that', 'dass')
-    ((2, 4), (5, 7), 'that he', ', dass er')
-    ((2, 4), (5, 7), 'that he', 'dass er')
-    ((2, 9), (5, 10), 'that he will stay in the house', ', dass er im haus bleibt')
-    ((2, 9), (5, 10), 'that he will stay in the house', 'dass er im haus bleibt')
-    ((3, 4), (6, 7), 'he', 'er')
-    ((3, 9), (6, 10), 'he will stay in the house', 'er im haus bleibt')
-    ((4, 6), (9, 10), 'will stay', 'bleibt')
-    ((4, 9), (7, 10), 'will stay in the house', 'im haus bleibt')
-    ((6, 8), (7, 8), 'in the', 'im')
-    ((6, 9), (7, 9), 'in the house', 'im haus')
-    ((8, 9), (8, 9), 'house', 'haus')
-    
-    :type srctext: str
-    :param srctext: The sentence string from the source language.
-    :type trgtext: str
-    :param trgtext: The sentence string from the target language.
-    :type alignment: str
-    :param alignment: The word alignment outputs as list of tuples, where
-        the first elements of tuples are the source words' indices and
-        second elements are the target words' indices. This is also the output
-        format of nltk.translate.ibm1
-    :rtype: list(tuple)
-    :return: A list of tuples, each element in a list is a phrase and each 
-        phrase is a tuple made up of (i) its source location, (ii) its target 
-        location, (iii) the source phrase and (iii) the target phrase. The phrase
-        list of tuples represents all the possible phrases extracted from the 
-        word alignments. 
-    :type max_phrase_length: int
-    :param max_phrase_length: maximal phrase length, if 0 or not specified
-        it is set to a length of the longer sentence (srctext or trgtext).
-    """
-
-    srctext = srctext.split()   # e
-    trgtext = trgtext.split()   # f
-    srclen = len(srctext)       # len(e)
-    trglen = len(trgtext)       # len(f)
-    # Keeps an index of which source/target words that are aligned.
-    f_aligned = [j for _,j in alignment]
-    max_phrase_length = max_phrase_length or max(srclen,trglen)
-
-    # set of phrase pairs BP
-    bp = set()
-
-    for e_start in range(srclen):
-        max_idx = min(srclen, e_start + max_phrase_length)
-        for e_end in range(e_start, max_idx):
-            # // find the minimally matching foreign phrase
-            # (f start , f end ) = ( length(f), 0 )
-            # f_start ∈ [0, len(f) - 1]; f_end ∈ [0, len(f) - 1]
-            f_start, f_end = trglen-1 , -1  #  0-based indexing
- 
-            for e,f in alignment:
-                if e_start <= e <= e_end:
-                    f_start = min(f, f_start)
-                    f_end = max(f, f_end)
-            # add extract (f start , f end , e start , e end ) to set BP
-            phrases = extract(f_start, f_end, e_start, e_end, 
-                              alignment, f_aligned,
-                              srctext, trgtext, srclen, trglen,
-                              max_phrase_length)
-            if phrases:
-                bp.update(phrases)
-    return bp
-
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Phrase Extraction Algorithm
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Authors: Liling Tan, Fredrik Hedman, Petra Barancikova
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+def extract(f_start, f_end, e_start, e_end, 
+            alignment, f_aligned,
+            srctext, trgtext, srclen, trglen, max_phrase_length):
+    """
+    This function checks for alignment point consistency and extracts 
+    phrases using the chunk of consistent phrases.
+    
+    A phrase pair (e, f ) is consistent with an alignment A if and only if:
+
+    (i) No English words in the phrase pair are aligned to words outside it.
+    
+           ∀e i ∈ e, (e i , f j ) ∈ A ⇒ f j ∈ f
+    
+    (ii) No Foreign words in the phrase pair are aligned to words outside it. 
+            
+            ∀f j ∈ f , (e i , f j ) ∈ A ⇒ e i ∈ e
+    
+    (iii) The phrase pair contains at least one alignment point. 
+            
+            ∃e i ∈ e  ̄ , f j ∈ f  ̄ s.t. (e i , f j ) ∈ A
+    
+    :type f_start: int
+    :param f_start: Starting index of the possible foreign language phrases
+    :type f_end: int
+    :param f_end: Starting index of the possible foreign language phrases
+    :type e_start: int
+    :param e_start: Starting index of the possible source language phrases
+    :type e_end: int
+    :param e_end: Starting index of the possible source language phrases
+    :type srctext: list
+    :param srctext: The source language tokens, a list of string.
+    :type trgtext: list
+    :param trgtext: The target language tokens, a list of string.
+    :type srclen: int
+    :param srclen: The number of tokens in the source language tokens.
+    :type trglen: int
+    :param trglen: The number of tokens in the target language tokens.
+    """
+
+    if f_end < 0:  # 0-based indexing.
+        return {}
+    # Check if alignment points are consistent.
+    for e,f in alignment:
+        if ((f_start <= f <= f_end) and (e < e_start or e > e_end)):
+            return {}
+
+    # Add phrase pairs (incl. additional unaligned f)
+    phrases = set()
+    fs = f_start
+    while True:
+        fe = min(f_end, f_start + max_phrase_length - 1)
+        while True:
+            # add phrase pair ([e_start, e_end], [fs, fe]) to set E
+            # Need to +1 in range  to include the end-point.
+            src_phrase = " ".join(srctext[e_start:e_end+1])
+            trg_phrase = " ".join(trgtext[fs:fe+1])
+            # Include more data for later ordering.
+            phrases.add(((e_start, e_end+1), (f_start, f_end+1), 
+                         src_phrase, trg_phrase))
+            fe += 1
+            if fe in f_aligned or fe == trglen:
+                break
+        fs -=1 
+        if fs in f_aligned or fs < 0:
+            break
+    return phrases
+
+def phrase_extraction(srctext, trgtext, alignment, max_phrase_length=0):
+    """
+    Phrase extraction algorithm extracts all consistent phrase pairs from 
+    a word-aligned sentence pair.
+
+    The idea is to loop over all possible source language (e) phrases and find 
+    the minimal foreign phrase (f) that matches each of them. Matching is done 
+    by identifying all alignment points for the source phrase and finding the 
+    shortest foreign phrase that includes all the foreign counterparts for the 
+    source words.
+
+    In short, a phrase alignment has to 
+    (a) contain all alignment points for all covered words
+    (b) contain at least one alignment point
+            
+    >>> srctext = "michael assumes that he will stay in the house"
+    >>> trgtext = "michael geht davon aus , dass er im haus bleibt"
+    >>> alignment = [(0,0), (1,1), (1,2), (1,3), (2,5), (3,6), (4,9), 
+    ... (5,9), (6,7), (7,7), (8,8)]
+    >>> phrases = phrase_extraction(srctext, trgtext, alignment)
+    >>> for i in sorted(phrases):
+    ...    print(i)
+    ...
+    ((0, 1), (0, 1), 'michael', 'michael')
+    ((0, 2), (0, 4), 'michael assumes', 'michael geht davon aus')
+    ((0, 2), (0, 4), 'michael assumes', 'michael geht davon aus ,')
+    ((0, 3), (0, 6), 'michael assumes that', 'michael geht davon aus , dass')
+    ((0, 4), (0, 7), 'michael assumes that he', 'michael geht davon aus , dass er')
+    ((0, 9), (0, 10), 'michael assumes that he will stay in the house', 'michael geht davon aus , dass er im haus bleibt')
+    ((1, 2), (1, 4), 'assumes', 'geht davon aus')
+    ((1, 2), (1, 4), 'assumes', 'geht davon aus ,')
+    ((1, 3), (1, 6), 'assumes that', 'geht davon aus , dass')
+    ((1, 4), (1, 7), 'assumes that he', 'geht davon aus , dass er')
+    ((1, 9), (1, 10), 'assumes that he will stay in the house', 'geht davon aus , dass er im haus bleibt')
+    ((2, 3), (5, 6), 'that', ', dass')
+    ((2, 3), (5, 6), 'that', 'dass')
+    ((2, 4), (5, 7), 'that he', ', dass er')
+    ((2, 4), (5, 7), 'that he', 'dass er')
+    ((2, 9), (5, 10), 'that he will stay in the house', ', dass er im haus bleibt')
+    ((2, 9), (5, 10), 'that he will stay in the house', 'dass er im haus bleibt')
+    ((3, 4), (6, 7), 'he', 'er')
+    ((3, 9), (6, 10), 'he will stay in the house', 'er im haus bleibt')
+    ((4, 6), (9, 10), 'will stay', 'bleibt')
+    ((4, 9), (7, 10), 'will stay in the house', 'im haus bleibt')
+    ((6, 8), (7, 8), 'in the', 'im')
+    ((6, 9), (7, 9), 'in the house', 'im haus')
+    ((8, 9), (8, 9), 'house', 'haus')
+    
+    :type srctext: str
+    :param srctext: The sentence string from the source language.
+    :type trgtext: str
+    :param trgtext: The sentence string from the target language.
+    :type alignment: str
+    :param alignment: The word alignment outputs as list of tuples, where
+        the first elements of tuples are the source words' indices and
+        second elements are the target words' indices. This is also the output
+        format of nltk.translate.ibm1
+    :rtype: list(tuple)
+    :return: A list of tuples, each element in a list is a phrase and each 
+        phrase is a tuple made up of (i) its source location, (ii) its target 
+        location, (iii) the source phrase and (iii) the target phrase. The phrase
+        list of tuples represents all the possible phrases extracted from the 
+        word alignments. 
+    :type max_phrase_length: int
+    :param max_phrase_length: maximal phrase length, if 0 or not specified
+        it is set to a length of the longer sentence (srctext or trgtext).
+    """
+
+    srctext = srctext.split()   # e
+    trgtext = trgtext.split()   # f
+    srclen = len(srctext)       # len(e)
+    trglen = len(trgtext)       # len(f)
+    # Keeps an index of which source/target words that are aligned.
+    f_aligned = [j for _,j in alignment]
+    max_phrase_length = max_phrase_length or max(srclen,trglen)
+
+    # set of phrase pairs BP
+    bp = set()
+
+    for e_start in range(srclen):
+        max_idx = min(srclen, e_start + max_phrase_length)
+        for e_end in range(e_start, max_idx):
+            # // find the minimally matching foreign phrase
+            # (f start , f end ) = ( length(f), 0 )
+            # f_start ∈ [0, len(f) - 1]; f_end ∈ [0, len(f) - 1]
+            f_start, f_end = trglen-1 , -1  #  0-based indexing
+ 
+            for e,f in alignment:
+                if e_start <= e <= e_end:
+                    f_start = min(f, f_start)
+                    f_end = max(f, f_end)
+            # add extract (f start , f end , e start , e end ) to set BP
+            phrases = extract(f_start, f_end, e_start, e_end, 
+                              alignment, f_aligned,
+                              srctext, trgtext, srclen, trglen,
+                              max_phrase_length)
+            if phrases:
+                bp.update(phrases)
+    return bp
+