nltk 3.2.1__zip → 3.3__zip

nltk-3.3/nltk/translate/nist_score.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: NIST Score
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Authors:
+# Contributors:
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""NIST score implementation."""
+from __future__ import division
+
+import math
+import fractions
+from collections import Counter
+
+from nltk.util import ngrams
+
+def sentence_nist(references, hypothesis, n=5):
+    """
+    Calculate NIST score from
+    George Doddington. 2002. "Automatic evaluation of machine translation quality
+    using n-gram co-occurrence statistics." Proceedings of HLT.
+    Morgan Kaufmann Publishers Inc. http://dl.acm.org/citation.cfm?id=1289189.1289273
+
+    DARPA commissioned NIST to develop an MT evaluation facility based on the BLEU
+    score. The official script used by NIST to compute BLEU and NIST score is
+    mteval-14.pl. The main differences are:
+
+     - BLEU uses geometric mean of the ngram overlaps, NIST uses arithmetic mean.
+     - NIST has a different brevity penalty
+     - NIST score from mteval-14.pl has a self-contained tokenizer
+
+    Note: The mteval-14.pl includes a smoothing function for BLEU score that is NOT
+          used in the NIST score computation.
+
+    >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
+    ...               'ensures', 'that', 'the', 'military', 'always',
+    ...               'obeys', 'the', 'commands', 'of', 'the', 'party']
+
+    >>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops',
+    ...               'forever', 'hearing', 'the', 'activity', 'guidebook',
+    ...               'that', 'party', 'direct']
+
+    >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
+    ...               'ensures', 'that', 'the', 'military', 'will', 'forever',
+    ...               'heed', 'Party', 'commands']
+
+    >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which',
+    ...               'guarantees', 'the', 'military', 'forces', 'always',
+    ...               'being', 'under', 'the', 'command', 'of', 'the',
+    ...               'Party']
+
+    >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
+    ...               'army', 'always', 'to', 'heed', 'the', 'directions',
+    ...               'of', 'the', 'party']
+
+    >>> sentence_nist([reference1, reference2, reference3], hypothesis1) # doctest: +ELLIPSIS
+    3.3709...
+
+    >>> sentence_nist([reference1, reference2, reference3], hypothesis2) # doctest: +ELLIPSIS
+    1.4619...
+
+    :param references: reference sentences
+    :type references: list(list(str))
+    :param hypothesis: a hypothesis sentence
+    :type hypothesis: list(str)
+    :param n: highest n-gram order
+    :type n: int
+    """
+    return corpus_nist([references], [hypothesis], n)
+
+def corpus_nist(list_of_references, hypotheses, n=5):
+    """
+    Calculate a single corpus-level NIST score (aka. system-level BLEU) for all
+    the hypotheses and their respective references.
+
+    :param references: a corpus of lists of reference sentences, w.r.t. hypotheses
+    :type references: list(list(list(str)))
+    :param hypotheses: a list of hypothesis sentences
+    :type hypotheses: list(list(str))
+    :param n: highest n-gram order
+    :type n: int
+    """
+    # Before proceeding to compute NIST, perform sanity checks.
+    assert len(list_of_references) == len(hypotheses), "The number of hypotheses and their reference(s) should be the same"
+
+    # Collect the ngram coounts from the reference sentences.
+    ngram_freq = Counter()
+    total_reference_words = 0
+    for references in list_of_references: # For each source sent, there's a list of reference sents.
+        for reference in references:
+            # For each order of ngram, count the ngram occurrences.
+            for i in range(1,n+1):
+                ngram_freq.update(ngrams(reference, i))
+            total_reference_words += len(reference)
+
+    # Compute the information weights based on the reference sentences.
+    # Eqn 2 in Doddington (2002):
+    # Info(w_1 ... w_n) = log_2 [ (# of occurrences of w_1 ... w_n-1) / (# of occurrences of w_1 ... w_n) ]
+    information_weights = {}
+    for _ngram in ngram_freq: # w_1 ... w_n
+        _mgram = _ngram[:-1] #  w_1 ... w_n-1
+        # From https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v13a.pl#L546
+        # it's computed as such:
+        #     denominator = ngram_freq[_mgram] if _mgram and _mgram in ngram_freq else denominator = total_reference_words
+        #     information_weights[_ngram] = -1 * math.log(ngram_freq[_ngram]/denominator) / math.log(2)
+        #
+        # Mathematically, it's equivalent to the our implementation:
+        if _mgram and _mgram in ngram_freq:
+            numerator = ngram_freq[_mgram]
+        else:
+            numerator = total_reference_words
+        information_weights[_ngram] = math.log(numerator/ngram_freq[_ngram], 2)
+
+    # Micro-average.
+    nist_precision_numerator_per_ngram = Counter()
+    nist_precision_denominator_per_ngram = Counter()
+    l_ref, l_sys = 0, 0
+    # For each order of ngram.
+    for i in range(1,n+1):
+        # Iterate through each hypothesis and their corresponding references.
+        for references, hypothesis in zip(list_of_references, hypotheses):
+            hyp_len = len(hypothesis)
+
+            # Find reference with the best NIST score.
+            nist_score_per_ref = []
+            for reference in references:
+                _ref_len = len(reference)
+                # Counter of ngrams in hypothesis.
+                hyp_ngrams = Counter(ngrams(hypothesis, i)) if len(hypothesis) >= i else Counter()
+                ref_ngrams = Counter(ngrams(reference, i)) if len(reference) >=i else Counter()
+                ngram_overlaps = hyp_ngrams & ref_ngrams
+                # Precision part of the score in Eqn 3
+                _numerator = sum(information_weights[_ngram] * count
+                                for _ngram, count in ngram_overlaps.items())
+                _denominator = sum(hyp_ngrams.values())
+                _precision = 0 if _denominator == 0 else _numerator/_denominator
+                nist_score_per_ref.append((_precision, _numerator, _denominator, _ref_len))
+            # Best reference.
+            precision, numerator, denominator, ref_len = max(nist_score_per_ref)
+            nist_precision_numerator_per_ngram[i] += numerator
+            nist_precision_denominator_per_ngram[i] += denominator
+            l_ref += ref_len
+            l_sys += hyp_len
+
+    # Final NIST micro-average mean aggregation.
+    nist_precision = 0
+    for i in nist_precision_numerator_per_ngram:
+        precision = nist_precision_numerator_per_ngram[i] / nist_precision_denominator_per_ngram[i]
+        nist_precision+= precision
+    # Eqn 3 in Doddington(2002)
+    return nist_precision * nist_length_penalty(l_ref, l_sys)
+
+
+def nist_length_penalty(ref_len, hyp_len):
+    """
+    Calculates the NIST length penalty, from Eq. 3 in Doddington (2002)
+
+        penalty = exp( beta * log( min( len(hyp)/len(ref) , 1.0 )))
+
+    where,
+
+        `beta` is chosen to make the brevity penalty factor = 0.5 when the
+        no. of words in the system output (hyp) is 2/3 of the average
+        no. of words in the reference translation (ref)
+
+    The NIST penalty is different from BLEU's such that it minimize the impact
+    of the score of small variations in the length of a translation.
+    See Fig. 4 in  Doddington (2002)
+    """
+    ratio = hyp_len / ref_len
+    if 0 < ratio < 1:
+        ratio_x, score_x = 1.5, 0.5
+        beta = math.log(score_x) / math.log(ratio_x)**2
+        return math.exp(beta * math.log(ratio)**2)
+    else: # ratio <= 0 or ratio >= 1
+        return max(min(ratio, 1.0), 0.0)

{nltk-3.2.1 → nltk-3.3}/nltk/translate/phrase_based.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Phrase Extraction Algorithm
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Authors: Liling Tan, Fredrik Hedman, Petra Barancikova
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/translate/ribes_score.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: RIBES Score
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Contributors: Katsuhito Sudoh, Liling Tan, Kasramvd, J.F.Sebastian
 #               Mark Byers, ekhumoro, P. Ortiz
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/translate/stack_decoder.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Stack decoder
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Tah Wei Hoon <hoon.tw@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -493,3 +493,7 @@ class _Stack(object):
 
     def __contains__(self, hypothesis):
         return hypothesis in self.items
+
+    def __bool__(self):
+        return len(self.items) != 0
+    __nonzero__=__bool__

{nltk-3.2.1 → nltk-3.3}/nltk/tree.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Text Trees
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 #         Peter Ljunglöf <peter.ljunglof@gu.se>
@@ -14,15 +14,19 @@ Class for representing hierarchical language structures, such as
 syntax trees and morphological trees.
 """
 from __future__ import print_function, unicode_literals
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
 
 # TODO: add LabelledTree (can be used for dependency trees)
 
 import re
 
+from six import string_types
+
 from nltk.grammar import Production, Nonterminal
 from nltk.probability import ProbabilisticMixIn
 from nltk.util import slice_bounds
-from nltk.compat import string_types, python_2_unicode_compatible, unicode_repr
+from nltk.compat import python_2_unicode_compatible, unicode_repr
 from nltk.internals import raise_unorderable_types
 
 ######################################################################
@@ -694,7 +698,7 @@ class Tree(list):
         from nltk.treeprettyprinter import TreePrettyPrinter
         print(TreePrettyPrinter(self, sentence, highlight).text(**kwargs),
               file=stream)
-        
+
     def __repr__(self):
         childstr = ", ".join(unicode_repr(c) for c in self)
         return '%s(%s, [%s])' % (type(self).__name__, unicode_repr(self._label), childstr)
@@ -877,7 +881,7 @@ class ImmutableTree(Tree):
 ######################################################################
 ## Parented trees
 ######################################################################
-
+@add_metaclass(ABCMeta)
 class AbstractParentedTree(Tree):
     """
     An abstract base class for a ``Tree`` that automatically maintains
@@ -921,7 +925,7 @@ class AbstractParentedTree(Tree):
     #////////////////////////////////////////////////////////////
     # Parent management
     #////////////////////////////////////////////////////////////
-
+    @abstractmethod
     def _setparent(self, child, index, dry_run=False):
         """
         Update the parent pointer of ``child`` to point to ``self``.  This
@@ -942,8 +946,8 @@ class AbstractParentedTree(Tree):
             parent pointer; just check for any error conditions, and
             raise an exception if one is found.
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def _delparent(self, child, index):
         """
         Update the parent pointer of ``child`` to not point to self.  This
@@ -956,7 +960,6 @@ class AbstractParentedTree(Tree):
         :type index: int
         :param index: The index of ``child`` in ``self``.
         """
-        raise NotImplementedError()
 
     #////////////////////////////////////////////////////////////
     # Methods that add/remove children
@@ -1600,4 +1603,3 @@ __all__ = ['ImmutableProbabilisticTree', 'ImmutableTree', 'ProbabilisticMixIn',
            'ProbabilisticTree', 'Tree', 'bracket_parse',
            'sinica_parse', 'ParentedTree', 'MultiParentedTree',
            'ImmutableParentedTree', 'ImmutableMultiParentedTree']
-

{nltk-3.2.1 → nltk-3.3}/nltk/treeprettyprinter.py

@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: ASCII visualization of NLTK trees
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Andreas van Cranenburgh <A.W.vanCranenburgh@uva.nl>
 #         Peter Ljunglöf <peter.ljunglof@gu.se>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
 """
-Pretty-printing of discontinuous trees. 
+Pretty-printing of discontinuous trees.
 Adapted from the disco-dop project, by Andreas van Cranenburgh.
 https://github.com/andreasvc/disco-dop
 
@@ -21,7 +21,7 @@ http://jgaa.info/accepted/2006/EschbachGuentherBecker2006.10.2.pdf
 from __future__ import division, print_function, unicode_literals
 
 from nltk.util import slice_bounds, OrderedDict
-from nltk.compat import string_types, python_2_unicode_compatible, unicode_repr
+from nltk.compat import python_2_unicode_compatible, unicode_repr
 from nltk.internals import raise_unorderable_types
 from nltk.tree import Tree
 
@@ -56,8 +56,8 @@ class TreePrettyPrinter(object):
     creates an object from which different visualizations can be created.
 
     :param tree: a Tree object.
-    :param sentence: a list of words (strings). If `sentence` is given, 
-        `tree` must contain integers as leaves, which are taken as indices 
+    :param sentence: a list of words (strings). If `sentence` is given,
+        `tree` must contain integers as leaves, which are taken as indices
         in `sentence`. Using this you can display a discontinuous tree.
     :param highlight: Optionally, a sequence of Tree objects in `tree` which
         should be highlighted. Has the effect of only applying colors to nodes
@@ -521,7 +521,7 @@ class TreePrettyPrinter(object):
             result += ['\t<text style="text-anchor: middle; fill: %s; '
                        'font-size: %dpx;" x="%g" y="%g">%s</text>' % (
                            color, fontsize, x, y,
-                           escape(node.label() if isinstance(node, Tree) 
+                           escape(node.label() if isinstance(node, Tree)
                                   else node))]
 
         result += ['</svg>']

{nltk-3.2.1 → nltk-3.3}/nltk/twitter/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Twitter
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/twitter/api.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Twitter API
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 #         Lorenzo Rubio <lrnzcig@gmail.com>
 # URL: <http://nltk.org/>
@@ -12,6 +12,8 @@ This module provides an interface for TweetHandlers, and support for timezone
 handling.
 """
 
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
 from datetime import tzinfo, timedelta, datetime
 from nltk.compat import UTC
 import time as _time
@@ -42,8 +44,11 @@ class LocalTimezoneOffsetWithUTC(tzinfo):
         """
         return self.DSTOFFSET
 
+
 LOCAL = LocalTimezoneOffsetWithUTC()
 
+
+@add_metaclass(ABCMeta)
 class BasicTweetHandler(object):
     """
     Minimal implementation of `TweetHandler`.
@@ -101,17 +106,17 @@ class TweetHandlerI(BasicTweetHandler):
 
         self.startingup = True
 
+    @abstractmethod
     def handle(self, data):
         """
         Deal appropriately with data returned by the Twitter API
         """
-        raise NotImplementedError
 
+    @abstractmethod
     def on_finish(self):
         """
         Actions when the tweet limit has been reached
         """
-        raise NotImplementedError
 
     def check_date_limit(self, data, verbose=False):
         """

{nltk-3.2.1 → nltk-3.3}/nltk/twitter/common.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Twitter client
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 #         Lorenzo Rubio <lrnzcig@gmail.com>
 # URL: <http://nltk.org/>
@@ -19,9 +19,9 @@ import json
 
 import nltk.compat as compat
 
-
 HIER_SEPARATOR = "."
 
+
 def extract_fields(tweet, fields):
     """
     Extract field values from a full tweet and return them as a list
@@ -38,6 +38,7 @@ def extract_fields(tweet, fields):
             raise RuntimeError('Fatal error when extracting fields. Cannot find field ', field)
     return out
 
+
 def _add_field_to_out(json, field, out):
     if _is_composed_key(field):
         key, value = _get_key_value_composed(field)
@@ -45,11 +46,13 @@ def _add_field_to_out(json, field, out):
     else:
         out += [json[field]]
 
+
 def _is_composed_key(field):
     if HIER_SEPARATOR in field:
         return True
     return False
 
+
 def _get_key_value_composed(field):
     out = field.split(HIER_SEPARATOR)
     # there could be up to 3 levels
@@ -57,6 +60,7 @@ def _get_key_value_composed(field):
     value = HIER_SEPARATOR.join(out[1:])
     return key, value
 
+
 def _get_entity_recursive(json, entity):
     if not json:
         return None
@@ -82,6 +86,7 @@ def _get_entity_recursive(json, entity):
     else:
         return None
 
+
 def json2csv(fp, outfile, fields, encoding='utf8', errors='replace',
              gzip_compress=False):
     """
@@ -207,6 +212,7 @@ def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fie
             _write_to_file(tweet_fields, items, entity_fields, writer)
     outf.close()
 
+
 def get_header_field_list(main_fields, entity_type, entity_fields):
     if _is_composed_key(entity_type):
         key, value = _get_key_value_composed(entity_type)
@@ -223,6 +229,7 @@ def get_header_field_list(main_fields, entity_type, entity_fields):
     output2 = [HIER_SEPARATOR.join([sub_entity, x]) for x in entity_fields]
     return output1 + output2
 
+
 def _write_to_file(object_fields, items, entity_fields, writer):
     if not items:
         # it could be that the entity is just not present for the tweet
@@ -255,4 +262,3 @@ def _write_to_file(object_fields, items, entity_fields, writer):
     for item in items:
         row = object_fields + extract_fields(item, entity_fields)
         writer.writerow(row)
-

{nltk-3.2.1 → nltk-3.3}/nltk/twitter/twitter_demo.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Twitter client
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 #         Lorenzo Rubio <lrnzcig@gmail.com>
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/twitter/twitterclient.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Twitter client
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 #         Lorenzo Rubio <lrnzcig@gmail.com>
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/twitter/util.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Twitter client
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 #         Lorenzo Rubio <lrnzcig@gmail.com>
 # URL: <http://nltk.org/>
@@ -17,6 +17,7 @@ import os
 import pprint
 from twython import Twython
 
+
 def credsfromfile(creds_file=None, subdir=None, verbose=False):
     """
     Convenience function for authentication
@@ -28,6 +29,7 @@ class Authenticate(object):
     """
     Methods for authenticating with Twitter.
     """
+
     def __init__(self):
         self.creds_file = 'credentials.txt'
         self.creds_fullpath = None
@@ -40,7 +42,6 @@ class Authenticate(object):
             self.twitter_dir = None
             self.creds_subdir = None
 
-
     def load_creds(self, creds_file=None, subdir=None, verbose=False):
         """
         Read OAuth credentials from a text file.
@@ -70,13 +71,13 @@ class Authenticate(object):
 
         if subdir is None:
             if self.creds_subdir is None:
-                msg = "Supply a value to the 'subdir' parameter or" +\
+                msg = "Supply a value to the 'subdir' parameter or" + \
                       " set the TWITTER environment variable."
                 raise ValueError(msg)
         else:
             self.creds_subdir = subdir
 
-        self.creds_fullpath =\
+        self.creds_fullpath = \
             os.path.normpath(os.path.join(self.creds_subdir, self.creds_file))
 
         if not os.path.isfile(self.creds_fullpath):