nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/classify/decisiontree.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Decision Tree Classifiers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -81,17 +81,17 @@ class DecisionTreeClassifier(ClassifierI):
         # [xx] display default!!
         if self._fname is None:
             n = width-len(prefix)-15
-            return '%s%s %s\n' % (prefix, '.'*n, self._label)
+            return '{0}{1} {2}\n'.format(prefix, '.'*n, self._label)
         s = ''
         for i, (fval, result) in enumerate(sorted(self._decisions.items())):
-            hdr = '%s%s=%s? ' % (prefix, self._fname, fval)
+            hdr = '{0}{1}={2}? '.format(prefix, self._fname, fval)
             n = width-15-len(hdr)
-            s += '%s%s %s\n' % (hdr, '.'*(n), result._label)
+            s += '{0}{1} {2}\n'.format(hdr, '.'*(n), result._label)
             if result._fname is not None and depth>1:
                 s += result.pretty_format(width, prefix+'  ', depth-1)
         if self._default is not None:
             n = width-len(prefix)-21
-            s += '%selse: %s %s\n' % (prefix, '.'*n, self._default._label)
+            s += '{0}else: {1} {2}\n'.format(prefix, '.'*n, self._default._label)
             if self._default._fname is not None and depth>1:
                 s += self._default.pretty_format(width, prefix+'  ', depth-1)
         return s
@@ -103,24 +103,24 @@ class DecisionTreeClassifier(ClassifierI):
         if statements.
         """
         if self._fname is None:
-            return "%sreturn %r\n" % (prefix, self._label)
+            return "{0}return {1!r}\n".format(prefix, self._label)
         s = ''
         for (fval, result) in sorted(self._decisions.items()):
-            s += '%sif %s == %r: ' % (prefix, self._fname, fval)
+            s += '{0}if {1} == {2!r}: '.format(prefix, self._fname, fval)
             if result._fname is not None and depth>1:
                 s += '\n'+result.pseudocode(prefix+'  ', depth-1)
             else:
-                s += 'return %r\n' % result._label
+                s += 'return {0!r}\n'.format(result._label)
         if self._default is not None:
             if len(self._decisions) == 1:
-                s += '%sif %s != %r: '% (prefix, self._fname,
+                s += '{0}if {1} != {2!r}: '.format(prefix, self._fname,
                                          list(self._decisions.keys())[0])
             else:
-                s += '%selse: ' % (prefix,)
+                s += '{0}else: '.format(prefix)
             if self._default._fname is not None and depth>1:
                 s += '\n'+self._default.pseudocode(prefix+'  ', depth-1)
             else:
-                s += 'return %r\n' % self._default._label
+                s += 'return {0!r}\n'.format(self._default._label)
         return s
 
     def __str__(self):
@@ -224,7 +224,7 @@ class DecisionTreeClassifier(ClassifierI):
                 best_error = stump_error
                 best_stump = stump
         if verbose:
-            print(('best stump for %6d toks uses %-20s err=%6.4f' %
+            print(('best stump for {:6d} toks uses {:20} err={:6.4f}'.format \
                    (len(labeled_featuresets), best_stump._fname, best_error)))
         return best_stump
 
@@ -266,13 +266,13 @@ class DecisionTreeClassifier(ClassifierI):
                 if stump_error < best_error:
                     best_error = stump_error
                     best_stump = stump
-        if best_stump._decisions:
-            descr = '%s=%s' % (best_stump._fname,
-                               list(best_stump._decisions.keys())[0])
-        else:
-            descr = '(default)'
         if verbose:
-            print(('best stump for %6d toks uses %-20s err=%6.4f' %
+            if best_stump._decisions:
+                descr = '{0}={1}'.format(best_stump._fname,
+                                         list(best_stump._decisions.keys())[0])
+            else:
+                descr = '(default)'
+            print(('best stump for {:6d} toks uses {:20} err={:6.4f}'.format \
                    (len(labeled_featuresets), descr, best_error)))
         return best_stump
 

{nltk-3.2.1 → nltk-3.3}/nltk/classify/maxent.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Maximum Entropy Classifiers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Dmitry Chichkov <dchichkov@gmail.com> (TypedMaxentFeatureEncoding)
 # URL: <http://nltk.org/>
@@ -52,7 +52,6 @@ performed by classes that implement the ``MaxentFeatureEncodingI``
 interface.
 """
 from __future__ import print_function, unicode_literals
-__docformat__ = 'epytext en'
 
 try:
     import numpy
@@ -61,8 +60,11 @@ except ImportError:
 
 import tempfile
 import os
+import re
 from collections import defaultdict
 
+from six import integer_types
+
 from nltk import compat
 from nltk.data import gzip_open_unicode
 from nltk.util import OrderedDict
@@ -74,6 +76,8 @@ from nltk.classify.megam import (call_megam,
                                  write_megam_file, parse_megam_weights)
 from nltk.classify.tadm import call_tadm, write_tadm_file, parse_tadm_weights
 
+__docformat__ = 'epytext en'
+
 ######################################################################
 #{ Classifier Model
 ######################################################################
@@ -200,13 +204,27 @@ class MaxentClassifier(ClassifierI):
         print('  PROBS:'.ljust(descr_width)+''.join(
             '%8.3f' % pdist.prob(l) for l in labels))
 
+    def most_informative_features(self, n=10):
+        """
+        Generates the ranked list of informative features from most to least.
+        """
+        if hasattr(self, '_most_informative_features'):
+            return self._most_informative_features[:n]
+        else:
+            self._most_informative_features = sorted(list(range(len(self._weights))),
+                                                key=lambda fid: abs(self._weights[fid]),
+                                                reverse=True)
+            return self._most_informative_features[:n]
+
     def show_most_informative_features(self, n=10, show='all'):
         """
         :param show: all, neg, or pos (for negative-only or positive-only)
+        :type show: str
+        :param n: The no. of top features
+        :type n: int
         """
-        fids = sorted(list(range(len(self._weights))),
-                      key=lambda fid: abs(self._weights[fid]),
-                      reverse=True)
+        # Use None the full list of ranked features.
+        fids = self.most_informative_features(None)
         if show == 'pos':
             fids = [fid for fid in fids if self._weights[fid] > 0]
         elif show == 'neg':
@@ -547,7 +565,7 @@ class BinaryMaxentFeatureEncoding(MaxentFeatureEncodingI):
 
     def describe(self, f_id):
         # Inherit docs.
-        if not isinstance(f_id, compat.integer_types):
+        if not isinstance(f_id, integer_types):
             raise TypeError('describe() expected an int')
         try:
             self._inv_mapping
@@ -854,7 +872,7 @@ class TypedMaxentFeatureEncoding(MaxentFeatureEncodingI):
 
         # Convert input-features to joint-features:
         for fname, fval in featureset.items():
-            if isinstance(fval, (compat.integer_types, float)):
+            if isinstance(fval, (integer_types, float)):
                 # Known feature name & value:
                 if (fname, type(fval), label) in self._mapping:
                     encoding.append((self._mapping[fname, type(fval),
@@ -884,7 +902,7 @@ class TypedMaxentFeatureEncoding(MaxentFeatureEncodingI):
 
     def describe(self, f_id):
         # Inherit docs.
-        if not isinstance(f_id, compat.integer_types):
+        if not isinstance(f_id, integer_types):
             raise TypeError('describe() expected an int')
         try:
             self._inv_mapping

{nltk-3.2.1 → nltk-3.3}/nltk/classify/megam.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Interface to Megam Classifier
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -26,6 +26,8 @@ from __future__ import print_function
 
 import subprocess
 
+from six import string_types
+
 from nltk import compat
 from nltk.internals import find_binary
 try:
@@ -155,7 +157,7 @@ def call_megam(args):
     """
     Call the ``megam`` binary with the given arguments.
     """
-    if isinstance(args, compat.string_types):
+    if isinstance(args, string_types):
         raise TypeError('args should be a list of strings')
     if _megam_bin is None:
         config_megam()
@@ -171,8 +173,7 @@ def call_megam(args):
         print(stderr)
         raise OSError('megam command failed!')
 
-    if isinstance(stdout, compat.string_types):
+    if isinstance(stdout, string_types):
         return stdout
     else:
         return stdout.decode('utf-8')
-

{nltk-3.2.1 → nltk-3.3}/nltk/classify/naivebayes.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Naive Bayes Classifiers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -21,8 +21,8 @@ independent, given the label:
 |  P(label|features) = --------------------------------------------
 |                                         P(features)
 
-Rather than computing P(featues) explicitly, the algorithm just
-calculates the denominator for each label, and normalizes them so they
+Rather than computing P(features) explicitly, the algorithm just
+calculates the numerator for each label, and normalizes them so they
 sum to one:
 
 |                       P(label) * P(f1|label) * ... * P(fn|label)
@@ -154,29 +154,32 @@ class NaiveBayesClassifier(ClassifierI):
 
         |  max[ P(fname=fval|label1) / P(fname=fval|label2) ]
         """
-        # The set of (fname, fval) pairs used by this classifier.
-        features = set()
-        # The max & min probability associated w/ each (fname, fval)
-        # pair.  Maps (fname,fval) -> float.
-        maxprob = defaultdict(lambda: 0.0)
-        minprob = defaultdict(lambda: 1.0)
-
-        for (label, fname), probdist in self._feature_probdist.items():
-            for fval in probdist.samples():
-                feature = (fname, fval)
-                features.add(feature)
-                p = probdist.prob(fval)
-                maxprob[feature] = max(p, maxprob[feature])
-                minprob[feature] = min(p, minprob[feature])
-                if minprob[feature] == 0:
-                    features.discard(feature)
-
-        # Convert features to a list, & sort it by how informative
-        # features are.
-        features = sorted(features,
-                          key=lambda feature_:
-                          minprob[feature_]/maxprob[feature_])
-        return features[:n]
+        if hasattr(self, '_most_informative_features'):
+            return self._most_informative_features[:n]
+        else:
+            # The set of (fname, fval) pairs used by this classifier.
+            features = set()
+            # The max & min probability associated w/ each (fname, fval)
+            # pair.  Maps (fname,fval) -> float.
+            maxprob = defaultdict(lambda: 0.0)
+            minprob = defaultdict(lambda: 1.0)
+
+            for (label, fname), probdist in self._feature_probdist.items():
+                for fval in probdist.samples():
+                    feature = (fname, fval)
+                    features.add(feature)
+                    p = probdist.prob(fval)
+                    maxprob[feature] = max(p, maxprob[feature])
+                    minprob[feature] = min(p, minprob[feature])
+                    if minprob[feature] == 0:
+                        features.discard(feature)
+
+            # Convert features to a list, & sort it by how informative
+            # features are.
+            self._most_informative_features = sorted(features,
+                                                key=lambda feature_:
+                                                minprob[feature_]/maxprob[feature_])
+        return self._most_informative_features[:n]
 
     @classmethod
     def train(cls, labeled_featuresets, estimator=ELEProbDist):

{nltk-3.2.1 → nltk-3.3}/nltk/classify/rte_classify.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: RTE Classifier
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -18,35 +18,16 @@ TO DO: add lemmatization
 """
 from __future__ import print_function
 
-import nltk
-from nltk.classify.util import accuracy
-
-def ne(token):
-    """
-    This just assumes that words in all caps or titles are
-    named entities.
-
-    :type token: str
-    """
-    if token.istitle() or token.isupper():
-        return True
-    return False
-
-def lemmatize(word):
-    """
-    Use morphy from WordNet to find the base form of verbs.
-    """
-    lemma = nltk.corpus.wordnet.morphy(word, pos=nltk.corpus.wordnet.VERB)
-    if lemma is not None:
-        return lemma
-    return word
+from nltk.tokenize import RegexpTokenizer
+from nltk.classify.util import accuracy, check_megam_config
+from nltk.classify.maxent import MaxentClassifier
 
 class RTEFeatureExtractor(object):
     """
     This builds a bag of words for both the text and the hypothesis after
     throwing away some stopwords, then calculates overlap and difference.
     """
-    def __init__(self, rtepair, stop=True, lemmatize=False):
+    def __init__(self, rtepair, stop=True, use_lemmatize=False):
         """
         :param rtepair: a ``RTEPair`` from which features should be extracted
         :param stop: if ``True``, stopwords are thrown away.
@@ -58,10 +39,9 @@ class RTEFeatureExtractor(object):
 
         self.negwords = set(['no', 'not', 'never', 'failed', 'rejected',
                              'denied'])
-        # Try to tokenize so that abbreviations like U.S.and monetary amounts
-        # like "$23.00" are kept as tokens.
-        from nltk.tokenize import RegexpTokenizer
-        tokenizer = RegexpTokenizer('([A-Z]\.)+|\w+|\$[\d\.]+')
+        # Try to tokenize so that abbreviations, monetary amounts, email
+        # addresses, URLs are single tokens.
+        tokenizer = RegexpTokenizer('[\w.@:/]+|\w+|\$[\d.]+')
 
         #Get the set of word types for text and hypothesis
         self.text_tokens = tokenizer.tokenize(rtepair.text)
@@ -69,9 +49,9 @@ class RTEFeatureExtractor(object):
         self.text_words = set(self.text_tokens)
         self.hyp_words = set(self.hyp_tokens)
 
-        if lemmatize:
-            self.text_words = set(lemmatize(token) for token in self.text_tokens)
-            self.hyp_words = set(lemmatize(token) for token in self.hyp_tokens)
+        if use_lemmatize:
+            self.text_words = set(self._lemmatize(token) for token in self.text_tokens)
+            self.hyp_words = set(self._lemmatize(token) for token in self.hyp_tokens)
 
         if self.stop:
             self.text_words = self.text_words - self.stopwords
@@ -89,7 +69,7 @@ class RTEFeatureExtractor(object):
         :param toktype: distinguish Named Entities from ordinary words
         :type toktype: 'ne' or 'word'
         """
-        ne_overlap = set(token for token in self._overlap if ne(token))
+        ne_overlap = set(token for token in self._overlap if self._ne(token))
         if toktype == 'ne':
             if debug:
                 print("ne overlap", ne_overlap)
@@ -108,7 +88,7 @@ class RTEFeatureExtractor(object):
         :param toktype: distinguish Named Entities from ordinary words
         :type toktype: 'ne' or 'word'
         """
-        ne_extra = set(token for token in self._hyp_extra if ne(token))
+        ne_extra = set(token for token in self._hyp_extra if self._ne(token))
         if toktype == 'ne':
             return ne_extra
         elif toktype == 'word':
@@ -116,6 +96,28 @@ class RTEFeatureExtractor(object):
         else:
             raise ValueError("Type not recognized: '%s'" % toktype)
 
+    @staticmethod
+    def _ne(token):
+        """
+        This just assumes that words in all caps or titles are
+        named entities.
+
+        :type token: str
+        """
+        if token.istitle() or token.isupper():
+            return True
+        return False
+
+    @staticmethod
+    def _lemmatize(word):
+        """
+        Use morphy from WordNet to find the base form of verbs.
+        """
+        lemma = nltk.corpus.wordnet.morphy(word, pos=nltk.corpus.wordnet.VERB)
+        if lemma is not None:
+            return lemma
+        return word
+
 
 def rte_features(rtepair):
     extractor = RTEFeatureExtractor(rtepair)
@@ -130,62 +132,29 @@ def rte_features(rtepair):
     return features
 
 
-def rte_classifier(trainer, features=rte_features):
-    """
-    Classify RTEPairs
-    """
-    train = ((pair, pair.value) for pair in
-             nltk.corpus.rte.pairs(['rte1_dev.xml', 'rte2_dev.xml',
-                                    'rte3_dev.xml']))
-    test = ((pair, pair.value) for pair in
-            nltk.corpus.rte.pairs(['rte1_test.xml', 'rte2_test.xml',
-                                   'rte3_test.xml']))
-
-    # Train up a classifier.
-    print('Training classifier...')
-    classifier = trainer([(features(pair), label) for (pair, label) in train])
+def rte_featurize(rte_pairs):
+    return [(rte_features(pair), pair.value) for pair in rte_pairs]
 
-    # Run the classifier on the test data.
+
+def rte_classifier(algorithm):
+    from nltk.corpus import rte as rte_corpus
+    train_set  = rte_corpus.pairs(['rte1_dev.xml', 'rte2_dev.xml', 'rte3_dev.xml'])
+    test_set = rte_corpus.pairs(['rte1_test.xml', 'rte2_test.xml', 'rte3_test.xml'])
+    featurized_train_set = rte_featurize(train_set)
+    featurized_test_set = rte_featurize(test_set)
+    # Train the classifier
+    print('Training classifier...')
+    if algorithm in ['megam', 'BFGS']: # MEGAM based algorithms.
+        # Ensure that MEGAM is configured first.
+        check_megam_config()
+        clf = lambda x: MaxentClassifier.train(featurized_train_set, algorithm)
+    elif algorithm in ['GIS', 'IIS']: # Use default GIS/IIS MaxEnt algorithm
+        clf = MaxentClassifier.train(featurized_train_set, algorithm)
+    else:
+        err_msg = str("RTEClassifier only supports these algorithms:\n "
+                      "'megam', 'BFGS', 'GIS', 'IIS'.\n")
+        raise Exception(err_msg)
     print('Testing classifier...')
-    acc = accuracy(classifier, [(features(pair), label)
-                                for (pair, label) in test])
+    acc = accuracy(clf, featurized_test_set)
     print('Accuracy: %6.4f' % acc)
-
-    # Return the classifier
-    return classifier
-
-
-def demo_features():
-    pairs = nltk.corpus.rte.pairs(['rte1_dev.xml'])[:6]
-    for pair in pairs:
-        print()
-        for key in sorted(rte_features(pair)):
-            print("%-15s => %s" % (key, rte_features(pair)[key]))
-
-
-def demo_feature_extractor():
-    rtepair = nltk.corpus.rte.pairs(['rte3_dev.xml'])[33]
-    extractor = RTEFeatureExtractor(rtepair)
-    print(extractor.hyp_words)
-    print(extractor.overlap('word'))
-    print(extractor.overlap('ne'))
-    print(extractor.hyp_extra('word'))
-
-
-def demo():
-    import nltk
-    try:
-        nltk.config_megam('/usr/local/bin/megam')
-        trainer = lambda x: nltk.MaxentClassifier.train(x, 'megam')
-    except ValueError:
-        try:
-            trainer = lambda x: nltk.MaxentClassifier.train(x, 'BFGS')
-        except ValueError:
-            trainer = nltk.MaxentClassifier.train
-    nltk.classify.rte_classifier(trainer)
-
-if __name__ == '__main__':
-    demo_features()
-    demo_feature_extractor()
-    demo()
-
+    return clf

{nltk-3.2.1 → nltk-3.3}/nltk/classify/scikitlearn.py

@@ -8,7 +8,7 @@ scikit-learn (http://scikit-learn.org) is a machine learning library for
 Python. It supports many classification algorithms, including SVMs,
 Naive Bayes, logistic regression (MaxEnt) and decision trees.
 
-This package implement a wrapper around scikit-learn classifiers. To use this
+This package implements a wrapper around scikit-learn classifiers. To use this
 wrapper, construct a scikit-learn estimator object, then use that to construct
 a SklearnClassifier. E.g., to wrap a linear SVM with default settings:
 
@@ -32,6 +32,8 @@ best 1000 features:
 """
 from __future__ import print_function, unicode_literals
 
+from six.moves import zip
+
 from nltk.classify.api import ClassifierI
 from nltk.probability import DictionaryProbDist
 from nltk import compat
@@ -111,7 +113,7 @@ class SklearnClassifier(ClassifierI):
             numbers, booleans or strings.
         """
 
-        X, y = list(compat.izip(*labeled_featuresets))
+        X, y = list(zip(*labeled_featuresets))
         X = self._vectorizer.fit_transform(X)
         y = self._encoder.fit_transform(y)
         self._clf.fit(X, y)

{nltk-3.2.1 → nltk-3.3}/nltk/classify/senna.py

@@ -1,7 +1,7 @@
 # encoding: utf-8
 # Natural Language Toolkit: Senna Interface
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Rami Al-Rfou' <ralrfou@cs.stonybrook.edu>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -22,17 +22,19 @@ system specific binary should be rebuilt. Otherwise this could introduce
 misalignment errors.
 
 The input is:
-- path to the directory that contains SENNA executables. If the path is incorrect, 
+- path to the directory that contains SENNA executables. If the path is incorrect,
    Senna will automatically search for executable file specified in SENNA environment variable
 - List of the operations needed to be performed.
 - (optionally) the encoding of the input data (default:utf-8)
 
+Note: Unit tests for this module can be found in test/unit/test_senna.py
+
     >>> from __future__ import unicode_literals
     >>> from nltk.classify import Senna
-    >>> pipeline = Senna('/usr/share/senna-v2.0', ['pos', 'chk', 'ner'])
+    >>> pipeline = Senna('/usr/share/senna-v3.0', ['pos', 'chk', 'ner'])
     >>> sent = 'Dusseldorf is an international business center'.split()
-    >>> [(token['word'], token['chk'], token['ner'], token['pos']) for token in pipeline.tag(sent)]
-    [('Dusseldorf', 'B-NP', 'B-LOC', 'NNP'), ('is', 'B-VP', 'O', 'VBZ'), ('an', 'B-NP', 'O', 'DT'), 
+    >>> [(token['word'], token['chk'], token['ner'], token['pos']) for token in pipeline.tag(sent)] # doctest: +SKIP
+    [('Dusseldorf', 'B-NP', 'B-LOC', 'NNP'), ('is', 'B-VP', 'O', 'VBZ'), ('an', 'B-NP', 'O', 'DT'),
     ('international', 'I-NP', 'O', 'JJ'), ('business', 'I-NP', 'O', 'NN'), ('center', 'I-NP', 'O', 'NN')]
 """
 
@@ -42,8 +44,10 @@ from os import path, sep, environ
 from subprocess import Popen, PIPE
 from platform import architecture, system
 
+from six import text_type
+
 from nltk.tag.api import TaggerI
-from nltk.compat import text_type, python_2_unicode_compatible
+from nltk.compat import python_2_unicode_compatible
 
 _senna_url = 'http://ml.nec-labs.com/senna/'
 
@@ -55,29 +59,29 @@ class Senna(TaggerI):
 
     def __init__(self, senna_path, operations, encoding='utf-8'):
         self._encoding = encoding
-        self._path = path.normpath(senna_path) + sep 
-        
-        # Verifies the existence of the executable on the self._path first    
+        self._path = path.normpath(senna_path) + sep
+
+        # Verifies the existence of the executable on the self._path first
         #senna_binary_file_1 = self.executable(self._path)
         exe_file_1 = self.executable(self._path)
         if not path.isfile(exe_file_1):
-            # Check for the system environment 
+            # Check for the system environment
             if 'SENNA' in environ:
-                #self._path = path.join(environ['SENNA'],'')  
-                self._path = path.normpath(environ['SENNA']) + sep 
+                #self._path = path.join(environ['SENNA'],'')
+                self._path = path.normpath(environ['SENNA']) + sep
                 exe_file_2 = self.executable(self._path)
                 if not path.isfile(exe_file_2):
                     raise OSError("Senna executable expected at %s or %s but not found" % (exe_file_1,exe_file_2))
-        
+
         self.operations = operations
 
-    
+
     def executable(self, base_path):
         """
         The function that determines the system specific binary that should be
         used in the pipeline. In case, the system is not known the default senna binary will
         be used.
-        """ 
+        """
         os_name = system()
         if os_name == 'Linux':
             bits = architecture()[0]
@@ -89,7 +93,7 @@ class Senna(TaggerI):
         if os_name == 'Darwin':
             return path.join(base_path, 'senna-osx')
         return path.join(base_path, 'senna')
-        
+
     def _map(self):
         """
         A method that calculates the order of the columns that SENNA pipeline
@@ -116,11 +120,11 @@ class Senna(TaggerI):
         calculated annotations/tags.
         """
         encoding = self._encoding
-        
+
         if not path.isfile(self.executable(self._path)):
             raise OSError("Senna executable expected at %s but not found" % self.executable(self._path))
-        
-         
+
+
         # Build the senna command to run the tagger
         _senna_cmd = [self.executable(self._path), '-path', self._path, '-usrtokens', '-iobtags']
         _senna_cmd.extend(['-'+op for op in self.operations])
@@ -174,8 +178,6 @@ class Senna(TaggerI):
 def setup_module(module):
     from nose import SkipTest
     try:
-        tagger = Senna('/usr/share/senna-v2.0', ['pos', 'chk', 'ner'])
+        tagger = Senna('/usr/share/senna-v3.0', ['pos', 'chk', 'ner'])
     except OSError:
         raise SkipTest("Senna executable not found")
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/classify/svm.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: SVM-based classifier
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Leon Derczynski <leon@dcs.shef.ac.uk>
 #
 # URL: <http://nltk.org/>