nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/classify/tadm.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Interface to TADM Classifier
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Joseph Frazee <jfrazee@mail.utexas.edu>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -9,7 +9,8 @@ from __future__ import print_function, unicode_literals
 import sys
 import subprocess
 
-from nltk import compat
+from six import string_types
+
 from nltk.internals import find_binary
 try:
     import numpy
@@ -73,7 +74,7 @@ def call_tadm(args):
     """
     Call the ``tadm`` binary with the given arguments.
     """
-    if isinstance(args, compat.string_types):
+    if isinstance(args, string_types):
         raise TypeError('args should be a list of strings')
     if _tadm_bin is None:
         config_tadm()

{nltk-3.2.1 → nltk-3.3}/nltk/classify/textcat.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Language ID module using TextCat algorithm
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Avital Pekker <avital.pekker@utoronto.ca>
 #
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/classify/util.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Classifier Utility Functions
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
 # URL: <http://nltk.org/>
@@ -310,3 +310,15 @@ def wsd_demo(trainer, word, features, n=1000):
     # Return the classifier
     return classifier
 
+
+
+def check_megam_config(self):
+    """
+    Checks whether the MEGAM binary is configured.
+    """
+    try:
+        _megam_bin
+    except NameError:
+        err_msg = str("Please configure your megam binary first, e.g.\n"
+                      ">>> nltk.config_megam('/usr/bin/local/megam')")
+        raise NameError(err_msg)

{nltk-3.2.1 → nltk-3.3}/nltk/classify/weka.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Interface to Weka Classsifiers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -15,9 +15,10 @@ import os
 import subprocess
 import re
 import zipfile
-
 from sys import stdin
-from nltk import compat
+
+from six import integer_types, string_types
+
 from nltk.probability import DictionaryProbDist
 from nltk.internals import java, config_java
 
@@ -265,9 +266,9 @@ class ARFF_Formatter:
             for (fname, fval) in tok.items():
                 if issubclass(type(fval), bool):
                     ftype = '{True, False}'
-                elif issubclass(type(fval), (compat.integer_types, float, bool)):
+                elif issubclass(type(fval), (integer_types, float, bool)):
                     ftype = 'NUMERIC'
-                elif issubclass(type(fval), compat.string_types):
+                elif issubclass(type(fval), string_types):
                     ftype = 'STRING'
                 elif fval is None:
                     continue # can't tell the type.
@@ -329,7 +330,7 @@ class ARFF_Formatter:
     def _fmt_arff_val(self, fval):
         if fval is None:
             return '?'
-        elif isinstance(fval, (bool, compat.integer_types)):
+        elif isinstance(fval, (bool, integer_types)):
             return '%s' % fval
         elif isinstance(fval, float):
             return '%r' % fval

{nltk-3.2.1 → nltk-3.3}/nltk/cluster/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Clusterers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/cluster/api.py

@@ -1,31 +1,34 @@
 # Natural Language Toolkit: Clusterer Interfaces
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
 # Porting: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
+
 from nltk.probability import DictionaryProbDist
 
+@add_metaclass(ABCMeta)
 class ClusterI(object):
     """
     Interface covering basic clustering functionality.
     """
-
+    @abstractmethod
     def cluster(self, vectors, assign_clusters=False):
         """
         Assigns the vectors to clusters, learning the clustering parameters
         from the data. Returns a cluster identifier for each vector.
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def classify(self, token):
         """
         Classifies the token into a cluster, setting the token's CLUSTER
         parameter to that cluster identifier.
         """
-        raise NotImplementedError()
 
     def likelihood(self, vector, label):
         """
@@ -51,15 +54,16 @@ class ClusterI(object):
             likelihoods[cluster] /= sum
         return DictionaryProbDist(likelihoods)
 
+    @abstractmethod
     def num_clusters(self):
         """
         Returns the number of clusters.
         """
-        raise NotImplementedError()
 
     def cluster_names(self):
         """
         Returns the names of the clusters.
+        :rtype: list
         """
         return list(range(self.num_clusters()))
 

{nltk-3.2.1 → nltk-3.3}/nltk/cluster/em.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Expectation Maximization Clusterer
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/cluster/gaac.py

@@ -1,10 +1,10 @@
 # Natural Language Toolkit: Group Average Agglomerative Clusterer
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
-from __future__ import print_function, unicode_literals
+from __future__ import print_function, unicode_literals, division
 
 try:
     import numpy
@@ -109,7 +109,7 @@ class GAAClusterer(VectorSpaceClusterer):
                     centroid += self._normalise(vector)
                 else:
                     centroid += vector
-            centroid /= float(len(cluster))
+            centroid /= len(cluster)
             self._centroids.append(centroid)
         self._num_clusters = len(self._centroids)
 

{nltk-3.2.1 → nltk-3.3}/nltk/cluster/kmeans.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: K-Means Clusterer
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/cluster/util.py

@@ -1,10 +1,12 @@
 # Natural Language Toolkit: Clusterer Utilities
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
+# Contributor: J Richard Snape
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
-from __future__ import print_function, unicode_literals
+from __future__ import print_function, unicode_literals, division
+from abc import abstractmethod
 
 import copy
 from sys import stdout
@@ -18,6 +20,7 @@ except ImportError:
 from nltk.cluster.api import ClusterI
 from nltk.compat import python_2_unicode_compatible
 
+
 class VectorSpaceClusterer(ClusterI):
     """
     Abstract clusterer which takes tokens and maps them into a vector space.
@@ -45,11 +48,12 @@ class VectorSpaceClusterer(ClusterI):
 
         # use SVD to reduce the dimensionality
         if self._svd_dimensions and self._svd_dimensions < len(vectors[0]):
-            [u, d, vt] = numpy.linalg.svd(numpy.transpose(numpy.array(vectors)))
+            [u, d, vt] = numpy.linalg.svd(numpy.transpose(
+                            numpy.array(vectors)))
             S = d[:self._svd_dimensions] * \
                 numpy.identity(self._svd_dimensions, numpy.float64)
-            T = u[:,:self._svd_dimensions]
-            Dt = vt[:self._svd_dimensions,:]
+            T = u[:, :self._svd_dimensions]
+            Dt = vt[:self._svd_dimensions, :]
             vectors = numpy.transpose(numpy.dot(S, Dt))
             self._Tt = numpy.transpose(T)
 
@@ -60,11 +64,11 @@ class VectorSpaceClusterer(ClusterI):
         if assign_clusters:
             return [self.classify(vector) for vector in vectors]
 
+    @abstractmethod
     def cluster_vectorspace(self, vectors, trace):
         """
         Finds the clusters using the given set of vectors.
         """
-        raise NotImplementedError()
 
     def classify(self, vector):
         if self._should_normalise:
@@ -74,11 +78,11 @@ class VectorSpaceClusterer(ClusterI):
         cluster = self.classify_vectorspace(vector)
         return self.cluster_name(cluster)
 
+    @abstractmethod
     def classify_vectorspace(self, vector):
         """
         Returns the index of the appropriate cluster for the vector.
         """
-        raise NotImplementedError()
 
     def likelihood(self, vector, label):
         if self._should_normalise:
@@ -110,6 +114,7 @@ class VectorSpaceClusterer(ClusterI):
         """
         return vector / sqrt(numpy.dot(vector, vector))
 
+
 def euclidean_distance(u, v):
     """
     Returns the euclidean distance between vectors u and v. This is equivalent
@@ -118,12 +123,15 @@ def euclidean_distance(u, v):
     diff = u - v
     return sqrt(numpy.dot(diff, diff))
 
+
 def cosine_distance(u, v):
     """
-    Returns 1 minus the cosine of the angle between vectors v and u. This is equal to
-    1 - (u.v / |u||v|).
+    Returns 1 minus the cosine of the angle between vectors v and u. This is
+    equal to 1 - (u.v / |u||v|).
     """
-    return 1 - (numpy.dot(u, v) / (sqrt(numpy.dot(u, u)) * sqrt(numpy.dot(v, v))))
+    return 1 - (numpy.dot(u, v) / (
+                sqrt(numpy.dot(u, u)) * sqrt(numpy.dot(v, v))))
+
 
 class _DendrogramNode(object):
     """ Tree node of a dendrogram. """
@@ -164,6 +172,9 @@ class _DendrogramNode(object):
             groups.append(node.leaves())
         return groups
 
+    def __lt__(self, comparator):
+        return cosine_distance(self._value, comparator._value) < 0
+
 
 @python_2_unicode_compatible
 class Dendrogram(object):
@@ -214,7 +225,8 @@ class Dendrogram(object):
     def show(self, leaf_labels=[]):
         """
         Print the dendrogram in ASCII art to standard out.
-        :param leaf_labels: an optional list of strings to use for labeling the leaves
+        :param leaf_labels: an optional list of strings to use for labeling the
+                            leaves
         :type leaf_labels: list
         """
 
@@ -235,30 +247,35 @@ class Dendrogram(object):
 
         # find the bottom row and the best cell width
         width = max(map(len, last_row)) + 1
-        lhalf = width / 2
-        rhalf = width - lhalf - 1
+        lhalf = width // 2
+        rhalf = int(width - lhalf - 1)
 
         # display functions
         def format(centre, left=' ', right=' '):
             return '%s%s%s' % (lhalf*left, centre, right*rhalf)
+
         def display(str):
             stdout.write(str)
 
         # for each merge, top down
         queue = [(root._value, root)]
-        verticals = [ format(' ') for leaf in leaves ]
+        verticals = [format(' ') for leaf in leaves]
         while queue:
             priority, node = queue.pop()
-            child_left_leaf = list(map(lambda c: c.leaves(False)[0], node._children))
+            child_left_leaf = list(map(
+                                lambda c: c.leaves(False)[0], node._children))
             indices = list(map(leaves.index, child_left_leaf))
             if child_left_leaf:
                 min_idx = min(indices)
                 max_idx = max(indices)
             for i in range(len(leaves)):
                 if leaves[i] in child_left_leaf:
-                    if i == min_idx:    display(format(JOIN, ' ', HLINK))
-                    elif i == max_idx:  display(format(JOIN, HLINK, ' '))
-                    else:               display(format(JOIN, HLINK, HLINK))
+                    if i == min_idx:
+                        display(format(JOIN, ' ', HLINK))
+                    elif i == max_idx:
+                        display(format(JOIN, HLINK, ' '))
+                    else:
+                        display(format(JOIN, HLINK, HLINK))
                     verticals[i] = format(VLINK)
                 elif min_idx <= i <= max_idx:
                     display(format(HLINK, HLINK, HLINK))
@@ -285,5 +302,3 @@ class Dendrogram(object):
             root = self._items[0]
         leaves = root.leaves(False)
         return '<Dendrogram with %d leaves>' % len(leaves)
-
-