nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/classify/__init__.py

@@ -1,98 +1,98 @@
-# Natural Language Toolkit: Classifiers
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Edward Loper <edloper@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Classes and interfaces for labeling tokens with category labels (or
-"class labels").  Typically, labels are represented with strings
-(such as ``'health'`` or ``'sports'``).  Classifiers can be used to
-perform a wide range of classification tasks.  For example,
-classifiers can be used...
-
-- to classify documents by topic
-- to classify ambiguous words by which word sense is intended
-- to classify acoustic signals by which phoneme they represent
-- to classify sentences by their author
-
-Features
-========
-In order to decide which category label is appropriate for a given
-token, classifiers examine one or more 'features' of the token.  These
-"features" are typically chosen by hand, and indicate which aspects
-of the token are relevant to the classification decision.  For
-example, a document classifier might use a separate feature for each
-word, recording how often that word occurred in the document.
-
-Featuresets
-===========
-The features describing a token are encoded using a "featureset",
-which is a dictionary that maps from "feature names" to "feature
-values".  Feature names are unique strings that indicate what aspect
-of the token is encoded by the feature.  Examples include
-``'prevword'``, for a feature whose value is the previous word; and
-``'contains-word(library)'`` for a feature that is true when a document
-contains the word ``'library'``.  Feature values are typically
-booleans, numbers, or strings, depending on which feature they
-describe.
-
-Featuresets are typically constructed using a "feature detector"
-(also known as a "feature extractor").  A feature detector is a
-function that takes a token (and sometimes information about its
-context) as its input, and returns a featureset describing that token.
-For example, the following feature detector converts a document
-(stored as a list of words) to a featureset describing the set of
-words included in the document:
-
-    >>> # Define a feature detector function.
-    >>> def document_features(document):
-    ...     return dict([('contains-word(%s)' % w, True) for w in document])
-
-Feature detectors are typically applied to each token before it is fed
-to the classifier:
-
-    >>> # Classify each Gutenberg document.
-    >>> from nltk.corpus import gutenberg
-    >>> for fileid in gutenberg.fileids(): # doctest: +SKIP
-    ...     doc = gutenberg.words(fileid) # doctest: +SKIP
-    ...     print fileid, classifier.classify(document_features(doc)) # doctest: +SKIP
-
-The parameters that a feature detector expects will vary, depending on
-the task and the needs of the feature detector.  For example, a
-feature detector for word sense disambiguation (WSD) might take as its
-input a sentence, and the index of a word that should be classified,
-and return a featureset for that word.  The following feature detector
-for WSD includes features describing the left and right contexts of
-the target word:
-
-    >>> def wsd_features(sentence, index):
-    ...     featureset = {}
-    ...     for i in range(max(0, index-3), index):
-    ...         featureset['left-context(%s)' % sentence[i]] = True
-    ...     for i in range(index, max(index+3, len(sentence))):
-    ...         featureset['right-context(%s)' % sentence[i]] = True
-    ...     return featureset
-
-Training Classifiers
-====================
-Most classifiers are built by training them on a list of hand-labeled
-examples, known as the "training set".  Training sets are represented
-as lists of ``(featuredict, label)`` tuples.
-"""
-
-from nltk.classify.api import ClassifierI, MultiClassifierI
-from nltk.classify.megam import config_megam, call_megam
-from nltk.classify.weka import WekaClassifier, config_weka
-from nltk.classify.naivebayes import NaiveBayesClassifier
-from nltk.classify.positivenaivebayes import PositiveNaiveBayesClassifier
-from nltk.classify.decisiontree import DecisionTreeClassifier
-from nltk.classify.rte_classify import rte_classifier, rte_features, RTEFeatureExtractor
-from nltk.classify.util import accuracy, apply_features, log_likelihood
-from nltk.classify.scikitlearn import SklearnClassifier
-from nltk.classify.maxent import (MaxentClassifier, BinaryMaxentFeatureEncoding,
-                                  TypedMaxentFeatureEncoding,
-                                  ConditionalExponentialClassifier)
-from nltk.classify.senna import Senna
-from nltk.classify.textcat import TextCat
+# Natural Language Toolkit: Classifiers
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classes and interfaces for labeling tokens with category labels (or
+"class labels").  Typically, labels are represented with strings
+(such as ``'health'`` or ``'sports'``).  Classifiers can be used to
+perform a wide range of classification tasks.  For example,
+classifiers can be used...
+
+- to classify documents by topic
+- to classify ambiguous words by which word sense is intended
+- to classify acoustic signals by which phoneme they represent
+- to classify sentences by their author
+
+Features
+========
+In order to decide which category label is appropriate for a given
+token, classifiers examine one or more 'features' of the token.  These
+"features" are typically chosen by hand, and indicate which aspects
+of the token are relevant to the classification decision.  For
+example, a document classifier might use a separate feature for each
+word, recording how often that word occurred in the document.
+
+Featuresets
+===========
+The features describing a token are encoded using a "featureset",
+which is a dictionary that maps from "feature names" to "feature
+values".  Feature names are unique strings that indicate what aspect
+of the token is encoded by the feature.  Examples include
+``'prevword'``, for a feature whose value is the previous word; and
+``'contains-word(library)'`` for a feature that is true when a document
+contains the word ``'library'``.  Feature values are typically
+booleans, numbers, or strings, depending on which feature they
+describe.
+
+Featuresets are typically constructed using a "feature detector"
+(also known as a "feature extractor").  A feature detector is a
+function that takes a token (and sometimes information about its
+context) as its input, and returns a featureset describing that token.
+For example, the following feature detector converts a document
+(stored as a list of words) to a featureset describing the set of
+words included in the document:
+
+    >>> # Define a feature detector function.
+    >>> def document_features(document):
+    ...     return dict([('contains-word(%s)' % w, True) for w in document])
+
+Feature detectors are typically applied to each token before it is fed
+to the classifier:
+
+    >>> # Classify each Gutenberg document.
+    >>> from nltk.corpus import gutenberg
+    >>> for fileid in gutenberg.fileids(): # doctest: +SKIP
+    ...     doc = gutenberg.words(fileid) # doctest: +SKIP
+    ...     print fileid, classifier.classify(document_features(doc)) # doctest: +SKIP
+
+The parameters that a feature detector expects will vary, depending on
+the task and the needs of the feature detector.  For example, a
+feature detector for word sense disambiguation (WSD) might take as its
+input a sentence, and the index of a word that should be classified,
+and return a featureset for that word.  The following feature detector
+for WSD includes features describing the left and right contexts of
+the target word:
+
+    >>> def wsd_features(sentence, index):
+    ...     featureset = {}
+    ...     for i in range(max(0, index-3), index):
+    ...         featureset['left-context(%s)' % sentence[i]] = True
+    ...     for i in range(index, max(index+3, len(sentence))):
+    ...         featureset['right-context(%s)' % sentence[i]] = True
+    ...     return featureset
+
+Training Classifiers
+====================
+Most classifiers are built by training them on a list of hand-labeled
+examples, known as the "training set".  Training sets are represented
+as lists of ``(featuredict, label)`` tuples.
+"""
+
+from nltk.classify.api import ClassifierI, MultiClassifierI
+from nltk.classify.megam import config_megam, call_megam
+from nltk.classify.weka import WekaClassifier, config_weka
+from nltk.classify.naivebayes import NaiveBayesClassifier
+from nltk.classify.positivenaivebayes import PositiveNaiveBayesClassifier
+from nltk.classify.decisiontree import DecisionTreeClassifier
+from nltk.classify.rte_classify import rte_classifier, rte_features, RTEFeatureExtractor
+from nltk.classify.util import accuracy, apply_features, log_likelihood
+from nltk.classify.scikitlearn import SklearnClassifier
+from nltk.classify.maxent import (MaxentClassifier, BinaryMaxentFeatureEncoding,
+                                  TypedMaxentFeatureEncoding,
+                                  ConditionalExponentialClassifier)
+from nltk.classify.senna import Senna
+from nltk.classify.textcat import TextCat

{nltk-3.2 → nltk-3.3}/nltk/classify/api.py

@@ -1,193 +1,193 @@
-# Natural Language Toolkit: Classifier Interface
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Edward Loper <edloper@gmail.com>
-#         Steven Bird <stevenbird1@gmail.com> (minor additions)
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Interfaces for labeling tokens with category labels (or "class labels").
-
-``ClassifierI`` is a standard interface for "single-category
-classification", in which the set of categories is known, the number
-of categories is finite, and each text belongs to exactly one
-category.
-
-``MultiClassifierI`` is a standard interface for "multi-category
-classification", which is like single-category classification except
-that each text belongs to zero or more categories.
-"""
-from nltk.internals import overridden
-
-##//////////////////////////////////////////////////////
-#{ Classification Interfaces
-##//////////////////////////////////////////////////////
-
-class ClassifierI(object):
-    """
-    A processing interface for labeling tokens with a single category
-    label (or "class").  Labels are typically strs or
-    ints, but can be any immutable type.  The set of labels
-    that the classifier chooses from must be fixed and finite.
-
-    Subclasses must define:
-      - ``labels()``
-      - either ``classify()`` or ``classify_many()`` (or both)
-
-    Subclasses may define:
-      - either ``prob_classify()`` or ``prob_classify_many()`` (or both)
-    """
-    def labels(self):
-        """
-        :return: the list of category labels used by this classifier.
-        :rtype: list of (immutable)
-        """
-        raise NotImplementedError()
-
-    def classify(self, featureset):
-        """
-        :return: the most appropriate label for the given featureset.
-        :rtype: label
-        """
-        if overridden(self.classify_many):
-            return self.classify_many([featureset])[0]
-        else:
-            raise NotImplementedError()
-
-    def prob_classify(self, featureset):
-        """
-        :return: a probability distribution over labels for the given
-            featureset.
-        :rtype: ProbDistI
-        """
-        if overridden(self.prob_classify_many):
-            return self.prob_classify_many([featureset])[0]
-        else:
-            raise NotImplementedError()
-
-    def classify_many(self, featuresets):
-        """
-        Apply ``self.classify()`` to each element of ``featuresets``.  I.e.:
-
-            return [self.classify(fs) for fs in featuresets]
-
-        :rtype: list(label)
-        """
-        return [self.classify(fs) for fs in featuresets]
-
-    def prob_classify_many(self, featuresets):
-        """
-        Apply ``self.prob_classify()`` to each element of ``featuresets``.  I.e.:
-
-            return [self.prob_classify(fs) for fs in featuresets]
-
-        :rtype: list(ProbDistI)
-        """
-        return [self.prob_classify(fs) for fs in featuresets]
-
-
-class MultiClassifierI(object):
-    """
-    A processing interface for labeling tokens with zero or more
-    category labels (or "labels").  Labels are typically strs
-    or ints, but can be any immutable type.  The set of labels
-    that the multi-classifier chooses from must be fixed and finite.
-
-    Subclasses must define:
-      - ``labels()``
-      - either ``classify()`` or ``classify_many()`` (or both)
-
-    Subclasses may define:
-      - either ``prob_classify()`` or ``prob_classify_many()`` (or both)
-    """
-    def labels(self):
-        """
-        :return: the list of category labels used by this classifier.
-        :rtype: list of (immutable)
-        """
-        raise NotImplementedError()
-
-    def classify(self, featureset):
-        """
-        :return: the most appropriate set of labels for the given featureset.
-        :rtype: set(label)
-        """
-        if overridden(self.classify_many):
-            return self.classify_many([featureset])[0]
-        else:
-            raise NotImplementedError()
-
-    def prob_classify(self, featureset):
-        """
-        :return: a probability distribution over sets of labels for the
-            given featureset.
-        :rtype: ProbDistI
-        """
-        if overridden(self.prob_classify_many):
-            return self.prob_classify_many([featureset])[0]
-        else:
-            raise NotImplementedError()
-
-    def classify_many(self, featuresets):
-        """
-        Apply ``self.classify()`` to each element of ``featuresets``.  I.e.:
-
-            return [self.classify(fs) for fs in featuresets]
-
-        :rtype: list(set(label))
-        """
-        return [self.classify(fs) for fs in featuresets]
-
-    def prob_classify_many(self, featuresets):
-        """
-        Apply ``self.prob_classify()`` to each element of ``featuresets``.  I.e.:
-
-            return [self.prob_classify(fs) for fs in featuresets]
-
-        :rtype: list(ProbDistI)
-        """
-        return [self.prob_classify(fs) for fs in featuresets]
-
-
-# # [XX] IN PROGRESS:
-# class SequenceClassifierI(object):
-#     """
-#     A processing interface for labeling sequences of tokens with a
-#     single category label (or "class").  Labels are typically
-#     strs or ints, but can be any immutable type.  The set
-#     of labels that the classifier chooses from must be fixed and
-#     finite.
-#     """
-#     def labels(self):
-#         """
-#         :return: the list of category labels used by this classifier.
-#         :rtype: list of (immutable)
-#         """
-#         raise NotImplementedError()
-
-#     def prob_classify(self, featureset):
-#         """
-#         Return a probability distribution over labels for the given
-#         featureset.
-
-#         If ``featureset`` is a list of featuresets, then return a
-#         corresponding list containing the probability distribution
-#         over labels for each of the given featuresets, where the
-#         *i*\ th element of this list is the most appropriate label for
-#         the *i*\ th element of ``featuresets``.
-#         """
-#         raise NotImplementedError()
-
-#     def classify(self, featureset):
-#         """
-#         Return the most appropriate label for the given featureset.
-
-#         If ``featureset`` is a list of featuresets, then return a
-#         corresponding list containing the most appropriate label for
-#         each of the given featuresets, where the *i*\ th element of
-#         this list is the most appropriate label for the *i*\ th element
-#         of ``featuresets``.
-#         """
-#         raise NotImplementedError()
-
+# Natural Language Toolkit: Classifier Interface
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interfaces for labeling tokens with category labels (or "class labels").
+
+``ClassifierI`` is a standard interface for "single-category
+classification", in which the set of categories is known, the number
+of categories is finite, and each text belongs to exactly one
+category.
+
+``MultiClassifierI`` is a standard interface for "multi-category
+classification", which is like single-category classification except
+that each text belongs to zero or more categories.
+"""
+from nltk.internals import overridden
+
+##//////////////////////////////////////////////////////
+#{ Classification Interfaces
+##//////////////////////////////////////////////////////
+
+class ClassifierI(object):
+    """
+    A processing interface for labeling tokens with a single category
+    label (or "class").  Labels are typically strs or
+    ints, but can be any immutable type.  The set of labels
+    that the classifier chooses from must be fixed and finite.
+
+    Subclasses must define:
+      - ``labels()``
+      - either ``classify()`` or ``classify_many()`` (or both)
+
+    Subclasses may define:
+      - either ``prob_classify()`` or ``prob_classify_many()`` (or both)
+    """
+    def labels(self):
+        """
+        :return: the list of category labels used by this classifier.
+        :rtype: list of (immutable)
+        """
+        raise NotImplementedError()
+
+    def classify(self, featureset):
+        """
+        :return: the most appropriate label for the given featureset.
+        :rtype: label
+        """
+        if overridden(self.classify_many):
+            return self.classify_many([featureset])[0]
+        else:
+            raise NotImplementedError()
+
+    def prob_classify(self, featureset):
+        """
+        :return: a probability distribution over labels for the given
+            featureset.
+        :rtype: ProbDistI
+        """
+        if overridden(self.prob_classify_many):
+            return self.prob_classify_many([featureset])[0]
+        else:
+            raise NotImplementedError()
+
+    def classify_many(self, featuresets):
+        """
+        Apply ``self.classify()`` to each element of ``featuresets``.  I.e.:
+
+            return [self.classify(fs) for fs in featuresets]
+
+        :rtype: list(label)
+        """
+        return [self.classify(fs) for fs in featuresets]
+
+    def prob_classify_many(self, featuresets):
+        """
+        Apply ``self.prob_classify()`` to each element of ``featuresets``.  I.e.:
+
+            return [self.prob_classify(fs) for fs in featuresets]
+
+        :rtype: list(ProbDistI)
+        """
+        return [self.prob_classify(fs) for fs in featuresets]
+
+
+class MultiClassifierI(object):
+    """
+    A processing interface for labeling tokens with zero or more
+    category labels (or "labels").  Labels are typically strs
+    or ints, but can be any immutable type.  The set of labels
+    that the multi-classifier chooses from must be fixed and finite.
+
+    Subclasses must define:
+      - ``labels()``
+      - either ``classify()`` or ``classify_many()`` (or both)
+
+    Subclasses may define:
+      - either ``prob_classify()`` or ``prob_classify_many()`` (or both)
+    """
+    def labels(self):
+        """
+        :return: the list of category labels used by this classifier.
+        :rtype: list of (immutable)
+        """
+        raise NotImplementedError()
+
+    def classify(self, featureset):
+        """
+        :return: the most appropriate set of labels for the given featureset.
+        :rtype: set(label)
+        """
+        if overridden(self.classify_many):
+            return self.classify_many([featureset])[0]
+        else:
+            raise NotImplementedError()
+
+    def prob_classify(self, featureset):
+        """
+        :return: a probability distribution over sets of labels for the
+            given featureset.
+        :rtype: ProbDistI
+        """
+        if overridden(self.prob_classify_many):
+            return self.prob_classify_many([featureset])[0]
+        else:
+            raise NotImplementedError()
+
+    def classify_many(self, featuresets):
+        """
+        Apply ``self.classify()`` to each element of ``featuresets``.  I.e.:
+
+            return [self.classify(fs) for fs in featuresets]
+
+        :rtype: list(set(label))
+        """
+        return [self.classify(fs) for fs in featuresets]
+
+    def prob_classify_many(self, featuresets):
+        """
+        Apply ``self.prob_classify()`` to each element of ``featuresets``.  I.e.:
+
+            return [self.prob_classify(fs) for fs in featuresets]
+
+        :rtype: list(ProbDistI)
+        """
+        return [self.prob_classify(fs) for fs in featuresets]
+
+
+# # [XX] IN PROGRESS:
+# class SequenceClassifierI(object):
+#     """
+#     A processing interface for labeling sequences of tokens with a
+#     single category label (or "class").  Labels are typically
+#     strs or ints, but can be any immutable type.  The set
+#     of labels that the classifier chooses from must be fixed and
+#     finite.
+#     """
+#     def labels(self):
+#         """
+#         :return: the list of category labels used by this classifier.
+#         :rtype: list of (immutable)
+#         """
+#         raise NotImplementedError()
+
+#     def prob_classify(self, featureset):
+#         """
+#         Return a probability distribution over labels for the given
+#         featureset.
+
+#         If ``featureset`` is a list of featuresets, then return a
+#         corresponding list containing the probability distribution
+#         over labels for each of the given featuresets, where the
+#         *i*\ th element of this list is the most appropriate label for
+#         the *i*\ th element of ``featuresets``.
+#         """
+#         raise NotImplementedError()
+
+#     def classify(self, featureset):
+#         """
+#         Return the most appropriate label for the given featureset.
+
+#         If ``featureset`` is a list of featuresets, then return a
+#         corresponding list containing the most appropriate label for
+#         each of the given featuresets, where the *i*\ th element of
+#         this list is the most appropriate label for the *i*\ th element
+#         of ``featuresets``.
+#         """
+#         raise NotImplementedError()
+