nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/classify/senna.py

@@ -1,181 +1,183 @@
-# encoding: utf-8
-# Natural Language Toolkit: Senna Interface
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Rami Al-Rfou' <ralrfou@cs.stonybrook.edu>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-A general interface to the SENNA pipeline that supports any of the
-operations specified in SUPPORTED_OPERATIONS.
-
-Applying multiple operations at once has the speed advantage. For example,
-Senna will automatically determine POS tags if you are extracting named
-entities. Applying both of the operations will cost only the time of
-extracting the named entities.
-
-The SENNA pipeline has a fixed maximum size of the sentences that it can read.
-By default it is 1024 token/sentence. If you have larger sentences, changing
-the MAX_SENTENCE_SIZE value in SENNA_main.c should be considered and your
-system specific binary should be rebuilt. Otherwise this could introduce
-misalignment errors.
-
-The input is:
-- path to the directory that contains SENNA executables. If the path is incorrect, 
-   Senna will automatically search for executable file specified in SENNA environment variable
-- List of the operations needed to be performed.
-- (optionally) the encoding of the input data (default:utf-8)
-
-    >>> from __future__ import unicode_literals
-    >>> from nltk.classify import Senna
-    >>> pipeline = Senna('/usr/share/senna-v2.0', ['pos', 'chk', 'ner'])
-    >>> sent = 'Dusseldorf is an international business center'.split()
-    >>> [(token['word'], token['chk'], token['ner'], token['pos']) for token in pipeline.tag(sent)]
-    [('Dusseldorf', 'B-NP', 'B-LOC', 'NNP'), ('is', 'B-VP', 'O', 'VBZ'), ('an', 'B-NP', 'O', 'DT'), 
-    ('international', 'I-NP', 'O', 'JJ'), ('business', 'I-NP', 'O', 'NN'), ('center', 'I-NP', 'O', 'NN')]
-"""
-
-
-from __future__ import unicode_literals
-from os import path, sep, environ
-from subprocess import Popen, PIPE
-from platform import architecture, system
-
-from nltk.tag.api import TaggerI
-from nltk.compat import text_type, python_2_unicode_compatible
-
-_senna_url = 'http://ml.nec-labs.com/senna/'
-
-
-@python_2_unicode_compatible
-class Senna(TaggerI):
-
-    SUPPORTED_OPERATIONS = ['pos', 'chk', 'ner']
-
-    def __init__(self, senna_path, operations, encoding='utf-8'):
-        self._encoding = encoding
-        self._path = path.normpath(senna_path) + sep 
-        
-        # Verifies the existence of the executable on the self._path first    
-        #senna_binary_file_1 = self.executable(self._path)
-        exe_file_1 = self.executable(self._path)
-        if not path.isfile(exe_file_1):
-            # Check for the system environment 
-            if 'SENNA' in environ:
-                #self._path = path.join(environ['SENNA'],'')  
-                self._path = path.normpath(environ['SENNA']) + sep 
-                exe_file_2 = self.executable(self._path)
-                if not path.isfile(exe_file_2):
-                    raise OSError("Senna executable expected at %s or %s but not found" % (exe_file_1,exe_file_2))
-        
-        self.operations = operations
-
-    
-    def executable(self, base_path):
-        """
-        The function that determines the system specific binary that should be
-        used in the pipeline. In case, the system is not known the default senna binary will
-        be used.
-        """ 
-        os_name = system()
-        if os_name == 'Linux':
-            bits = architecture()[0]
-            if bits == '64bit':
-                return path.join(base_path, 'senna-linux64')
-            return path.join(base_path, 'senna-linux32')
-        if os_name == 'Windows':
-            return path.join(base_path, 'senna-win32.exe')
-        if os_name == 'Darwin':
-            return path.join(base_path, 'senna-osx')
-        return path.join(base_path, 'senna')
-        
-    def _map(self):
-        """
-        A method that calculates the order of the columns that SENNA pipeline
-        will output the tags into. This depends on the operations being ordered.
-        """
-        _map = {}
-        i = 1
-        for operation in Senna.SUPPORTED_OPERATIONS:
-            if operation in self.operations:
-                _map[operation] = i
-                i+= 1
-        return _map
-
-    def tag(self, tokens):
-        """
-        Applies the specified operation(s) on a list of tokens.
-        """
-        return self.tag_sents([tokens])[0]
-
-    def tag_sents(self, sentences):
-        """
-        Applies the tag method over a list of sentences. This method will return a
-        list of dictionaries. Every dictionary will contain a word with its
-        calculated annotations/tags.
-        """
-        encoding = self._encoding
-        
-        if not path.isfile(self.executable(self._path)):
-            raise OSError("Senna executable expected at %s but not found" % self.executable(self._path))
-        
-         
-        # Build the senna command to run the tagger
-        _senna_cmd = [self.executable(self._path), '-path', self._path, '-usrtokens', '-iobtags']
-        _senna_cmd.extend(['-'+op for op in self.operations])
-
-        # Serialize the actual sentences to a temporary string
-        _input = '\n'.join((' '.join(x) for x in sentences))+'\n'
-        if isinstance(_input, text_type) and encoding:
-            _input = _input.encode(encoding)
-
-        # Run the tagger and get the output
-        p = Popen(_senna_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
-        (stdout, stderr) = p.communicate(input=_input)
-        senna_output = stdout
-
-        # Check the return code.
-        if p.returncode != 0:
-            raise RuntimeError('Senna command failed! Details: %s' % stderr)
-
-        if encoding:
-            senna_output = stdout.decode(encoding)
-
-        # Output the tagged sentences
-        map_ = self._map()
-        tagged_sentences = [[]]
-        sentence_index = 0
-        token_index = 0
-        for tagged_word in senna_output.strip().split("\n"):
-            if not tagged_word:
-                tagged_sentences.append([])
-                sentence_index += 1
-                token_index = 0
-                continue
-            tags = tagged_word.split('\t')
-            result = {}
-            for tag in map_:
-              result[tag] = tags[map_[tag]].strip()
-            try:
-              result['word'] = sentences[sentence_index][token_index]
-            except IndexError:
-              raise IndexError(
-                "Misalignment error occurred at sentence number %d. Possible reason"
-                " is that the sentence size exceeded the maximum size. Check the "
-                "documentation of Senna class for more information."
-                % sentence_index)
-            tagged_sentences[-1].append(result)
-            token_index += 1
-        return tagged_sentences
-
-
-# skip doctests if Senna is not installed
-def setup_module(module):
-    from nose import SkipTest
-    try:
-        tagger = Senna('/usr/share/senna-v2.0', ['pos', 'chk', 'ner'])
-    except OSError:
-        raise SkipTest("Senna executable not found")
-
-
+# encoding: utf-8
+# Natural Language Toolkit: Senna Interface
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Rami Al-Rfou' <ralrfou@cs.stonybrook.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A general interface to the SENNA pipeline that supports any of the
+operations specified in SUPPORTED_OPERATIONS.
+
+Applying multiple operations at once has the speed advantage. For example,
+Senna will automatically determine POS tags if you are extracting named
+entities. Applying both of the operations will cost only the time of
+extracting the named entities.
+
+The SENNA pipeline has a fixed maximum size of the sentences that it can read.
+By default it is 1024 token/sentence. If you have larger sentences, changing
+the MAX_SENTENCE_SIZE value in SENNA_main.c should be considered and your
+system specific binary should be rebuilt. Otherwise this could introduce
+misalignment errors.
+
+The input is:
+- path to the directory that contains SENNA executables. If the path is incorrect,
+   Senna will automatically search for executable file specified in SENNA environment variable
+- List of the operations needed to be performed.
+- (optionally) the encoding of the input data (default:utf-8)
+
+Note: Unit tests for this module can be found in test/unit/test_senna.py
+
+    >>> from __future__ import unicode_literals
+    >>> from nltk.classify import Senna
+    >>> pipeline = Senna('/usr/share/senna-v3.0', ['pos', 'chk', 'ner'])
+    >>> sent = 'Dusseldorf is an international business center'.split()
+    >>> [(token['word'], token['chk'], token['ner'], token['pos']) for token in pipeline.tag(sent)] # doctest: +SKIP
+    [('Dusseldorf', 'B-NP', 'B-LOC', 'NNP'), ('is', 'B-VP', 'O', 'VBZ'), ('an', 'B-NP', 'O', 'DT'),
+    ('international', 'I-NP', 'O', 'JJ'), ('business', 'I-NP', 'O', 'NN'), ('center', 'I-NP', 'O', 'NN')]
+"""
+
+
+from __future__ import unicode_literals
+from os import path, sep, environ
+from subprocess import Popen, PIPE
+from platform import architecture, system
+
+from six import text_type
+
+from nltk.tag.api import TaggerI
+from nltk.compat import python_2_unicode_compatible
+
+_senna_url = 'http://ml.nec-labs.com/senna/'
+
+
+@python_2_unicode_compatible
+class Senna(TaggerI):
+
+    SUPPORTED_OPERATIONS = ['pos', 'chk', 'ner']
+
+    def __init__(self, senna_path, operations, encoding='utf-8'):
+        self._encoding = encoding
+        self._path = path.normpath(senna_path) + sep
+
+        # Verifies the existence of the executable on the self._path first
+        #senna_binary_file_1 = self.executable(self._path)
+        exe_file_1 = self.executable(self._path)
+        if not path.isfile(exe_file_1):
+            # Check for the system environment
+            if 'SENNA' in environ:
+                #self._path = path.join(environ['SENNA'],'')
+                self._path = path.normpath(environ['SENNA']) + sep
+                exe_file_2 = self.executable(self._path)
+                if not path.isfile(exe_file_2):
+                    raise OSError("Senna executable expected at %s or %s but not found" % (exe_file_1,exe_file_2))
+
+        self.operations = operations
+
+
+    def executable(self, base_path):
+        """
+        The function that determines the system specific binary that should be
+        used in the pipeline. In case, the system is not known the default senna binary will
+        be used.
+        """
+        os_name = system()
+        if os_name == 'Linux':
+            bits = architecture()[0]
+            if bits == '64bit':
+                return path.join(base_path, 'senna-linux64')
+            return path.join(base_path, 'senna-linux32')
+        if os_name == 'Windows':
+            return path.join(base_path, 'senna-win32.exe')
+        if os_name == 'Darwin':
+            return path.join(base_path, 'senna-osx')
+        return path.join(base_path, 'senna')
+
+    def _map(self):
+        """
+        A method that calculates the order of the columns that SENNA pipeline
+        will output the tags into. This depends on the operations being ordered.
+        """
+        _map = {}
+        i = 1
+        for operation in Senna.SUPPORTED_OPERATIONS:
+            if operation in self.operations:
+                _map[operation] = i
+                i+= 1
+        return _map
+
+    def tag(self, tokens):
+        """
+        Applies the specified operation(s) on a list of tokens.
+        """
+        return self.tag_sents([tokens])[0]
+
+    def tag_sents(self, sentences):
+        """
+        Applies the tag method over a list of sentences. This method will return a
+        list of dictionaries. Every dictionary will contain a word with its
+        calculated annotations/tags.
+        """
+        encoding = self._encoding
+
+        if not path.isfile(self.executable(self._path)):
+            raise OSError("Senna executable expected at %s but not found" % self.executable(self._path))
+
+
+        # Build the senna command to run the tagger
+        _senna_cmd = [self.executable(self._path), '-path', self._path, '-usrtokens', '-iobtags']
+        _senna_cmd.extend(['-'+op for op in self.operations])
+
+        # Serialize the actual sentences to a temporary string
+        _input = '\n'.join((' '.join(x) for x in sentences))+'\n'
+        if isinstance(_input, text_type) and encoding:
+            _input = _input.encode(encoding)
+
+        # Run the tagger and get the output
+        p = Popen(_senna_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        (stdout, stderr) = p.communicate(input=_input)
+        senna_output = stdout
+
+        # Check the return code.
+        if p.returncode != 0:
+            raise RuntimeError('Senna command failed! Details: %s' % stderr)
+
+        if encoding:
+            senna_output = stdout.decode(encoding)
+
+        # Output the tagged sentences
+        map_ = self._map()
+        tagged_sentences = [[]]
+        sentence_index = 0
+        token_index = 0
+        for tagged_word in senna_output.strip().split("\n"):
+            if not tagged_word:
+                tagged_sentences.append([])
+                sentence_index += 1
+                token_index = 0
+                continue
+            tags = tagged_word.split('\t')
+            result = {}
+            for tag in map_:
+              result[tag] = tags[map_[tag]].strip()
+            try:
+              result['word'] = sentences[sentence_index][token_index]
+            except IndexError:
+              raise IndexError(
+                "Misalignment error occurred at sentence number %d. Possible reason"
+                " is that the sentence size exceeded the maximum size. Check the "
+                "documentation of Senna class for more information."
+                % sentence_index)
+            tagged_sentences[-1].append(result)
+            token_index += 1
+        return tagged_sentences
+
+
+# skip doctests if Senna is not installed
+def setup_module(module):
+    from nose import SkipTest
+    try:
+        tagger = Senna('/usr/share/senna-v3.0', ['pos', 'chk', 'ner'])
+    except OSError:
+        raise SkipTest("Senna executable not found")

{nltk-3.2 → nltk-3.3}/nltk/classify/svm.py

@@ -1,15 +1,15 @@
-# Natural Language Toolkit: SVM-based classifier
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Leon Derczynski <leon@dcs.shef.ac.uk>
-#
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-"""
-nltk.classify.svm was deprecated. For classification based
-on support vector machines SVMs use nltk.classify.scikitlearn
-(or `scikit-learn <http://scikit-learn.org>`_ directly).
-"""
-class SvmClassifier(object):
-    def __init__(self, *args, **kwargs):
-        raise NotImplementedError(__doc__)
+# Natural Language Toolkit: SVM-based classifier
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Leon Derczynski <leon@dcs.shef.ac.uk>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+nltk.classify.svm was deprecated. For classification based
+on support vector machines SVMs use nltk.classify.scikitlearn
+(or `scikit-learn <http://scikit-learn.org>`_ directly).
+"""
+class SvmClassifier(object):
+    def __init__(self, *args, **kwargs):
+        raise NotImplementedError(__doc__)

{nltk-3.2 → nltk-3.3}/nltk/classify/tadm.py

@@ -1,112 +1,113 @@
-# Natural Language Toolkit: Interface to TADM Classifier
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Joseph Frazee <jfrazee@mail.utexas.edu>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-from __future__ import print_function, unicode_literals
-
-import sys
-import subprocess
-
-from nltk import compat
-from nltk.internals import find_binary
-try:
-    import numpy
-except ImportError:
-    pass
-
-_tadm_bin = None
-def config_tadm(bin=None):
-    global _tadm_bin
-    _tadm_bin = find_binary(
-        'tadm', bin,
-        env_vars=['TADM'],
-        binary_names=['tadm'],
-        url='http://tadm.sf.net')
-
-def write_tadm_file(train_toks, encoding, stream):
-    """
-    Generate an input file for ``tadm`` based on the given corpus of
-    classified tokens.
-
-    :type train_toks: list(tuple(dict, str))
-    :param train_toks: Training data, represented as a list of
-        pairs, the first member of which is a feature dictionary,
-        and the second of which is a classification label.
-    :type encoding: TadmEventMaxentFeatureEncoding
-    :param encoding: A feature encoding, used to convert featuresets
-        into feature vectors.
-    :type stream: stream
-    :param stream: The stream to which the ``tadm`` input file should be
-        written.
-    """
-    # See the following for a file format description:
-    #
-    # http://sf.net/forum/forum.php?thread_id=1391502&forum_id=473054
-    # http://sf.net/forum/forum.php?thread_id=1675097&forum_id=473054
-    labels = encoding.labels()
-    for featureset, label in train_toks:
-        length_line = '%d\n' % len(labels)
-        stream.write(length_line)
-        for known_label in labels:
-            v = encoding.encode(featureset, known_label)
-            line = '%d %d %s\n' % (
-                int(label == known_label),
-                len(v),
-                ' '.join('%d %d' % u for u in v)
-            )
-            stream.write(line)
-
-def parse_tadm_weights(paramfile):
-    """
-    Given the stdout output generated by ``tadm`` when training a
-    model, return a ``numpy`` array containing the corresponding weight
-    vector.
-    """
-    weights = []
-    for line in paramfile:
-        weights.append(float(line.strip()))
-    return numpy.array(weights, 'd')
-
-def call_tadm(args):
-    """
-    Call the ``tadm`` binary with the given arguments.
-    """
-    if isinstance(args, compat.string_types):
-        raise TypeError('args should be a list of strings')
-    if _tadm_bin is None:
-        config_tadm()
-
-    # Call tadm via a subprocess
-    cmd = [_tadm_bin] + args
-    p = subprocess.Popen(cmd, stdout=sys.stdout)
-    (stdout, stderr) = p.communicate()
-
-    # Check the return code.
-    if p.returncode != 0:
-        print()
-        print(stderr)
-        raise OSError('tadm command failed!')
-
-def names_demo():
-    from nltk.classify.util import names_demo
-    from nltk.classify.maxent import TadmMaxentClassifier
-    classifier = names_demo(TadmMaxentClassifier.train)
-
-def encoding_demo():
-    import sys
-    from nltk.classify.maxent import TadmEventMaxentFeatureEncoding
-    tokens = [({'f0':1, 'f1':1, 'f3':1}, 'A'),
-              ({'f0':1, 'f2':1, 'f4':1}, 'B'),
-              ({'f0':2, 'f2':1, 'f3':1, 'f4':1}, 'A')]
-    encoding = TadmEventMaxentFeatureEncoding.train(tokens)
-    write_tadm_file(tokens, encoding, sys.stdout)
-    print()
-    for i in range(encoding.length()):
-        print('%s --> %d' % (encoding.describe(i), i))
-    print()
-
-if __name__ == '__main__':
-    encoding_demo()
-    names_demo()
+# Natural Language Toolkit: Interface to TADM Classifier
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Joseph Frazee <jfrazee@mail.utexas.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+import sys
+import subprocess
+
+from six import string_types
+
+from nltk.internals import find_binary
+try:
+    import numpy
+except ImportError:
+    pass
+
+_tadm_bin = None
+def config_tadm(bin=None):
+    global _tadm_bin
+    _tadm_bin = find_binary(
+        'tadm', bin,
+        env_vars=['TADM'],
+        binary_names=['tadm'],
+        url='http://tadm.sf.net')
+
+def write_tadm_file(train_toks, encoding, stream):
+    """
+    Generate an input file for ``tadm`` based on the given corpus of
+    classified tokens.
+
+    :type train_toks: list(tuple(dict, str))
+    :param train_toks: Training data, represented as a list of
+        pairs, the first member of which is a feature dictionary,
+        and the second of which is a classification label.
+    :type encoding: TadmEventMaxentFeatureEncoding
+    :param encoding: A feature encoding, used to convert featuresets
+        into feature vectors.
+    :type stream: stream
+    :param stream: The stream to which the ``tadm`` input file should be
+        written.
+    """
+    # See the following for a file format description:
+    #
+    # http://sf.net/forum/forum.php?thread_id=1391502&forum_id=473054
+    # http://sf.net/forum/forum.php?thread_id=1675097&forum_id=473054
+    labels = encoding.labels()
+    for featureset, label in train_toks:
+        length_line = '%d\n' % len(labels)
+        stream.write(length_line)
+        for known_label in labels:
+            v = encoding.encode(featureset, known_label)
+            line = '%d %d %s\n' % (
+                int(label == known_label),
+                len(v),
+                ' '.join('%d %d' % u for u in v)
+            )
+            stream.write(line)
+
+def parse_tadm_weights(paramfile):
+    """
+    Given the stdout output generated by ``tadm`` when training a
+    model, return a ``numpy`` array containing the corresponding weight
+    vector.
+    """
+    weights = []
+    for line in paramfile:
+        weights.append(float(line.strip()))
+    return numpy.array(weights, 'd')
+
+def call_tadm(args):
+    """
+    Call the ``tadm`` binary with the given arguments.
+    """
+    if isinstance(args, string_types):
+        raise TypeError('args should be a list of strings')
+    if _tadm_bin is None:
+        config_tadm()
+
+    # Call tadm via a subprocess
+    cmd = [_tadm_bin] + args
+    p = subprocess.Popen(cmd, stdout=sys.stdout)
+    (stdout, stderr) = p.communicate()
+
+    # Check the return code.
+    if p.returncode != 0:
+        print()
+        print(stderr)
+        raise OSError('tadm command failed!')
+
+def names_demo():
+    from nltk.classify.util import names_demo
+    from nltk.classify.maxent import TadmMaxentClassifier
+    classifier = names_demo(TadmMaxentClassifier.train)
+
+def encoding_demo():
+    import sys
+    from nltk.classify.maxent import TadmEventMaxentFeatureEncoding
+    tokens = [({'f0':1, 'f1':1, 'f3':1}, 'A'),
+              ({'f0':1, 'f2':1, 'f4':1}, 'B'),
+              ({'f0':2, 'f2':1, 'f3':1, 'f4':1}, 'A')]
+    encoding = TadmEventMaxentFeatureEncoding.train(tokens)
+    write_tadm_file(tokens, encoding, sys.stdout)
+    print()
+    for i in range(encoding.length()):
+        print('%s --> %d' % (encoding.describe(i), i))
+    print()
+
+if __name__ == '__main__':
+    encoding_demo()
+    names_demo()