nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/tokenize/stanford.py

@@ -1,113 +1,117 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: Interface to the Stanford Tokenizer
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Steven Xu <xxu@student.unimelb.edu.au>
-#
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-from __future__ import unicode_literals, print_function
-
-import tempfile
-import os
-import json
-from subprocess import PIPE
-
-from nltk import compat
-from nltk.internals import find_jar, config_java, java, _java_options, find_jars_within_path
-
-from nltk.tokenize.api import TokenizerI
-
-_stanford_url = 'http://nlp.stanford.edu/software/tokenizer.shtml'
-
-class StanfordTokenizer(TokenizerI):
-    r"""
-    Interface to the Stanford Tokenizer
-
-    >>> from nltk.tokenize import StanfordTokenizer
-    >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\nThanks."
-    >>> StanfordTokenizer().tokenize(s)
-    ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
-    >>> s = "The colour of the wall is blue."
-    >>> StanfordTokenizer(options={"americanize": True}).tokenize(s)
-    ['The', 'color', 'of', 'the', 'wall', 'is', 'blue', '.']
-    """
-
-    _JAR = 'stanford-postagger.jar'
-
-    def __init__(self, path_to_jar=None, encoding='utf8', options=None, verbose=False, java_options='-mx1000m'):
-        self._stanford_jar = find_jar(
-            self._JAR, path_to_jar,
-            env_vars=('STANFORD_POSTAGGER',),
-            searchpath=(), url=_stanford_url,
-            verbose=verbose
-        )
-        
-        # Adding logging jar files to classpath 
-        stanford_dir = os.path.split(self._stanford_jar)[0]
-        self._stanford_jar = tuple(find_jars_within_path(stanford_dir))
-        
-        self._encoding = encoding
-        self.java_options = java_options
-
-        options = {} if options is None else options
-        self._options_cmd = ','.join('{0}={1}'.format(key, val) for key, val in options.items())
-
-    @staticmethod
-    def _parse_tokenized_output(s):
-        return s.splitlines()
-
-    def tokenize(self, s):
-        """
-        Use stanford tokenizer's PTBTokenizer to tokenize multiple sentences.
-        """
-        cmd = [
-            'edu.stanford.nlp.process.PTBTokenizer',
-        ]
-        return self._parse_tokenized_output(self._execute(cmd, s))
-
-    def _execute(self, cmd, input_, verbose=False):
-        encoding = self._encoding
-        cmd.extend(['-charset', encoding])
-        _options_cmd = self._options_cmd
-        if _options_cmd:
-            cmd.extend(['-options', self._options_cmd])
-
-        default_options = ' '.join(_java_options)
-
-        # Configure java.
-        config_java(options=self.java_options, verbose=verbose)
-
-        # Windows is incompatible with NamedTemporaryFile() without passing in delete=False.
-        with tempfile.NamedTemporaryFile(mode='wb', delete=False) as input_file:
-            # Write the actual sentences to the temporary input file
-            if isinstance(input_, compat.text_type) and encoding:
-                input_ = input_.encode(encoding)
-            input_file.write(input_)
-            input_file.flush()
-
-            cmd.append(input_file.name)
-
-            # Run the tagger and get the output.
-            stdout, stderr = java(cmd, classpath=self._stanford_jar,
-                                  stdout=PIPE, stderr=PIPE)
-            stdout = stdout.decode(encoding)
-
-        os.unlink(input_file.name)
-
-        # Return java configurations to their default values.
-        config_java(options=default_options, verbose=False)
-
-        return stdout
-
-
-def setup_module(module):
-    from nose import SkipTest
-
-    try:
-        StanfordTokenizer()
-    except LookupError:
-        raise SkipTest('doctests from nltk.tokenize.stanford are skipped because the stanford postagger jar doesn\'t exist')
-
-
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the Stanford Tokenizer
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Steven Xu <xxu@student.unimelb.edu.au>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import unicode_literals, print_function
+
+import tempfile
+import os
+import json
+from subprocess import PIPE
+import warnings
+
+from six import text_type
+
+from nltk.internals import find_jar, config_java, java, _java_options
+from nltk.tokenize.api import TokenizerI
+from nltk.parse.corenlp import CoreNLPParser
+
+_stanford_url = 'https://nlp.stanford.edu/software/tokenizer.shtml'
+
+
+class StanfordTokenizer(TokenizerI):
+    r"""
+    Interface to the Stanford Tokenizer
+
+    >>> from nltk.tokenize.stanford import StanfordTokenizer
+    >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\nThanks."
+    >>> StanfordTokenizer().tokenize(s)
+    ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+    >>> s = "The colour of the wall is blue."
+    >>> StanfordTokenizer(options={"americanize": True}).tokenize(s)
+    ['The', 'color', 'of', 'the', 'wall', 'is', 'blue', '.']
+    """
+
+    _JAR = 'stanford-postagger.jar'
+
+    def __init__(self, path_to_jar=None, encoding='utf8', options=None, verbose=False, java_options='-mx1000m'):
+        # Raise deprecation warning.
+        warnings.warn(str("\nThe StanfordTokenizer will "
+                          "be deprecated in version 3.2.5.\n"
+                          "Please use \033[91mnltk.parse.corenlp.CoreNLPParser\033[0m instead.'"),
+                      DeprecationWarning, stacklevel=2)
+
+        self._stanford_jar = find_jar(
+            self._JAR, path_to_jar,
+            env_vars=('STANFORD_POSTAGGER',),
+            searchpath=(), url=_stanford_url,
+            verbose=verbose
+        )
+
+        self._encoding = encoding
+        self.java_options = java_options
+
+        options = {} if options is None else options
+        self._options_cmd = ','.join('{0}={1}'.format(key, val) for key, val in options.items())
+
+    @staticmethod
+    def _parse_tokenized_output(s):
+        return s.splitlines()
+
+    def tokenize(self, s):
+        """
+        Use stanford tokenizer's PTBTokenizer to tokenize multiple sentences.
+        """
+        cmd = [
+            'edu.stanford.nlp.process.PTBTokenizer',
+        ]
+        return self._parse_tokenized_output(self._execute(cmd, s))
+
+    def _execute(self, cmd, input_, verbose=False):
+        encoding = self._encoding
+        cmd.extend(['-charset', encoding])
+        _options_cmd = self._options_cmd
+        if _options_cmd:
+            cmd.extend(['-options', self._options_cmd])
+
+        default_options = ' '.join(_java_options)
+
+        # Configure java.
+        config_java(options=self.java_options, verbose=verbose)
+
+        # Windows is incompatible with NamedTemporaryFile() without passing in delete=False.
+        with tempfile.NamedTemporaryFile(mode='wb', delete=False) as input_file:
+            # Write the actual sentences to the temporary input file
+            if isinstance(input_, text_type) and encoding:
+                input_ = input_.encode(encoding)
+            input_file.write(input_)
+            input_file.flush()
+
+            cmd.append(input_file.name)
+
+            # Run the tagger and get the output.
+            stdout, stderr = java(cmd, classpath=self._stanford_jar,
+                                  stdout=PIPE, stderr=PIPE)
+            stdout = stdout.decode(encoding)
+
+        os.unlink(input_file.name)
+
+        # Return java configurations to their default values.
+        config_java(options=default_options, verbose=False)
+
+        return stdout
+
+
+def setup_module(module):
+    from nose import SkipTest
+
+    try:
+        StanfordTokenizer()
+    except LookupError:
+        raise SkipTest(
+            'doctests from nltk.tokenize.stanford are skipped because the stanford postagger jar doesn\'t exist')

nltk-3.3/nltk/tokenize/stanford_segmenter.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the Stanford Segmenter
+# for Chinese and Arabic
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: 52nlp <52nlpcn@gmail.com>
+#         Casper Lehmann-Strøm <casperlehmann@gmail.com>
+#         Alex Constantin <alex@keyworder.ch>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import unicode_literals, print_function
+
+import tempfile
+import os
+import json
+from subprocess import PIPE
+import warnings
+
+from nltk import compat
+from nltk.internals import find_jar, find_file, find_dir, \
+                           config_java, java, _java_options
+from nltk.tokenize.api import TokenizerI
+
+from six import text_type
+
+_stanford_url = 'https://nlp.stanford.edu/software'
+
+
+class StanfordSegmenter(TokenizerI):
+    """Interface to the Stanford Segmenter
+
+    If stanford-segmenter version is older than 2016-10-31, then path_to_slf4j
+    should be provieded, for example::
+
+        seg = StanfordSegmenter(path_to_slf4j='/YOUR_PATH/slf4j-api.jar')
+
+    >>> from nltk.tokenize.stanford_segmenter import StanfordSegmenter
+    >>> seg = StanfordSegmenter()
+    >>> seg.default_config('zh')
+    >>> sent = u'这是斯坦福中文分词器测试'
+    >>> print(seg.segment(sent))
+    \u8fd9 \u662f \u65af\u5766\u798f \u4e2d\u6587 \u5206\u8bcd\u5668 \u6d4b\u8bd5
+    <BLANKLINE>
+    >>> seg.default_config('ar')
+    >>> sent = u'هذا هو تصنيف ستانفورد العربي للكلمات'
+    >>> print(seg.segment(sent.split()))
+    \u0647\u0630\u0627 \u0647\u0648 \u062a\u0635\u0646\u064a\u0641 \u0633\u062a\u0627\u0646\u0641\u0648\u0631\u062f \u0627\u0644\u0639\u0631\u0628\u064a \u0644 \u0627\u0644\u0643\u0644\u0645\u0627\u062a
+    <BLANKLINE>
+    """
+
+    _JAR = 'stanford-segmenter.jar'
+
+    def __init__(self,
+                 path_to_jar=None,
+                 path_to_slf4j=None,
+                 java_class=None,
+                 path_to_model=None,
+                 path_to_dict=None,
+                 path_to_sihan_corpora_dict=None,
+                 sihan_post_processing='false',
+                 keep_whitespaces='false',
+                 encoding='UTF-8', options=None,
+                 verbose=False, java_options='-mx2g'):
+        # Raise deprecation warning.
+        warnings.simplefilter('always', DeprecationWarning)
+        warnings.warn(str("\nThe StanfordTokenizer will "
+                          "be deprecated in version 3.2.5.\n"
+                          "Please use \033[91mnltk.parse.corenlp.CoreNLPTokenizer\033[0m instead.'"),
+                      DeprecationWarning, stacklevel=2)
+        warnings.simplefilter('ignore', DeprecationWarning)
+
+        stanford_segmenter = find_jar(
+                self._JAR, path_to_jar,
+                env_vars=('STANFORD_SEGMENTER',),
+                searchpath=(), url=_stanford_url,
+                verbose=verbose)
+        if path_to_slf4j is not None:
+            slf4j = find_jar(
+                'slf4j-api.jar', path_to_slf4j,
+                env_vars=('SLF4J', 'STANFORD_SEGMENTER',),
+                searchpath=(), url=_stanford_url,
+                verbose=verbose)
+        else:
+            slf4j = None
+
+        # This is passed to java as the -cp option, the old version of segmenter needs slf4j.
+        # The new version of stanford-segmenter-2016-10-31 doesn't need slf4j
+        self._stanford_jar = os.pathsep.join(
+            _ for _ in [stanford_segmenter, slf4j] if _ is not None
+        )
+
+        self._java_class = java_class
+        self._model = path_to_model
+        self._sihan_corpora_dict = path_to_sihan_corpora_dict
+        self._sihan_post_processing = sihan_post_processing
+        self._keep_whitespaces = keep_whitespaces
+        self._dict = path_to_dict
+
+        self._encoding = encoding
+        self.java_options = java_options
+        options = {} if options is None else options
+        self._options_cmd = ','.join('{0}={1}'.format(key, json.dumps(val)) for key, val in options.items())
+
+    def default_config(self, lang):
+        """
+        Attempt to intialize Stanford Word Segmenter for the specified language
+        using the STANFORD_SEGMENTER and STANFORD_MODELS environment variables
+        """
+
+        search_path = ()
+        if os.environ.get('STANFORD_SEGMENTER'):
+            search_path = {os.path.join(os.environ.get('STANFORD_SEGMENTER'), 'data')}
+
+        # init for Chinese-specific files
+        self._dict = None
+        self._sihan_corpora_dict = None
+        self._sihan_post_processing = 'false'
+
+        if lang == 'ar':
+            self._java_class = 'edu.stanford.nlp.international.arabic.process.ArabicSegmenter'
+            model = 'arabic-segmenter-atb+bn+arztrain.ser.gz'
+
+        elif lang == 'zh':
+            self._java_class = 'edu.stanford.nlp.ie.crf.CRFClassifier'
+            model = 'pku.gz'
+            self._sihan_post_processing = 'true'
+
+            path_to_dict = 'dict-chris6.ser.gz'
+            try:
+                self._dict = find_file(path_to_dict, searchpath=search_path,
+                                       url=_stanford_url, verbose=False,
+                                       env_vars=('STANFORD_MODELS',))
+            except LookupError:
+                raise LookupError("Could not find '%s' (tried using env. "
+                    "variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)" % path_to_dict)
+
+            sihan_dir = './data/'
+            try:
+                path_to_sihan_dir = find_dir(sihan_dir,
+                                             url=_stanford_url, verbose=False,
+                                             env_vars=('STANFORD_SEGMENTER',))
+                self._sihan_corpora_dict = os.path.join(path_to_sihan_dir, sihan_dir)
+            except LookupError:
+                raise LookupError("Could not find '%s' (tried using the "
+                    "STANFORD_SEGMENTER environment variable)" % sihan_dir)
+        else:
+            raise LookupError("Unsupported language '%'" % lang)
+
+        try:
+            self._model = find_file(model, searchpath=search_path,
+                                    url=_stanford_url, verbose=False,
+                                    env_vars=('STANFORD_MODELS', 'STANFORD_SEGMENTER',))
+        except LookupError:
+            raise LookupError("Could not find '%s' (tried using env. "
+                "variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)" % model)
+
+    def tokenize(self, s):
+        super().tokenize(s)
+
+    def segment_file(self, input_file_path):
+        """
+        """
+        cmd = [
+            self._java_class,
+            '-loadClassifier', self._model,
+            '-keepAllWhitespaces', self._keep_whitespaces,
+            '-textFile', input_file_path
+        ]
+        if self._sihan_corpora_dict is not None:
+            cmd.extend(['-serDictionary', self._dict,
+                        '-sighanCorporaDict', self._sihan_corpora_dict,
+                        '-sighanPostProcessing', self._sihan_post_processing])
+
+        stdout = self._execute(cmd)
+
+        return stdout
+
+    def segment(self, tokens):
+        return self.segment_sents([tokens])
+
+    def segment_sents(self, sentences):
+        """
+        """
+        encoding = self._encoding
+        # Create a temporary input file
+        _input_fh, self._input_file_path = tempfile.mkstemp(text=True)
+
+        # Write the actural sentences to the temporary input file
+        _input_fh = os.fdopen(_input_fh, 'wb')
+        _input = '\n'.join((' '.join(x) for x in sentences))
+        if isinstance(_input, text_type) and encoding:
+            _input = _input.encode(encoding)
+        _input_fh.write(_input)
+        _input_fh.close()
+
+        cmd = [
+            self._java_class,
+            '-loadClassifier', self._model,
+            '-keepAllWhitespaces', self._keep_whitespaces,
+            '-textFile', self._input_file_path
+        ]
+        if self._sihan_corpora_dict is not None:
+            cmd.extend(['-serDictionary', self._dict,
+                        '-sighanCorporaDict', self._sihan_corpora_dict,
+                        '-sighanPostProcessing', self._sihan_post_processing])
+
+        stdout = self._execute(cmd)
+
+        # Delete the temporary file
+        os.unlink(self._input_file_path)
+
+        return stdout
+
+    def _execute(self, cmd, verbose=False):
+        encoding = self._encoding
+        cmd.extend(['-inputEncoding', encoding])
+        _options_cmd = self._options_cmd
+        if _options_cmd:
+            cmd.extend(['-options', self._options_cmd])
+
+        default_options = ' '.join(_java_options)
+
+        # Configure java.
+        config_java(options=self.java_options, verbose=verbose)
+
+        stdout, _stderr = java(cmd, classpath=self._stanford_jar, stdout=PIPE, stderr=PIPE)
+        stdout = stdout.decode(encoding)
+
+        # Return java configurations to their default values.
+        config_java(options=default_options, verbose=False)
+
+        return stdout
+
+
+def setup_module(module):
+    from nose import SkipTest
+
+    try:
+        seg = StanfordSegmenter()
+        seg.default_config('ar')
+        seg.default_config('zh')
+    except LookupError as e:
+        raise SkipTest('Tests for nltk.tokenize.stanford_segmenter skipped: %s' % str(e))