nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/regexp.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Tokenizers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 #         Trevor Cohn <tacohn@csse.unimelb.edu.au>
@@ -73,6 +73,7 @@ from nltk.tokenize.api import TokenizerI
 from nltk.tokenize.util import regexp_span_tokenize
 from nltk.compat import python_2_unicode_compatible
 
+
 @python_2_unicode_compatible
 class RegexpTokenizer(TokenizerI):
     """
@@ -100,6 +101,7 @@ class RegexpTokenizer(TokenizerI):
         used: `re.UNICODE | re.MULTILINE | re.DOTALL`.
 
     """
+
     def __init__(self, pattern, gaps=False, discard_empty=True,
                  flags=re.UNICODE | re.MULTILINE | re.DOTALL):
         # If they gave us a regexp object, extract the pattern.
@@ -110,11 +112,11 @@ class RegexpTokenizer(TokenizerI):
         self._discard_empty = discard_empty
         self._flags = flags
         self._regexp = None
-        
+
     def _check_regexp(self):
         if self._regexp is None:
             self._regexp = re.compile(self._pattern, self._flags)
-        
+
     def tokenize(self, text):
         self._check_regexp()
         # If our regexp matches gaps, use re.split:
@@ -144,6 +146,7 @@ class RegexpTokenizer(TokenizerI):
                 (self.__class__.__name__, self._pattern, self._gaps,
                  self._discard_empty, self._flags))
 
+
 class WhitespaceTokenizer(RegexpTokenizer):
     r"""
     Tokenize a string on whitespace (space, tab, newline).
@@ -159,15 +162,18 @@ class WhitespaceTokenizer(RegexpTokenizer):
     def __init__(self):
         RegexpTokenizer.__init__(self, r'\s+', gaps=True)
 
+
 class BlanklineTokenizer(RegexpTokenizer):
     """
     Tokenize a string, treating any sequence of blank lines as a delimiter.
     Blank lines are defined as lines containing no characters, except for
     space or tab characters.
     """
+
     def __init__(self):
         RegexpTokenizer.__init__(self, r'\s*\n\s*\n\s*', gaps=True)
 
+
 class WordPunctTokenizer(RegexpTokenizer):
     """
     Tokenize a text into a sequence of alphabetic and
@@ -179,11 +185,13 @@ class WordPunctTokenizer(RegexpTokenizer):
         ['Good', 'muffins', 'cost', '$', '3', '.', '88', 'in', 'New', 'York',
         '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
     """
+
     def __init__(self):
         RegexpTokenizer.__init__(self, r'\w+|[^\w\s]+')
 
+
 ######################################################################
-#{ Tokenization Functions
+# { Tokenization Functions
 ######################################################################
 
 def regexp_tokenize(text, pattern, gaps=False, discard_empty=True,
@@ -195,8 +203,6 @@ def regexp_tokenize(text, pattern, gaps=False, discard_empty=True,
     tokenizer = RegexpTokenizer(pattern, gaps, discard_empty, flags)
     return tokenizer.tokenize(text)
 
+
 blankline_tokenize = BlanklineTokenizer().tokenize
 wordpunct_tokenize = WordPunctTokenizer().tokenize
-
-
-

nltk-3.3/nltk/tokenize/repp.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the Repp Tokenizer
+#
+# Copyright (C) 2001-2015 NLTK Project
+# Authors: Rebecca Dridan and Stephan Oepen
+# Contributors: Liling Tan
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import unicode_literals, print_function
+from six import text_type
+
+import os
+import re
+import sys
+import subprocess
+import tempfile
+
+
+from nltk.data import ZipFilePathPointer
+from nltk.internals import find_dir
+
+from nltk.tokenize.api import TokenizerI
+
+class ReppTokenizer(TokenizerI):
+    """
+    A class for word tokenization using the REPP parser described in
+    Rebecca Dridan and Stephan Oepen (2012) Tokenization: Returning to a 
+    Long Solved Problem - A Survey, Contrastive  Experiment, Recommendations, 
+    and Toolkit. In ACL. http://anthology.aclweb.org/P/P12/P12-2.pdf#page=406
+
+    >>> sents = ['Tokenization is widely regarded as a solved problem due to the high accuracy that rulebased tokenizers achieve.' ,
+    ... 'But rule-based tokenizers are hard to maintain and their rules language specific.' ,
+    ... 'We evaluated our method on three languages and obtained error rates of 0.27% (English), 0.35% (Dutch) and 0.76% (Italian) for our best models.'
+    ... ]
+    >>> tokenizer = ReppTokenizer('/home/alvas/repp/') # doctest: +SKIP
+    >>> for sent in sents:                             # doctest: +SKIP
+    ...     tokenizer.tokenize(sent)                   # doctest: +SKIP
+    ... 
+    (u'Tokenization', u'is', u'widely', u'regarded', u'as', u'a', u'solved', u'problem', u'due', u'to', u'the', u'high', u'accuracy', u'that', u'rulebased', u'tokenizers', u'achieve', u'.')
+    (u'But', u'rule-based', u'tokenizers', u'are', u'hard', u'to', u'maintain', u'and', u'their', u'rules', u'language', u'specific', u'.')
+    (u'We', u'evaluated', u'our', u'method', u'on', u'three', u'languages', u'and', u'obtained', u'error', u'rates', u'of', u'0.27', u'%', u'(', u'English', u')', u',', u'0.35', u'%', u'(', u'Dutch', u')', u'and', u'0.76', u'%', u'(', u'Italian', u')', u'for', u'our', u'best', u'models', u'.')
+
+    >>> for sent in tokenizer.tokenize_sents(sents): # doctest: +SKIP
+    ...     print sent                               # doctest: +SKIP
+    ... 
+    (u'Tokenization', u'is', u'widely', u'regarded', u'as', u'a', u'solved', u'problem', u'due', u'to', u'the', u'high', u'accuracy', u'that', u'rulebased', u'tokenizers', u'achieve', u'.')
+    (u'But', u'rule-based', u'tokenizers', u'are', u'hard', u'to', u'maintain', u'and', u'their', u'rules', u'language', u'specific', u'.')
+    (u'We', u'evaluated', u'our', u'method', u'on', u'three', u'languages', u'and', u'obtained', u'error', u'rates', u'of', u'0.27', u'%', u'(', u'English', u')', u',', u'0.35', u'%', u'(', u'Dutch', u')', u'and', u'0.76', u'%', u'(', u'Italian', u')', u'for', u'our', u'best', u'models', u'.')
+    >>> for sent in tokenizer.tokenize_sents(sents, keep_token_positions=True): # doctest: +SKIP
+    ...     print sent                                                          # doctest: +SKIP
+    ... 
+    [(u'Tokenization', 0, 12), (u'is', 13, 15), (u'widely', 16, 22), (u'regarded', 23, 31), (u'as', 32, 34), (u'a', 35, 36), (u'solved', 37, 43), (u'problem', 44, 51), (u'due', 52, 55), (u'to', 56, 58), (u'the', 59, 62), (u'high', 63, 67), (u'accuracy', 68, 76), (u'that', 77, 81), (u'rulebased', 82, 91), (u'tokenizers', 92, 102), (u'achieve', 103, 110), (u'.', 110, 111)]
+    [(u'But', 0, 3), (u'rule-based', 4, 14), (u'tokenizers', 15, 25), (u'are', 26, 29), (u'hard', 30, 34), (u'to', 35, 37), (u'maintain', 38, 46), (u'and', 47, 50), (u'their', 51, 56), (u'rules', 57, 62), (u'language', 63, 71), (u'specific', 72, 80), (u'.', 80, 81)]
+    [(u'We', 0, 2), (u'evaluated', 3, 12), (u'our', 13, 16), (u'method', 17, 23), (u'on', 24, 26), (u'three', 27, 32), (u'languages', 33, 42), (u'and', 43, 46), (u'obtained', 47, 55), (u'error', 56, 61), (u'rates', 62, 67), (u'of', 68, 70), (u'0.27', 71, 75), (u'%', 75, 76), (u'(', 77, 78), (u'English', 78, 85), (u')', 85, 86), (u',', 86, 87), (u'0.35', 88, 92), (u'%', 92, 93), (u'(', 94, 95), (u'Dutch', 95, 100), (u')', 100, 101), (u'and', 102, 105), (u'0.76', 106, 110), (u'%', 110, 111), (u'(', 112, 113), (u'Italian', 113, 120), (u')', 120, 121), (u'for', 122, 125), (u'our', 126, 129), (u'best', 130, 134), (u'models', 135, 141), (u'.', 141, 142)]
+    """
+    def __init__(self, repp_dir, encoding='utf8'):
+        self.repp_dir = self.find_repptokenizer(repp_dir)
+        # Set a directory to store the temporary files. 
+        self.working_dir = tempfile.gettempdir()
+        # Set an encoding for the input strings.
+        self.encoding = encoding
+        
+    def tokenize(self, sentence):
+        """
+        Use Repp to tokenize a single sentence.  
+        
+        :param sentence: A single sentence string.
+        :type sentence: str
+        :return: A tuple of tokens. 
+        :rtype: tuple(str)
+        """
+        return next(self.tokenize_sents([sentence]))
+    
+    def tokenize_sents(self, sentences, keep_token_positions=False):
+        """
+        Tokenize multiple sentences using Repp.
+                
+        :param sentences: A list of sentence strings.
+        :type sentences: list(str)
+        :return: A list of tuples of tokens
+        :rtype: iter(tuple(str))
+        """
+        with tempfile.NamedTemporaryFile(prefix='repp_input.', 
+            dir=self.working_dir, mode='w', delete=False) as input_file:
+            # Write sentences to temporary input file.
+            for sent in sentences:
+                input_file.write(text_type(sent) + '\n')
+            input_file.close()
+            # Generate command to run REPP. 
+            cmd =self.generate_repp_command(input_file.name)
+            # Decode the stdout and strips the ending newline.
+            repp_output = self._execute(cmd).decode(self.encoding).strip()
+            for tokenized_sent in self.parse_repp_outputs(repp_output):
+                if not keep_token_positions:
+                    # Removes token position information.
+                    tokenized_sent, starts, ends = zip(*tokenized_sent)
+                yield tokenized_sent      
+        
+    def generate_repp_command(self, inputfilename):
+        """
+        This module generates the REPP command to be used at the terminal.
+        
+        :param inputfilename: path to the input file
+        :type inputfilename: str
+        """
+        cmd = [self.repp_dir + '/src/repp']
+        cmd+= ['-c', self.repp_dir + '/erg/repp.set']
+        cmd+= ['--format', 'triple']
+        cmd+= [inputfilename]
+        return cmd  
+
+    @staticmethod
+    def _execute(cmd):
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        return stdout
+    
+    @staticmethod    
+    def parse_repp_outputs(repp_output):
+        """
+        This module parses the tri-tuple format that REPP outputs using the
+        "--format triple" option and returns an generator with tuple of string
+        tokens.
+        
+        :param repp_output:
+        :type repp_output: type
+        :return: an iterable of the tokenized sentences as tuples of strings
+        :rtype: iter(tuple)
+        """
+        line_regex = re.compile('^\((\d+), (\d+), (.+)\)$', re.MULTILINE)
+        for section in repp_output.split('\n\n'):
+            words_with_positions = [(token, int(start), int(end))
+                                    for start, end, token in 
+                                    line_regex.findall(section)]
+            words = tuple(t[2] for t in words_with_positions)
+            yield words_with_positions
+    
+    def find_repptokenizer(self, repp_dirname):
+        """
+        A module to find REPP tokenizer binary and its *repp.set* config file.
+        """
+        if os.path.exists(repp_dirname): # If a full path is given.
+            _repp_dir = repp_dirname
+        else: # Try to find path to REPP directory in environment variables.
+            _repp_dir = find_dir(repp_dirname, env_vars=('REPP_TOKENIZER',))
+        # Checks for the REPP binary and erg/repp.set config file.
+        assert os.path.exists(_repp_dir+'/src/repp')
+        assert os.path.exists(_repp_dir+'/erg/repp.set')
+        return _repp_dir

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/sexpr.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Tokenizers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Yoav Goldberg <yoavg@cs.bgu.ac.il>
 #         Steven Bird <stevenbird1@gmail.com> (minor edits)
 # URL: <http://nltk.sourceforge.net>
@@ -53,6 +53,7 @@ import re
 
 from nltk.tokenize.api import TokenizerI
 
+
 class SExprTokenizer(TokenizerI):
     """
     A tokenizer that divides strings into s-expressions.
@@ -125,7 +126,7 @@ class SExprTokenizer(TokenizerI):
                 if self._strict and depth == 0:
                     raise ValueError('Un-matched close paren at char %d'
                                      % m.start())
-                depth = max(0, depth-1)
+                depth = max(0, depth - 1)
                 if depth == 0:
                     result.append(text[pos:m.end()])
                     pos = m.end()
@@ -135,8 +136,5 @@ class SExprTokenizer(TokenizerI):
             result.append(text[pos:])
         return result
 
-sexpr_tokenize = SExprTokenizer().tokenize
-
-
-
 
+sexpr_tokenize = SExprTokenizer().tokenize

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/simple.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Simple Tokenizers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.sourceforge.net>
@@ -38,6 +38,7 @@ from __future__ import unicode_literals
 from nltk.tokenize.api import TokenizerI, StringTokenizer
 from nltk.tokenize.util import string_span_tokenize, regexp_span_tokenize
 
+
 class SpaceTokenizer(StringTokenizer):
     r"""Tokenize a string using the space character as a delimiter,
     which is the same as ``s.split(' ')``.
@@ -51,6 +52,7 @@ class SpaceTokenizer(StringTokenizer):
 
     _string = ' '
 
+
 class TabTokenizer(StringTokenizer):
     r"""Tokenize a string use the tab character as a delimiter,
     the same as ``s.split('\t')``.
@@ -62,6 +64,7 @@ class TabTokenizer(StringTokenizer):
 
     _string = '\t'
 
+
 class CharTokenizer(StringTokenizer):
     """Tokenize a string into individual characters.  If this functionality
     is ever required directly, use ``for char in string``.
@@ -74,6 +77,7 @@ class CharTokenizer(StringTokenizer):
         for i, j in enumerate(range(1, len(s) + 1)):
             yield i, j
 
+
 class LineTokenizer(TokenizerI):
     r"""Tokenize a string into its lines, optionally discarding blank lines.
     This is similar to ``s.split('\n')``.
@@ -124,13 +128,11 @@ class LineTokenizer(TokenizerI):
             for span in regexp_span_tokenize(s, r'\n(\s+\n)*'):
                 yield span
 
+
 ######################################################################
-#{ Tokenization Functions
+# { Tokenization Functions
 ######################################################################
 # XXX: it is stated in module docs that there is no function versions
 
 def line_tokenize(text, blanklines='discard'):
     return LineTokenizer(blanklines).tokenize(text)
-
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/tokenize/stanford.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Interface to the Stanford Tokenizer
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Xu <xxu@student.unimelb.edu.au>
 #
 # URL: <http://nltk.org/>
@@ -13,19 +13,22 @@ import tempfile
 import os
 import json
 from subprocess import PIPE
+import warnings
 
-from nltk import compat
-from nltk.internals import find_jar, config_java, java, _java_options, find_jars_within_path
+from six import text_type
 
+from nltk.internals import find_jar, config_java, java, _java_options
 from nltk.tokenize.api import TokenizerI
+from nltk.parse.corenlp import CoreNLPParser
+
+_stanford_url = 'https://nlp.stanford.edu/software/tokenizer.shtml'
 
-_stanford_url = 'http://nlp.stanford.edu/software/tokenizer.shtml'
 
 class StanfordTokenizer(TokenizerI):
     r"""
     Interface to the Stanford Tokenizer
 
-    >>> from nltk.tokenize import StanfordTokenizer
+    >>> from nltk.tokenize.stanford import StanfordTokenizer
     >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\nThanks."
     >>> StanfordTokenizer().tokenize(s)
     ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
@@ -37,17 +40,19 @@ class StanfordTokenizer(TokenizerI):
     _JAR = 'stanford-postagger.jar'
 
     def __init__(self, path_to_jar=None, encoding='utf8', options=None, verbose=False, java_options='-mx1000m'):
+        # Raise deprecation warning.
+        warnings.warn(str("\nThe StanfordTokenizer will "
+                          "be deprecated in version 3.2.5.\n"
+                          "Please use \033[91mnltk.parse.corenlp.CoreNLPParser\033[0m instead.'"),
+                      DeprecationWarning, stacklevel=2)
+
         self._stanford_jar = find_jar(
             self._JAR, path_to_jar,
             env_vars=('STANFORD_POSTAGGER',),
             searchpath=(), url=_stanford_url,
             verbose=verbose
         )
-        
-        # Adding logging jar files to classpath 
-        stanford_dir = os.path.split(self._stanford_jar)[0]
-        self._stanford_jar = tuple(find_jars_within_path(stanford_dir))
-        
+
         self._encoding = encoding
         self.java_options = java_options
 
@@ -82,7 +87,7 @@ class StanfordTokenizer(TokenizerI):
         # Windows is incompatible with NamedTemporaryFile() without passing in delete=False.
         with tempfile.NamedTemporaryFile(mode='wb', delete=False) as input_file:
             # Write the actual sentences to the temporary input file
-            if isinstance(input_, compat.text_type) and encoding:
+            if isinstance(input_, text_type) and encoding:
                 input_ = input_.encode(encoding)
             input_file.write(input_)
             input_file.flush()
@@ -108,6 +113,5 @@ def setup_module(module):
     try:
         StanfordTokenizer()
     except LookupError:
-        raise SkipTest('doctests from nltk.tokenize.stanford are skipped because the stanford postagger jar doesn\'t exist')
-
-
+        raise SkipTest(
+            'doctests from nltk.tokenize.stanford are skipped because the stanford postagger jar doesn\'t exist')

nltk-3.3/nltk/tokenize/stanford_segmenter.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the Stanford Segmenter
+# for Chinese and Arabic
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: 52nlp <52nlpcn@gmail.com>
+#         Casper Lehmann-Strøm <casperlehmann@gmail.com>
+#         Alex Constantin <alex@keyworder.ch>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import unicode_literals, print_function
+
+import tempfile
+import os
+import json
+from subprocess import PIPE
+import warnings
+
+from nltk import compat
+from nltk.internals import find_jar, find_file, find_dir, \
+                           config_java, java, _java_options
+from nltk.tokenize.api import TokenizerI
+
+from six import text_type
+
+_stanford_url = 'https://nlp.stanford.edu/software'
+
+
+class StanfordSegmenter(TokenizerI):
+    """Interface to the Stanford Segmenter
+
+    If stanford-segmenter version is older than 2016-10-31, then path_to_slf4j
+    should be provieded, for example::
+
+        seg = StanfordSegmenter(path_to_slf4j='/YOUR_PATH/slf4j-api.jar')
+
+    >>> from nltk.tokenize.stanford_segmenter import StanfordSegmenter
+    >>> seg = StanfordSegmenter()
+    >>> seg.default_config('zh')
+    >>> sent = u'这是斯坦福中文分词器测试'
+    >>> print(seg.segment(sent))
+    \u8fd9 \u662f \u65af\u5766\u798f \u4e2d\u6587 \u5206\u8bcd\u5668 \u6d4b\u8bd5
+    <BLANKLINE>
+    >>> seg.default_config('ar')
+    >>> sent = u'هذا هو تصنيف ستانفورد العربي للكلمات'
+    >>> print(seg.segment(sent.split()))
+    \u0647\u0630\u0627 \u0647\u0648 \u062a\u0635\u0646\u064a\u0641 \u0633\u062a\u0627\u0646\u0641\u0648\u0631\u062f \u0627\u0644\u0639\u0631\u0628\u064a \u0644 \u0627\u0644\u0643\u0644\u0645\u0627\u062a
+    <BLANKLINE>
+    """
+
+    _JAR = 'stanford-segmenter.jar'
+
+    def __init__(self,
+                 path_to_jar=None,
+                 path_to_slf4j=None,
+                 java_class=None,
+                 path_to_model=None,
+                 path_to_dict=None,
+                 path_to_sihan_corpora_dict=None,
+                 sihan_post_processing='false',
+                 keep_whitespaces='false',
+                 encoding='UTF-8', options=None,
+                 verbose=False, java_options='-mx2g'):
+        # Raise deprecation warning.
+        warnings.simplefilter('always', DeprecationWarning)
+        warnings.warn(str("\nThe StanfordTokenizer will "
+                          "be deprecated in version 3.2.5.\n"
+                          "Please use \033[91mnltk.parse.corenlp.CoreNLPTokenizer\033[0m instead.'"),
+                      DeprecationWarning, stacklevel=2)
+        warnings.simplefilter('ignore', DeprecationWarning)
+
+        stanford_segmenter = find_jar(
+                self._JAR, path_to_jar,
+                env_vars=('STANFORD_SEGMENTER',),
+                searchpath=(), url=_stanford_url,
+                verbose=verbose)
+        if path_to_slf4j is not None:
+            slf4j = find_jar(
+                'slf4j-api.jar', path_to_slf4j,
+                env_vars=('SLF4J', 'STANFORD_SEGMENTER',),
+                searchpath=(), url=_stanford_url,
+                verbose=verbose)
+        else:
+            slf4j = None
+
+        # This is passed to java as the -cp option, the old version of segmenter needs slf4j.
+        # The new version of stanford-segmenter-2016-10-31 doesn't need slf4j
+        self._stanford_jar = os.pathsep.join(
+            _ for _ in [stanford_segmenter, slf4j] if _ is not None
+        )
+
+        self._java_class = java_class
+        self._model = path_to_model
+        self._sihan_corpora_dict = path_to_sihan_corpora_dict
+        self._sihan_post_processing = sihan_post_processing
+        self._keep_whitespaces = keep_whitespaces
+        self._dict = path_to_dict
+
+        self._encoding = encoding
+        self.java_options = java_options
+        options = {} if options is None else options
+        self._options_cmd = ','.join('{0}={1}'.format(key, json.dumps(val)) for key, val in options.items())
+
+    def default_config(self, lang):
+        """
+        Attempt to intialize Stanford Word Segmenter for the specified language
+        using the STANFORD_SEGMENTER and STANFORD_MODELS environment variables
+        """
+
+        search_path = ()
+        if os.environ.get('STANFORD_SEGMENTER'):
+            search_path = {os.path.join(os.environ.get('STANFORD_SEGMENTER'), 'data')}
+
+        # init for Chinese-specific files
+        self._dict = None
+        self._sihan_corpora_dict = None
+        self._sihan_post_processing = 'false'
+
+        if lang == 'ar':
+            self._java_class = 'edu.stanford.nlp.international.arabic.process.ArabicSegmenter'
+            model = 'arabic-segmenter-atb+bn+arztrain.ser.gz'
+
+        elif lang == 'zh':
+            self._java_class = 'edu.stanford.nlp.ie.crf.CRFClassifier'
+            model = 'pku.gz'
+            self._sihan_post_processing = 'true'
+
+            path_to_dict = 'dict-chris6.ser.gz'
+            try:
+                self._dict = find_file(path_to_dict, searchpath=search_path,
+                                       url=_stanford_url, verbose=False,
+                                       env_vars=('STANFORD_MODELS',))
+            except LookupError:
+                raise LookupError("Could not find '%s' (tried using env. "
+                    "variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)" % path_to_dict)
+
+            sihan_dir = './data/'
+            try:
+                path_to_sihan_dir = find_dir(sihan_dir,
+                                             url=_stanford_url, verbose=False,
+                                             env_vars=('STANFORD_SEGMENTER',))
+                self._sihan_corpora_dict = os.path.join(path_to_sihan_dir, sihan_dir)
+            except LookupError:
+                raise LookupError("Could not find '%s' (tried using the "
+                    "STANFORD_SEGMENTER environment variable)" % sihan_dir)
+        else:
+            raise LookupError("Unsupported language '%'" % lang)
+
+        try:
+            self._model = find_file(model, searchpath=search_path,
+                                    url=_stanford_url, verbose=False,
+                                    env_vars=('STANFORD_MODELS', 'STANFORD_SEGMENTER',))
+        except LookupError:
+            raise LookupError("Could not find '%s' (tried using env. "
+                "variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)" % model)
+
+    def tokenize(self, s):
+        super().tokenize(s)
+
+    def segment_file(self, input_file_path):
+        """
+        """
+        cmd = [
+            self._java_class,
+            '-loadClassifier', self._model,
+            '-keepAllWhitespaces', self._keep_whitespaces,
+            '-textFile', input_file_path
+        ]
+        if self._sihan_corpora_dict is not None:
+            cmd.extend(['-serDictionary', self._dict,
+                        '-sighanCorporaDict', self._sihan_corpora_dict,
+                        '-sighanPostProcessing', self._sihan_post_processing])
+
+        stdout = self._execute(cmd)
+
+        return stdout
+
+    def segment(self, tokens):
+        return self.segment_sents([tokens])
+
+    def segment_sents(self, sentences):
+        """
+        """
+        encoding = self._encoding
+        # Create a temporary input file
+        _input_fh, self._input_file_path = tempfile.mkstemp(text=True)
+
+        # Write the actural sentences to the temporary input file
+        _input_fh = os.fdopen(_input_fh, 'wb')
+        _input = '\n'.join((' '.join(x) for x in sentences))
+        if isinstance(_input, text_type) and encoding:
+            _input = _input.encode(encoding)
+        _input_fh.write(_input)
+        _input_fh.close()
+
+        cmd = [
+            self._java_class,
+            '-loadClassifier', self._model,
+            '-keepAllWhitespaces', self._keep_whitespaces,
+            '-textFile', self._input_file_path
+        ]
+        if self._sihan_corpora_dict is not None:
+            cmd.extend(['-serDictionary', self._dict,
+                        '-sighanCorporaDict', self._sihan_corpora_dict,
+                        '-sighanPostProcessing', self._sihan_post_processing])
+
+        stdout = self._execute(cmd)
+
+        # Delete the temporary file
+        os.unlink(self._input_file_path)
+
+        return stdout
+
+    def _execute(self, cmd, verbose=False):
+        encoding = self._encoding
+        cmd.extend(['-inputEncoding', encoding])
+        _options_cmd = self._options_cmd
+        if _options_cmd:
+            cmd.extend(['-options', self._options_cmd])
+
+        default_options = ' '.join(_java_options)
+
+        # Configure java.
+        config_java(options=self.java_options, verbose=verbose)
+
+        stdout, _stderr = java(cmd, classpath=self._stanford_jar, stdout=PIPE, stderr=PIPE)
+        stdout = stdout.decode(encoding)
+
+        # Return java configurations to their default values.
+        config_java(options=default_options, verbose=False)
+
+        return stdout
+
+
+def setup_module(module):
+    from nose import SkipTest
+
+    try:
+        seg = StanfordSegmenter()
+        seg.default_config('ar')
+        seg.default_config('zh')
+    except LookupError as e:
+        raise SkipTest('Tests for nltk.tokenize.stanford_segmenter skipped: %s' % str(e))