nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/tokenize/sexpr.py

@@ -1,142 +1,140 @@
-# Natural Language Toolkit: Tokenizers
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Yoav Goldberg <yoavg@cs.bgu.ac.il>
-#         Steven Bird <stevenbird1@gmail.com> (minor edits)
-# URL: <http://nltk.sourceforge.net>
-# For license information, see LICENSE.TXT
-
-"""
-S-Expression Tokenizer
-
-``SExprTokenizer`` is used to find parenthesized expressions in a
-string.  In particular, it divides a string into a sequence of
-substrings that are either parenthesized expressions (including any
-nested parenthesized expressions), or other whitespace-separated
-tokens.
-
-    >>> from nltk.tokenize import SExprTokenizer
-    >>> SExprTokenizer().tokenize('(a b (c d)) e f (g)')
-    ['(a b (c d))', 'e', 'f', '(g)']
-
-By default, `SExprTokenizer` will raise a ``ValueError`` exception if
-used to tokenize an expression with non-matching parentheses:
-
-    >>> SExprTokenizer().tokenize('c) d) e (f (g')
-    Traceback (most recent call last):
-      ...
-    ValueError: Un-matched close paren at char 1
-
-The ``strict`` argument can be set to False to allow for
-non-matching parentheses.  Any unmatched close parentheses will be
-listed as their own s-expression; and the last partial sexpr with
-unmatched open parentheses will be listed as its own sexpr:
-
-    >>> SExprTokenizer(strict=False).tokenize('c) d) e (f (g')
-    ['c', ')', 'd', ')', 'e', '(f (g']
-
-The characters used for open and close parentheses may be customized
-using the ``parens`` argument to the `SExprTokenizer` constructor:
-
-    >>> SExprTokenizer(parens='{}').tokenize('{a b {c d}} e f {g}')
-    ['{a b {c d}}', 'e', 'f', '{g}']
-
-The s-expression tokenizer is also available as a function:
-
-    >>> from nltk.tokenize import sexpr_tokenize
-    >>> sexpr_tokenize('(a b (c d)) e f (g)')
-    ['(a b (c d))', 'e', 'f', '(g)']
-
-"""
-
-import re
-
-from nltk.tokenize.api import TokenizerI
-
-class SExprTokenizer(TokenizerI):
-    """
-    A tokenizer that divides strings into s-expressions.
-    An s-expresion can be either:
-
-      - a parenthesized expression, including any nested parenthesized
-        expressions, or
-      - a sequence of non-whitespace non-parenthesis characters.
-
-    For example, the string ``(a (b c)) d e (f)`` consists of four
-    s-expressions: ``(a (b c))``, ``d``, ``e``, and ``(f)``.
-
-    By default, the characters ``(`` and ``)`` are treated as open and
-    close parentheses, but alternative strings may be specified.
-
-    :param parens: A two-element sequence specifying the open and close parentheses
-        that should be used to find sexprs.  This will typically be either a
-        two-character string, or a list of two strings.
-    :type parens: str or list
-    :param strict: If true, then raise an exception when tokenizing an ill-formed sexpr.
-    """
-
-    def __init__(self, parens='()', strict=True):
-        if len(parens) != 2:
-            raise ValueError('parens must contain exactly two strings')
-        self._strict = strict
-        self._open_paren = parens[0]
-        self._close_paren = parens[1]
-        self._paren_regexp = re.compile('%s|%s' % (re.escape(parens[0]),
-                                                   re.escape(parens[1])))
-
-    def tokenize(self, text):
-        """
-        Return a list of s-expressions extracted from *text*.
-        For example:
-
-            >>> SExprTokenizer().tokenize('(a b (c d)) e f (g)')
-            ['(a b (c d))', 'e', 'f', '(g)']
-
-        All parentheses are assumed to mark s-expressions.
-        (No special processing is done to exclude parentheses that occur
-        inside strings, or following backslash characters.)
-
-        If the given expression contains non-matching parentheses,
-        then the behavior of the tokenizer depends on the ``strict``
-        parameter to the constructor.  If ``strict`` is ``True``, then
-        raise a ``ValueError``.  If ``strict`` is ``False``, then any
-        unmatched close parentheses will be listed as their own
-        s-expression; and the last partial s-expression with unmatched open
-        parentheses will be listed as its own s-expression:
-
-            >>> SExprTokenizer(strict=False).tokenize('c) d) e (f (g')
-            ['c', ')', 'd', ')', 'e', '(f (g']
-
-        :param text: the string to be tokenized
-        :type text: str or iter(str)
-        :rtype: iter(str)
-        """
-        result = []
-        pos = 0
-        depth = 0
-        for m in self._paren_regexp.finditer(text):
-            paren = m.group()
-            if depth == 0:
-                result += text[pos:m.start()].split()
-                pos = m.start()
-            if paren == self._open_paren:
-                depth += 1
-            if paren == self._close_paren:
-                if self._strict and depth == 0:
-                    raise ValueError('Un-matched close paren at char %d'
-                                     % m.start())
-                depth = max(0, depth-1)
-                if depth == 0:
-                    result.append(text[pos:m.end()])
-                    pos = m.end()
-        if self._strict and depth > 0:
-            raise ValueError('Un-matched open paren at char %d' % pos)
-        if pos < len(text):
-            result.append(text[pos:])
-        return result
-
-sexpr_tokenize = SExprTokenizer().tokenize
-
-
-
-
+# Natural Language Toolkit: Tokenizers
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Yoav Goldberg <yoavg@cs.bgu.ac.il>
+#         Steven Bird <stevenbird1@gmail.com> (minor edits)
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+"""
+S-Expression Tokenizer
+
+``SExprTokenizer`` is used to find parenthesized expressions in a
+string.  In particular, it divides a string into a sequence of
+substrings that are either parenthesized expressions (including any
+nested parenthesized expressions), or other whitespace-separated
+tokens.
+
+    >>> from nltk.tokenize import SExprTokenizer
+    >>> SExprTokenizer().tokenize('(a b (c d)) e f (g)')
+    ['(a b (c d))', 'e', 'f', '(g)']
+
+By default, `SExprTokenizer` will raise a ``ValueError`` exception if
+used to tokenize an expression with non-matching parentheses:
+
+    >>> SExprTokenizer().tokenize('c) d) e (f (g')
+    Traceback (most recent call last):
+      ...
+    ValueError: Un-matched close paren at char 1
+
+The ``strict`` argument can be set to False to allow for
+non-matching parentheses.  Any unmatched close parentheses will be
+listed as their own s-expression; and the last partial sexpr with
+unmatched open parentheses will be listed as its own sexpr:
+
+    >>> SExprTokenizer(strict=False).tokenize('c) d) e (f (g')
+    ['c', ')', 'd', ')', 'e', '(f (g']
+
+The characters used for open and close parentheses may be customized
+using the ``parens`` argument to the `SExprTokenizer` constructor:
+
+    >>> SExprTokenizer(parens='{}').tokenize('{a b {c d}} e f {g}')
+    ['{a b {c d}}', 'e', 'f', '{g}']
+
+The s-expression tokenizer is also available as a function:
+
+    >>> from nltk.tokenize import sexpr_tokenize
+    >>> sexpr_tokenize('(a b (c d)) e f (g)')
+    ['(a b (c d))', 'e', 'f', '(g)']
+
+"""
+
+import re
+
+from nltk.tokenize.api import TokenizerI
+
+
+class SExprTokenizer(TokenizerI):
+    """
+    A tokenizer that divides strings into s-expressions.
+    An s-expresion can be either:
+
+      - a parenthesized expression, including any nested parenthesized
+        expressions, or
+      - a sequence of non-whitespace non-parenthesis characters.
+
+    For example, the string ``(a (b c)) d e (f)`` consists of four
+    s-expressions: ``(a (b c))``, ``d``, ``e``, and ``(f)``.
+
+    By default, the characters ``(`` and ``)`` are treated as open and
+    close parentheses, but alternative strings may be specified.
+
+    :param parens: A two-element sequence specifying the open and close parentheses
+        that should be used to find sexprs.  This will typically be either a
+        two-character string, or a list of two strings.
+    :type parens: str or list
+    :param strict: If true, then raise an exception when tokenizing an ill-formed sexpr.
+    """
+
+    def __init__(self, parens='()', strict=True):
+        if len(parens) != 2:
+            raise ValueError('parens must contain exactly two strings')
+        self._strict = strict
+        self._open_paren = parens[0]
+        self._close_paren = parens[1]
+        self._paren_regexp = re.compile('%s|%s' % (re.escape(parens[0]),
+                                                   re.escape(parens[1])))
+
+    def tokenize(self, text):
+        """
+        Return a list of s-expressions extracted from *text*.
+        For example:
+
+            >>> SExprTokenizer().tokenize('(a b (c d)) e f (g)')
+            ['(a b (c d))', 'e', 'f', '(g)']
+
+        All parentheses are assumed to mark s-expressions.
+        (No special processing is done to exclude parentheses that occur
+        inside strings, or following backslash characters.)
+
+        If the given expression contains non-matching parentheses,
+        then the behavior of the tokenizer depends on the ``strict``
+        parameter to the constructor.  If ``strict`` is ``True``, then
+        raise a ``ValueError``.  If ``strict`` is ``False``, then any
+        unmatched close parentheses will be listed as their own
+        s-expression; and the last partial s-expression with unmatched open
+        parentheses will be listed as its own s-expression:
+
+            >>> SExprTokenizer(strict=False).tokenize('c) d) e (f (g')
+            ['c', ')', 'd', ')', 'e', '(f (g']
+
+        :param text: the string to be tokenized
+        :type text: str or iter(str)
+        :rtype: iter(str)
+        """
+        result = []
+        pos = 0
+        depth = 0
+        for m in self._paren_regexp.finditer(text):
+            paren = m.group()
+            if depth == 0:
+                result += text[pos:m.start()].split()
+                pos = m.start()
+            if paren == self._open_paren:
+                depth += 1
+            if paren == self._close_paren:
+                if self._strict and depth == 0:
+                    raise ValueError('Un-matched close paren at char %d'
+                                     % m.start())
+                depth = max(0, depth - 1)
+                if depth == 0:
+                    result.append(text[pos:m.end()])
+                    pos = m.end()
+        if self._strict and depth > 0:
+            raise ValueError('Un-matched open paren at char %d' % pos)
+        if pos < len(text):
+            result.append(text[pos:])
+        return result
+
+
+sexpr_tokenize = SExprTokenizer().tokenize

{nltk-3.2 → nltk-3.3}/nltk/tokenize/simple.py

@@ -1,136 +1,138 @@
-# Natural Language Toolkit: Simple Tokenizers
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Edward Loper <edloper@gmail.com>
-#         Steven Bird <stevenbird1@gmail.com>
-# URL: <http://nltk.sourceforge.net>
-# For license information, see LICENSE.TXT
-
-r"""
-Simple Tokenizers
-
-These tokenizers divide strings into substrings using the string
-``split()`` method.
-When tokenizing using a particular delimiter string, use
-the string ``split()`` method directly, as this is more efficient.
-
-The simple tokenizers are *not* available as separate functions;
-instead, you should just use the string ``split()`` method directly:
-
-    >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\n\nThanks."
-    >>> s.split()
-    ['Good', 'muffins', 'cost', '$3.88', 'in', 'New', 'York.',
-    'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks.']
-    >>> s.split(' ')
-    ['Good', 'muffins', 'cost', '$3.88\nin', 'New', 'York.', '',
-    'Please', 'buy', 'me\ntwo', 'of', 'them.\n\nThanks.']
-    >>> s.split('\n')
-    ['Good muffins cost $3.88', 'in New York.  Please buy me',
-    'two of them.', '', 'Thanks.']
-
-The simple tokenizers are mainly useful because they follow the
-standard ``TokenizerI`` interface, and so can be used with any code
-that expects a tokenizer.  For example, these tokenizers can be used
-to specify the tokenization conventions when building a `CorpusReader`.
-
-"""
-from __future__ import unicode_literals
-from nltk.tokenize.api import TokenizerI, StringTokenizer
-from nltk.tokenize.util import string_span_tokenize, regexp_span_tokenize
-
-class SpaceTokenizer(StringTokenizer):
-    r"""Tokenize a string using the space character as a delimiter,
-    which is the same as ``s.split(' ')``.
-
-        >>> from nltk.tokenize import SpaceTokenizer
-        >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\n\nThanks."
-        >>> SpaceTokenizer().tokenize(s)
-        ['Good', 'muffins', 'cost', '$3.88\nin', 'New', 'York.', '',
-        'Please', 'buy', 'me\ntwo', 'of', 'them.\n\nThanks.']
-    """
-
-    _string = ' '
-
-class TabTokenizer(StringTokenizer):
-    r"""Tokenize a string use the tab character as a delimiter,
-    the same as ``s.split('\t')``.
-
-        >>> from nltk.tokenize import TabTokenizer
-        >>> TabTokenizer().tokenize('a\tb c\n\t d')
-        ['a', 'b c\n', ' d']
-    """
-
-    _string = '\t'
-
-class CharTokenizer(StringTokenizer):
-    """Tokenize a string into individual characters.  If this functionality
-    is ever required directly, use ``for char in string``.
-    """
-
-    def tokenize(self, s):
-        return list(s)
-
-    def span_tokenize(self, s):
-        for i, j in enumerate(range(1, len(s) + 1)):
-            yield i, j
-
-class LineTokenizer(TokenizerI):
-    r"""Tokenize a string into its lines, optionally discarding blank lines.
-    This is similar to ``s.split('\n')``.
-
-        >>> from nltk.tokenize import LineTokenizer
-        >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\n\nThanks."
-        >>> LineTokenizer(blanklines='keep').tokenize(s)
-        ['Good muffins cost $3.88', 'in New York.  Please buy me',
-        'two of them.', '', 'Thanks.']
-        >>> # same as [l for l in s.split('\n') if l.strip()]:
-        >>> LineTokenizer(blanklines='discard').tokenize(s)
-        ['Good muffins cost $3.88', 'in New York.  Please buy me',
-        'two of them.', 'Thanks.']
-
-    :param blanklines: Indicates how blank lines should be handled.  Valid values are:
-
-        - ``discard``: strip blank lines out of the token list before returning it.
-           A line is considered blank if it contains only whitespace characters.
-        - ``keep``: leave all blank lines in the token list.
-        - ``discard-eof``: if the string ends with a newline, then do not generate
-           a corresponding token ``''`` after that newline.
-    """
-
-    def __init__(self, blanklines='discard'):
-        valid_blanklines = ('discard', 'keep', 'discard-eof')
-        if blanklines not in valid_blanklines:
-            raise ValueError('Blank lines must be one of: %s' %
-                             ' '.join(valid_blanklines))
-
-        self._blanklines = blanklines
-
-    def tokenize(self, s):
-        lines = s.splitlines()
-        # If requested, strip off blank lines.
-        if self._blanklines == 'discard':
-            lines = [l for l in lines if l.rstrip()]
-        elif self._blanklines == 'discard-eof':
-            if lines and not lines[-1].strip():
-                lines.pop()
-        return lines
-
-    # discard-eof not implemented
-    def span_tokenize(self, s):
-        if self._blanklines == 'keep':
-            for span in string_span_tokenize(s, r'\n'):
-                yield span
-        else:
-            for span in regexp_span_tokenize(s, r'\n(\s+\n)*'):
-                yield span
-
-######################################################################
-#{ Tokenization Functions
-######################################################################
-# XXX: it is stated in module docs that there is no function versions
-
-def line_tokenize(text, blanklines='discard'):
-    return LineTokenizer(blanklines).tokenize(text)
-
-
-
+# Natural Language Toolkit: Simple Tokenizers
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+r"""
+Simple Tokenizers
+
+These tokenizers divide strings into substrings using the string
+``split()`` method.
+When tokenizing using a particular delimiter string, use
+the string ``split()`` method directly, as this is more efficient.
+
+The simple tokenizers are *not* available as separate functions;
+instead, you should just use the string ``split()`` method directly:
+
+    >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\n\nThanks."
+    >>> s.split()
+    ['Good', 'muffins', 'cost', '$3.88', 'in', 'New', 'York.',
+    'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks.']
+    >>> s.split(' ')
+    ['Good', 'muffins', 'cost', '$3.88\nin', 'New', 'York.', '',
+    'Please', 'buy', 'me\ntwo', 'of', 'them.\n\nThanks.']
+    >>> s.split('\n')
+    ['Good muffins cost $3.88', 'in New York.  Please buy me',
+    'two of them.', '', 'Thanks.']
+
+The simple tokenizers are mainly useful because they follow the
+standard ``TokenizerI`` interface, and so can be used with any code
+that expects a tokenizer.  For example, these tokenizers can be used
+to specify the tokenization conventions when building a `CorpusReader`.
+
+"""
+from __future__ import unicode_literals
+from nltk.tokenize.api import TokenizerI, StringTokenizer
+from nltk.tokenize.util import string_span_tokenize, regexp_span_tokenize
+
+
+class SpaceTokenizer(StringTokenizer):
+    r"""Tokenize a string using the space character as a delimiter,
+    which is the same as ``s.split(' ')``.
+
+        >>> from nltk.tokenize import SpaceTokenizer
+        >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\n\nThanks."
+        >>> SpaceTokenizer().tokenize(s)
+        ['Good', 'muffins', 'cost', '$3.88\nin', 'New', 'York.', '',
+        'Please', 'buy', 'me\ntwo', 'of', 'them.\n\nThanks.']
+    """
+
+    _string = ' '
+
+
+class TabTokenizer(StringTokenizer):
+    r"""Tokenize a string use the tab character as a delimiter,
+    the same as ``s.split('\t')``.
+
+        >>> from nltk.tokenize import TabTokenizer
+        >>> TabTokenizer().tokenize('a\tb c\n\t d')
+        ['a', 'b c\n', ' d']
+    """
+
+    _string = '\t'
+
+
+class CharTokenizer(StringTokenizer):
+    """Tokenize a string into individual characters.  If this functionality
+    is ever required directly, use ``for char in string``.
+    """
+
+    def tokenize(self, s):
+        return list(s)
+
+    def span_tokenize(self, s):
+        for i, j in enumerate(range(1, len(s) + 1)):
+            yield i, j
+
+
+class LineTokenizer(TokenizerI):
+    r"""Tokenize a string into its lines, optionally discarding blank lines.
+    This is similar to ``s.split('\n')``.
+
+        >>> from nltk.tokenize import LineTokenizer
+        >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\n\nThanks."
+        >>> LineTokenizer(blanklines='keep').tokenize(s)
+        ['Good muffins cost $3.88', 'in New York.  Please buy me',
+        'two of them.', '', 'Thanks.']
+        >>> # same as [l for l in s.split('\n') if l.strip()]:
+        >>> LineTokenizer(blanklines='discard').tokenize(s)
+        ['Good muffins cost $3.88', 'in New York.  Please buy me',
+        'two of them.', 'Thanks.']
+
+    :param blanklines: Indicates how blank lines should be handled.  Valid values are:
+
+        - ``discard``: strip blank lines out of the token list before returning it.
+           A line is considered blank if it contains only whitespace characters.
+        - ``keep``: leave all blank lines in the token list.
+        - ``discard-eof``: if the string ends with a newline, then do not generate
+           a corresponding token ``''`` after that newline.
+    """
+
+    def __init__(self, blanklines='discard'):
+        valid_blanklines = ('discard', 'keep', 'discard-eof')
+        if blanklines not in valid_blanklines:
+            raise ValueError('Blank lines must be one of: %s' %
+                             ' '.join(valid_blanklines))
+
+        self._blanklines = blanklines
+
+    def tokenize(self, s):
+        lines = s.splitlines()
+        # If requested, strip off blank lines.
+        if self._blanklines == 'discard':
+            lines = [l for l in lines if l.rstrip()]
+        elif self._blanklines == 'discard-eof':
+            if lines and not lines[-1].strip():
+                lines.pop()
+        return lines
+
+    # discard-eof not implemented
+    def span_tokenize(self, s):
+        if self._blanklines == 'keep':
+            for span in string_span_tokenize(s, r'\n'):
+                yield span
+        else:
+            for span in regexp_span_tokenize(s, r'\n(\s+\n)*'):
+                yield span
+
+
+######################################################################
+# { Tokenization Functions
+######################################################################
+# XXX: it is stated in module docs that there is no function versions
+
+def line_tokenize(text, blanklines='discard'):
+    return LineTokenizer(blanklines).tokenize(text)