nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/chunk/regexp.py

@@ -1,1384 +1,1390 @@
-# Natural Language Toolkit: Regular Expression Chunkers
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Edward Loper <edloper@gmail.com>
-#         Steven Bird <stevenbird1@gmail.com> (minor additions)
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-from __future__ import print_function, unicode_literals
-from __future__ import division
-
-import re
-
-from nltk.tree import Tree
-from nltk.chunk.api import ChunkParserI
-from nltk.compat import python_2_unicode_compatible, string_types, unicode_repr
-
-##//////////////////////////////////////////////////////
-##  ChunkString
-##//////////////////////////////////////////////////////
-
-@python_2_unicode_compatible
-class ChunkString(object):
-    """
-    A string-based encoding of a particular chunking of a text.
-    Internally, the ``ChunkString`` class uses a single string to
-    encode the chunking of the input text.  This string contains a
-    sequence of angle-bracket delimited tags, with chunking indicated
-    by braces.  An example of this encoding is::
-
-        {<DT><JJ><NN>}<VBN><IN>{<DT><NN>}<.>{<DT><NN>}<VBD><.>
-
-    ``ChunkString`` are created from tagged texts (i.e., lists of
-    ``tokens`` whose type is ``TaggedType``).  Initially, nothing is
-    chunked.
-
-    The chunking of a ``ChunkString`` can be modified with the ``xform()``
-    method, which uses a regular expression to transform the string
-    representation.  These transformations should only add and remove
-    braces; they should *not* modify the sequence of angle-bracket
-    delimited tags.
-
-    :type _str: str
-    :ivar _str: The internal string representation of the text's
-        encoding.  This string representation contains a sequence of
-        angle-bracket delimited tags, with chunking indicated by
-        braces.  An example of this encoding is::
-
-            {<DT><JJ><NN>}<VBN><IN>{<DT><NN>}<.>{<DT><NN>}<VBD><.>
-
-    :type _pieces: list(tagged tokens and chunks)
-    :ivar _pieces: The tagged tokens and chunks encoded by this ``ChunkString``.
-    :ivar _debug: The debug level.  See the constructor docs.
-
-    :cvar IN_CHUNK_PATTERN: A zero-width regexp pattern string that
-        will only match positions that are in chunks.
-    :cvar IN_CHINK_PATTERN: A zero-width regexp pattern string that
-        will only match positions that are in chinks.
-    """
-    CHUNK_TAG_CHAR = r'[^\{\}<>]'
-    CHUNK_TAG = r'(<%s+?>)' % CHUNK_TAG_CHAR
-
-    IN_CHUNK_PATTERN = r'(?=[^\{]*\})'
-    IN_CHINK_PATTERN = r'(?=[^\}]*(\{|$))'
-
-    # These are used by _verify
-    _CHUNK = r'(\{%s+?\})+?' % CHUNK_TAG
-    _CHINK = r'(%s+?)+?' % CHUNK_TAG
-    _VALID = re.compile(r'^(\{?%s\}?)*?$' % CHUNK_TAG)
-    _BRACKETS = re.compile('[^\{\}]+')
-    _BALANCED_BRACKETS = re.compile(r'(\{\})*$')
-
-    def __init__(self, chunk_struct, debug_level=1):
-        """
-        Construct a new ``ChunkString`` that encodes the chunking of
-        the text ``tagged_tokens``.
-
-        :type chunk_struct: Tree
-        :param chunk_struct: The chunk structure to be further chunked.
-        :type debug_level: int
-        :param debug_level: The level of debugging which should be
-            applied to transformations on the ``ChunkString``.  The
-            valid levels are:
-                - 0: no checks
-                - 1: full check on to_chunkstruct
-                - 2: full check on to_chunkstruct and cursory check after
-                   each transformation.
-                - 3: full check on to_chunkstruct and full check after
-                   each transformation.
-            We recommend you use at least level 1.  You should
-            probably use level 3 if you use any non-standard
-            subclasses of ``RegexpChunkRule``.
-        """
-        self._root_label = chunk_struct.label()
-        self._pieces = chunk_struct[:]
-        tags = [self._tag(tok) for tok in self._pieces]
-        self._str = '<' + '><'.join(tags) + '>'
-        self._debug = debug_level
-
-    def _tag(self, tok):
-        if isinstance(tok, tuple):
-            return tok[1]
-        elif isinstance(tok, Tree):
-            return tok.label()
-        else:
-            raise ValueError('chunk structures must contain tagged '
-                             'tokens or trees')
-
-    def _verify(self, s, verify_tags):
-        """
-        Check to make sure that ``s`` still corresponds to some chunked
-        version of ``_pieces``.
-
-        :type verify_tags: bool
-        :param verify_tags: Whether the individual tags should be
-            checked.  If this is false, ``_verify`` will check to make
-            sure that ``_str`` encodes a chunked version of *some*
-            list of tokens.  If this is true, then ``_verify`` will
-            check to make sure that the tags in ``_str`` match those in
-            ``_pieces``.
-
-        :raise ValueError: if the internal string representation of
-            this ``ChunkString`` is invalid or not consistent with _pieces.
-        """
-        # Check overall form
-        if not ChunkString._VALID.match(s):
-            raise ValueError('Transformation generated invalid '
-                             'chunkstring:\n  %s' % s)
-
-        # Check that parens are balanced.  If the string is long, we
-        # have to do this in pieces, to avoid a maximum recursion
-        # depth limit for regular expressions.
-        brackets = ChunkString._BRACKETS.sub('', s)
-        for i in range(1 + len(brackets) // 5000):
-            substr = brackets[i*5000:i*5000+5000]
-            if not ChunkString._BALANCED_BRACKETS.match(substr):
-                raise ValueError('Transformation generated invalid '
-                                 'chunkstring:\n  %s' % s)
-
-        if verify_tags<=0: return
-
-        tags1 = (re.split(r'[\{\}<>]+', s))[1:-1]
-        tags2 = [self._tag(piece) for piece in self._pieces]
-        if tags1 != tags2:
-            raise ValueError('Transformation generated invalid '
-                             'chunkstring: tag changed')
-
-    def to_chunkstruct(self, chunk_label='CHUNK'):
-        """
-        Return the chunk structure encoded by this ``ChunkString``.
-
-        :rtype: Tree
-        :raise ValueError: If a transformation has generated an
-            invalid chunkstring.
-        """
-        if self._debug > 0: self._verify(self._str, 1)
-
-        # Use this alternating list to create the chunkstruct.
-        pieces = []
-        index = 0
-        piece_in_chunk = 0
-        for piece in re.split('[{}]', self._str):
-
-            # Find the list of tokens contained in this piece.
-            length = piece.count('<')
-            subsequence = self._pieces[index:index+length]
-
-            # Add this list of tokens to our pieces.
-            if piece_in_chunk:
-                pieces.append(Tree(chunk_label, subsequence))
-            else:
-                pieces += subsequence
-
-            # Update index, piece_in_chunk
-            index += length
-            piece_in_chunk = not piece_in_chunk
-
-        return Tree(self._root_label, pieces)
-
-    def xform(self, regexp, repl):
-        """
-        Apply the given transformation to the string encoding of this
-        ``ChunkString``.  In particular, find all occurrences that match
-        ``regexp``, and replace them using ``repl`` (as done by
-        ``re.sub``).
-
-        This transformation should only add and remove braces; it
-        should *not* modify the sequence of angle-bracket delimited
-        tags.  Furthermore, this transformation may not result in
-        improper bracketing.  Note, in particular, that bracketing may
-        not be nested.
-
-        :type regexp: str or regexp
-        :param regexp: A regular expression matching the substring
-            that should be replaced.  This will typically include a
-            named group, which can be used by ``repl``.
-        :type repl: str
-        :param repl: An expression specifying what should replace the
-            matched substring.  Typically, this will include a named
-            replacement group, specified by ``regexp``.
-        :rtype: None
-        :raise ValueError: If this transformation generated an
-            invalid chunkstring.
-        """
-        # Do the actual substitution
-        s = re.sub(regexp, repl, self._str)
-
-        # The substitution might have generated "empty chunks"
-        # (substrings of the form "{}").  Remove them, so they don't
-        # interfere with other transformations.
-        s = re.sub('\{\}', '', s)
-
-        # Make sure that the transformation was legal.
-        if self._debug > 1: self._verify(s, self._debug-2)
-
-        # Commit the transformation.
-        self._str = s
-
-    def __repr__(self):
-        """
-        Return a string representation of this ``ChunkString``.
-        It has the form::
-
-            <ChunkString: '{<DT><JJ><NN>}<VBN><IN>{<DT><NN>}'>
-
-        :rtype: str
-        """
-        return '<ChunkString: %s>' % unicode_repr(self._str)
-
-    def __str__(self):
-        """
-        Return a formatted representation of this ``ChunkString``.
-        This representation will include extra spaces to ensure that
-        tags will line up with the representation of other
-        ``ChunkStrings`` for the same text, regardless of the chunking.
-
-       :rtype: str
-        """
-        # Add spaces to make everything line up.
-        str = re.sub(r'>(?!\})', r'> ', self._str)
-        str = re.sub(r'([^\{])<', r'\1 <', str)
-        if str[0] == '<': str = ' ' + str
-        return str
-
-##//////////////////////////////////////////////////////
-##  Chunking Rules
-##//////////////////////////////////////////////////////
-
-@python_2_unicode_compatible
-class RegexpChunkRule(object):
-    """
-    A rule specifying how to modify the chunking in a ``ChunkString``,
-    using a transformational regular expression.  The
-    ``RegexpChunkRule`` class itself can be used to implement any
-    transformational rule based on regular expressions.  There are
-    also a number of subclasses, which can be used to implement
-    simpler types of rules, based on matching regular expressions.
-
-    Each ``RegexpChunkRule`` has a regular expression and a
-    replacement expression.  When a ``RegexpChunkRule`` is "applied"
-    to a ``ChunkString``, it searches the ``ChunkString`` for any
-    substring that matches the regular expression, and replaces it
-    using the replacement expression.  This search/replace operation
-    has the same semantics as ``re.sub``.
-
-    Each ``RegexpChunkRule`` also has a description string, which
-    gives a short (typically less than 75 characters) description of
-    the purpose of the rule.
-
-    This transformation defined by this ``RegexpChunkRule`` should
-    only add and remove braces; it should *not* modify the sequence
-    of angle-bracket delimited tags.  Furthermore, this transformation
-    may not result in nested or mismatched bracketing.
-    """
-    def __init__(self, regexp, repl, descr):
-        """
-        Construct a new RegexpChunkRule.
-
-        :type regexp: regexp or str
-        :param regexp: The regular expression for this ``RegexpChunkRule``.
-            When this rule is applied to a ``ChunkString``, any
-            substring that matches ``regexp`` will be replaced using
-            the replacement string ``repl``.  Note that this must be a
-            normal regular expression, not a tag pattern.
-        :type repl: str
-        :param repl: The replacement expression for this ``RegexpChunkRule``.
-            When this rule is applied to a ``ChunkString``, any substring
-            that matches ``regexp`` will be replaced using ``repl``.
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        if isinstance(regexp, string_types):
-            regexp = re.compile(regexp)
-        self._repl = repl
-        self._descr = descr
-        self._regexp = regexp
-
-    def apply(self, chunkstr):
-        # Keep docstring generic so we can inherit it.
-        """
-        Apply this rule to the given ``ChunkString``.  See the
-        class reference documentation for a description of what it
-        means to apply a rule.
-
-        :type chunkstr: ChunkString
-        :param chunkstr: The chunkstring to which this rule is applied.
-        :rtype: None
-        :raise ValueError: If this transformation generated an
-            invalid chunkstring.
-        """
-        chunkstr.xform(self._regexp, self._repl)
-
-    def descr(self):
-        """
-        Return a short description of the purpose and/or effect of
-        this rule.
-
-        :rtype: str
-        """
-        return self._descr
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <RegexpChunkRule: '{<IN|VB.*>}'->'<IN>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-        :rtype: str
-        """
-        return ('<RegexpChunkRule: '+unicode_repr(self._regexp.pattern)+
-                '->'+unicode_repr(self._repl)+'>')
-
-    @staticmethod
-    def fromstring(s):
-        """
-        Create a RegexpChunkRule from a string description.
-        Currently, the following formats are supported::
-
-          {regexp}         # chunk rule
-          }regexp{         # chink rule
-          regexp}{regexp   # split rule
-          regexp{}regexp   # merge rule
-
-        Where ``regexp`` is a regular expression for the rule.  Any
-        text following the comment marker (``#``) will be used as
-        the rule's description:
-
-        >>> from nltk.chunk.regexp import RegexpChunkRule
-        >>> RegexpChunkRule.fromstring('{<DT>?<NN.*>+}')
-        <ChunkRule: '<DT>?<NN.*>+'>
-        """
-        # Split off the comment (but don't split on '\#')
-        m = re.match(r'(?P<rule>(\\.|[^#])*)(?P<comment>#.*)?', s)
-        rule = m.group('rule').strip()
-        comment = (m.group('comment') or '')[1:].strip()
-
-        # Pattern bodies: chunk, chink, split, merge
-        try:
-            if not rule:
-                raise ValueError('Empty chunk pattern')
-            if rule[0] == '{' and rule[-1] == '}':
-                return ChunkRule(rule[1:-1], comment)
-            elif rule[0] == '}' and rule[-1] == '{':
-                return ChinkRule(rule[1:-1], comment)
-            elif '}{' in rule:
-                left, right = rule.split('}{')
-                return SplitRule(left, right, comment)
-            elif '{}' in rule:
-                left, right = rule.split('{}')
-                return MergeRule(left, right, comment)
-            elif re.match('[^{}]*{[^{}]*}[^{}]*', rule):
-                left, chunk, right = re.split('[{}]', rule)
-                return ChunkRuleWithContext(left, chunk, right, comment)
-            else:
-                raise ValueError('Illegal chunk pattern: %s' % rule)
-        except (ValueError, re.error):
-            raise ValueError('Illegal chunk pattern: %s' % rule)
-
-
-@python_2_unicode_compatible
-class ChunkRule(RegexpChunkRule):
-    """
-    A rule specifying how to add chunks to a ``ChunkString``, using a
-    matching tag pattern.  When applied to a ``ChunkString``, it will
-    find any substring that matches this tag pattern and that is not
-    already part of a chunk, and create a new chunk containing that
-    substring.
-    """
-    def __init__(self, tag_pattern, descr):
-
-        """
-        Construct a new ``ChunkRule``.
-
-        :type tag_pattern: str
-        :param tag_pattern: This rule's tag pattern.  When
-            applied to a ``ChunkString``, this rule will
-            chunk any substring that matches this tag pattern and that
-            is not already part of a chunk.
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        self._pattern = tag_pattern
-        regexp = re.compile('(?P<chunk>%s)%s' %
-                            (tag_pattern2re_pattern(tag_pattern),
-                             ChunkString.IN_CHINK_PATTERN))
-        RegexpChunkRule.__init__(self, regexp, '{\g<chunk>}', descr)
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <ChunkRule: '<IN|VB.*>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-        :rtype: str
-        """
-        return '<ChunkRule: '+unicode_repr(self._pattern)+'>'
-
-@python_2_unicode_compatible
-class ChinkRule(RegexpChunkRule):
-    """
-    A rule specifying how to remove chinks to a ``ChunkString``,
-    using a matching tag pattern.  When applied to a
-    ``ChunkString``, it will find any substring that matches this
-    tag pattern and that is contained in a chunk, and remove it
-    from that chunk, thus creating two new chunks.
-    """
-    def __init__(self, tag_pattern, descr):
-        """
-        Construct a new ``ChinkRule``.
-
-        :type tag_pattern: str
-        :param tag_pattern: This rule's tag pattern.  When
-            applied to a ``ChunkString``, this rule will
-            find any substring that matches this tag pattern and that
-            is contained in a chunk, and remove it from that chunk,
-            thus creating two new chunks.
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        self._pattern = tag_pattern
-        regexp = re.compile('(?P<chink>%s)%s' %
-                            (tag_pattern2re_pattern(tag_pattern),
-                             ChunkString.IN_CHUNK_PATTERN))
-        RegexpChunkRule.__init__(self, regexp, '}\g<chink>{', descr)
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <ChinkRule: '<IN|VB.*>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-        :rtype: str
-        """
-        return '<ChinkRule: '+unicode_repr(self._pattern)+'>'
-
-
-@python_2_unicode_compatible
-class UnChunkRule(RegexpChunkRule):
-    """
-    A rule specifying how to remove chunks to a ``ChunkString``,
-    using a matching tag pattern.  When applied to a
-    ``ChunkString``, it will find any complete chunk that matches this
-    tag pattern, and un-chunk it.
-    """
-    def __init__(self, tag_pattern, descr):
-        """
-        Construct a new ``UnChunkRule``.
-
-        :type tag_pattern: str
-        :param tag_pattern: This rule's tag pattern.  When
-            applied to a ``ChunkString``, this rule will
-            find any complete chunk that matches this tag pattern,
-            and un-chunk it.
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        self._pattern = tag_pattern
-        regexp = re.compile('\{(?P<chunk>%s)\}' %
-                            tag_pattern2re_pattern(tag_pattern))
-        RegexpChunkRule.__init__(self, regexp, '\g<chunk>', descr)
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <UnChunkRule: '<IN|VB.*>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-        :rtype: str
-        """
-        return '<UnChunkRule: '+unicode_repr(self._pattern)+'>'
-
-
-@python_2_unicode_compatible
-class MergeRule(RegexpChunkRule):
-    """
-    A rule specifying how to merge chunks in a ``ChunkString``, using
-    two matching tag patterns: a left pattern, and a right pattern.
-    When applied to a ``ChunkString``, it will find any chunk whose end
-    matches left pattern, and immediately followed by a chunk whose
-    beginning matches right pattern.  It will then merge those two
-    chunks into a single chunk.
-    """
-    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
-        """
-        Construct a new ``MergeRule``.
-
-        :type right_tag_pattern: str
-        :param right_tag_pattern: This rule's right tag
-            pattern.  When applied to a ``ChunkString``, this
-            rule will find any chunk whose end matches
-            ``left_tag_pattern``, and immediately followed by a chunk
-            whose beginning matches this pattern.  It will
-            then merge those two chunks into a single chunk.
-        :type left_tag_pattern: str
-        :param left_tag_pattern: This rule's left tag
-            pattern.  When applied to a ``ChunkString``, this
-            rule will find any chunk whose end matches
-            this pattern, and immediately followed by a chunk
-            whose beginning matches ``right_tag_pattern``.  It will
-            then merge those two chunks into a single chunk.
-
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        # Ensure that the individual patterns are coherent.  E.g., if
-        # left='(' and right=')', then this will raise an exception:
-        re.compile(tag_pattern2re_pattern(left_tag_pattern))
-        re.compile(tag_pattern2re_pattern(right_tag_pattern))
-
-        self._left_tag_pattern = left_tag_pattern
-        self._right_tag_pattern = right_tag_pattern
-        regexp = re.compile('(?P<left>%s)}{(?=%s)' %
-                            (tag_pattern2re_pattern(left_tag_pattern),
-                             tag_pattern2re_pattern(right_tag_pattern)))
-        RegexpChunkRule.__init__(self, regexp, '\g<left>', descr)
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <MergeRule: '<NN|DT|JJ>', '<NN|JJ>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-        :rtype: str
-        """
-        return ('<MergeRule: '+unicode_repr(self._left_tag_pattern)+', '+
-                unicode_repr(self._right_tag_pattern)+'>')
-
-
-@python_2_unicode_compatible
-class SplitRule(RegexpChunkRule):
-    """
-    A rule specifying how to split chunks in a ``ChunkString``, using
-    two matching tag patterns: a left pattern, and a right pattern.
-    When applied to a ``ChunkString``, it will find any chunk that
-    matches the left pattern followed by the right pattern.  It will
-    then split the chunk into two new chunks, at the point between the
-    two pattern matches.
-    """
-    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
-        """
-        Construct a new ``SplitRule``.
-
-        :type right_tag_pattern: str
-        :param right_tag_pattern: This rule's right tag
-            pattern.  When applied to a ``ChunkString``, this rule will
-            find any chunk containing a substring that matches
-            ``left_tag_pattern`` followed by this pattern.  It will
-            then split the chunk into two new chunks at the point
-            between these two matching patterns.
-        :type left_tag_pattern: str
-        :param left_tag_pattern: This rule's left tag
-            pattern.  When applied to a ``ChunkString``, this rule will
-            find any chunk containing a substring that matches this
-            pattern followed by ``right_tag_pattern``.  It will then
-            split the chunk into two new chunks at the point between
-            these two matching patterns.
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        # Ensure that the individual patterns are coherent.  E.g., if
-        # left='(' and right=')', then this will raise an exception:
-        re.compile(tag_pattern2re_pattern(left_tag_pattern))
-        re.compile(tag_pattern2re_pattern(right_tag_pattern))
-
-        self._left_tag_pattern = left_tag_pattern
-        self._right_tag_pattern = right_tag_pattern
-        regexp = re.compile('(?P<left>%s)(?=%s)' %
-                            (tag_pattern2re_pattern(left_tag_pattern),
-                             tag_pattern2re_pattern(right_tag_pattern)))
-        RegexpChunkRule.__init__(self, regexp, r'\g<left>}{', descr)
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <SplitRule: '<NN>', '<DT>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-       :rtype: str
-        """
-        return ('<SplitRule: '+unicode_repr(self._left_tag_pattern)+', '+
-                unicode_repr(self._right_tag_pattern)+'>')
-
-
-@python_2_unicode_compatible
-class ExpandLeftRule(RegexpChunkRule):
-    """
-    A rule specifying how to expand chunks in a ``ChunkString`` to the left,
-    using two matching tag patterns: a left pattern, and a right pattern.
-    When applied to a ``ChunkString``, it will find any chunk whose beginning
-    matches right pattern, and immediately preceded by a chink whose
-    end matches left pattern.  It will then expand the chunk to incorporate
-    the new material on the left.
-    """
-    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
-        """
-        Construct a new ``ExpandRightRule``.
-
-        :type right_tag_pattern: str
-        :param right_tag_pattern: This rule's right tag
-            pattern.  When applied to a ``ChunkString``, this
-            rule will find any chunk whose beginning matches
-            ``right_tag_pattern``, and immediately preceded by a chink
-            whose end matches this pattern.  It will
-            then merge those two chunks into a single chunk.
-        :type left_tag_pattern: str
-        :param left_tag_pattern: This rule's left tag
-            pattern.  When applied to a ``ChunkString``, this
-            rule will find any chunk whose beginning matches
-            this pattern, and immediately preceded by a chink
-            whose end matches ``left_tag_pattern``.  It will
-            then expand the chunk to incorporate the new material on the left.
-
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        # Ensure that the individual patterns are coherent.  E.g., if
-        # left='(' and right=')', then this will raise an exception:
-        re.compile(tag_pattern2re_pattern(left_tag_pattern))
-        re.compile(tag_pattern2re_pattern(right_tag_pattern))
-
-        self._left_tag_pattern = left_tag_pattern
-        self._right_tag_pattern = right_tag_pattern
-        regexp = re.compile('(?P<left>%s)\{(?P<right>%s)' %
-                            (tag_pattern2re_pattern(left_tag_pattern),
-                             tag_pattern2re_pattern(right_tag_pattern)))
-        RegexpChunkRule.__init__(self, regexp, '{\g<left>\g<right>', descr)
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <ExpandLeftRule: '<NN|DT|JJ>', '<NN|JJ>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-        :rtype: str
-        """
-        return ('<ExpandLeftRule: '+unicode_repr(self._left_tag_pattern)+', '+
-                unicode_repr(self._right_tag_pattern)+'>')
-
-
-@python_2_unicode_compatible
-class ExpandRightRule(RegexpChunkRule):
-    """
-    A rule specifying how to expand chunks in a ``ChunkString`` to the
-    right, using two matching tag patterns: a left pattern, and a
-    right pattern.  When applied to a ``ChunkString``, it will find any
-    chunk whose end matches left pattern, and immediately followed by
-    a chink whose beginning matches right pattern.  It will then
-    expand the chunk to incorporate the new material on the right.
-    """
-    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
-        """
-        Construct a new ``ExpandRightRule``.
-
-        :type right_tag_pattern: str
-        :param right_tag_pattern: This rule's right tag
-            pattern.  When applied to a ``ChunkString``, this
-            rule will find any chunk whose end matches
-            ``left_tag_pattern``, and immediately followed by a chink
-            whose beginning matches this pattern.  It will
-            then merge those two chunks into a single chunk.
-        :type left_tag_pattern: str
-        :param left_tag_pattern: This rule's left tag
-            pattern.  When applied to a ``ChunkString``, this
-            rule will find any chunk whose end matches
-            this pattern, and immediately followed by a chink
-            whose beginning matches ``right_tag_pattern``.  It will
-            then expand the chunk to incorporate the new material on the right.
-
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        # Ensure that the individual patterns are coherent.  E.g., if
-        # left='(' and right=')', then this will raise an exception:
-        re.compile(tag_pattern2re_pattern(left_tag_pattern))
-        re.compile(tag_pattern2re_pattern(right_tag_pattern))
-
-        self._left_tag_pattern = left_tag_pattern
-        self._right_tag_pattern = right_tag_pattern
-        regexp = re.compile('(?P<left>%s)\}(?P<right>%s)' %
-                            (tag_pattern2re_pattern(left_tag_pattern),
-                             tag_pattern2re_pattern(right_tag_pattern)))
-        RegexpChunkRule.__init__(self, regexp, '\g<left>\g<right>}', descr)
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <ExpandRightRule: '<NN|DT|JJ>', '<NN|JJ>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-        :rtype: str
-        """
-        return ('<ExpandRightRule: '+unicode_repr(self._left_tag_pattern)+', '+
-                unicode_repr(self._right_tag_pattern)+'>')
-
-
-@python_2_unicode_compatible
-class ChunkRuleWithContext(RegexpChunkRule):
-    """
-    A rule specifying how to add chunks to a ``ChunkString``, using
-    three matching tag patterns: one for the left context, one for the
-    chunk, and one for the right context.  When applied to a
-    ``ChunkString``, it will find any substring that matches the chunk
-    tag pattern, is surrounded by substrings that match the two
-    context patterns, and is not already part of a chunk; and create a
-    new chunk containing the substring that matched the chunk tag
-    pattern.
-
-    Caveat: Both the left and right context are consumed when this
-    rule matches; therefore, if you need to find overlapping matches,
-    you will need to apply your rule more than once.
-    """
-    def __init__(self, left_context_tag_pattern, chunk_tag_pattern,
-                 right_context_tag_pattern, descr):
-        """
-        Construct a new ``ChunkRuleWithContext``.
-
-        :type left_context_tag_pattern: str
-        :param left_context_tag_pattern: A tag pattern that must match
-            the left context of ``chunk_tag_pattern`` for this rule to
-            apply.
-        :type chunk_tag_pattern: str
-        :param chunk_tag_pattern: A tag pattern that must match for this
-            rule to apply.  If the rule does apply, then this pattern
-            also identifies the substring that will be made into a chunk.
-        :type right_context_tag_pattern: str
-        :param right_context_tag_pattern: A tag pattern that must match
-            the right context of ``chunk_tag_pattern`` for this rule to
-            apply.
-        :type descr: str
-        :param descr: A short description of the purpose and/or effect
-            of this rule.
-        """
-        # Ensure that the individual patterns are coherent.  E.g., if
-        # left='(' and right=')', then this will raise an exception:
-        re.compile(tag_pattern2re_pattern(left_context_tag_pattern))
-        re.compile(tag_pattern2re_pattern(chunk_tag_pattern))
-        re.compile(tag_pattern2re_pattern(right_context_tag_pattern))
-
-        self._left_context_tag_pattern = left_context_tag_pattern
-        self._chunk_tag_pattern = chunk_tag_pattern
-        self._right_context_tag_pattern = right_context_tag_pattern
-        regexp = re.compile('(?P<left>%s)(?P<chunk>%s)(?P<right>%s)%s' %
-                            (tag_pattern2re_pattern(left_context_tag_pattern),
-                             tag_pattern2re_pattern(chunk_tag_pattern),
-                             tag_pattern2re_pattern(right_context_tag_pattern),
-                             ChunkString.IN_CHINK_PATTERN))
-        replacement = r'\g<left>{\g<chunk>}\g<right>'
-        RegexpChunkRule.__init__(self, regexp, replacement, descr)
-
-    def __repr__(self):
-        """
-        Return a string representation of this rule.  It has the form::
-
-            <ChunkRuleWithContext: '<IN>', '<NN>', '<DT>'>
-
-        Note that this representation does not include the
-        description string; that string can be accessed
-        separately with the ``descr()`` method.
-
-        :rtype: str
-        """
-        return '<ChunkRuleWithContext:  %r, %r, %r>' % (
-            self._left_context_tag_pattern, self._chunk_tag_pattern,
-            self._right_context_tag_pattern)
-
-##//////////////////////////////////////////////////////
-##  Tag Pattern Format Conversion
-##//////////////////////////////////////////////////////
-
-# this should probably be made more strict than it is -- e.g., it
-# currently accepts 'foo'.
-CHUNK_TAG_PATTERN = re.compile(r'^((%s|<%s>)*)$' %
-                                ('[^\{\}<>]+',
-                                 '[^\{\}<>]+'))
-
-def tag_pattern2re_pattern(tag_pattern):
-    """
-    Convert a tag pattern to a regular expression pattern.  A "tag
-    pattern" is a modified version of a regular expression, designed
-    for matching sequences of tags.  The differences between regular
-    expression patterns and tag patterns are:
-
-        - In tag patterns, ``'<'`` and ``'>'`` act as parentheses; so
-          ``'<NN>+'`` matches one or more repetitions of ``'<NN>'``, not
-          ``'<NN'`` followed by one or more repetitions of ``'>'``.
-        - Whitespace in tag patterns is ignored.  So
-          ``'<DT> | <NN>'`` is equivalant to ``'<DT>|<NN>'``
-        - In tag patterns, ``'.'`` is equivalant to ``'[^{}<>]'``; so
-          ``'<NN.*>'`` matches any single tag starting with ``'NN'``.
-
-    In particular, ``tag_pattern2re_pattern`` performs the following
-    transformations on the given pattern:
-
-        - Replace '.' with '[^<>{}]'
-        - Remove any whitespace
-        - Add extra parens around '<' and '>', to make '<' and '>' act
-          like parentheses.  E.g., so that in '<NN>+', the '+' has scope
-          over the entire '<NN>'; and so that in '<NN|IN>', the '|' has
-          scope over 'NN' and 'IN', but not '<' or '>'.
-        - Check to make sure the resulting pattern is valid.
-
-    :type tag_pattern: str
-    :param tag_pattern: The tag pattern to convert to a regular
-        expression pattern.
-    :raise ValueError: If ``tag_pattern`` is not a valid tag pattern.
-        In particular, ``tag_pattern`` should not include braces; and it
-        should not contain nested or mismatched angle-brackets.
-    :rtype: str
-    :return: A regular expression pattern corresponding to
-        ``tag_pattern``.
-    """
-    # Clean up the regular expression
-    tag_pattern = re.sub(r'\s', '', tag_pattern)
-    tag_pattern = re.sub(r'<', '(<(', tag_pattern)
-    tag_pattern = re.sub(r'>', ')>)', tag_pattern)
-
-    # Check the regular expression
-    if not CHUNK_TAG_PATTERN.match(tag_pattern):
-        raise ValueError('Bad tag pattern: %r' % tag_pattern)
-
-    # Replace "." with CHUNK_TAG_CHAR.
-    # We have to do this after, since it adds {}[]<>s, which would
-    # confuse CHUNK_TAG_PATTERN.
-    # PRE doesn't have lookback assertions, so reverse twice, and do
-    # the pattern backwards (with lookahead assertions).  This can be
-    # made much cleaner once we can switch back to SRE.
-    def reverse_str(str):
-        lst = list(str)
-        lst.reverse()
-        return ''.join(lst)
-    tc_rev = reverse_str(ChunkString.CHUNK_TAG_CHAR)
-    reversed = reverse_str(tag_pattern)
-    reversed = re.sub(r'\.(?!\\(\\\\)*($|[^\\]))', tc_rev, reversed)
-    tag_pattern = reverse_str(reversed)
-
-    return tag_pattern
-
-
-##//////////////////////////////////////////////////////
-##  RegexpChunkParser
-##//////////////////////////////////////////////////////
-
-@python_2_unicode_compatible
-class RegexpChunkParser(ChunkParserI):
-    """
-    A regular expression based chunk parser.  ``RegexpChunkParser`` uses a
-    sequence of "rules" to find chunks of a single type within a
-    text.  The chunking of the text is encoded using a ``ChunkString``,
-    and each rule acts by modifying the chunking in the
-    ``ChunkString``.  The rules are all implemented using regular
-    expression matching and substitution.
-
-    The ``RegexpChunkRule`` class and its subclasses (``ChunkRule``,
-    ``ChinkRule``, ``UnChunkRule``, ``MergeRule``, and ``SplitRule``)
-    define the rules that are used by ``RegexpChunkParser``.  Each rule
-    defines an ``apply()`` method, which modifies the chunking encoded
-    by a given ``ChunkString``.
-
-    :type _rules: list(RegexpChunkRule)
-    :ivar _rules: The list of rules that should be applied to a text.
-    :type _trace: int
-    :ivar _trace: The default level of tracing.
-
-    """
-    def __init__(self, rules, chunk_label='NP', root_label='S', trace=0):
-        """
-        Construct a new ``RegexpChunkParser``.
-
-        :type rules: list(RegexpChunkRule)
-        :param rules: The sequence of rules that should be used to
-            generate the chunking for a tagged text.
-        :type chunk_label: str
-        :param chunk_label: The node value that should be used for
-            chunk subtrees.  This is typically a short string
-            describing the type of information contained by the chunk,
-            such as ``"NP"`` for base noun phrases.
-        :type root_label: str
-        :param root_label: The node value that should be used for the
-            top node of the chunk structure.
-        :type trace: int
-        :param trace: The level of tracing that should be used when
-            parsing a text.  ``0`` will generate no tracing output;
-            ``1`` will generate normal tracing output; and ``2`` or
-            higher will generate verbose tracing output.
-        """
-        self._rules = rules
-        self._trace = trace
-        self._chunk_label = chunk_label
-        self._root_label = root_label
-
-    def _trace_apply(self, chunkstr, verbose):
-        """
-        Apply each rule of this ``RegexpChunkParser`` to ``chunkstr``, in
-        turn.  Generate trace output between each rule.  If ``verbose``
-        is true, then generate verbose output.
-
-        :type chunkstr: ChunkString
-        :param chunkstr: The chunk string to which each rule should be
-            applied.
-        :type verbose: bool
-        :param verbose: Whether output should be verbose.
-        :rtype: None
-        """
-        print('# Input:')
-        print(chunkstr)
-        for rule in self._rules:
-            rule.apply(chunkstr)
-            if verbose:
-                print('#', rule.descr()+' ('+unicode_repr(rule)+'):')
-            else:
-                print('#', rule.descr()+':')
-            print(chunkstr)
-
-    def _notrace_apply(self, chunkstr):
-        """
-        Apply each rule of this ``RegexpChunkParser`` to ``chunkstr``, in
-        turn.
-
-        :param chunkstr: The chunk string to which each rule should be
-            applied.
-        :type chunkstr: ChunkString
-        :rtype: None
-        """
-
-        for rule in self._rules:
-            rule.apply(chunkstr)
-
-    def parse(self, chunk_struct, trace=None):
-        """
-        :type chunk_struct: Tree
-        :param chunk_struct: the chunk structure to be (further) chunked
-        :type trace: int
-        :param trace: The level of tracing that should be used when
-            parsing a text.  ``0`` will generate no tracing output;
-            ``1`` will generate normal tracing output; and ``2`` or
-            highter will generate verbose tracing output.  This value
-            overrides the trace level value that was given to the
-            constructor.
-        :rtype: Tree
-        :return: a chunk structure that encodes the chunks in a given
-            tagged sentence.  A chunk is a non-overlapping linguistic
-            group, such as a noun phrase.  The set of chunks
-            identified in the chunk structure depends on the rules
-            used to define this ``RegexpChunkParser``.
-        """
-        if len(chunk_struct) == 0:
-            print('Warning: parsing empty text')
-            return Tree(self._root_label, [])
-
-        try:
-            chunk_struct.label()
-        except AttributeError:
-            chunk_struct = Tree(self._root_label, chunk_struct)
-
-        # Use the default trace value?
-        if trace is None: trace = self._trace
-
-        chunkstr = ChunkString(chunk_struct)
-
-        # Apply the sequence of rules to the chunkstring.
-        if trace:
-            verbose = (trace>1)
-            self._trace_apply(chunkstr, verbose)
-        else:
-            self._notrace_apply(chunkstr)
-
-        # Use the chunkstring to create a chunk structure.
-        return chunkstr.to_chunkstruct(self._chunk_label)
-
-    def rules(self):
-        """
-        :return: the sequence of rules used by ``RegexpChunkParser``.
-        :rtype: list(RegexpChunkRule)
-        """
-        return self._rules
-
-    def __repr__(self):
-        """
-        :return: a concise string representation of this
-            ``RegexpChunkParser``.
-        :rtype: str
-        """
-        return "<RegexpChunkParser with %d rules>" % len(self._rules)
-
-    def __str__(self):
-        """
-        :return: a verbose string representation of this ``RegexpChunkParser``.
-        :rtype: str
-        """
-        s = "RegexpChunkParser with %d rules:\n" % len(self._rules)
-        margin = 0
-        for rule in self._rules:
-            margin = max(margin, len(rule.descr()))
-        if margin < 35:
-            format = "    %" + repr(-(margin+3)) + "s%s\n"
-        else:
-            format = "    %s\n      %s\n"
-        for rule in self._rules:
-            s += format % (rule.descr(), unicode_repr(rule))
-        return s[:-1]
-
-##//////////////////////////////////////////////////////
-##  Chunk Grammar
-##//////////////////////////////////////////////////////
-
-@python_2_unicode_compatible
-class RegexpParser(ChunkParserI):
-    """
-    A grammar based chunk parser.  ``chunk.RegexpParser`` uses a set of
-    regular expression patterns to specify the behavior of the parser.
-    The chunking of the text is encoded using a ``ChunkString``, and
-    each rule acts by modifying the chunking in the ``ChunkString``.
-    The rules are all implemented using regular expression matching
-    and substitution.
-
-    A grammar contains one or more clauses in the following form::
-
-     NP:
-       {<DT|JJ>}          # chunk determiners and adjectives
-       }<[\.VI].*>+{      # chink any tag beginning with V, I, or .
-       <.*>}{<DT>         # split a chunk at a determiner
-       <DT|JJ>{}<NN.*>    # merge chunk ending with det/adj
-                          # with one starting with a noun
-
-    The patterns of a clause are executed in order.  An earlier
-    pattern may introduce a chunk boundary that prevents a later
-    pattern from executing.  Sometimes an individual pattern will
-    match on multiple, overlapping extents of the input.  As with
-    regular expression substitution more generally, the chunker will
-    identify the first match possible, then continue looking for matches
-    after this one has ended.
-
-    The clauses of a grammar are also executed in order.  A cascaded
-    chunk parser is one having more than one clause.  The maximum depth
-    of a parse tree created by this chunk parser is the same as the
-    number of clauses in the grammar.
-
-    When tracing is turned on, the comment portion of a line is displayed
-    each time the corresponding pattern is applied.
-
-    :type _start: str
-    :ivar _start: The start symbol of the grammar (the root node of
-        resulting trees)
-    :type _stages: int
-    :ivar _stages: The list of parsing stages corresponding to the grammar
-
-    """
-    def __init__(self, grammar, root_label='S', loop=1, trace=0):
-        """
-        Create a new chunk parser, from the given start state
-        and set of chunk patterns.
-
-        :param grammar: The grammar, or a list of RegexpChunkParser objects
-        :type grammar: str or list(RegexpChunkParser)
-        :param root_label: The top node of the tree being created
-        :type root_label: str or Nonterminal
-        :param loop: The number of times to run through the patterns
-        :type loop: int
-        :type trace: int
-        :param trace: The level of tracing that should be used when
-            parsing a text.  ``0`` will generate no tracing output;
-            ``1`` will generate normal tracing output; and ``2`` or
-            higher will generate verbose tracing output.
-        """
-        self._trace = trace
-        self._stages = []
-        self._grammar = grammar
-        self._loop = loop
-
-        if isinstance(grammar, string_types):
-            self._read_grammar(grammar, root_label, trace)
-        else:
-            # Make sur the grammar looks like it has the right type:
-            type_err = ('Expected string or list of RegexpChunkParsers '
-                        'for the grammar.')
-            try: grammar = list(grammar)
-            except: raise TypeError(type_err)
-            for elt in grammar:
-                if not isinstance(elt, RegexpChunkParser):
-                    raise TypeError(type_err)
-            self._stages = grammar
-
-    def _read_grammar(self, grammar, root_label, trace):
-        """
-        Helper function for __init__: read the grammar if it is a
-        string.
-        """
-        rules = []
-        lhs = None
-        for line in grammar.split('\n'):
-            line = line.strip()
-
-            # New stage begins if there's an unescaped ':'
-            m = re.match('(?P<nonterminal>(\\.|[^:])*)(:(?P<rule>.*))', line)
-            if m:
-                # Record the stage that we just completed.
-                self._add_stage(rules, lhs, root_label, trace)
-                # Start a new stage.
-                lhs = m.group('nonterminal').strip()
-                rules = []
-                line = m.group('rule').strip()
-
-            # Skip blank & comment-only lines
-            if line=='' or line.startswith('#'): continue
-
-            # Add the rule
-            rules.append(RegexpChunkRule.fromstring(line))
-
-        # Record the final stage
-        self._add_stage(rules, lhs, root_label, trace)
-
-    def _add_stage(self, rules, lhs, root_label, trace):
-        """
-        Helper function for __init__: add a new stage to the parser.
-        """
-        if rules != []:
-            if not lhs:
-                raise ValueError('Expected stage marker (eg NP:)')
-            parser = RegexpChunkParser(rules, chunk_label=lhs,
-                                       root_label=root_label, trace=trace)
-            self._stages.append(parser)
-
-    def parse(self, chunk_struct, trace=None):
-        """
-        Apply the chunk parser to this input.
-
-        :type chunk_struct: Tree
-        :param chunk_struct: the chunk structure to be (further) chunked
-            (this tree is modified, and is also returned)
-        :type trace: int
-        :param trace: The level of tracing that should be used when
-            parsing a text.  ``0`` will generate no tracing output;
-            ``1`` will generate normal tracing output; and ``2`` or
-            highter will generate verbose tracing output.  This value
-            overrides the trace level value that was given to the
-            constructor.
-        :return: the chunked output.
-        :rtype: Tree
-        """
-        if trace is None: trace = self._trace
-        for i in range(self._loop):
-            for parser in self._stages:
-                chunk_struct = parser.parse(chunk_struct, trace=trace)
-        return chunk_struct
-
-    def __repr__(self):
-        """
-        :return: a concise string representation of this ``chunk.RegexpParser``.
-        :rtype: str
-        """
-        return "<chunk.RegexpParser with %d stages>" % len(self._stages)
-
-    def __str__(self):
-        """
-        :return: a verbose string representation of this
-            ``RegexpParser``.
-        :rtype: str
-        """
-        s = "chunk.RegexpParser with %d stages:\n" % len(self._stages)
-        margin = 0
-        for parser in self._stages:
-            s += "%s\n" % parser
-        return s[:-1]
-
-##//////////////////////////////////////////////////////
-##  Demonstration code
-##//////////////////////////////////////////////////////
-
-def demo_eval(chunkparser, text):
-    """
-    Demonstration code for evaluating a chunk parser, using a
-    ``ChunkScore``.  This function assumes that ``text`` contains one
-    sentence per line, and that each sentence has the form expected by
-    ``tree.chunk``.  It runs the given chunk parser on each sentence in
-    the text, and scores the result.  It prints the final score
-    (precision, recall, and f-measure); and reports the set of chunks
-    that were missed and the set of chunks that were incorrect.  (At
-    most 10 missing chunks and 10 incorrect chunks are reported).
-
-    :param chunkparser: The chunkparser to be tested
-    :type chunkparser: ChunkParserI
-    :param text: The chunked tagged text that should be used for
-        evaluation.
-    :type text: str
-    """
-    from nltk import chunk
-    from nltk.tree import Tree
-
-    # Evaluate our chunk parser.
-    chunkscore = chunk.ChunkScore()
-
-    for sentence in text.split('\n'):
-        print(sentence)
-        sentence = sentence.strip()
-        if not sentence: continue
-        gold = chunk.tagstr2tree(sentence)
-        tokens = gold.leaves()
-        test = chunkparser.parse(Tree('S', tokens), trace=1)
-        chunkscore.score(gold, test)
-        print()
-
-    print('/'+('='*75)+'\\')
-    print('Scoring', chunkparser)
-    print(('-'*77))
-    print('Precision: %5.1f%%' % (chunkscore.precision()*100), ' '*4, end=' ')
-    print('Recall: %5.1f%%' % (chunkscore.recall()*100), ' '*6, end=' ')
-    print('F-Measure: %5.1f%%' % (chunkscore.f_measure()*100))
-
-
-    # Missed chunks.
-    if chunkscore.missed():
-        print('Missed:')
-        missed = chunkscore.missed()
-        for chunk in missed[:10]:
-            print('  ', ' '.join(map(str,chunk)))
-        if len(chunkscore.missed()) > 10:
-            print('  ...')
-
-    # Incorrect chunks.
-    if chunkscore.incorrect():
-        print('Incorrect:')
-        incorrect = chunkscore.incorrect()
-        for chunk in incorrect[:10]:
-            print('  ', ' '.join(map(str,chunk)))
-        if len(chunkscore.incorrect()) > 10:
-            print('  ...')
-
-    print('\\'+('='*75)+'/')
-    print()
-
-def demo():
-    """
-    A demonstration for the ``RegexpChunkParser`` class.  A single text is
-    parsed with four different chunk parsers, using a variety of rules
-    and strategies.
-    """
-
-    from nltk import chunk, Tree
-
-    text = """\
-    [ the/DT little/JJ cat/NN ] sat/VBD on/IN [ the/DT mat/NN ] ./.
-    [ John/NNP ] saw/VBD [the/DT cats/NNS] [the/DT dog/NN] chased/VBD ./.
-    [ John/NNP ] thinks/VBZ [ Mary/NN ] saw/VBD [ the/DT cat/NN ] sit/VB on/IN [ the/DT mat/NN ]./.
-    """
-
-    print('*'*75)
-    print('Evaluation text:')
-    print(text)
-    print('*'*75)
-    print()
-
-    grammar = r"""
-    NP:                   # NP stage
-      {<DT>?<JJ>*<NN>}    # chunk determiners, adjectives and nouns
-      {<NNP>+}            # chunk proper nouns
-    """
-    cp = chunk.RegexpParser(grammar)
-    demo_eval(cp, text)
-
-    grammar = r"""
-    NP:
-      {<.*>}              # start by chunking each tag
-      }<[\.VI].*>+{       # unchunk any verbs, prepositions or periods
-      <DT|JJ>{}<NN.*>     # merge det/adj with nouns
-    """
-    cp = chunk.RegexpParser(grammar)
-    demo_eval(cp, text)
-
-    grammar = r"""
-    NP: {<DT>?<JJ>*<NN>}    # chunk determiners, adjectives and nouns
-    VP: {<TO>?<VB.*>}       # VP = verb words
-    """
-    cp = chunk.RegexpParser(grammar)
-    demo_eval(cp, text)
-
-    grammar = r"""
-    NP: {<.*>*}             # start by chunking everything
-        }<[\.VI].*>+{       # chink any verbs, prepositions or periods
-        <.*>}{<DT>          # separate on determiners
-    PP: {<IN><NP>}          # PP = preposition + noun phrase
-    VP: {<VB.*><NP|PP>*}    # VP = verb words + NPs and PPs
-    """
-    cp = chunk.RegexpParser(grammar)
-    demo_eval(cp, text)
-
-# Evaluation
-
-    from nltk.corpus import conll2000
-
-    print()
-    print("Demonstration of empty grammar:")
-
-    cp = chunk.RegexpParser("")
-    print(chunk.accuracy(cp, conll2000.chunked_sents('test.txt',
-                                                     chunk_types=('NP',))))
-
-    print()
-    print("Demonstration of accuracy evaluation using CoNLL tags:")
-
-    grammar = r"""
-    NP:
-      {<.*>}              # start by chunking each tag
-      }<[\.VI].*>+{       # unchunk any verbs, prepositions or periods
-      <DT|JJ>{}<NN.*>     # merge det/adj with nouns
-    """
-    cp = chunk.RegexpParser(grammar)
-    print(chunk.accuracy(cp, conll2000.chunked_sents('test.txt')[:5]))
-
-    print()
-    print("Demonstration of tagged token input")
-
-    grammar = r"""
-    NP: {<.*>*}             # start by chunking everything
-        }<[\.VI].*>+{       # chink any verbs, prepositions or periods
-        <.*>}{<DT>          # separate on determiners
-    PP: {<IN><NP>}          # PP = preposition + noun phrase
-    VP: {<VB.*><NP|PP>*}    # VP = verb words + NPs and PPs
-    """
-    cp = chunk.RegexpParser(grammar)
-    print(cp.parse([("the","DT"), ("little","JJ"), ("cat", "NN"),
-                    ("sat", "VBD"), ("on", "IN"), ("the", "DT"),
-                    ("mat", "NN"), (".", ".")]))
-
-if __name__ == '__main__':
-    demo()
+# Natural Language Toolkit: Regular Expression Chunkers
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+from __future__ import division
+
+import re
+
+from six import string_types
+
+from nltk.tree import Tree
+from nltk.chunk.api import ChunkParserI
+from nltk.compat import python_2_unicode_compatible, unicode_repr
+
+##//////////////////////////////////////////////////////
+##  ChunkString
+##//////////////////////////////////////////////////////
+
+@python_2_unicode_compatible
+class ChunkString(object):
+    """
+    A string-based encoding of a particular chunking of a text.
+    Internally, the ``ChunkString`` class uses a single string to
+    encode the chunking of the input text.  This string contains a
+    sequence of angle-bracket delimited tags, with chunking indicated
+    by braces.  An example of this encoding is::
+
+        {<DT><JJ><NN>}<VBN><IN>{<DT><NN>}<.>{<DT><NN>}<VBD><.>
+
+    ``ChunkString`` are created from tagged texts (i.e., lists of
+    ``tokens`` whose type is ``TaggedType``).  Initially, nothing is
+    chunked.
+
+    The chunking of a ``ChunkString`` can be modified with the ``xform()``
+    method, which uses a regular expression to transform the string
+    representation.  These transformations should only add and remove
+    braces; they should *not* modify the sequence of angle-bracket
+    delimited tags.
+
+    :type _str: str
+    :ivar _str: The internal string representation of the text's
+        encoding.  This string representation contains a sequence of
+        angle-bracket delimited tags, with chunking indicated by
+        braces.  An example of this encoding is::
+
+            {<DT><JJ><NN>}<VBN><IN>{<DT><NN>}<.>{<DT><NN>}<VBD><.>
+
+    :type _pieces: list(tagged tokens and chunks)
+    :ivar _pieces: The tagged tokens and chunks encoded by this ``ChunkString``.
+    :ivar _debug: The debug level.  See the constructor docs.
+
+    :cvar IN_CHUNK_PATTERN: A zero-width regexp pattern string that
+        will only match positions that are in chunks.
+    :cvar IN_CHINK_PATTERN: A zero-width regexp pattern string that
+        will only match positions that are in chinks.
+    """
+    CHUNK_TAG_CHAR = r'[^\{\}<>]'
+    CHUNK_TAG = r'(<%s+?>)' % CHUNK_TAG_CHAR
+
+    IN_CHUNK_PATTERN = r'(?=[^\{]*\})'
+    IN_CHINK_PATTERN = r'(?=[^\}]*(\{|$))'
+
+    # These are used by _verify
+    _CHUNK = r'(\{%s+?\})+?' % CHUNK_TAG
+    _CHINK = r'(%s+?)+?' % CHUNK_TAG
+    _VALID = re.compile(r'^(\{?%s\}?)*?$' % CHUNK_TAG)
+    _BRACKETS = re.compile('[^\{\}]+')
+    _BALANCED_BRACKETS = re.compile(r'(\{\})*$')
+
+    def __init__(self, chunk_struct, debug_level=1):
+        """
+        Construct a new ``ChunkString`` that encodes the chunking of
+        the text ``tagged_tokens``.
+
+        :type chunk_struct: Tree
+        :param chunk_struct: The chunk structure to be further chunked.
+        :type debug_level: int
+        :param debug_level: The level of debugging which should be
+            applied to transformations on the ``ChunkString``.  The
+            valid levels are:
+                - 0: no checks
+                - 1: full check on to_chunkstruct
+                - 2: full check on to_chunkstruct and cursory check after
+                   each transformation.
+                - 3: full check on to_chunkstruct and full check after
+                   each transformation.
+            We recommend you use at least level 1.  You should
+            probably use level 3 if you use any non-standard
+            subclasses of ``RegexpChunkRule``.
+        """
+        self._root_label = chunk_struct.label()
+        self._pieces = chunk_struct[:]
+        tags = [self._tag(tok) for tok in self._pieces]
+        self._str = '<' + '><'.join(tags) + '>'
+        self._debug = debug_level
+
+    def _tag(self, tok):
+        if isinstance(tok, tuple):
+            return tok[1]
+        elif isinstance(tok, Tree):
+            return tok.label()
+        else:
+            raise ValueError('chunk structures must contain tagged '
+                             'tokens or trees')
+
+    def _verify(self, s, verify_tags):
+        """
+        Check to make sure that ``s`` still corresponds to some chunked
+        version of ``_pieces``.
+
+        :type verify_tags: bool
+        :param verify_tags: Whether the individual tags should be
+            checked.  If this is false, ``_verify`` will check to make
+            sure that ``_str`` encodes a chunked version of *some*
+            list of tokens.  If this is true, then ``_verify`` will
+            check to make sure that the tags in ``_str`` match those in
+            ``_pieces``.
+
+        :raise ValueError: if the internal string representation of
+            this ``ChunkString`` is invalid or not consistent with _pieces.
+        """
+        # Check overall form
+        if not ChunkString._VALID.match(s):
+            raise ValueError('Transformation generated invalid '
+                             'chunkstring:\n  %s' % s)
+
+        # Check that parens are balanced.  If the string is long, we
+        # have to do this in pieces, to avoid a maximum recursion
+        # depth limit for regular expressions.
+        brackets = ChunkString._BRACKETS.sub('', s)
+        for i in range(1 + len(brackets) // 5000):
+            substr = brackets[i*5000:i*5000+5000]
+            if not ChunkString._BALANCED_BRACKETS.match(substr):
+                raise ValueError('Transformation generated invalid '
+                                 'chunkstring:\n  %s' % s)
+
+        if verify_tags<=0: return
+
+        tags1 = (re.split(r'[\{\}<>]+', s))[1:-1]
+        tags2 = [self._tag(piece) for piece in self._pieces]
+        if tags1 != tags2:
+            raise ValueError('Transformation generated invalid '
+                             'chunkstring: tag changed')
+
+    def to_chunkstruct(self, chunk_label='CHUNK'):
+        """
+        Return the chunk structure encoded by this ``ChunkString``.
+
+        :rtype: Tree
+        :raise ValueError: If a transformation has generated an
+            invalid chunkstring.
+        """
+        if self._debug > 0: self._verify(self._str, 1)
+
+        # Use this alternating list to create the chunkstruct.
+        pieces = []
+        index = 0
+        piece_in_chunk = 0
+        for piece in re.split('[{}]', self._str):
+
+            # Find the list of tokens contained in this piece.
+            length = piece.count('<')
+            subsequence = self._pieces[index:index+length]
+
+            # Add this list of tokens to our pieces.
+            if piece_in_chunk:
+                pieces.append(Tree(chunk_label, subsequence))
+            else:
+                pieces += subsequence
+
+            # Update index, piece_in_chunk
+            index += length
+            piece_in_chunk = not piece_in_chunk
+
+        return Tree(self._root_label, pieces)
+
+    def xform(self, regexp, repl):
+        """
+        Apply the given transformation to the string encoding of this
+        ``ChunkString``.  In particular, find all occurrences that match
+        ``regexp``, and replace them using ``repl`` (as done by
+        ``re.sub``).
+
+        This transformation should only add and remove braces; it
+        should *not* modify the sequence of angle-bracket delimited
+        tags.  Furthermore, this transformation may not result in
+        improper bracketing.  Note, in particular, that bracketing may
+        not be nested.
+
+        :type regexp: str or regexp
+        :param regexp: A regular expression matching the substring
+            that should be replaced.  This will typically include a
+            named group, which can be used by ``repl``.
+        :type repl: str
+        :param repl: An expression specifying what should replace the
+            matched substring.  Typically, this will include a named
+            replacement group, specified by ``regexp``.
+        :rtype: None
+        :raise ValueError: If this transformation generated an
+            invalid chunkstring.
+        """
+        # Do the actual substitution
+        s = re.sub(regexp, repl, self._str)
+
+        # The substitution might have generated "empty chunks"
+        # (substrings of the form "{}").  Remove them, so they don't
+        # interfere with other transformations.
+        s = re.sub('\{\}', '', s)
+
+        # Make sure that the transformation was legal.
+        if self._debug > 1: self._verify(s, self._debug-2)
+
+        # Commit the transformation.
+        self._str = s
+
+    def __repr__(self):
+        """
+        Return a string representation of this ``ChunkString``.
+        It has the form::
+
+            <ChunkString: '{<DT><JJ><NN>}<VBN><IN>{<DT><NN>}'>
+
+        :rtype: str
+        """
+        return '<ChunkString: %s>' % unicode_repr(self._str)
+
+    def __str__(self):
+        """
+        Return a formatted representation of this ``ChunkString``.
+        This representation will include extra spaces to ensure that
+        tags will line up with the representation of other
+        ``ChunkStrings`` for the same text, regardless of the chunking.
+
+       :rtype: str
+        """
+        # Add spaces to make everything line up.
+        str = re.sub(r'>(?!\})', r'> ', self._str)
+        str = re.sub(r'([^\{])<', r'\1 <', str)
+        if str[0] == '<': str = ' ' + str
+        return str
+
+##//////////////////////////////////////////////////////
+##  Chunking Rules
+##//////////////////////////////////////////////////////
+
+@python_2_unicode_compatible
+class RegexpChunkRule(object):
+    """
+    A rule specifying how to modify the chunking in a ``ChunkString``,
+    using a transformational regular expression.  The
+    ``RegexpChunkRule`` class itself can be used to implement any
+    transformational rule based on regular expressions.  There are
+    also a number of subclasses, which can be used to implement
+    simpler types of rules, based on matching regular expressions.
+
+    Each ``RegexpChunkRule`` has a regular expression and a
+    replacement expression.  When a ``RegexpChunkRule`` is "applied"
+    to a ``ChunkString``, it searches the ``ChunkString`` for any
+    substring that matches the regular expression, and replaces it
+    using the replacement expression.  This search/replace operation
+    has the same semantics as ``re.sub``.
+
+    Each ``RegexpChunkRule`` also has a description string, which
+    gives a short (typically less than 75 characters) description of
+    the purpose of the rule.
+
+    This transformation defined by this ``RegexpChunkRule`` should
+    only add and remove braces; it should *not* modify the sequence
+    of angle-bracket delimited tags.  Furthermore, this transformation
+    may not result in nested or mismatched bracketing.
+    """
+    def __init__(self, regexp, repl, descr):
+        """
+        Construct a new RegexpChunkRule.
+
+        :type regexp: regexp or str
+        :param regexp: The regular expression for this ``RegexpChunkRule``.
+            When this rule is applied to a ``ChunkString``, any
+            substring that matches ``regexp`` will be replaced using
+            the replacement string ``repl``.  Note that this must be a
+            normal regular expression, not a tag pattern.
+        :type repl: str
+        :param repl: The replacement expression for this ``RegexpChunkRule``.
+            When this rule is applied to a ``ChunkString``, any substring
+            that matches ``regexp`` will be replaced using ``repl``.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        if isinstance(regexp, string_types):
+            regexp = re.compile(regexp)
+        self._repl = repl
+        self._descr = descr
+        self._regexp = regexp
+
+    def apply(self, chunkstr):
+        # Keep docstring generic so we can inherit it.
+        """
+        Apply this rule to the given ``ChunkString``.  See the
+        class reference documentation for a description of what it
+        means to apply a rule.
+
+        :type chunkstr: ChunkString
+        :param chunkstr: The chunkstring to which this rule is applied.
+        :rtype: None
+        :raise ValueError: If this transformation generated an
+            invalid chunkstring.
+        """
+        chunkstr.xform(self._regexp, self._repl)
+
+    def descr(self):
+        """
+        Return a short description of the purpose and/or effect of
+        this rule.
+
+        :rtype: str
+        """
+        return self._descr
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <RegexpChunkRule: '{<IN|VB.*>}'->'<IN>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return ('<RegexpChunkRule: '+unicode_repr(self._regexp.pattern)+
+                '->'+unicode_repr(self._repl)+'>')
+
+    @staticmethod
+    def fromstring(s):
+        """
+        Create a RegexpChunkRule from a string description.
+        Currently, the following formats are supported::
+
+          {regexp}         # chunk rule
+          }regexp{         # chink rule
+          regexp}{regexp   # split rule
+          regexp{}regexp   # merge rule
+
+        Where ``regexp`` is a regular expression for the rule.  Any
+        text following the comment marker (``#``) will be used as
+        the rule's description:
+
+        >>> from nltk.chunk.regexp import RegexpChunkRule
+        >>> RegexpChunkRule.fromstring('{<DT>?<NN.*>+}')
+        <ChunkRule: '<DT>?<NN.*>+'>
+        """
+        # Split off the comment (but don't split on '\#')
+        m = re.match(r'(?P<rule>(\\.|[^#])*)(?P<comment>#.*)?', s)
+        rule = m.group('rule').strip()
+        comment = (m.group('comment') or '')[1:].strip()
+
+        # Pattern bodies: chunk, chink, split, merge
+        try:
+            if not rule:
+                raise ValueError('Empty chunk pattern')
+            if rule[0] == '{' and rule[-1] == '}':
+                return ChunkRule(rule[1:-1], comment)
+            elif rule[0] == '}' and rule[-1] == '{':
+                return ChinkRule(rule[1:-1], comment)
+            elif '}{' in rule:
+                left, right = rule.split('}{')
+                return SplitRule(left, right, comment)
+            elif '{}' in rule:
+                left, right = rule.split('{}')
+                return MergeRule(left, right, comment)
+            elif re.match('[^{}]*{[^{}]*}[^{}]*', rule):
+                left, chunk, right = re.split('[{}]', rule)
+                return ChunkRuleWithContext(left, chunk, right, comment)
+            else:
+                raise ValueError('Illegal chunk pattern: %s' % rule)
+        except (ValueError, re.error):
+            raise ValueError('Illegal chunk pattern: %s' % rule)
+
+
+@python_2_unicode_compatible
+class ChunkRule(RegexpChunkRule):
+    """
+    A rule specifying how to add chunks to a ``ChunkString``, using a
+    matching tag pattern.  When applied to a ``ChunkString``, it will
+    find any substring that matches this tag pattern and that is not
+    already part of a chunk, and create a new chunk containing that
+    substring.
+    """
+    def __init__(self, tag_pattern, descr):
+
+        """
+        Construct a new ``ChunkRule``.
+
+        :type tag_pattern: str
+        :param tag_pattern: This rule's tag pattern.  When
+            applied to a ``ChunkString``, this rule will
+            chunk any substring that matches this tag pattern and that
+            is not already part of a chunk.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        self._pattern = tag_pattern
+        regexp = re.compile('(?P<chunk>%s)%s' %
+                            (tag_pattern2re_pattern(tag_pattern),
+                             ChunkString.IN_CHINK_PATTERN))
+        RegexpChunkRule.__init__(self, regexp, '{\g<chunk>}', descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ChunkRule: '<IN|VB.*>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return '<ChunkRule: '+unicode_repr(self._pattern)+'>'
+
+@python_2_unicode_compatible
+class ChinkRule(RegexpChunkRule):
+    """
+    A rule specifying how to remove chinks to a ``ChunkString``,
+    using a matching tag pattern.  When applied to a
+    ``ChunkString``, it will find any substring that matches this
+    tag pattern and that is contained in a chunk, and remove it
+    from that chunk, thus creating two new chunks.
+    """
+    def __init__(self, tag_pattern, descr):
+        """
+        Construct a new ``ChinkRule``.
+
+        :type tag_pattern: str
+        :param tag_pattern: This rule's tag pattern.  When
+            applied to a ``ChunkString``, this rule will
+            find any substring that matches this tag pattern and that
+            is contained in a chunk, and remove it from that chunk,
+            thus creating two new chunks.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        self._pattern = tag_pattern
+        regexp = re.compile('(?P<chink>%s)%s' %
+                            (tag_pattern2re_pattern(tag_pattern),
+                             ChunkString.IN_CHUNK_PATTERN))
+        RegexpChunkRule.__init__(self, regexp, '}\g<chink>{', descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ChinkRule: '<IN|VB.*>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return '<ChinkRule: '+unicode_repr(self._pattern)+'>'
+
+
+@python_2_unicode_compatible
+class UnChunkRule(RegexpChunkRule):
+    """
+    A rule specifying how to remove chunks to a ``ChunkString``,
+    using a matching tag pattern.  When applied to a
+    ``ChunkString``, it will find any complete chunk that matches this
+    tag pattern, and un-chunk it.
+    """
+    def __init__(self, tag_pattern, descr):
+        """
+        Construct a new ``UnChunkRule``.
+
+        :type tag_pattern: str
+        :param tag_pattern: This rule's tag pattern.  When
+            applied to a ``ChunkString``, this rule will
+            find any complete chunk that matches this tag pattern,
+            and un-chunk it.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        self._pattern = tag_pattern
+        regexp = re.compile('\{(?P<chunk>%s)\}' %
+                            tag_pattern2re_pattern(tag_pattern))
+        RegexpChunkRule.__init__(self, regexp, '\g<chunk>', descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <UnChunkRule: '<IN|VB.*>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return '<UnChunkRule: '+unicode_repr(self._pattern)+'>'
+
+
+@python_2_unicode_compatible
+class MergeRule(RegexpChunkRule):
+    """
+    A rule specifying how to merge chunks in a ``ChunkString``, using
+    two matching tag patterns: a left pattern, and a right pattern.
+    When applied to a ``ChunkString``, it will find any chunk whose end
+    matches left pattern, and immediately followed by a chunk whose
+    beginning matches right pattern.  It will then merge those two
+    chunks into a single chunk.
+    """
+    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+        """
+        Construct a new ``MergeRule``.
+
+        :type right_tag_pattern: str
+        :param right_tag_pattern: This rule's right tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose end matches
+            ``left_tag_pattern``, and immediately followed by a chunk
+            whose beginning matches this pattern.  It will
+            then merge those two chunks into a single chunk.
+        :type left_tag_pattern: str
+        :param left_tag_pattern: This rule's left tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose end matches
+            this pattern, and immediately followed by a chunk
+            whose beginning matches ``right_tag_pattern``.  It will
+            then merge those two chunks into a single chunk.
+
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+        self._left_tag_pattern = left_tag_pattern
+        self._right_tag_pattern = right_tag_pattern
+        regexp = re.compile('(?P<left>%s)}{(?=%s)' %
+                            (tag_pattern2re_pattern(left_tag_pattern),
+                             tag_pattern2re_pattern(right_tag_pattern)))
+        RegexpChunkRule.__init__(self, regexp, '\g<left>', descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <MergeRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return ('<MergeRule: '+unicode_repr(self._left_tag_pattern)+', '+
+                unicode_repr(self._right_tag_pattern)+'>')
+
+
+@python_2_unicode_compatible
+class SplitRule(RegexpChunkRule):
+    """
+    A rule specifying how to split chunks in a ``ChunkString``, using
+    two matching tag patterns: a left pattern, and a right pattern.
+    When applied to a ``ChunkString``, it will find any chunk that
+    matches the left pattern followed by the right pattern.  It will
+    then split the chunk into two new chunks, at the point between the
+    two pattern matches.
+    """
+    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+        """
+        Construct a new ``SplitRule``.
+
+        :type right_tag_pattern: str
+        :param right_tag_pattern: This rule's right tag
+            pattern.  When applied to a ``ChunkString``, this rule will
+            find any chunk containing a substring that matches
+            ``left_tag_pattern`` followed by this pattern.  It will
+            then split the chunk into two new chunks at the point
+            between these two matching patterns.
+        :type left_tag_pattern: str
+        :param left_tag_pattern: This rule's left tag
+            pattern.  When applied to a ``ChunkString``, this rule will
+            find any chunk containing a substring that matches this
+            pattern followed by ``right_tag_pattern``.  It will then
+            split the chunk into two new chunks at the point between
+            these two matching patterns.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+        self._left_tag_pattern = left_tag_pattern
+        self._right_tag_pattern = right_tag_pattern
+        regexp = re.compile('(?P<left>%s)(?=%s)' %
+                            (tag_pattern2re_pattern(left_tag_pattern),
+                             tag_pattern2re_pattern(right_tag_pattern)))
+        RegexpChunkRule.__init__(self, regexp, r'\g<left>}{', descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <SplitRule: '<NN>', '<DT>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+       :rtype: str
+        """
+        return ('<SplitRule: '+unicode_repr(self._left_tag_pattern)+', '+
+                unicode_repr(self._right_tag_pattern)+'>')
+
+
+@python_2_unicode_compatible
+class ExpandLeftRule(RegexpChunkRule):
+    """
+    A rule specifying how to expand chunks in a ``ChunkString`` to the left,
+    using two matching tag patterns: a left pattern, and a right pattern.
+    When applied to a ``ChunkString``, it will find any chunk whose beginning
+    matches right pattern, and immediately preceded by a chink whose
+    end matches left pattern.  It will then expand the chunk to incorporate
+    the new material on the left.
+    """
+    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+        """
+        Construct a new ``ExpandRightRule``.
+
+        :type right_tag_pattern: str
+        :param right_tag_pattern: This rule's right tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose beginning matches
+            ``right_tag_pattern``, and immediately preceded by a chink
+            whose end matches this pattern.  It will
+            then merge those two chunks into a single chunk.
+        :type left_tag_pattern: str
+        :param left_tag_pattern: This rule's left tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose beginning matches
+            this pattern, and immediately preceded by a chink
+            whose end matches ``left_tag_pattern``.  It will
+            then expand the chunk to incorporate the new material on the left.
+
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+        self._left_tag_pattern = left_tag_pattern
+        self._right_tag_pattern = right_tag_pattern
+        regexp = re.compile('(?P<left>%s)\{(?P<right>%s)' %
+                            (tag_pattern2re_pattern(left_tag_pattern),
+                             tag_pattern2re_pattern(right_tag_pattern)))
+        RegexpChunkRule.__init__(self, regexp, '{\g<left>\g<right>', descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ExpandLeftRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return ('<ExpandLeftRule: '+unicode_repr(self._left_tag_pattern)+', '+
+                unicode_repr(self._right_tag_pattern)+'>')
+
+
+@python_2_unicode_compatible
+class ExpandRightRule(RegexpChunkRule):
+    """
+    A rule specifying how to expand chunks in a ``ChunkString`` to the
+    right, using two matching tag patterns: a left pattern, and a
+    right pattern.  When applied to a ``ChunkString``, it will find any
+    chunk whose end matches left pattern, and immediately followed by
+    a chink whose beginning matches right pattern.  It will then
+    expand the chunk to incorporate the new material on the right.
+    """
+    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+        """
+        Construct a new ``ExpandRightRule``.
+
+        :type right_tag_pattern: str
+        :param right_tag_pattern: This rule's right tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose end matches
+            ``left_tag_pattern``, and immediately followed by a chink
+            whose beginning matches this pattern.  It will
+            then merge those two chunks into a single chunk.
+        :type left_tag_pattern: str
+        :param left_tag_pattern: This rule's left tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose end matches
+            this pattern, and immediately followed by a chink
+            whose beginning matches ``right_tag_pattern``.  It will
+            then expand the chunk to incorporate the new material on the right.
+
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+        self._left_tag_pattern = left_tag_pattern
+        self._right_tag_pattern = right_tag_pattern
+        regexp = re.compile('(?P<left>%s)\}(?P<right>%s)' %
+                            (tag_pattern2re_pattern(left_tag_pattern),
+                             tag_pattern2re_pattern(right_tag_pattern)))
+        RegexpChunkRule.__init__(self, regexp, '\g<left>\g<right>}', descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ExpandRightRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return ('<ExpandRightRule: '+unicode_repr(self._left_tag_pattern)+', '+
+                unicode_repr(self._right_tag_pattern)+'>')
+
+
+@python_2_unicode_compatible
+class ChunkRuleWithContext(RegexpChunkRule):
+    """
+    A rule specifying how to add chunks to a ``ChunkString``, using
+    three matching tag patterns: one for the left context, one for the
+    chunk, and one for the right context.  When applied to a
+    ``ChunkString``, it will find any substring that matches the chunk
+    tag pattern, is surrounded by substrings that match the two
+    context patterns, and is not already part of a chunk; and create a
+    new chunk containing the substring that matched the chunk tag
+    pattern.
+
+    Caveat: Both the left and right context are consumed when this
+    rule matches; therefore, if you need to find overlapping matches,
+    you will need to apply your rule more than once.
+    """
+    def __init__(self, left_context_tag_pattern, chunk_tag_pattern,
+                 right_context_tag_pattern, descr):
+        """
+        Construct a new ``ChunkRuleWithContext``.
+
+        :type left_context_tag_pattern: str
+        :param left_context_tag_pattern: A tag pattern that must match
+            the left context of ``chunk_tag_pattern`` for this rule to
+            apply.
+        :type chunk_tag_pattern: str
+        :param chunk_tag_pattern: A tag pattern that must match for this
+            rule to apply.  If the rule does apply, then this pattern
+            also identifies the substring that will be made into a chunk.
+        :type right_context_tag_pattern: str
+        :param right_context_tag_pattern: A tag pattern that must match
+            the right context of ``chunk_tag_pattern`` for this rule to
+            apply.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_context_tag_pattern))
+        re.compile(tag_pattern2re_pattern(chunk_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_context_tag_pattern))
+
+        self._left_context_tag_pattern = left_context_tag_pattern
+        self._chunk_tag_pattern = chunk_tag_pattern
+        self._right_context_tag_pattern = right_context_tag_pattern
+        regexp = re.compile('(?P<left>%s)(?P<chunk>%s)(?P<right>%s)%s' %
+                            (tag_pattern2re_pattern(left_context_tag_pattern),
+                             tag_pattern2re_pattern(chunk_tag_pattern),
+                             tag_pattern2re_pattern(right_context_tag_pattern),
+                             ChunkString.IN_CHINK_PATTERN))
+        replacement = r'\g<left>{\g<chunk>}\g<right>'
+        RegexpChunkRule.__init__(self, regexp, replacement, descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ChunkRuleWithContext: '<IN>', '<NN>', '<DT>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return '<ChunkRuleWithContext:  %r, %r, %r>' % (
+            self._left_context_tag_pattern, self._chunk_tag_pattern,
+            self._right_context_tag_pattern)
+
+##//////////////////////////////////////////////////////
+##  Tag Pattern Format Conversion
+##//////////////////////////////////////////////////////
+
+# this should probably be made more strict than it is -- e.g., it
+# currently accepts 'foo'.
+CHUNK_TAG_PATTERN = re.compile(r'^((%s|<%s>)*)$' %
+                                ('([^\{\}<>]|\{\d+,?\}|\{\d*,\d+\})+',
+                                 '[^\{\}<>]+'))
+
+
+
+
+
+def tag_pattern2re_pattern(tag_pattern):
+    """
+    Convert a tag pattern to a regular expression pattern.  A "tag
+    pattern" is a modified version of a regular expression, designed
+    for matching sequences of tags.  The differences between regular
+    expression patterns and tag patterns are:
+
+        - In tag patterns, ``'<'`` and ``'>'`` act as parentheses; so
+          ``'<NN>+'`` matches one or more repetitions of ``'<NN>'``, not
+          ``'<NN'`` followed by one or more repetitions of ``'>'``.
+        - Whitespace in tag patterns is ignored.  So
+          ``'<DT> | <NN>'`` is equivalant to ``'<DT>|<NN>'``
+        - In tag patterns, ``'.'`` is equivalant to ``'[^{}<>]'``; so
+          ``'<NN.*>'`` matches any single tag starting with ``'NN'``.
+
+    In particular, ``tag_pattern2re_pattern`` performs the following
+    transformations on the given pattern:
+
+        - Replace '.' with '[^<>{}]'
+        - Remove any whitespace
+        - Add extra parens around '<' and '>', to make '<' and '>' act
+          like parentheses.  E.g., so that in '<NN>+', the '+' has scope
+          over the entire '<NN>'; and so that in '<NN|IN>', the '|' has
+          scope over 'NN' and 'IN', but not '<' or '>'.
+        - Check to make sure the resulting pattern is valid.
+
+    :type tag_pattern: str
+    :param tag_pattern: The tag pattern to convert to a regular
+        expression pattern.
+    :raise ValueError: If ``tag_pattern`` is not a valid tag pattern.
+        In particular, ``tag_pattern`` should not include braces; and it
+        should not contain nested or mismatched angle-brackets.
+    :rtype: str
+    :return: A regular expression pattern corresponding to
+        ``tag_pattern``.
+    """
+    # Clean up the regular expression
+    tag_pattern = re.sub(r'\s', '', tag_pattern)
+    tag_pattern = re.sub(r'<', '(<(', tag_pattern)
+    tag_pattern = re.sub(r'>', ')>)', tag_pattern)
+
+    # Check the regular expression
+    if not CHUNK_TAG_PATTERN.match(tag_pattern):
+        raise ValueError('Bad tag pattern: %r' % tag_pattern)
+
+    # Replace "." with CHUNK_TAG_CHAR.
+    # We have to do this after, since it adds {}[]<>s, which would
+    # confuse CHUNK_TAG_PATTERN.
+    # PRE doesn't have lookback assertions, so reverse twice, and do
+    # the pattern backwards (with lookahead assertions).  This can be
+    # made much cleaner once we can switch back to SRE.
+    def reverse_str(str):
+        lst = list(str)
+        lst.reverse()
+        return ''.join(lst)
+    tc_rev = reverse_str(ChunkString.CHUNK_TAG_CHAR)
+    reversed = reverse_str(tag_pattern)
+    reversed = re.sub(r'\.(?!\\(\\\\)*($|[^\\]))', tc_rev, reversed)
+    tag_pattern = reverse_str(reversed)
+
+    return tag_pattern
+
+
+##//////////////////////////////////////////////////////
+##  RegexpChunkParser
+##//////////////////////////////////////////////////////
+
+@python_2_unicode_compatible
+class RegexpChunkParser(ChunkParserI):
+    """
+    A regular expression based chunk parser.  ``RegexpChunkParser`` uses a
+    sequence of "rules" to find chunks of a single type within a
+    text.  The chunking of the text is encoded using a ``ChunkString``,
+    and each rule acts by modifying the chunking in the
+    ``ChunkString``.  The rules are all implemented using regular
+    expression matching and substitution.
+
+    The ``RegexpChunkRule`` class and its subclasses (``ChunkRule``,
+    ``ChinkRule``, ``UnChunkRule``, ``MergeRule``, and ``SplitRule``)
+    define the rules that are used by ``RegexpChunkParser``.  Each rule
+    defines an ``apply()`` method, which modifies the chunking encoded
+    by a given ``ChunkString``.
+
+    :type _rules: list(RegexpChunkRule)
+    :ivar _rules: The list of rules that should be applied to a text.
+    :type _trace: int
+    :ivar _trace: The default level of tracing.
+
+    """
+    def __init__(self, rules, chunk_label='NP', root_label='S', trace=0):
+        """
+        Construct a new ``RegexpChunkParser``.
+
+        :type rules: list(RegexpChunkRule)
+        :param rules: The sequence of rules that should be used to
+            generate the chunking for a tagged text.
+        :type chunk_label: str
+        :param chunk_label: The node value that should be used for
+            chunk subtrees.  This is typically a short string
+            describing the type of information contained by the chunk,
+            such as ``"NP"`` for base noun phrases.
+        :type root_label: str
+        :param root_label: The node value that should be used for the
+            top node of the chunk structure.
+        :type trace: int
+        :param trace: The level of tracing that should be used when
+            parsing a text.  ``0`` will generate no tracing output;
+            ``1`` will generate normal tracing output; and ``2`` or
+            higher will generate verbose tracing output.
+        """
+        self._rules = rules
+        self._trace = trace
+        self._chunk_label = chunk_label
+        self._root_label = root_label
+
+    def _trace_apply(self, chunkstr, verbose):
+        """
+        Apply each rule of this ``RegexpChunkParser`` to ``chunkstr``, in
+        turn.  Generate trace output between each rule.  If ``verbose``
+        is true, then generate verbose output.
+
+        :type chunkstr: ChunkString
+        :param chunkstr: The chunk string to which each rule should be
+            applied.
+        :type verbose: bool
+        :param verbose: Whether output should be verbose.
+        :rtype: None
+        """
+        print('# Input:')
+        print(chunkstr)
+        for rule in self._rules:
+            rule.apply(chunkstr)
+            if verbose:
+                print('#', rule.descr()+' ('+unicode_repr(rule)+'):')
+            else:
+                print('#', rule.descr()+':')
+            print(chunkstr)
+
+    def _notrace_apply(self, chunkstr):
+        """
+        Apply each rule of this ``RegexpChunkParser`` to ``chunkstr``, in
+        turn.
+
+        :param chunkstr: The chunk string to which each rule should be
+            applied.
+        :type chunkstr: ChunkString
+        :rtype: None
+        """
+
+        for rule in self._rules:
+            rule.apply(chunkstr)
+
+    def parse(self, chunk_struct, trace=None):
+        """
+        :type chunk_struct: Tree
+        :param chunk_struct: the chunk structure to be (further) chunked
+        :type trace: int
+        :param trace: The level of tracing that should be used when
+            parsing a text.  ``0`` will generate no tracing output;
+            ``1`` will generate normal tracing output; and ``2`` or
+            highter will generate verbose tracing output.  This value
+            overrides the trace level value that was given to the
+            constructor.
+        :rtype: Tree
+        :return: a chunk structure that encodes the chunks in a given
+            tagged sentence.  A chunk is a non-overlapping linguistic
+            group, such as a noun phrase.  The set of chunks
+            identified in the chunk structure depends on the rules
+            used to define this ``RegexpChunkParser``.
+        """
+        if len(chunk_struct) == 0:
+            print('Warning: parsing empty text')
+            return Tree(self._root_label, [])
+
+        try:
+            chunk_struct.label()
+        except AttributeError:
+            chunk_struct = Tree(self._root_label, chunk_struct)
+
+        # Use the default trace value?
+        if trace is None: trace = self._trace
+
+        chunkstr = ChunkString(chunk_struct)
+
+        # Apply the sequence of rules to the chunkstring.
+        if trace:
+            verbose = (trace>1)
+            self._trace_apply(chunkstr, verbose)
+        else:
+            self._notrace_apply(chunkstr)
+
+        # Use the chunkstring to create a chunk structure.
+        return chunkstr.to_chunkstruct(self._chunk_label)
+
+    def rules(self):
+        """
+        :return: the sequence of rules used by ``RegexpChunkParser``.
+        :rtype: list(RegexpChunkRule)
+        """
+        return self._rules
+
+    def __repr__(self):
+        """
+        :return: a concise string representation of this
+            ``RegexpChunkParser``.
+        :rtype: str
+        """
+        return "<RegexpChunkParser with %d rules>" % len(self._rules)
+
+    def __str__(self):
+        """
+        :return: a verbose string representation of this ``RegexpChunkParser``.
+        :rtype: str
+        """
+        s = "RegexpChunkParser with %d rules:\n" % len(self._rules)
+        margin = 0
+        for rule in self._rules:
+            margin = max(margin, len(rule.descr()))
+        if margin < 35:
+            format = "    %" + repr(-(margin+3)) + "s%s\n"
+        else:
+            format = "    %s\n      %s\n"
+        for rule in self._rules:
+            s += format % (rule.descr(), unicode_repr(rule))
+        return s[:-1]
+
+##//////////////////////////////////////////////////////
+##  Chunk Grammar
+##//////////////////////////////////////////////////////
+
+@python_2_unicode_compatible
+class RegexpParser(ChunkParserI):
+    """
+    A grammar based chunk parser.  ``chunk.RegexpParser`` uses a set of
+    regular expression patterns to specify the behavior of the parser.
+    The chunking of the text is encoded using a ``ChunkString``, and
+    each rule acts by modifying the chunking in the ``ChunkString``.
+    The rules are all implemented using regular expression matching
+    and substitution.
+
+    A grammar contains one or more clauses in the following form::
+
+     NP:
+       {<DT|JJ>}          # chunk determiners and adjectives
+       }<[\.VI].*>+{      # chink any tag beginning with V, I, or .
+       <.*>}{<DT>         # split a chunk at a determiner
+       <DT|JJ>{}<NN.*>    # merge chunk ending with det/adj
+                          # with one starting with a noun
+
+    The patterns of a clause are executed in order.  An earlier
+    pattern may introduce a chunk boundary that prevents a later
+    pattern from executing.  Sometimes an individual pattern will
+    match on multiple, overlapping extents of the input.  As with
+    regular expression substitution more generally, the chunker will
+    identify the first match possible, then continue looking for matches
+    after this one has ended.
+
+    The clauses of a grammar are also executed in order.  A cascaded
+    chunk parser is one having more than one clause.  The maximum depth
+    of a parse tree created by this chunk parser is the same as the
+    number of clauses in the grammar.
+
+    When tracing is turned on, the comment portion of a line is displayed
+    each time the corresponding pattern is applied.
+
+    :type _start: str
+    :ivar _start: The start symbol of the grammar (the root node of
+        resulting trees)
+    :type _stages: int
+    :ivar _stages: The list of parsing stages corresponding to the grammar
+
+    """
+    def __init__(self, grammar, root_label='S', loop=1, trace=0):
+        """
+        Create a new chunk parser, from the given start state
+        and set of chunk patterns.
+
+        :param grammar: The grammar, or a list of RegexpChunkParser objects
+        :type grammar: str or list(RegexpChunkParser)
+        :param root_label: The top node of the tree being created
+        :type root_label: str or Nonterminal
+        :param loop: The number of times to run through the patterns
+        :type loop: int
+        :type trace: int
+        :param trace: The level of tracing that should be used when
+            parsing a text.  ``0`` will generate no tracing output;
+            ``1`` will generate normal tracing output; and ``2`` or
+            higher will generate verbose tracing output.
+        """
+        self._trace = trace
+        self._stages = []
+        self._grammar = grammar
+        self._loop = loop
+
+        if isinstance(grammar, string_types):
+            self._read_grammar(grammar, root_label, trace)
+        else:
+            # Make sur the grammar looks like it has the right type:
+            type_err = ('Expected string or list of RegexpChunkParsers '
+                        'for the grammar.')
+            try: grammar = list(grammar)
+            except: raise TypeError(type_err)
+            for elt in grammar:
+                if not isinstance(elt, RegexpChunkParser):
+                    raise TypeError(type_err)
+            self._stages = grammar
+
+    def _read_grammar(self, grammar, root_label, trace):
+        """
+        Helper function for __init__: read the grammar if it is a
+        string.
+        """
+        rules = []
+        lhs = None
+        for line in grammar.split('\n'):
+            line = line.strip()
+
+            # New stage begins if there's an unescaped ':'
+            m = re.match('(?P<nonterminal>(\\.|[^:])*)(:(?P<rule>.*))', line)
+            if m:
+                # Record the stage that we just completed.
+                self._add_stage(rules, lhs, root_label, trace)
+                # Start a new stage.
+                lhs = m.group('nonterminal').strip()
+                rules = []
+                line = m.group('rule').strip()
+
+            # Skip blank & comment-only lines
+            if line=='' or line.startswith('#'): continue
+
+            # Add the rule
+            rules.append(RegexpChunkRule.fromstring(line))
+
+        # Record the final stage
+        self._add_stage(rules, lhs, root_label, trace)
+
+    def _add_stage(self, rules, lhs, root_label, trace):
+        """
+        Helper function for __init__: add a new stage to the parser.
+        """
+        if rules != []:
+            if not lhs:
+                raise ValueError('Expected stage marker (eg NP:)')
+            parser = RegexpChunkParser(rules, chunk_label=lhs,
+                                       root_label=root_label, trace=trace)
+            self._stages.append(parser)
+
+    def parse(self, chunk_struct, trace=None):
+        """
+        Apply the chunk parser to this input.
+
+        :type chunk_struct: Tree
+        :param chunk_struct: the chunk structure to be (further) chunked
+            (this tree is modified, and is also returned)
+        :type trace: int
+        :param trace: The level of tracing that should be used when
+            parsing a text.  ``0`` will generate no tracing output;
+            ``1`` will generate normal tracing output; and ``2`` or
+            highter will generate verbose tracing output.  This value
+            overrides the trace level value that was given to the
+            constructor.
+        :return: the chunked output.
+        :rtype: Tree
+        """
+        if trace is None: trace = self._trace
+        for i in range(self._loop):
+            for parser in self._stages:
+                chunk_struct = parser.parse(chunk_struct, trace=trace)
+        return chunk_struct
+
+    def __repr__(self):
+        """
+        :return: a concise string representation of this ``chunk.RegexpParser``.
+        :rtype: str
+        """
+        return "<chunk.RegexpParser with %d stages>" % len(self._stages)
+
+    def __str__(self):
+        """
+        :return: a verbose string representation of this
+            ``RegexpParser``.
+        :rtype: str
+        """
+        s = "chunk.RegexpParser with %d stages:\n" % len(self._stages)
+        margin = 0
+        for parser in self._stages:
+            s += "%s\n" % parser
+        return s[:-1]
+
+##//////////////////////////////////////////////////////
+##  Demonstration code
+##//////////////////////////////////////////////////////
+
+def demo_eval(chunkparser, text):
+    """
+    Demonstration code for evaluating a chunk parser, using a
+    ``ChunkScore``.  This function assumes that ``text`` contains one
+    sentence per line, and that each sentence has the form expected by
+    ``tree.chunk``.  It runs the given chunk parser on each sentence in
+    the text, and scores the result.  It prints the final score
+    (precision, recall, and f-measure); and reports the set of chunks
+    that were missed and the set of chunks that were incorrect.  (At
+    most 10 missing chunks and 10 incorrect chunks are reported).
+
+    :param chunkparser: The chunkparser to be tested
+    :type chunkparser: ChunkParserI
+    :param text: The chunked tagged text that should be used for
+        evaluation.
+    :type text: str
+    """
+    from nltk import chunk
+    from nltk.tree import Tree
+
+    # Evaluate our chunk parser.
+    chunkscore = chunk.ChunkScore()
+
+    for sentence in text.split('\n'):
+        print(sentence)
+        sentence = sentence.strip()
+        if not sentence: continue
+        gold = chunk.tagstr2tree(sentence)
+        tokens = gold.leaves()
+        test = chunkparser.parse(Tree('S', tokens), trace=1)
+        chunkscore.score(gold, test)
+        print()
+
+    print('/'+('='*75)+'\\')
+    print('Scoring', chunkparser)
+    print(('-'*77))
+    print('Precision: %5.1f%%' % (chunkscore.precision()*100), ' '*4, end=' ')
+    print('Recall: %5.1f%%' % (chunkscore.recall()*100), ' '*6, end=' ')
+    print('F-Measure: %5.1f%%' % (chunkscore.f_measure()*100))
+
+
+    # Missed chunks.
+    if chunkscore.missed():
+        print('Missed:')
+        missed = chunkscore.missed()
+        for chunk in missed[:10]:
+            print('  ', ' '.join(map(str,chunk)))
+        if len(chunkscore.missed()) > 10:
+            print('  ...')
+
+    # Incorrect chunks.
+    if chunkscore.incorrect():
+        print('Incorrect:')
+        incorrect = chunkscore.incorrect()
+        for chunk in incorrect[:10]:
+            print('  ', ' '.join(map(str,chunk)))
+        if len(chunkscore.incorrect()) > 10:
+            print('  ...')
+
+    print('\\'+('='*75)+'/')
+    print()
+
+def demo():
+    """
+    A demonstration for the ``RegexpChunkParser`` class.  A single text is
+    parsed with four different chunk parsers, using a variety of rules
+    and strategies.
+    """
+
+    from nltk import chunk, Tree
+
+    text = """\
+    [ the/DT little/JJ cat/NN ] sat/VBD on/IN [ the/DT mat/NN ] ./.
+    [ John/NNP ] saw/VBD [the/DT cats/NNS] [the/DT dog/NN] chased/VBD ./.
+    [ John/NNP ] thinks/VBZ [ Mary/NN ] saw/VBD [ the/DT cat/NN ] sit/VB on/IN [ the/DT mat/NN ]./.
+    """
+
+    print('*'*75)
+    print('Evaluation text:')
+    print(text)
+    print('*'*75)
+    print()
+
+    grammar = r"""
+    NP:                   # NP stage
+      {<DT>?<JJ>*<NN>}    # chunk determiners, adjectives and nouns
+      {<NNP>+}            # chunk proper nouns
+    """
+    cp = chunk.RegexpParser(grammar)
+    demo_eval(cp, text)
+
+    grammar = r"""
+    NP:
+      {<.*>}              # start by chunking each tag
+      }<[\.VI].*>+{       # unchunk any verbs, prepositions or periods
+      <DT|JJ>{}<NN.*>     # merge det/adj with nouns
+    """
+    cp = chunk.RegexpParser(grammar)
+    demo_eval(cp, text)
+
+    grammar = r"""
+    NP: {<DT>?<JJ>*<NN>}    # chunk determiners, adjectives and nouns
+    VP: {<TO>?<VB.*>}       # VP = verb words
+    """
+    cp = chunk.RegexpParser(grammar)
+    demo_eval(cp, text)
+
+    grammar = r"""
+    NP: {<.*>*}             # start by chunking everything
+        }<[\.VI].*>+{       # chink any verbs, prepositions or periods
+        <.*>}{<DT>          # separate on determiners
+    PP: {<IN><NP>}          # PP = preposition + noun phrase
+    VP: {<VB.*><NP|PP>*}    # VP = verb words + NPs and PPs
+    """
+    cp = chunk.RegexpParser(grammar)
+    demo_eval(cp, text)
+
+# Evaluation
+
+    from nltk.corpus import conll2000
+
+    print()
+    print("Demonstration of empty grammar:")
+
+    cp = chunk.RegexpParser("")
+    print(chunk.accuracy(cp, conll2000.chunked_sents('test.txt',
+                                                     chunk_types=('NP',))))
+
+    print()
+    print("Demonstration of accuracy evaluation using CoNLL tags:")
+
+    grammar = r"""
+    NP:
+      {<.*>}              # start by chunking each tag
+      }<[\.VI].*>+{       # unchunk any verbs, prepositions or periods
+      <DT|JJ>{}<NN.*>     # merge det/adj with nouns
+    """
+    cp = chunk.RegexpParser(grammar)
+    print(chunk.accuracy(cp, conll2000.chunked_sents('test.txt')[:5]))
+
+    print()
+    print("Demonstration of tagged token input")
+
+    grammar = r"""
+    NP: {<.*>*}             # start by chunking everything
+        }<[\.VI].*>+{       # chink any verbs, prepositions or periods
+        <.*>}{<DT>          # separate on determiners
+    PP: {<IN><NP>}          # PP = preposition + noun phrase
+    VP: {<VB.*><NP|PP>*}    # VP = verb words + NPs and PPs
+    """
+    cp = chunk.RegexpParser(grammar)
+    print(cp.parse([("the","DT"), ("little","JJ"), ("cat", "NN"),
+                    ("sat", "VBD"), ("on", "IN"), ("the", "DT"),
+                    ("mat", "NN"), (".", ".")]))
+
+if __name__ == '__main__':
+    demo()