nltk 3.2__zip → 3.3__zip

nltk-3.3/nltk/tokenize/toktok.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Python port of the tok-tok.pl tokenizer.
+#
+# Copyright (C) 2001-2015 NLTK Project
+# Author: Jon Dehdari
+# Contributors: Liling Tan, Selcuk Ayguney, ikegami, Martijn Pieters
+#
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+"""
+The tok-tok tokenizer is a simple, general tokenizer, where the input has one 
+sentence per line; thus only final period is tokenized.
+
+Tok-tok has been tested on, and gives reasonably good results for English, 
+Persian, Russian, Czech, French, German, Vietnamese, Tajik, and a few others. 
+The input should be in UTF-8 encoding.
+
+Reference:
+Jon Dehdari. 2014. A Neurophysiologically-Inspired Statistical Language 
+Model (Doctoral dissertation). Columbus, OH, USA: The Ohio State University. 
+"""
+
+import re
+from six import text_type
+
+from nltk.tokenize.api import TokenizerI
+
+class ToktokTokenizer(TokenizerI):
+    """
+    This is a Python port of the tok-tok.pl from
+    https://github.com/jonsafari/tok-tok/blob/master/tok-tok.pl
+    
+    >>> toktok = ToktokTokenizer()
+    >>> text = u'Is 9.5 or 525,600 my favorite number?'
+    >>> print (toktok.tokenize(text, return_str=True))
+    Is 9.5 or 525,600 my favorite number ?
+    >>> text = u'The https://github.com/jonsafari/tok-tok/blob/master/tok-tok.pl is a website with/and/or slashes and sort of weird : things'
+    >>> print (toktok.tokenize(text, return_str=True))
+    The https://github.com/jonsafari/tok-tok/blob/master/tok-tok.pl is a website with/and/or slashes and sort of weird : things
+    >>> text = u'\xa1This, is a sentence with weird\xbb symbols\u2026 appearing everywhere\xbf'
+    >>> expected = u'\xa1 This , is a sentence with weird \xbb symbols \u2026 appearing everywhere \xbf'
+    >>> assert toktok.tokenize(text, return_str=True) == expected
+    >>> toktok.tokenize(text) == [u'\xa1', u'This', u',', u'is', u'a', u'sentence', u'with', u'weird', u'\xbb', u'symbols', u'\u2026', u'appearing', u'everywhere', u'\xbf']
+    True
+    """
+    # Replace non-breaking spaces with normal spaces.
+    NON_BREAKING = re.compile(u"\u00A0"), " "
+    
+    # Pad some funky punctuation.
+    FUNKY_PUNCT_1 = re.compile(u'([،;؛¿!"\])}»›”؟¡%٪°±©®।॥…])'), r" \1 "
+    # Pad more funky punctuation.
+    FUNKY_PUNCT_2 = re.compile(u'([({\[“‘„‚«‹「『])'), r" \1 "
+    # Pad En dash and em dash
+    EN_EM_DASHES = re.compile(u'([–—])'), r" \1 "
+    
+    # Replace problematic character with numeric character reference.
+    AMPERCENT = re.compile('& '), '&amp; '
+    TAB = re.compile('\t'), ' &#9; '
+    PIPE = re.compile('\|'), ' &#124; '
+    
+    # Pad numbers with commas to keep them from further tokenization. 
+    COMMA_IN_NUM = re.compile(r'(?<!,)([,،])(?![,\d])'), r' \1 '
+    
+    # Just pad problematic (often neurotic) hyphen/single quote, etc.
+    PROB_SINGLE_QUOTES = re.compile(r"(['’`])"), r' \1 '
+    # Group ` ` stupid quotes ' ' into a single token.
+    STUPID_QUOTES_1 = re.compile(r" ` ` "), r" `` "
+    STUPID_QUOTES_2 = re.compile(r" ' ' "), r" '' "
+    
+    # Don't tokenize period unless it ends the line and that it isn't 
+    # preceded by another period, e.g.  
+    # "something ..." -> "something ..." 
+    # "something." -> "something ." 
+    FINAL_PERIOD_1 = re.compile(r"(?<!\.)\.$"), r" ."
+    # Don't tokenize period unless it ends the line eg. 
+    # " ... stuff." ->  "... stuff ."
+    FINAL_PERIOD_2 = re.compile(r"""(?<!\.)\.\s*(["'’»›”]) *$"""), r" . \1"
+
+    # Treat continuous commas as fake German,Czech, etc.: „
+    MULTI_COMMAS = re.compile(r'(,{2,})'), r' \1 '
+    # Treat continuous dashes as fake en-dash, etc.
+    MULTI_DASHES = re.compile(r'(-{2,})'), r' \1 '
+    # Treat multiple periods as a thing (eg. ellipsis)
+    MULTI_DOTS = re.compile(r'(\.{2,})'), r' \1 '
+
+    # This is the \p{Open_Punctuation} from Perl's perluniprops
+    # see http://perldoc.perl.org/perluniprops.html
+    OPEN_PUNCT = text_type(u'([{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d'
+                            u'\u208d\u2329\u2768\u276a\u276c\u276e\u2770\u2772'
+                            u'\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983'
+                            u'\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993'
+                            u'\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26'
+                            u'\u2e28\u3008\u300a\u300c\u300e\u3010\u3014\u3016'
+                            u'\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39'
+                            u'\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b'
+                            u'\ufe5d\uff08\uff3b\uff5b\uff5f\uff62')
+    # This is the \p{Close_Punctuation} from Perl's perluniprops
+    CLOSE_PUNCT = text_type(u')]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u232a'
+                            u'\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6'
+                            u'\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988'
+                            u'\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998'
+                            u'\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009'
+                            u'\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b'
+                            u'\u301e\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c'
+                            u'\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e'
+                            u'\uff09\uff3d\uff5d\uff60\uff63')
+    # This is the \p{Close_Punctuation} from Perl's perluniprops
+    CURRENCY_SYM = text_type(u'$\xa2\xa3\xa4\xa5\u058f\u060b\u09f2\u09f3\u09fb'
+                             u'\u0af1\u0bf9\u0e3f\u17db\u20a0\u20a1\u20a2\u20a3'
+                             u'\u20a4\u20a5\u20a6\u20a7\u20a8\u20a9\u20aa\u20ab'
+                             u'\u20ac\u20ad\u20ae\u20af\u20b0\u20b1\u20b2\u20b3'
+                             u'\u20b4\u20b5\u20b6\u20b7\u20b8\u20b9\u20ba\ua838'
+                             u'\ufdfc\ufe69\uff04\uffe0\uffe1\uffe5\uffe6')
+    
+    # Pad spaces after opening punctuations.
+    OPEN_PUNCT_RE = re.compile(u'([{}])'.format(OPEN_PUNCT)), r'\1 '
+    # Pad spaces before closing punctuations.
+    CLOSE_PUNCT_RE = re.compile(u'([{}])'.format(CLOSE_PUNCT)), r'\1 '
+    # Pad spaces after currency symbols.
+    CURRENCY_SYM_RE = re.compile(u'([{}])'.format(CURRENCY_SYM)), r'\1 '
+    
+    # Use for tokenizing URL-unfriendly characters: [:/?#]
+    URL_FOE_1 = re.compile(r':(?!//)'), r' : ' # in perl s{:(?!//)}{ : }g;
+    URL_FOE_2 = re.compile(r'\?(?!\S)'), r' ? ' # in perl s{\?(?!\S)}{ ? }g;
+    # in perl: m{://} or m{\S+\.\S+/\S+} or s{/}{ / }g;
+    URL_FOE_3 = re.compile(r'(:\/\/)[\S+\.\S+\/\S+][\/]'), ' / '
+    URL_FOE_4 = re.compile(r' /'), r' / ' # s{ /}{ / }g;
+    
+    # Left/Right strip, i.e. remove heading/trailing spaces.
+    # These strip regexes should NOT be used,
+    # instead use str.lstrip(), str.rstrip() or str.strip() 
+    # (They are kept for reference purposes to the original toktok.pl code)  
+    LSTRIP = re.compile(r'^ +'), ''
+    RSTRIP = re.compile(r'\s+$'),'\n' 
+    # Merge multiple spaces.
+    ONE_SPACE = re.compile(r' {2,}'), ' '
+    
+    TOKTOK_REGEXES = [NON_BREAKING, FUNKY_PUNCT_1, 
+                      URL_FOE_1, URL_FOE_2, URL_FOE_3, URL_FOE_4,
+                      AMPERCENT, TAB, PIPE,
+                      OPEN_PUNCT_RE, CLOSE_PUNCT_RE, 
+                      MULTI_COMMAS, COMMA_IN_NUM, FINAL_PERIOD_2,
+                      PROB_SINGLE_QUOTES, STUPID_QUOTES_1, STUPID_QUOTES_2,
+                      CURRENCY_SYM_RE, EN_EM_DASHES, MULTI_DASHES, MULTI_DOTS,
+                      FINAL_PERIOD_1, FINAL_PERIOD_2, ONE_SPACE]
+    
+    def tokenize(self, text, return_str=False):
+        text = text_type(text) # Converts input string into unicode.
+        for regexp, subsitution in self.TOKTOK_REGEXES:
+            text = regexp.sub(subsitution, text)
+        # Finally, strips heading and trailing spaces
+        # and converts output string into unicode.
+        text = text_type(text.strip()) 
+        return text if return_str else text.split()

nltk-3.3/nltk/tokenize/treebank.py

@@ -0,0 +1,363 @@
+# Natural Language Toolkit: Tokenizers
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Michael Heilman <mheilman@cmu.edu> (re-port from http://www.cis.upenn.edu/~treebank/tokenizer.sed)
+#
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+r"""
+
+Penn Treebank Tokenizer
+
+The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
+This implementation is a port of the tokenizer sed script written by Robert McIntyre
+and available at http://www.cis.upenn.edu/~treebank/tokenizer.sed.
+"""
+
+import re
+from nltk.tokenize.api import TokenizerI
+from nltk.tokenize.util import align_tokens
+
+
+class MacIntyreContractions:
+    """
+    List of contractions adapted from Robert MacIntyre's tokenizer.
+    """
+    CONTRACTIONS2 = [r"(?i)\b(can)(?#X)(not)\b",
+                     r"(?i)\b(d)(?#X)('ye)\b",
+                     r"(?i)\b(gim)(?#X)(me)\b",
+                     r"(?i)\b(gon)(?#X)(na)\b",
+                     r"(?i)\b(got)(?#X)(ta)\b",
+                     r"(?i)\b(lem)(?#X)(me)\b",
+                     r"(?i)\b(mor)(?#X)('n)\b",
+                     r"(?i)\b(wan)(?#X)(na)\s"]
+    CONTRACTIONS3 = [r"(?i) ('t)(?#X)(is)\b", r"(?i) ('t)(?#X)(was)\b"]
+    CONTRACTIONS4 = [r"(?i)\b(whad)(dd)(ya)\b",
+                     r"(?i)\b(wha)(t)(cha)\b"]
+
+
+class TreebankWordTokenizer(TokenizerI):
+    """
+    The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
+    This is the method that is invoked by ``word_tokenize()``.  It assumes that the
+    text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
+
+    This tokenizer performs the following steps:
+
+    - split standard contractions, e.g. ``don't`` -> ``do n't`` and ``they'll`` -> ``they 'll``
+    - treat most punctuation characters as separate tokens
+    - split off commas and single quotes, when followed by whitespace
+    - separate periods that appear at the end of line
+
+        >>> from nltk.tokenize import TreebankWordTokenizer
+        >>> s = '''Good muffins cost $3.88\\nin New York.  Please buy me\\ntwo of them.\\nThanks.'''
+        >>> TreebankWordTokenizer().tokenize(s)
+        ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks', '.']
+        >>> s = "They'll save and invest more."
+        >>> TreebankWordTokenizer().tokenize(s)
+        ['They', "'ll", 'save', 'and', 'invest', 'more', '.']
+        >>> s = "hi, my name can't hello,"
+        >>> TreebankWordTokenizer().tokenize(s)
+        ['hi', ',', 'my', 'name', 'ca', "n't", 'hello', ',']
+    """
+
+    # starting quotes
+    STARTING_QUOTES = [
+        (re.compile(r'^\"'), r'``'),
+        (re.compile(r'(``)'), r' \1 '),
+        (re.compile(r"([ \(\[{<])(\"|\'{2})"), r'\1 `` '),
+    ]
+
+    # punctuation
+    PUNCTUATION = [
+        (re.compile(r'([:,])([^\d])'), r' \1 \2'),
+        (re.compile(r'([:,])$'), r' \1 '),
+        (re.compile(r'\.\.\.'), r' ... '),
+        (re.compile(r'[;@#$%&]'), r' \g<0> '),
+        (re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'), r'\1 \2\3 '),  # Handles the final period.
+        (re.compile(r'[?!]'), r' \g<0> '),
+
+        (re.compile(r"([^'])' "), r"\1 ' "),
+    ]
+
+    # Pads parentheses
+    PARENS_BRACKETS = (re.compile(r'[\]\[\(\)\{\}\<\>]'), r' \g<0> ')
+
+    # Optionally: Convert parentheses, brackets and converts them to PTB symbols.
+    CONVERT_PARENTHESES = [
+        (re.compile(r'\('), '-LRB-'), (re.compile(r'\)'), '-RRB-'),
+        (re.compile(r'\['), '-LSB-'), (re.compile(r'\]'), '-RSB-'),
+        (re.compile(r'\{'), '-LCB-'), (re.compile(r'\}'), '-RCB-')
+    ]
+
+    DOUBLE_DASHES = (re.compile(r'--'), r' -- ')
+
+    # ending quotes
+    ENDING_QUOTES = [
+        (re.compile(r'"'), " '' "),
+        (re.compile(r'(\S)(\'\')'), r'\1 \2 '),
+        (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
+        (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
+    ]
+
+    # List of contractions adapted from Robert MacIntyre's tokenizer.
+    _contractions = MacIntyreContractions()
+    CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
+    CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))
+
+    def tokenize(self, text, convert_parentheses=False, return_str=False):
+        for regexp, substitution in self.STARTING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        for regexp, substitution in self.PUNCTUATION:
+            text = regexp.sub(substitution, text)
+
+        # Handles parentheses.
+        regexp, substitution = self.PARENS_BRACKETS
+        text = regexp.sub(substitution, text)
+        # Optionally convert parentheses
+        if convert_parentheses:
+            for regexp, substitution in self.CONVERT_PARENTHESES:
+                text = regexp.sub(substitution, text)
+
+        # Handles double dash.
+        regexp, substitution = self.DOUBLE_DASHES
+        text = regexp.sub(substitution, text)
+
+        # add extra space to make things easier
+        text = " " + text + " "
+
+        for regexp, substitution in self.ENDING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        for regexp in self.CONTRACTIONS2:
+            text = regexp.sub(r' \1 \2 ', text)
+        for regexp in self.CONTRACTIONS3:
+            text = regexp.sub(r' \1 \2 ', text)
+
+        # We are not using CONTRACTIONS4 since
+        # they are also commented out in the SED scripts
+        # for regexp in self._contractions.CONTRACTIONS4:
+        #     text = regexp.sub(r' \1 \2 \3 ', text)
+
+        return text if return_str else text.split()
+
+    def span_tokenize(self, text):
+        """
+        Uses the post-hoc nltk.tokens.align_tokens to return the offset spans.
+
+            >>> from nltk.tokenize import TreebankWordTokenizer
+            >>> s = '''Good muffins cost $3.88\\nin New (York).  Please (buy) me\\ntwo of them.\\n(Thanks).'''
+            >>> expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
+            ... (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38),
+            ... (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59),
+            ... (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
+            >>> list(TreebankWordTokenizer().span_tokenize(s)) == expected
+            True
+            >>> expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
+            ... 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')',
+            ... 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
+            >>> [s[start:end] for start, end in TreebankWordTokenizer().span_tokenize(s)] == expected
+            True
+
+            Additional example
+            >>> from nltk.tokenize import TreebankWordTokenizer
+            >>> s = '''I said, "I'd like to buy some ''good muffins" which cost $3.88\\n each in New (York)."'''
+            >>> expected = [(0, 1), (2, 6), (6, 7), (8, 9), (9, 10), (10, 12),
+            ... (13, 17), (18, 20), (21, 24), (25, 29), (30, 32), (32, 36),
+            ... (37, 44), (44, 45), (46, 51), (52, 56), (57, 58), (58, 62),
+            ... (64, 68), (69, 71), (72, 75), (76, 77), (77, 81), (81, 82),
+            ... (82, 83), (83, 84)]
+            >>> list(TreebankWordTokenizer().span_tokenize(s)) == expected
+            True
+            >>> expected = ['I', 'said', ',', '"', 'I', "'d", 'like', 'to',
+            ... 'buy', 'some', "''", "good", 'muffins', '"', 'which', 'cost',
+            ... '$', '3.88', 'each', 'in', 'New', '(', 'York', ')', '.', '"']
+            >>> [s[start:end] for start, end in TreebankWordTokenizer().span_tokenize(s)] == expected
+            True
+
+        """
+        raw_tokens = self.tokenize(text)
+
+        # Convert converted quotes back to original double quotes
+        # Do this only if original text contains double quote(s) or double
+        # single-quotes (because '' might be transformed to `` if it is
+        # treated as starting quotes).
+        if ('"' in text) or ("''" in text):
+            # Find double quotes and converted quotes
+            matched = [m.group() for m in re.finditer(r"``|'{2}|\"", text)]
+
+            # Replace converted quotes back to double quotes
+            tokens = [matched.pop(0) if tok in ['"', "``", "''"] else tok
+                      for tok in raw_tokens]
+        else:
+            tokens = raw_tokens
+
+        for tok in align_tokens(tokens, text):
+            yield tok
+
+
+class TreebankWordDetokenizer(TokenizerI):
+    """
+    The Treebank detokenizer uses the reverse regex operations corresponding to
+    the Treebank tokenizer's regexes.
+
+    Note:
+    - There're additional assumption mades when undoing the padding of [;@#$%&]
+      punctuation symbols that isn't presupposed in the TreebankTokenizer.
+    - There're additional regexes added in reversing the parentheses tokenization,
+       - the r'([\]\)\}\>])\s([:;,.])' removes the additional right padding added
+         to the closing parentheses precedding [:;,.].
+    - It's not possible to return the original whitespaces as they were because
+      there wasn't explicit records of where '\n', '\t' or '\s' were removed at
+      the text.split() operation.
+
+        >>> from nltk.tokenize.treebank import TreebankWordTokenizer, TreebankWordDetokenizer
+        >>> s = '''Good muffins cost $3.88\\nin New York.  Please buy me\\ntwo of them.\\nThanks.'''
+        >>> d = TreebankWordDetokenizer()
+        >>> t = TreebankWordTokenizer()
+        >>> toks = t.tokenize(s)
+        >>> d.detokenize(toks)
+        'Good muffins cost $3.88 in New York. Please buy me two of them. Thanks.'
+
+    The MXPOST parentheses substitution can be undone using the `convert_parentheses`
+    parameter:
+
+    >>> s = '''Good muffins cost $3.88\\nin New (York).  Please (buy) me\\ntwo of them.\\n(Thanks).'''
+    >>> expected_tokens = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
+    ... 'New', '-LRB-', 'York', '-RRB-', '.', 'Please', '-LRB-', 'buy',
+    ... '-RRB-', 'me', 'two', 'of', 'them.', '-LRB-', 'Thanks', '-RRB-', '.']
+    >>> expected_tokens == t.tokenize(s, convert_parentheses=True)
+    True
+    >>> expected_detoken = 'Good muffins cost $3.88 in New (York). Please (buy) me two of them. (Thanks).'
+    >>> expected_detoken == d.detokenize(t.tokenize(s, convert_parentheses=True), convert_parentheses=True)
+    True
+
+    During tokenization it's safe to add more spaces but during detokenization,
+    simply undoing the padding doesn't really help.
+
+    - During tokenization, left and right pad is added to [!?], when
+      detokenizing, only left shift the [!?] is needed.
+      Thus (re.compile(r'\s([?!])'), r'\g<1>')
+
+    - During tokenization [:,] are left and right padded but when detokenizing,
+      only left shift is necessary and we keep right pad after comma/colon
+      if the string after is a non-digit.
+      Thus (re.compile(r'\s([:,])\s([^\d])'), r'\1 \2')
+
+    >>> from nltk.tokenize.treebank import TreebankWordDetokenizer
+    >>> toks = ['hello', ',', 'i', 'ca', "n't", 'feel', 'my', 'feet', '!', 'Help', '!', '!']
+    >>> twd = TreebankWordDetokenizer()
+    >>> twd.detokenize(toks)
+    "hello, i can't feel my feet! Help!!"
+
+    >>> toks = ['hello', ',', 'i', "can't", 'feel', ';', 'my', 'feet', '!',
+    ... 'Help', '!', '!', 'He', 'said', ':', 'Help', ',', 'help', '?', '!']
+    >>> twd.detokenize(toks)
+    "hello, i can't feel; my feet! Help!! He said: Help, help?!"
+    """
+    _contractions = MacIntyreContractions()
+    CONTRACTIONS2 = [re.compile(pattern.replace('(?#X)', '\s'))
+                     for pattern in _contractions.CONTRACTIONS2]
+    CONTRACTIONS3 = [re.compile(pattern.replace('(?#X)', '\s'))
+                     for pattern in _contractions.CONTRACTIONS3]
+
+    # ending quotes
+    ENDING_QUOTES = [
+        (re.compile(r"([^' ])\s('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1\2 "),
+        (re.compile(r"([^' ])\s('[sS]|'[mM]|'[dD]|') "), r"\1\2 "),
+        (re.compile(r'(\S)(\'\')'), r'\1\2 '),
+        (re.compile(r" '' "), '"')
+    ]
+
+    # Handles double dashes
+    DOUBLE_DASHES = (re.compile(r' -- '), r'--')
+
+    # Optionally: Convert parentheses, brackets and converts them from PTB symbols.
+    CONVERT_PARENTHESES = [
+        (re.compile('-LRB-'), '('), (re.compile('-RRB-'), ')'),
+        (re.compile('-LSB-'), '['), (re.compile('-RSB-'), ']'),
+        (re.compile('-LCB-'), '{'), (re.compile('-RCB-'), '}')
+    ]
+
+    # Undo padding on parentheses.
+    PARENS_BRACKETS = [(re.compile(r'\s([\[\(\{\<])\s'), r' \g<1>'),
+                       (re.compile(r'\s([\]\)\}\>])\s'), r'\g<1> '),
+                       (re.compile(r'([\]\)\}\>])\s([:;,.])'), r'\1\2')]
+
+    # punctuation
+    PUNCTUATION = [
+        (re.compile(r"([^'])\s'\s"), r"\1' "),
+        (re.compile(r'\s([?!])'), r'\g<1>'),  # Strip left pad for [?!]
+        # (re.compile(r'\s([?!])\s'), r'\g<1>'),
+        (re.compile(r'([^\.])\s(\.)([\]\)}>"\']*)\s*$'), r'\1\2\3'),
+        # When tokenizing, [;@#$%&] are padded with whitespace regardless of
+        # whether there are spaces before or after them.
+        # But during detokenization, we need to distinguish between left/right
+        # pad, so we split this up.
+        (re.compile(r'\s([#$])\s'), r' \g<1>'),  # Left pad.
+        (re.compile(r'\s([;%])\s'), r'\g<1> '),  # Right pad.
+        (re.compile(r'\s([&])\s'), r' \g<1> '),  # Unknown pad.
+        (re.compile(r'\s\.\.\.\s'), r'...'),
+        (re.compile(r'\s([:,])\s$'), r'\1'),
+        (re.compile(r'\s([:,])\s([^\d])'), r'\1 \2')  # Keep right pad after comma/colon before non-digits.
+        # (re.compile(r'\s([:,])\s([^\d])'), r'\1\2')
+    ]
+
+    # starting quotes
+    STARTING_QUOTES = [
+        (re.compile(r'([ (\[{<])\s``'), r'\1"'),
+        (re.compile(r'\s(``)\s'), r'\1'),
+        (re.compile(r'^``'), r'\"'),
+    ]
+
+    def tokenize(self, tokens, convert_parentheses=False):
+        """
+        Python port of the Moses detokenizer.
+
+        :param tokens: A list of strings, i.e. tokenized text.
+        :type tokens: list(str)
+        :return: str
+        """
+        text = ' '.join(tokens)
+        # Reverse the contractions regexes.
+        # Note: CONTRACTIONS4 are not used in tokenization.
+        for regexp in self.CONTRACTIONS3:
+            text = regexp.sub(r'\1\2', text)
+        for regexp in self.CONTRACTIONS2:
+            text = regexp.sub(r'\1\2', text)
+
+        # Reverse the regexes applied for ending quotes.
+        for regexp, substitution in self.ENDING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        # Undo the space padding.
+        text = text.strip()
+
+        # Reverse the padding on double dashes.
+        regexp, substitution = self.DOUBLE_DASHES
+        text = regexp.sub(substitution, text)
+
+        if convert_parentheses:
+            for regexp, substitution in self.CONVERT_PARENTHESES:
+                text = regexp.sub(substitution, text)
+
+        # Reverse the padding regexes applied for parenthesis/brackets.
+        for regexp, substitution in self.PARENS_BRACKETS:
+            text = regexp.sub(substitution, text)
+
+        # Reverse the regexes applied for punctuations.
+        for regexp, substitution in self.PUNCTUATION:
+            text = regexp.sub(substitution, text)
+
+        # Reverse the regexes applied for starting quotes.
+        for regexp, substitution in self.STARTING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        return text.strip()
+
+    def detokenize(self, tokens, convert_parentheses=False):
+        """ Duck-typing the abstract *tokenize()*."""
+        return self.tokenize(tokens, convert_parentheses)