enmerkar-underscore 2.4.2__py3-none-any.whl

enmerkar_underscore/__init__.py

@@ -0,0 +1,96 @@
+from importlib.metadata import version
+
+import django
+from django.template.base import Lexer
+
+if django.VERSION[:2] >= (2, 1):
+    from django.template.base import TokenType
+    TOKEN_TEXT = TokenType.TEXT
+else:  # django < 2.1
+    from django.template.base import TOKEN_TEXT
+
+from django.utils.encoding import force_str
+from .vendor.enmerkar.enmerkar_extract import extract_django
+
+from .vendor.markey import underscore
+from .vendor.markey.machine import parse_arguments, tokenize
+from .vendor.markey.tools import TokenStream
+
+__version__ = version("enmerkar-underscore")
+
+
+def extract(fileobj, keywords, comment_tags, options):
+    """Extracts translation messages from underscore template files.
+
+    This method does also extract django templates. If a template does not
+    contain any django translation tags we always fallback to underscore extraction.
+
+    This is a plugin to Babel, written according to
+    http://babel.pocoo.org/docs/messages/#writing-extraction-methods
+
+    :param fileobj: the file-like object the messages should be extracted
+                    from
+    :param keywords: a list of keywords (i.e. function names) that should
+                     be recognized as translation functions
+    :param comment_tags: a list of translator tags to search for and
+                         include in the results
+    :param options: a dictionary of additional options (optional)
+    :return: an iterator over ``(lineno, funcname, message, comments)``
+             tuples
+    :rtype: ``iterator``
+    """
+    encoding = options.get('encoding', 'utf-8')
+
+    original_position = fileobj.tell()
+
+    text = fileobj.read().decode(encoding)
+
+    if django.VERSION[:2] >= (1, 9):
+        tokens = Lexer(text).tokenize()
+    else:
+        tokens = Lexer(text, None).tokenize()
+
+    vars = [token.token_type != TOKEN_TEXT for token in tokens]
+
+    could_be_django = any(list(vars))
+
+    if could_be_django:
+        fileobj.seek(original_position)
+        iterator = extract_django(fileobj, keywords, comment_tags, options)
+        for lineno, funcname, message, comments in iterator:
+            yield lineno, funcname, message, comments
+    else:
+        # Underscore template extraction
+        comments = []
+
+        fileobj.seek(original_position)
+
+        for lineno, line in enumerate(fileobj, 1):
+            funcname = None
+
+            stream = TokenStream.from_tuple_iter(tokenize(line, underscore.rules))
+            while not stream.eof:
+                if stream.current.type == 'gettext_begin':
+                    stream.expect('gettext_begin')
+                    funcname = stream.expect('func_name').value
+                    args, kwargs = parse_arguments(stream, 'gettext_end')
+
+                    strings = []
+
+                    for arg, argtype in args:
+                        if argtype == 'func_string_arg':
+                            strings.append(force_str(arg))
+                        else:
+                            strings.append(None)
+
+                    for arg in kwargs:
+                        strings.append(None)
+
+                    if len(strings) == 1:
+                        strings = strings[0]
+                    else:
+                        strings = tuple(strings)
+
+                    yield lineno, funcname, strings, []
+
+                stream.next()

enmerkar_underscore/vendor/enmerkar/LICENSE

@@ -0,0 +1,30 @@
+Copyright (C) 2013-2014 django-babel Team
+Copyright (C) 2019 Extracover Holdings Limited
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+ 3. The name of the author may not be used to endorse or promote
+    products derived from this software without specific prior
+    written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

enmerkar_underscore/vendor/enmerkar/enmerkar_extract.py

@@ -0,0 +1,174 @@
+# Vendored from https://github.com/Zegocover/enmerkar (enmerkar 0.7.1)
+# enmerkar/__init__.py uses pkg_resources (a setuptools artifact not guaranteed
+# to be present in Python 3.12+ environments). We only need extract_django, and
+# enmerkar/extract.py has no such dependency, so we vendor it here directly.
+from django.template.base import Lexer
+from django.template.base import TokenType
+
+from django.utils.translation import trim_whitespace
+from django.utils.encoding import smart_str
+
+from django.utils.translation.template import (
+    inline_re, block_re, endblock_re, plural_re, constant_re)
+
+
+TOKEN_TEXT = TokenType.TEXT
+TOKEN_VAR = TokenType.VAR
+TOKEN_BLOCK = TokenType.BLOCK
+
+
+def join_tokens(tokens, trim=False):
+    message = ''.join(tokens)
+    if trim:
+        message = trim_whitespace(message)
+    return message
+
+
+def strip_quotes(s):
+    if (s[0] == s[-1]) and s.startswith(("'", '"')):
+        return s[1:-1]
+    return s
+
+
+def extract_django(fileobj, keywords, comment_tags, options):
+    """Extract messages from Django template files.
+
+    :param fileobj: the file-like object the messages should be extracted from
+    :param keywords: a list of keywords (i.e. function names) that should
+                     be recognized as translation functions
+    :param comment_tags: a list of translator tags to search for and
+                         include in the results
+    :param options: a dictionary of additional options (optional)
+    :return: an iterator over ``(lineno, funcname, message, comments)``
+             tuples
+    :rtype: ``iterator``
+    """
+    intrans = False
+    inplural = False
+    trimmed = False
+    message_context = None
+    singular = []
+    plural = []
+    lineno = 1
+
+    encoding = options.get('encoding', 'utf8')
+    text = fileobj.read().decode(encoding)
+
+    try:
+        text_lexer = Lexer(text)
+    except TypeError:
+        # Django 1.9 changed the way we invoke Lexer; older versions
+        # require two parameters.
+        text_lexer = Lexer(text, None)
+
+    for t in text_lexer.tokenize():
+        lineno += t.contents.count('\n')
+        if intrans:
+            if t.token_type == TOKEN_BLOCK:
+                endbmatch = endblock_re.match(t.contents)
+                pluralmatch = plural_re.match(t.contents)
+                if endbmatch:
+                    if inplural:
+                        if message_context:
+                            yield (
+                                lineno,
+                                'npgettext',
+                                [smart_str(message_context),
+                                 smart_str(join_tokens(singular, trimmed)),
+                                 smart_str(join_tokens(plural, trimmed))],
+                                [],
+                            )
+                        else:
+                            yield (
+                                lineno,
+                                'ngettext',
+                                (smart_str(join_tokens(singular, trimmed)),
+                                 smart_str(join_tokens(plural, trimmed))),
+                                [])
+                    else:
+                        if message_context:
+                            yield (
+                                lineno,
+                                'pgettext',
+                                [smart_str(message_context),
+                                 smart_str(join_tokens(singular, trimmed))],
+                                [],
+                            )
+                        else:
+                            yield (
+                                lineno,
+                                None,
+                                smart_str(join_tokens(singular, trimmed)),
+                                [])
+
+                    intrans = False
+                    inplural = False
+                    message_context = None
+                    singular = []
+                    plural = []
+                elif pluralmatch:
+                    inplural = True
+                else:
+                    raise SyntaxError('Translation blocks must not include '
+                                      'other block tags: %s' % t.contents)
+            elif t.token_type == TOKEN_VAR:
+                if inplural:
+                    plural.append('%%(%s)s' % t.contents)
+                else:
+                    singular.append('%%(%s)s' % t.contents)
+            elif t.token_type == TOKEN_TEXT:
+                if inplural:
+                    plural.append(t.contents)
+                else:
+                    singular.append(t.contents)
+        else:
+            if t.token_type == TOKEN_BLOCK:
+                imatch = inline_re.match(t.contents)
+                bmatch = block_re.match(t.contents)
+                cmatches = constant_re.findall(t.contents)
+                if imatch:
+                    g = imatch.group(1)
+                    g = strip_quotes(g)
+                    message_context = imatch.group(3)
+                    if message_context:
+                        # strip quotes
+                        message_context = message_context[1:-1]
+                        yield (
+                            lineno,
+                            'pgettext',
+                            [smart_str(message_context), smart_str(g)],
+                            [],
+                        )
+                        message_context = None
+                    else:
+                        yield lineno, None, smart_str(g), []
+                elif bmatch:
+                    if bmatch.group(2):
+                        message_context = bmatch.group(2)[1:-1]
+                    for fmatch in constant_re.findall(t.contents):
+                        stripped_fmatch = strip_quotes(fmatch)
+                        yield lineno, None, smart_str(stripped_fmatch), []
+                    intrans = True
+                    inplural = False
+                    trimmed = 'trimmed' in t.split_contents()
+                    singular = []
+                    plural = []
+                elif cmatches:
+                    for cmatch in cmatches:
+                        stripped_cmatch = strip_quotes(cmatch)
+                        yield lineno, None, smart_str(stripped_cmatch), []
+            elif t.token_type == TOKEN_VAR:
+                parts = t.contents.split('|')
+                cmatch = constant_re.match(parts[0])
+                if cmatch:
+                    stripped_cmatch = strip_quotes(cmatch.group(1))
+                    yield lineno, None, smart_str(stripped_cmatch), []
+                for p in parts[1:]:
+                    if p.find(':_(') >= 0:
+                        p1 = p.split(':', 1)[1]
+                        if p1[0] == '_':
+                            p1 = p1[1:]
+                        if p1[0] == '(':
+                            p1 = p1.strip('()')
+                        p1 = strip_quotes(p1)
+                        yield lineno, None, smart_str(p1), []

enmerkar_underscore/vendor/markey/COPYING

@@ -0,0 +1,28 @@
+Copyright (C) 2014 Christopher Grebs.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+ 3. The name of the author may not be used to endorse or promote
+    products derived from this software without specific prior
+    written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

enmerkar_underscore/vendor/markey/README.rst

@@ -0,0 +1,11 @@
+These files were copied here from https://github.com/EnTeQuAk/markey which is a
+project by the same original owner as the project this repo was forked from
+(django-babel-underscore).
+
+The original project hasn't had updates in 8 years and while a PR was made to
+make the same fix upstream (https://github.com/EnTeQuAk/markey/pull/6) we can't
+wait for it to get merged before the Redwood release cut (May 2024).
+
+To reduce the number of forks we have in play for frontend tech that we have
+already deprecated, we copy the bits we need here and make the updates we need
+to get support for Python 3.11 and 3.12. The content was copied from https://github.com/EnTeQuAk/markey/commit/d9fc042b147824164908f00a24da6ce2c26dcd8e before we added modifications to allow for Python 3.11 support.

enmerkar_underscore/vendor/markey/_compat.py

@@ -0,0 +1,87 @@
+import sys
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+
+_identity = lambda x: x
+
+
+if not PY2:
+    text_type = str
+    string_types = (str,)
+    integer_types = (int, )
+    unichr = chr
+
+    text_to_native = lambda s, enc: s
+
+    iterkeys = lambda d: iter(d.keys())
+    itervalues = lambda d: iter(d.values())
+    iteritems = lambda d: iter(d.items())
+
+    from io import StringIO, BytesIO
+    import pickle
+
+    izip = zip
+    imap = map
+    range_type = range
+
+    cmp = lambda a, b: (a > b) - (a < b)
+
+else:
+    text_type = unicode
+    string_types = (str, unicode)
+    integer_types = (int, long)
+
+    text_to_native = lambda s, enc: s.encode(enc)
+    unichr = unichr
+
+    iterkeys = lambda d: d.iterkeys()
+    itervalues = lambda d: d.itervalues()
+    iteritems = lambda d: d.iteritems()
+
+    from cStringIO import StringIO as BytesIO
+    from StringIO import StringIO
+    import cPickle as pickle
+
+    from itertools import izip, imap
+    range_type = xrange
+
+    cmp = cmp
+
+
+number_types = integer_types + (float,)
+
+
+# Mostly taken from Django.
+def force_text(s, encoding='utf-8', errors='strict'):
+    # Handle the common case first for performance reasons.
+    if isinstance(s, text_type):
+        return s
+    try:
+        if not isinstance(s, string_types):
+            if PY3:
+                if isinstance(s, bytes):
+                    s = text_type(s, encoding, errors)
+                else:
+                    s = text_type(s)
+            elif hasattr(s, '__unicode__'):
+                s = text_type(s)
+            else:
+                s = text_type(bytes(s), encoding, errors)
+        else:
+            # Note: We use .decode() here, instead of six.text_type(s, encoding,
+            # errors), so that if s is a SafeBytes, it ends up being a
+            # SafeText at the end.
+            s = s.decode(encoding, errors)
+    except UnicodeDecodeError as e:
+        if not isinstance(s, Exception):
+            raise UnicodeDecodeError(s, *e.args)
+        else:
+            # If we get to here, the caller has passed in an Exception
+            # subclass populated with non-ASCII bytestring data without a
+            # working unicode method. Try to handle this without raising a
+            # further exception by individually forcing the exception args
+            # to unicode.
+            s = ' '.join([force_text(arg, encoding, errors) for arg in s])
+    return s

enmerkar_underscore/vendor/markey/machine.py

@@ -0,0 +1,119 @@
+from .rules import include
+from ._compat import force_text
+
+
+def iter_rules(x, rules):
+    for rule in rules[x]:
+        if rule.__class__ is include:
+            for item in iter_rules(rule, rules):
+                yield item
+        else:
+            yield rule
+
+
+def tokenize(string, rules, encoding='utf-8'):
+    string = force_text(string)
+
+    pos = 0
+    end = len(string)
+    stack = [(None, 'everything')]
+    rule_cache = {}
+    text_buffer = []
+    add_text = text_buffer.append
+    push = stack.append
+    flatten = ''.join
+
+    while pos < end:
+        state = stack[-1][1]
+        if state not in rule_cache:
+            rule_cache[state] = list(iter_rules(state, rules))
+        for rule in rule_cache[state]:
+            m = rule.match(string, pos)
+            if m is not None:
+                # first flush text that is left in the buffer
+                if text_buffer:
+                    text = flatten(text_buffer)
+                    if text:
+                        yield 'text', text
+                    del text_buffer[:]
+
+                # now enter the new scopes if entered in a
+                # non silent way
+                if rule.enter is not None:
+                    push((rule.enter + '_end', rule.enter))
+                    yield rule.enter + '_begin', m.group()
+
+                # now process the data
+                if callable(rule.token):
+                    for item in rule.token(m):
+                        yield item
+                elif rule.token is not None:
+                    yield rule.token, m.group()
+
+                # now check if we leave something. if the state was
+                # entered non silent, send a close token.
+                pos = m.end()
+                for x in range(rule.leave):
+                    announce, item = stack.pop()
+                    if announce is not None:
+                        yield announce, m.group()
+
+                break
+        else:
+            char = string[pos]
+            add_text(char)
+            pos += 1
+
+    # if the text buffer is left filled, we flush it
+    if text_buffer:
+        text = flatten(text_buffer)
+        if text:
+            yield 'text', text
+
+    # if there are things in the stack left we should
+    # emit the end tokens
+    for announce, item in reversed(stack):
+        if announce is not None:
+            yield announce, u''
+
+
+def parse_arguments(stream, end_token):
+    """
+    Helper function for function argument parsing.  Pass it a
+    `TokenStream` and the delimiter token for the argument section and
+    it will extract all position and keyword arguments as well as
+    each argument's type info (string literal or not).
+
+    Returns a ``(args, kwargs)`` tuple.
+    """
+    args = []
+    kwargs = {}
+    keywords = []
+    while stream.current.type != end_token:
+        if stream.current.type == 'func_string_arg':
+            value = stream.current.value.strip()
+            for char in ['\'', '"']:
+                if value.startswith(char) and value.endswith(char):
+                    value = value[1: -1]
+                    break
+            stream.next()
+            if keywords:
+                for keyword in keywords:
+                    kwargs[keyword] = value
+                del keywords[:]
+            else:
+                args.append((value, 'func_string_arg'))
+        elif stream.current.type == 'text':
+            args.append((stream.current.value, 'text'))
+            stream.next()
+        elif stream.current.type == 'func_kwarg':
+            keywords.append(stream.current.value)
+            stream.next()
+        elif stream.current.type == 'func_argument_delimiter':
+            stream.next()
+        else:
+            break
+    for keyword in keywords:
+        args.append((keyword, 'func_kwarg'))
+
+    return tuple(args), kwargs

enmerkar_underscore/vendor/markey/rules.py

@@ -0,0 +1,34 @@
+import re
+from ._compat import izip
+
+
+def bygroups(*args):
+    """
+    Callback creator for bygroup yielding.
+    """
+    return lambda m: izip(args, m.groups())
+
+
+class include(str):
+    """
+    Tells the lexer to include tokens from another set.
+    """
+    __slots__ = ()
+
+
+class ruleset(tuple):
+    __slots__ = ()
+
+    def __new__(cls, *args):
+        return tuple.__new__(cls, args)
+
+
+class rule(object):
+    """This represents a parsing rule."""
+    __slots__ = ('match', 'token', 'enter', 'leave')
+
+    def __init__(self, regexp, token=None, enter=None, leave=0):
+        self.match = re.compile(regexp, re.U).match
+        self.token = token
+        self.enter = enter
+        self.leave = leave

enmerkar_underscore/vendor/markey/tests/test_machine.py

@@ -0,0 +1,15 @@
+from enmerkar_underscore.vendor.markey.machine import tokenize, parse_arguments
+from enmerkar_underscore.vendor.markey.tools import TokenStream
+from enmerkar_underscore.vendor.markey.underscore import rules as underscore_rules
+
+
+def test_parse_arguments():
+    line = '<% gettext(varible, "some string", str="foo")'
+    stream = TokenStream.from_tuple_iter(tokenize(line, underscore_rules))
+    stream.next()
+    stream.next()
+    stream.expect('gettext_begin')
+    stream.expect('func_name').value
+    args, kwargs = parse_arguments(stream, 'gettext_end')
+    assert args == (('varible', 'text'), ('some string', 'func_string_arg'),)
+    assert kwargs == {'str': 'foo'}

enmerkar_underscore/vendor/markey/tests/test_rules.py

@@ -0,0 +1,21 @@
+import re
+
+from enmerkar_underscore.vendor.markey.rules import bygroups, rule, ruleset
+
+
+def test_bygroups():
+    match = re.match(r'(1)(2)', '12')
+    assert list(bygroups(('one',))(match)) == [(('one',), '1')]
+
+
+class test_rule():
+    r = rule(r'(\d)', 'num', 'do_number')
+    assert r.match('1').group() == '1'
+    assert r.token == 'num'
+    assert r.enter == 'do_number'
+    assert r.leave == 0
+
+
+class test_ruleset():
+    rs = ruleset(rule(r'(\d)', 'num', 'do_number'),)
+    assert isinstance(rs, tuple)

enmerkar_underscore/vendor/markey/tests/test_tools.py

@@ -0,0 +1,96 @@
+import sys
+from six import StringIO
+
+import pytest
+
+from enmerkar_underscore.vendor.markey.tools import Token, TokenStream, TokenStreamIterator
+
+
+def test_tokenstream():
+    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
+    assert s.current == Token('a', 1)
+
+
+def test_tokenstream_from_tuple_iter():
+    # from_tuple_iter
+    s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3))))
+    assert s.current == Token('a', 1)
+
+    # iter
+    assert isinstance(iter(s), TokenStreamIterator)
+    assert tuple(iter(s)) == (Token('a', 1), Token('b', 2), Token('c', 3))
+
+
+def test_tokenstream_eof():
+    # eof
+    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
+    assert not s.eof
+    list(s)
+    assert s.eof
+
+
+def test_tokenstream_look_push():
+    # look, push
+    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
+    assert s.current == Token('a', 1)
+    assert s.look() == Token('b', 2)
+    s.next()
+    assert s.look() == Token('c', 3)
+    s.push(Token('b', 2))
+    assert s.look() == Token('b', 2)
+    s.push(Token('e', 4), current=True)
+    assert s.current == Token('e', 4)
+    assert s.look() == Token('b', 2)
+
+
+def test_tokenstream_skip_next():
+    # skip, next
+    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
+    s.skip(1)
+    assert s.current == Token('b', 2)
+    s.next()
+    assert s.current == Token('c', 3)
+    s.push(Token('e', 4))
+    assert s.current == Token('c', 3)
+    s.next()
+    assert s.current == Token('e', 4)
+    s.next()
+    assert s.current == Token('eof', None)
+
+
+def test_tokenstream_expect():
+    # expect
+    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
+    assert s.expect('a') == Token('a', 1)
+    assert s.expect('b', 2) == Token('b', 2)
+    pytest.raises(AssertionError, s.expect, 'e')
+    pytest.raises(AssertionError, s.expect, 'c', 5)
+
+
+def test_tokenstream_test_shift():
+    # test
+    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
+    assert s.test('a')
+    s.next()
+    assert s.test('b', 2)
+
+    # shift
+    assert s.current == Token('b', 2)
+    s.shift(Token('f', 5))
+    assert s.current == Token('f', 5)
+    s.next()
+    assert s.current == Token('b', 2)
+
+
+def test_tokenstream_debug():
+    stream = StringIO()
+
+    _original_stdout = sys.stdout
+    sys.stdout = stream
+
+    try:
+        s = TokenStream(iter((Token('a', 1),)))
+        s.debug()
+        assert stream.getvalue() == "Token(type='a', value=1)\n"
+    finally:
+        sys.stdout = _original_stdout

enmerkar_underscore/vendor/markey/tools.py

@@ -0,0 +1,130 @@
+import sys
+from collections import namedtuple
+
+#: Represents one token.
+Token = namedtuple('Token', ('type', 'value'))
+
+
+class TokenStreamIterator(object):
+    """
+    The iterator for tokenstreams.  Iterate over the stream
+    until the eof token is reached.
+    """
+
+    def __init__(self, stream):
+        self._stream = stream
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        token = self._stream.current
+        if token.type == 'eof':
+            raise StopIteration()
+        self._stream.next()
+        return token
+
+    # Python 2.x
+    next = __next__
+
+
+class TokenStream(object):
+    """
+    A token stream wraps a generator and supports pushing tokens back.
+    It also provides some functions to expect tokens and similar stuff.
+
+    Important note: Do never push more than one token back to the
+                    stream.  Although the stream object won't stop you
+                    from doing so, the behavior is undefined.  Multiple
+                    pushed tokens are only used internally!
+    """
+
+    def __init__(self, generator):
+        self._next = lambda: next(generator)
+        self._pushed = []
+        self.current = Token('initial', '')
+        self.next()
+
+    @classmethod
+    def from_tuple_iter(cls, tupleiter):
+        return cls(Token(*a) for a in tupleiter)
+
+    def __iter__(self):
+        return TokenStreamIterator(self)
+
+    @property
+    def eof(self):
+        """Are we at the end of the tokenstream?"""
+        return not bool(self._pushed) and self.current.type == 'eof'
+
+    def debug(self, stream=None):
+        """Displays the tokenized code on the stream provided or stdout."""
+        if stream is None:
+            stream = sys.stdout
+        for token in self:
+            stream.write(repr(token) + '\n')
+
+    def look(self):
+        """See what's the next token."""
+        if self._pushed:
+            return self._pushed[-1]
+        old_token = self.current
+        self.next()
+        new_token = self.current
+        self.current = old_token
+        self.push(new_token)
+        return new_token
+
+    def push(self, token, current=False):
+        """Push a token back to the stream (only one!)."""
+        self._pushed.append(token)
+        if current:
+            self.next()
+
+    def skip(self, n):
+        """Got n tokens ahead."""
+        for x in range(n):
+            self.next()
+
+    def next(self):
+        """Go one token ahead."""
+        if self._pushed:
+            self.current = self._pushed.pop()
+        else:
+            try:
+                self.current = self._next()
+            except StopIteration:
+                if self.current.type != 'eof':
+                    self.current = Token('eof', None)
+
+    def expect(self, type, value=None):
+        """expect a given token."""
+        assert self.current.type == type, "expect failed (%s, %s)" % (
+            self.current.type, type)
+        if value is not None:
+            assert self.current.value == value or \
+                (value.__class__ is tuple and
+                    self.current.value in value), "%s != %s" % (type, value)
+        try:
+            return self.current
+        finally:
+            self.next()
+
+    def test(self, type, value=Ellipsis):
+        """Test the current token."""
+        return (
+            self.current.type == type and
+            (value is Ellipsis or self.current.value == value or
+             value.__class__ is tuple and
+             self.current.value in value))
+
+    def shift(self, token):
+        """
+        Push one token into the stream.
+        """
+        old_current = self.current
+        self.next()
+        self.push(self.current)
+        self.push(old_current)
+        self.push(token)
+        self.next()

enmerkar_underscore/vendor/markey/underscore.py

@@ -0,0 +1,54 @@
+# Full functioning underscore template parser.
+
+from .rules import ruleset, include, rule, bygroups
+
+
+keywords = frozenset((
+    'pluralidx', 'gettext', 'ngettext', 'gettext_noop', 'pgettext',
+    'npgettext', '_'
+))
+
+
+rules = {
+    'everything': ruleset(
+        include('inline'),
+        include('function'),
+    ),
+    'inline': ruleset(
+        rule(r'<%=', enter='interpolate'),
+        rule(r'<%-', enter='escape'),
+        rule(r'<%', enter='evaluate'),
+    ),
+    'inline_with_func': ruleset(
+        include('inline'),
+        include('function')
+    ),
+    'function': ruleset(
+        rule(r'({keywords})\('.format(keywords='|'.join(keywords)),
+             bygroups('func_name'), enter='gettext')
+    ),
+    'gettext': ruleset(
+        rule(r'([\"\'])?\)', leave=1),
+        include('function_call')
+    ),
+    'evaluate': ruleset(
+        rule(r'%>', leave=1),
+        include('inline_with_func'),
+    ),
+    'interpolate': ruleset(
+        rule(r'%>', leave=1),
+        include('inline_with_func'),
+    ),
+    'escape': ruleset(
+        rule(r'%>', leave=1),
+        include('inline_with_func'),
+    ),
+    # function calls (parse string arguments and implicit strings)
+    'function_call': ruleset(
+        rule(',', 'func_argument_delimiter'),
+        rule('\s+', None),
+        rule(r"(?s)('([^'\\]*(?:\\.[^'\\]*)*)'|"
+             r'"([^"\\]*(?:\\.[^"\\]*)*)")', 'func_string_arg'),
+        rule(r'([\w_]+)\s*=', bygroups('func_kwarg'))
+    )
+}

enmerkar_underscore-2.4.2.dist-info/METADATA

@@ -0,0 +1,127 @@
+Metadata-Version: 2.4
+Name: enmerkar-underscore
+Version: 2.4.2
+Summary: Implements a underscore extractor for django-babel.
+Author-email: Open edX Project <oscm@openedx.org>
+License-Expression: BSD-3-Clause
+Project-URL: Homepage, https://github.com/openedx/enmerkar-underscore
+Project-URL: Repository, https://github.com/openedx/enmerkar-underscore
+Keywords: enmerkar-underscore,Python,edx
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 4.2
+Classifier: Framework :: Django :: 5.2
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: django
+Requires-Dist: babel>=1.3
+Dynamic: license-file
+
+=================================
+Django Babel Underscore Extractor
+=================================
+
+.. image:: https://badge.fury.io/py/django-babel-underscore.png
+    :target: http://badge.fury.io/py/django-babel-underscore
+
+.. image:: https://github.com/openedx/enmerkar-underscore/workflows/Python%20CI/badge.svg?branch=master
+        :target: https://github.com/openedx/enmerkar-underscore/actions?query=workflow%3A%22Python+CI%22
+
+
+Implements a underscore extractor for django-babel.
+
+* Free software: BSD license
+* Documentation: https://github.com/openedx/enmerkar-underscore
+
+Features
+--------
+
+* TODO
+
+.. :changelog:
+
+History
+-------
+
+2.4.0 (2025-04-22)
+------------------
+
+* Add support for django 5.2
+
+2.0.0 (2020-05-29)
+------------------
+
+* Drop support for Python 3.8
+
+
+1.0.0 (2020-05-29)
+------------------
+
+* Rename the repo
+* Using latest version of enmerkar.
+* Release of pypi
+
+0.6.0 (2020-01-07)
+------------------
+
+* Adds support for Python 3.6, 3.7
+* Adds support for Django 2.0, 2.1, 2.2 and 3.0.
+* Drop support for Django < 1.11
+
+
+0.5.2 (2018-08-18)
+------------------
+
+* Adds support for Django 1.10 and 1.11.
+
+
+0.5.1 (2016-07-15)
+------------------
+
+* Fix release
+
+
+0.5.0 (2016-07-15)
+------------------
+
+* Remove support for Python 2.6 and Python 3.3
+* Remove support for Django < 1.8
+* Add support for Python 3.5
+* Add support for Django 1.9
+* Now requires markey 0.8
+
+0.4.2 (2015-08-28)
+------------------
+
+* Update to markey 0.7
+
+0.4.1 (2015-08-28)
+------------------
+
+* Fix release
+
+0.4.0 (2015-08-28)
+------------------
+
+* Add support for Django 1.8
+
+0.3.0 (2015-03-15)
+------------------
+
+* Added `underscore` entry_point for babel.extractors (#1)
+
+0.2.0 (2014-12-24)
+------------------
+
+* Fix dependency names.
+
+0.1.0 (2014-05-25)
+------------------
+
+* First release on PyPI.

enmerkar_underscore-2.4.2.dist-info/RECORD

@@ -0,0 +1,22 @@
+enmerkar_underscore/__init__.py,sha256=eM7M4vra7qcytA8VXhN6SNj-PgjMI4Lwx1o-sQ1am0o,3322
+enmerkar_underscore/vendor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+enmerkar_underscore/vendor/enmerkar/LICENSE,sha256=65Z5NCGyb2yADuQ2IAQy0wAPIgQEW7cuXhOwLhY5V34,1465
+enmerkar_underscore/vendor/enmerkar/enmerkar_extract.py,sha256=0cLhMs7ITNLLkwkmqy7taUyjy0YB1OrAqO7HRi7-uvo,6957
+enmerkar_underscore/vendor/markey/COPYING,sha256=zsVkBdUL9ZVl2LNG-wNmF5TKtwiR9qxIgD7zcDbF6PY,1413
+enmerkar_underscore/vendor/markey/README.rst,sha256=BrJFx727YSybIuZUYRUqUGUKeDH7nNvMCovWlOExco8,784
+enmerkar_underscore/vendor/markey/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+enmerkar_underscore/vendor/markey/_compat.py,sha256=51WsDkLYOK99-MZgx0HvpTzPoWixcmoWTPB7mNC3X-A,2409
+enmerkar_underscore/vendor/markey/machine.py,sha256=B8Qe96X4tYimUl8cknjEcFzQKhDPczcPpk9AZR3kLY0,3807
+enmerkar_underscore/vendor/markey/rules.py,sha256=_uHGDO1wczBiKNV4BeG9M4dGI0PRSefxPtnWiT-LXHk,704
+enmerkar_underscore/vendor/markey/tools.py,sha256=zG2sik1k6Jyb83dVv6xoZ0Sh3kbaUA9UrIfqzHflLtw,3735
+enmerkar_underscore/vendor/markey/underscore.py,sha256=aNLTO_iAxobOB6ZWtBOciyNFTiS3rIZ1gShVW7q5TGA,1464
+enmerkar_underscore/vendor/markey/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+enmerkar_underscore/vendor/markey/tests/test_machine.py,sha256=rNFe7Bmyjkh_CKR_Ih5PMSrlR2bkJ9rvkcUFpw_f4zA,671
+enmerkar_underscore/vendor/markey/tests/test_rules.py,sha256=1_ZA60aow0H5AYLmE8GKXCnQR_3Oz5IYj5GDcp3R2GI,510
+enmerkar_underscore/vendor/markey/tests/test_tools.py,sha256=RFZtkziapmJpzuQ8julXSMI1q34kB9U4WhANHr-yNQ4,2552
+enmerkar_underscore-2.4.2.dist-info/licenses/LICENSE,sha256=OXqUbEDFudXLg2c9yUtD5V5DOJI2aA1Q0_CoAs2DEIY,1498
+enmerkar_underscore-2.4.2.dist-info/METADATA,sha256=iFbEeYO4idwn-_ezXPQOUwh5NVU4FuAwq75VsXXx6wU,2748
+enmerkar_underscore-2.4.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+enmerkar_underscore-2.4.2.dist-info/entry_points.txt,sha256=RrEDKrwUU3vz5RRVRFhECDEcubpmkUvaAaqWkv4HOHM,60
+enmerkar_underscore-2.4.2.dist-info/top_level.txt,sha256=OQ-esflDonOuDESY9Lh7nOhxOpceH_6CI3lbopUABH4,20
+enmerkar_underscore-2.4.2.dist-info/RECORD,,

enmerkar_underscore-2.4.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

enmerkar_underscore-2.4.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[babel.extractors]
+underscore = enmerkar_underscore:extract

enmerkar_underscore-2.4.2.dist-info/licenses/LICENSE

@@ -0,0 +1,12 @@
+Copyright (c) 2015, Christopher Grebs
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of Django Babel Underscore Extractor nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

enmerkar_underscore-2.4.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+enmerkar_underscore