Cython 3.1.0__py3-none-any.whl

Cython/Compiler/Tests/Utils.py

@@ -0,0 +1,36 @@
+import copy
+
+from .. import Options
+
+
+def backup_Options():
+    backup = {}
+    for name, value in vars(Options).items():
+        # we need a deep copy of _directive_defaults, because they can be changed
+        if name == '_directive_defaults':
+            value = copy.deepcopy(value)
+        backup[name] = value
+    return backup
+
+
+def restore_Options(backup):
+    no_value = object()
+    for name, orig_value in backup.items():
+        if getattr(Options, name, no_value) != orig_value:
+            setattr(Options, name, orig_value)
+    # strip Options from new keys that might have been added:
+    for name in vars(Options).keys():
+        if name not in backup:
+            delattr(Options, name)
+
+
+def check_global_options(expected_options, white_list=[]):
+    """
+    returns error message of "" if check Ok
+    """
+    no_value = object()
+    for name, orig_value in expected_options.items():
+        if name not in white_list:
+            if getattr(Options, name, no_value) != orig_value:
+                return "error in option " + name
+    return ""

Cython/Compiler/Tests/__init__.py

@@ -0,0 +1 @@
+# empty file

Cython/Compiler/TreeFragment.py

@@ -0,0 +1,278 @@
+#
+# TreeFragments - parsing of strings to trees
+#
+
+"""
+Support for parsing strings into code trees.
+"""
+
+
+import re
+from io import StringIO
+
+from .Scanning import PyrexScanner, StringSourceDescriptor
+from .Symtab import ModuleScope
+from . import PyrexTypes
+from .Visitor import VisitorTransform
+from .Nodes import Node, StatListNode
+from .ExprNodes import NameNode
+from . import Parsing
+from . import Main
+from . import UtilNodes
+
+
+class StringParseContext(Main.Context):
+    def __init__(self, name, include_directories=None, compiler_directives=None, cpp=False):
+        if include_directories is None:
+            include_directories = []
+        if compiler_directives is None:
+            compiler_directives = {}
+        Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level='3')
+        self.module_name = name
+
+    def find_module(self, module_name, from_module=None, pos=None, need_pxd=1, absolute_fallback=True, relative_import=False):
+        if module_name not in (self.module_name, 'cython'):
+            raise AssertionError("Not yet supporting any cimports/includes from string code snippets")
+        return ModuleScope(module_name, parent_module=None, context=self)
+
+
+def parse_from_strings(name, code, pxds=None, level=None, initial_pos=None,
+                       context=None, allow_struct_enum_decorator=False,
+                       in_utility_code=False):
+    """
+    Utility method to parse a (unicode) string of code. This is mostly
+    used for internal Cython compiler purposes (creating code snippets
+    that transforms should emit, as well as unit testing).
+
+    code - a unicode string containing Cython (module-level) code
+    name - a descriptive name for the code source (to use in error messages etc.)
+    in_utility_code - used to suppress some messages from utility code. False by default
+                      because some generated code snippets like properties and dataclasses
+                      probably want to see those messages.
+
+    RETURNS
+
+    The tree, i.e. a ModuleNode. The ModuleNode's scope attribute is
+    set to the scope used when parsing.
+    """
+    if context is None:
+        context = StringParseContext(name)
+    # Since source files carry an encoding, it makes sense in this context
+    # to use a unicode string so that code fragments don't have to bother
+    # with encoding. This means that test code passed in should not have an
+    # encoding header.
+    assert isinstance(code, str), "unicode code snippets only please"
+    encoding = "UTF-8"
+
+    module_name = name
+    if initial_pos is None:
+        initial_pos = (name, 1, 0)
+    code_source = StringSourceDescriptor(name, code)
+    if in_utility_code:
+        code_source.in_utility_code = True
+
+    scope = context.find_module(module_name, pos=initial_pos, need_pxd=False)
+
+    buf = StringIO(code)
+
+    scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
+                     scope = scope, context = context, initial_pos = initial_pos)
+    ctx = Parsing.Ctx(allow_struct_enum_decorator=allow_struct_enum_decorator)
+
+    if level is None:
+        tree = Parsing.p_module(scanner, 0, module_name, ctx=ctx)
+        tree.scope = scope
+        tree.is_pxd = False
+    else:
+        tree = Parsing.p_code(scanner, level=level, ctx=ctx)
+
+    tree.scope = scope
+    return tree
+
+
+class TreeCopier(VisitorTransform):
+    def visit_Node(self, node):
+        if node is None:
+            return node
+        else:
+            c = node.clone_node()
+            self.visitchildren(c)
+            return c
+
+
+class ApplyPositionAndCopy(TreeCopier):
+    def __init__(self, pos):
+        super().__init__()
+        self.pos = pos
+
+    def visit_Node(self, node):
+        copy = super().visit_Node(node)
+        copy.pos = self.pos
+        return copy
+
+
+class TemplateTransform(VisitorTransform):
+    """
+    Makes a copy of a template tree while doing substitutions.
+
+    A dictionary "substitutions" should be passed in when calling
+    the transform; mapping names to replacement nodes. Then replacement
+    happens like this:
+     - If an ExprStatNode contains a single NameNode, whose name is
+       a key in the substitutions dictionary, the ExprStatNode is
+       replaced with a copy of the tree given in the dictionary.
+       It is the responsibility of the caller that the replacement
+       node is a valid statement.
+     - If a single NameNode is otherwise encountered, it is replaced
+       if its name is listed in the substitutions dictionary in the
+       same way. It is the responsibility of the caller to make sure
+       that the replacement nodes is a valid expression.
+
+    Also a list "temps" should be passed. Any names listed will
+    be transformed into anonymous, temporary names.
+
+    Currently supported for tempnames is:
+    NameNode
+    (various function and class definition nodes etc. should be added to this)
+
+    Each replacement node gets the position of the substituted node
+    recursively applied to every member node.
+    """
+
+    temp_name_counter = 0
+
+    def __call__(self, node, substitutions, temps, pos):
+        self.substitutions = substitutions
+        self.pos = pos
+        tempmap = {}
+        temphandles = []
+        for temp in temps:
+            TemplateTransform.temp_name_counter += 1
+            handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)
+            tempmap[temp] = handle
+            temphandles.append(handle)
+        self.tempmap = tempmap
+        result = super().__call__(node)
+        if temps:
+            result = UtilNodes.TempsBlockNode(self.get_pos(node),
+                                              temps=temphandles,
+                                              body=result)
+        return result
+
+    def get_pos(self, node):
+        if self.pos:
+            return self.pos
+        else:
+            return node.pos
+
+    def visit_Node(self, node):
+        if node is None:
+            return None
+        else:
+            c = node.clone_node()
+            if self.pos is not None:
+                c.pos = self.pos
+            self.visitchildren(c)
+            return c
+
+    def try_substitution(self, node, key):
+        sub = self.substitutions.get(key)
+        if sub is not None:
+            pos = self.pos
+            if pos is None: pos = node.pos
+            return ApplyPositionAndCopy(pos)(sub)
+        else:
+            return self.visit_Node(node)  # make copy as usual
+
+    def visit_NameNode(self, node):
+        temphandle = self.tempmap.get(node.name)
+        if temphandle:
+            # Replace name with temporary
+            return temphandle.ref(self.get_pos(node))
+        else:
+            return self.try_substitution(node, node.name)
+
+    def visit_ExprStatNode(self, node):
+        # If an expression-as-statement consists of only a replaceable
+        # NameNode, we replace the entire statement, not only the NameNode
+        if isinstance(node.expr, NameNode):
+            return self.try_substitution(node, node.expr.name)
+        else:
+            return self.visit_Node(node)
+
+
+def copy_code_tree(node):
+    return TreeCopier()(node)
+
+
+_match_indent = re.compile("^ *").match
+
+
+def strip_common_indent(lines):
+    """Strips empty lines and common indentation from the list of strings given in lines"""
+    # TODO: Facilitate textwrap.indent instead
+    lines = [x for x in lines if x.strip() != ""]
+    if lines:
+        minindent = min([len(_match_indent(x).group(0)) for x in lines])
+        lines = [x[minindent:] for x in lines]
+    return lines
+
+
+class TreeFragment:
+    def __init__(self, code, name=None, pxds=None, temps=None, pipeline=None, level=None, initial_pos=None):
+        if pxds is None:
+            pxds = {}
+        if temps is None:
+            temps = []
+        if pipeline is None:
+            pipeline = []
+        if not name:
+            name = "(tree fragment)"
+
+        if isinstance(code, str):
+            def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n")))
+
+            fmt_code = fmt(code)
+            fmt_pxds = {}
+            for key, value in pxds.items():
+                fmt_pxds[key] = fmt(value)
+            mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level, initial_pos=initial_pos)
+            if level is None:
+                t = t.body  # Make sure a StatListNode is at the top
+            if not isinstance(t, StatListNode):
+                t = StatListNode(pos=mod.pos, stats=[t])
+            for transform in pipeline:
+                if transform is None:
+                    continue
+                t = transform(t)
+            self.root = t
+        elif isinstance(code, Node):
+            if pxds:
+                raise NotImplementedError()
+            self.root = code
+        else:
+            raise ValueError("Unrecognized code format (accepts unicode and Node)")
+        self.temps = temps
+
+    def copy(self):
+        return copy_code_tree(self.root)
+
+    def substitute(self, nodes=None, temps=None, pos = None):
+        if nodes is None:
+            nodes = {}
+        if temps is None:
+            temps = []
+        return TemplateTransform()(self.root,
+                                   substitutions = nodes,
+                                   temps = self.temps + temps, pos = pos)
+
+
+class SetPosTransform(VisitorTransform):
+    def __init__(self, pos):
+        super().__init__()
+        self.pos = pos
+
+    def visit_Node(self, node):
+        node.pos = self.pos
+        self.visitchildren(node)
+        return node

Cython/Compiler/TreePath.py

@@ -0,0 +1,303 @@
+"""
+A simple XPath-like language for tree traversal.
+
+This works by creating a filter chain of generator functions.  Each
+function selects a part of the expression, e.g. a child node, a
+specific descendant or a node that holds an attribute.
+"""
+
+
+import re
+import operator
+
+path_tokenizer = re.compile(
+    r"("
+    r"b?'[^']*'|b?\"[^\"]*\"|"
+    r"//?|"
+    r"\(\)|"
+    r"==?|"
+    r"[/.*\[\]()@])|"
+    r"([^/\[\]()@=\s]+)|"
+    r"\s+"
+    ).findall
+
+def iterchildren(node, attr_name):
+    # returns an iterable of all child nodes of that name
+    child = getattr(node, attr_name)
+    if child is not None:
+        if type(child) is list:
+            return child
+        else:
+            return [child]
+    else:
+        return ()
+
+def _get_first_or_none(it):
+    try:
+        return next(it)
+    except StopIteration:
+        return None
+
+def type_name(node):
+    return node.__class__.__name__.split('.')[-1]
+
+def parse_func(next, token):
+    name = token[1]
+    token = next()
+    if token[0] != '(':
+        raise ValueError("Expected '(' after function name '%s'" % name)
+    predicate = handle_predicate(next, token)
+    return name, predicate
+
+def handle_func_not(next, token):
+    """
+    not(...)
+    """
+    name, predicate = parse_func(next, token)
+
+    def select(result):
+        for node in result:
+            if _get_first_or_none(predicate([node])) is None:
+                yield node
+    return select
+
+def handle_name(next, token):
+    """
+    /NodeName/
+    or
+    func(...)
+    """
+    name = token[1]
+    if name in functions:
+        return functions[name](next, token)
+    def select(result):
+        for node in result:
+            for attr_name in node.child_attrs:
+                for child in iterchildren(node, attr_name):
+                    if type_name(child) == name:
+                        yield child
+    return select
+
+def handle_star(next, token):
+    """
+    /*/
+    """
+    def select(result):
+        for node in result:
+            for name in node.child_attrs:
+                yield from iterchildren(node, name)
+    return select
+
+def handle_dot(next, token):
+    """
+    /./
+    """
+    def select(result):
+        return result
+    return select
+
+def handle_descendants(next, token):
+    """
+    //...
+    """
+    token = next()
+    if token[0] == "*":
+        def iter_recursive(node):
+            for name in node.child_attrs:
+                for child in iterchildren(node, name):
+                    yield child
+                    yield from iter_recursive(child)
+    elif not token[0]:
+        node_name = token[1]
+        def iter_recursive(node):
+            for name in node.child_attrs:
+                for child in iterchildren(node, name):
+                    if type_name(child) == node_name:
+                        yield child
+                    yield from iter_recursive(child)
+    else:
+        raise ValueError("Expected node name after '//'")
+
+    def select(result):
+        for node in result:
+            yield from iter_recursive(node)
+
+    return select
+
+
+def handle_attribute(next, token):
+    token = next()
+    if token[0]:
+        raise ValueError("Expected attribute name")
+    name = token[1]
+    value = None
+    token = next.peek()
+    if token[0] == '=':
+        next()
+        value = parse_path_value(next)
+
+    readattr = operator.attrgetter(name)
+    if value is None:
+        def select(result):
+            for node in result:
+                try:
+                    attr_value = readattr(node)
+                except AttributeError:
+                    continue
+                if attr_value is not None:
+                    yield attr_value
+    else:
+        def select(result):
+            for node in result:
+                try:
+                    attr_value = readattr(node)
+                except AttributeError:
+                    continue
+                if attr_value == value:
+                    yield attr_value
+                elif (isinstance(attr_value, bytes) and isinstance(value, str) and
+                        attr_value == value.encode()):
+                    # allow a bytes-to-string comparison too
+                    yield attr_value
+
+    return select
+
+
+def parse_path_value(next):
+    token = next()
+    value = token[0]
+    if value:
+        if value[:1] == "'" or value[:1] == '"':
+            assert value[-1] == value[0]
+            return value[1:-1]
+        if value[:2] == "b'" or value[:2] == 'b"':
+            assert value[-1] == value[1]
+            return value[2:-1].encode('UTF-8')
+        try:
+            return int(value)
+        except ValueError:
+            pass
+    elif token[1].isdigit():
+        return int(token[1])
+    else:
+        name = token[1].lower()
+        if name == 'true':
+            return True
+        elif name == 'false':
+            return False
+    raise ValueError(f"Invalid attribute predicate: '{value}'")
+
+
+def handle_predicate(next, token):
+    token = next()
+
+    and_conditions = [[]]
+    or_conditions = [and_conditions]
+
+    while token[0] not in (']', ')'):
+        and_conditions[-1].append( operations[token[0]](next, token) )
+        try:
+            token = next()
+        except StopIteration:
+            break
+        else:
+            if token[0] == "/":
+                token = next()
+
+        if not token[0]:
+            if token[1] == 'and':
+                and_conditions.append([])
+                token = next()
+            elif token[1] == 'or':
+                and_conditions = [[]]
+                or_conditions.append(and_conditions)
+                token = next()
+
+    if not and_conditions[-1]:
+        raise ValueError("Incomplete predicate")
+
+    def select(result):
+        for node in result:
+            node_base = (node,)
+            for and_conditions in or_conditions:
+                for condition in and_conditions:
+                    subresult = iter(node_base)
+                    for select in condition:
+                        subresult = select(subresult)
+                    predicate_result = _get_first_or_none(subresult)
+                    if predicate_result is None:
+                        # Fail current 'and' condition and skip to next 'or' condition.
+                        break
+                else:
+                    # All 'and' conditions matched, report and skip following 'or' conditions.
+                    yield node
+                    break
+
+    return select
+
+
+operations = {
+    "@":  handle_attribute,
+    "":   handle_name,
+    "*":  handle_star,
+    ".":  handle_dot,
+    "//": handle_descendants,
+    "[":  handle_predicate,
+    }
+
+functions = {
+    'not' : handle_func_not
+    }
+
+
+class _LookAheadTokenizer:
+    def __init__(self, path):
+        self._tokens = [
+            (special, text)
+            for (special, text) in path_tokenizer(path)
+            if special or text
+        ]
+        self._tokens.reverse()  # allow efficient .pop()
+
+    def peek(self, default=(None, None)):
+        return self._tokens[-1] if self._tokens else default
+
+    def __call__(self):
+        try:
+            return self._tokens.pop()
+        except IndexError:
+            raise StopIteration from None
+
+
+def _build_path_iterator(path):
+    # parse pattern
+    _next = _LookAheadTokenizer(path)
+    token = _next()
+    selector = []
+    while 1:
+        try:
+            selector.append(operations[token[0]](_next, token))
+        except StopIteration:
+            raise ValueError("invalid path")
+        try:
+            token = _next()
+            if token[0] == "/":
+                token = _next()
+        except StopIteration:
+            break
+    return selector
+
+# main module API
+
+def iterfind(node, path):
+    selector_chain = _build_path_iterator(path)
+    result = iter((node,))
+    for select in selector_chain:
+        result = select(result)
+    return result
+
+def find_first(node, path):
+    return _get_first_or_none(iterfind(node, path))
+
+def find_all(node, path):
+    return list(iterfind(node, path))