cinderx 2026.1.16.2__cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl

cinderx/compiler/unparse.py

@@ -0,0 +1,414 @@
+# Portions copyright (c) Meta Platforms, Inc. and affiliates.
+# pyre-strict
+
+import ast
+import sys
+import warnings
+from collections.abc import Callable
+from typing import Optional
+
+
+PR_TUPLE = 0
+PR_TEST = 1  # 'if'-'else', 'lambda'
+PR_OR = 2  # 'or'
+PR_AND = 3  # 'and'
+PR_NOT = 4  # 'not'
+PR_CMP = 5  # '<', '>', '==', '>=', '<=', '!=' 'in', 'not in', 'is', 'is not'
+PR_EXPR = 6
+PR_BOR: int = PR_EXPR  # '|'
+PR_BXOR = 7  # '^'
+PR_BAND = 8  # '&'
+PR_SHIFT = 9  # '<<', '>>'
+PR_ARITH = 10  # '+', '-'
+PR_TERM = 11  # '*', '@', '/', '%', '//'
+PR_FACTOR = 12  # unary '+', '-', '~'
+PR_POWER = 13  # '**'
+PR_AWAIT = 14  # 'await'
+PR_ATOM = 15
+
+
+def get_op(node: ast.cmpop) -> str:
+    if isinstance(node, ast.Is):
+        return " is "
+    elif isinstance(node, ast.IsNot):
+        return " is not "
+    elif isinstance(node, ast.In):
+        return " in "
+    elif isinstance(node, ast.NotIn):
+        return " not in "
+    elif isinstance(node, ast.Lt):
+        return " < "
+    elif isinstance(node, ast.Gt):
+        return " > "
+    elif isinstance(node, ast.LtE):
+        return " <= "
+    elif isinstance(node, ast.GtE):
+        return " >= "
+    elif isinstance(node, ast.Eq):
+        return " == "
+    elif isinstance(node, ast.NotEq):
+        return " != "
+    else:
+        return "unknown op: " + type(node).__name__
+
+
+def get_unop(node: ast.unaryop) -> str:
+    if isinstance(node, ast.UAdd):
+        return "+"
+    elif isinstance(node, ast.USub):
+        return "-"
+    elif isinstance(node, ast.Not):
+        return "not "
+    elif isinstance(node, ast.Invert):
+        return "~"
+    return "<unknown unary op>"
+
+
+def _format_name(node: ast.Name, level: int) -> str:
+    return node.id
+
+
+def _format_compare(node: ast.Compare, level: int) -> str:
+    return parens(
+        level,
+        PR_CMP,
+        to_expr(node.left, PR_CMP + 1)
+        + "".join(
+            (
+                get_op(op) + to_expr(comp, PR_CMP + 1)
+                for comp, op in zip(node.comparators, node.ops)
+            )
+        ),
+    )
+
+
+def _format_attribute(node: ast.Attribute, level: int) -> str:
+    value = to_expr(node.value, PR_ATOM)
+    const = node.value
+    if isinstance(const, ast.Constant) and isinstance(const.value, int):
+        value += " ."
+    else:
+        value += "."
+    return value + node.attr
+
+
+def _format_tuple(node: ast.Tuple, level: int) -> str:
+    if not node.elts:
+        return "()"
+    elif len(node.elts) == 1:
+        return parens(level, PR_TUPLE, to_expr(node.elts[0]) + ",")
+
+    return parens(level, PR_TUPLE, ", ".join(to_expr(elm) for elm in node.elts))
+
+
+def _format_list(node: ast.List, level: int) -> str:
+    return "[" + ", ".join(to_expr(elm) for elm in node.elts) + "]"
+
+
+def _format_kw(node: ast.keyword) -> str:
+    if node.arg:
+        return f"{node.arg}={to_expr(node.value)}"
+    return f"**{to_expr(node.value)}"
+
+
+def _format_call(node: ast.Call, level: int) -> str:
+    args = [to_expr(arg) for arg in node.args] + [
+        _format_kw(arg) for arg in node.keywords
+    ]
+    return to_expr(node.func, PR_TEST) + "(" + ", ".join(args) + ")"
+
+
+def _format_unaryop(node: ast.UnaryOp, level: int) -> str:
+    tgt_level = PR_FACTOR
+    if isinstance(node.op, ast.Not):
+        tgt_level = PR_NOT
+    return parens(
+        level, tgt_level, get_unop(node.op) + to_expr(node.operand, tgt_level)
+    )
+
+
+BIN_OPS: dict[type[ast.AST], tuple[str, int]] = {
+    ast.Add: (" + ", PR_ARITH),
+    ast.Sub: (" - ", PR_ARITH),
+    ast.Mult: (" * ", PR_TERM),
+    ast.MatMult: (" @ ", PR_TERM),
+    ast.Div: (" / ", PR_TERM),
+    ast.Mod: (" % ", PR_TERM),
+    ast.LShift: (" << ", PR_SHIFT),
+    ast.RShift: (" >> ", PR_SHIFT),
+    ast.BitOr: (" | ", PR_BOR),
+    ast.BitXor: (" ^ ", PR_BXOR),
+    ast.BitAnd: (" & ", PR_BAND),
+    ast.FloorDiv: (" // ", PR_TERM),
+    ast.Pow: (" ** ", PR_POWER),
+}
+
+
+def _format_binaryop(node: ast.BinOp, level: int) -> str:
+    tgt_level = PR_FACTOR
+
+    # pyre-fixme[6]: For 1st argument expected `Type[Union[Add, BitAnd, BitOr,
+    #  BitXor, Div, FloorDiv, LShift, MatMult, Mod, Mult, Pow, RShift, Sub]]` but got
+    #  `Type[operator]`.
+    op, tgt_level = BIN_OPS[type(node.op)]
+    rassoc = 0
+    if isinstance(node.op, ast.Pow):
+        rassoc = 1
+    return parens(
+        level,
+        tgt_level,
+        to_expr(node.left, tgt_level + rassoc)
+        + op
+        + to_expr(node.right, tgt_level + (1 - rassoc)),
+    )
+
+
+def _format_subscript(node: ast.Subscript, level: int) -> str:
+    return f"{to_expr(node.value, PR_ATOM)}[{to_expr(node.slice, PR_TUPLE)}]"
+
+
+def _format_yield(node: ast.Yield, level: int) -> str:
+    raise SyntaxError("'yield expression' can not be used within an annotation")
+
+
+def _format_yield_from(node: ast.YieldFrom, level: int) -> str:
+    raise SyntaxError("'yield expression' can not be used within an annotation")
+
+
+def _format_dict(node: ast.Dict, level: int) -> str:
+    return (
+        "{"
+        + ", ".join(
+            to_expr(k) + ": " + to_expr(v) for k, v in zip(node.keys, node.values)
+        )
+        + "}"
+    )
+
+
+def _format_comprehension(node: ast.comprehension) -> str:
+    header = " for "
+    if node.is_async:
+        header = " async for "
+
+    res = (
+        header
+        + to_expr(node.target, PR_TUPLE)
+        + " in "
+        + to_expr(node.iter, PR_TEST + 1)
+    )
+    for if_ in node.ifs:
+        res += " if " + to_expr(if_, PR_TEST + 1)
+    return res
+
+
+def parens(level: int, target_lvl: int, value: str) -> str:
+    if level > target_lvl:
+        return f"({value})"
+    return value
+
+
+def _format_await(node: ast.Await, level: int) -> str:
+    return parens(level, PR_AWAIT, "await " + to_expr(node.value, PR_ATOM))
+
+
+def _format_starred(node: ast.Starred, level: int) -> str:
+    return "*" + to_expr(node.value, PR_EXPR)
+
+
+def _format_boolop(node: ast.BoolOp, level: int) -> str:
+    if isinstance(node.op, ast.And):
+        name = " and "
+        tgt_level = PR_AND
+    else:
+        name = " or "
+        tgt_level = PR_OR
+
+    return parens(
+        level, tgt_level, name.join(to_expr(n, tgt_level + 1) for n in node.values)
+    )
+
+
+def _format_arguments(node: ast.arguments) -> str:
+    res = []
+    for i, arg in enumerate(node.args):
+        if i:
+            res.append(", ")
+        res.append(arg.arg)
+        if i < len(node.defaults):
+            res.append("=")
+            res.append(to_expr(node.defaults[i]))
+
+    if node.vararg or node.kwonlyargs:
+        if node.args:
+            res.append(", ")
+        res.append("*")
+        vararg = node.vararg
+        if vararg:
+            res.append(vararg.arg)
+
+    for i, arg in enumerate(node.kwonlyargs):
+        if res:
+            res.append(", ")
+        res.append(arg.arg)
+        if i < len(node.kw_defaults) and node.kw_defaults[i]:
+            res.append("=")
+            res.append(to_expr(node.kw_defaults[i]))
+
+    return "".join(res)
+
+
+def _format_lambda(node: ast.Lambda, level: int) -> str:
+    value = "lambda "
+    if not node.args.args:
+        value = "lambda"
+    value += _format_arguments(node.args)
+    value += ": " + to_expr(node.body, PR_TEST)
+
+    return parens(level, PR_TEST, value)
+
+
+def _format_if_exp(node: ast.IfExp, level: int) -> str:
+    body = to_expr(node.body, PR_TEST + 1)
+    orelse = to_expr(node.orelse, PR_TEST)
+    test = to_expr(node.test, PR_TEST + 1)
+    return parens(level, PR_TEST, f"{body} if {test} else {orelse}")
+
+
+def _format_set(node: ast.Set, level: int) -> str:
+    return "{" + ", ".join(to_expr(elt, PR_TEST) for elt in node.elts) + "}"
+
+
+def _format_comprehensions(nodes: list[ast.comprehension]) -> str:
+    return "".join(_format_comprehension(n) for n in nodes)
+
+
+def _format_set_comp(node: ast.SetComp, level: int) -> str:
+    return "{" + to_expr(node.elt) + _format_comprehensions(node.generators) + "}"
+
+
+def _format_list_comp(node: ast.ListComp, level: int) -> str:
+    return "[" + to_expr(node.elt) + _format_comprehensions(node.generators) + "]"
+
+
+def _format_dict_comp(node: ast.DictComp, level: int) -> str:
+    return (
+        "{"
+        + to_expr(node.key)
+        + ": "
+        + to_expr(node.value)
+        + _format_comprehensions(node.generators)
+        + "}"
+    )
+
+
+def _format_gen_exp(node: ast.GeneratorExp, level: int) -> str:
+    return "(" + to_expr(node.elt) + _format_comprehensions(node.generators) + ")"
+
+
+def format_fstring_elt(res: list[str], elt: ast.expr, is_format_spec: bool) -> None:
+    if isinstance(elt, ast.Constant):
+        res.append(elt.value)
+    elif isinstance(elt, ast.JoinedStr):
+        res.append(format_joinedstr(elt, PR_TEST, is_format_spec))
+    elif isinstance(elt, ast.FormattedValue):
+        expr = to_expr(elt.value, PR_TEST + 1)
+        if expr.startswith("{"):
+            # Expression starts with a brace, we need an extra space
+            res.append("{ ")
+        else:
+            res.append("{")
+        res.append(expr)
+        conversion = elt.conversion
+        if conversion is not None and conversion != -1:
+            res.append("!")
+            res.append(chr(conversion))
+        format_spec = elt.format_spec
+        if format_spec is not None:
+            res.append(":")
+            format_fstring_elt(res, format_spec, True)
+        res.append("}")
+
+
+def format_joinedstr(
+    node: ast.JoinedStr, level: int, is_format_spec: bool = False
+) -> str:
+    res = []
+    for elt in node.values:
+        format_fstring_elt(res, elt, is_format_spec)
+
+    joined = "".join(res)
+    if is_format_spec:
+        return joined
+    return f"f{repr(joined)}"
+
+
+def _format_slice(node: ast.Slice, level: int) -> str:
+    res = ""
+    if node.lower is not None:
+        res += to_expr(node.lower)
+    res += ":"
+    if node.upper is not None:
+        res += to_expr(node.upper)
+    if node.step:
+        res += ":"
+        res += to_expr(node.step)
+    return res
+
+
+def _format_constant(node: ast.Constant, level: int) -> str:
+    if node.value is Ellipsis:
+        return "..."
+
+    return repr(node.value)
+
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", category=DeprecationWarning)
+
+    # pyre-ignore[9]: Pyre tries to union all the keys and values into concrete types.
+    _FORMATTERS: dict[type[ast.AST], Callable[[ast.AST, int], str]] = {
+        ast.Attribute: _format_attribute,
+        ast.Await: _format_await,
+        ast.BinOp: _format_binaryop,
+        ast.BoolOp: _format_boolop,
+        ast.Call: _format_call,
+        ast.Compare: _format_compare,
+        ast.Constant: _format_constant,
+        ast.Dict: _format_dict,
+        ast.DictComp: _format_dict_comp,
+        ast.FormattedValue: None,
+        ast.GeneratorExp: _format_gen_exp,
+        ast.IfExp: _format_if_exp,
+        ast.JoinedStr: format_joinedstr,
+        ast.Lambda: _format_lambda,
+        ast.List: _format_list,
+        ast.ListComp: _format_list_comp,
+        ast.Name: _format_name,
+        ast.Set: _format_set,
+        ast.SetComp: _format_set_comp,
+        ast.Slice: _format_slice,
+        ast.Starred: _format_starred,
+        ast.Subscript: _format_subscript,
+        ast.Tuple: _format_tuple,
+        ast.UnaryOp: _format_unaryop,
+        ast.Yield: _format_yield,
+        ast.YieldFrom: _format_yield_from,
+        **(
+            {
+                ast.Bytes: lambda node, level: repr(node.s),
+                ast.Ellipsis: lambda node, level: "...",
+            }
+            if sys.version_info < (3, 14)
+            else {}
+        ),
+    }
+
+
+def to_expr(node: Optional[ast.AST], level: int = PR_TEST) -> str:
+    if node is None:
+        return ""
+    formatter = _FORMATTERS.get(type(node))
+    if formatter is not None:
+        return formatter(node, level)
+
+    return "<unsupported node: " + type(node).__name__ + ">"

cinderx/compiler/visitor.py

@@ -0,0 +1,194 @@
+# Portions copyright (c) Meta Platforms, Inc. and affiliates.
+# pyre-strict
+
+from ast import AST, copy_location, iter_fields
+from typing import Sequence, TypeVar
+
+# XXX should probably rename ASTVisitor to ASTWalker
+# XXX can it be made even more generic?
+
+
+TAst = TypeVar("TAst", bound=AST)
+
+
+class ASTVisitor:
+    """Performs a depth-first walk of the AST
+
+    The ASTVisitor is responsible for walking over the tree in the
+    correct order.  For each node, it checks the visitor argument for
+    a method named 'visitNodeType' where NodeType is the name of the
+    node's class, e.g. Class.  If the method exists, it is called
+    with the node as its sole argument.
+
+    This is basically the same as the built-in ast.NodeVisitor except
+    for the following differences:
+        It accepts extra parameters through the visit methods for flowing state
+        It uses "visitNodeName" instead of "visit_NodeName"
+        It accepts a list to the generic_visit function rather than just nodes
+    """
+
+    def __init__(self) -> None:
+        self.node: AST | None = None
+        self._cache: dict[object, object] = {}
+
+    def generic_visit(self, node: TAst, *args: object) -> object:
+        """Called if no explicit visitor function exists for a node."""
+        for _field, value in iter_fields(node):
+            if isinstance(value, list):
+                for item in value:
+                    if isinstance(item, AST):
+                        self.visit(item, *args)
+            elif isinstance(value, AST):
+                self.visit(value, *args)
+
+    def visit(self, node: TAst, *args: object) -> object:
+        if not isinstance(node, AST):
+            raise TypeError(f"Expected AST node, got {node!r}")
+
+        self.node = node
+        klass = node.__class__
+        meth = self._cache.get(klass, None)
+        if meth is None:
+            className = klass.__name__
+            meth = getattr(type(self), "visit" + className, type(self).generic_visit)
+            self._cache[klass] = meth
+        if not args:
+            return meth(self, node)
+        return meth(self, node, *args)
+
+    def visit_list(self, nodes: Sequence[TAst], *args: object) -> None:
+        if not isinstance(nodes, list):
+            raise TypeError(f"Expected a list of AST nodes, got {nodes!r}")
+
+        for node in nodes:
+            self.visit(node, *args)
+
+
+class ASTRewriter(ASTVisitor):
+    """performs rewrites on the AST, rewriting parent nodes when child nodes
+    are replaced."""
+
+    @staticmethod
+    def update_node(node: TAst, **replacement: object) -> TAst:
+        res = node
+        for name, val in replacement.items():
+            existing = getattr(res, name)
+            if existing is val:
+                continue
+
+            if node is res:
+                res = ASTRewriter.clone_node(node)
+
+            setattr(res, name, val)
+        return res
+
+    @staticmethod
+    def clone_node(node: TAst) -> TAst:
+        attrs = []
+        for name in node._fields:
+            attr = getattr(node, name, None)
+            if isinstance(attr, list):
+                attr = list(attr)
+            attrs.append(attr)
+
+        new = type(node)(*attrs)
+        return copy_location(new, node)
+
+    def skip_field(self, node: TAst, field: str) -> bool:
+        return False
+
+    def generic_visit(self, node: TAst, *args: object) -> TAst:
+        ret_node = node
+        for field, old_value in iter_fields(node):
+            if self.skip_field(node, field):
+                continue
+
+            if isinstance(old_value, AST):
+                new_value = self.visit(old_value, *args)
+            elif isinstance(old_value, list):
+                new_value = self.walk_list(old_value, *args)
+            else:
+                continue
+
+            assert new_value is not None, (
+                f"can't remove AST nodes that aren't part of a list {old_value!r}"
+            )
+            if new_value is not old_value:
+                if ret_node is node:
+                    ret_node = self.clone_node(node)
+
+                setattr(ret_node, field, new_value)
+
+        return ret_node
+
+    def walk_list(self, values: Sequence[object], *args: object) -> Sequence[object]:
+        """
+        Like visit_list(), but it also walks values returned by ast.iter_fields()
+        so it has to handle non-AST node values.
+        """
+
+        new_values = []
+        changed = False
+
+        for value in values:
+            if not isinstance(value, AST):
+                new_values.append(value)
+                continue
+
+            new_value = self.visit(value, *args)
+            if new_value is not None:
+                new_values.append(new_value)
+                changed = True
+
+        # Reuse the existing list when possible.
+        return new_values if changed else values
+
+
+class ExampleASTVisitor(ASTVisitor):
+    """Prints examples of the nodes that aren't visited
+
+    This visitor-driver is only useful for development, when it's
+    helpful to develop a visitor incrementally, and get feedback on what
+    you still have to do.
+    """
+
+    VERBOSE: int = 0
+
+    examples: dict[object, object] = {}
+
+    def visit(self, node: TAst, *args: object) -> object:
+        self.node = node
+        meth = self._cache.get(node.__class__, None)
+        className = node.__class__.__name__
+        if meth is None:
+            meth = getattr(self, "visit" + className, 0)
+            self._cache[node.__class__] = meth
+        if self.VERBOSE > 1:
+            print("visit", className, meth and meth.__name__ or "")
+        if meth:
+            meth(node, *args)
+        elif self.VERBOSE > 0:
+            klass = node.__class__
+            if klass not in self.examples:
+                self.examples[klass] = klass
+                print()
+                print(self)
+                print(klass)
+                for attr in dir(node):
+                    if attr[0] != "_":
+                        print("\t", "%-12.12s" % attr, getattr(node, attr))
+                print()
+            return self.default(node, *args)
+
+    def default(self, node: TAst, *args: object) -> TAst:
+        return node
+
+
+# XXX this is an API change
+
+
+def dumpNode(node: object) -> None:
+    print(node.__class__)
+    for attr in dir(node):
+        if attr[0] != "_":
+            print("\t", "%-10.10s" % attr, getattr(node, attr))