cinderx 2026.1.16.2__cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl

cinderx/compiler/__init__.py

@@ -0,0 +1,40 @@
+# Portions copyright (c) Meta Platforms, Inc. and affiliates.
+
+# pyre-strict
+
+"""Package for compiling Python source code
+
+There are several functions defined at the top level that are imported
+from modules contained in the package.
+
+compile(source, filename, mode, flags=None, dont_inherit=None)
+    Returns a code object.  A replacement for the builtin compile() function.
+
+compileFile(filename)
+    Generates a .pyc file by compiling filename.
+"""
+
+import ast
+from types import CodeType
+from typing import Any
+
+from .pycodegen import CinderCodeGenerator, compile, compile_code, compileFile
+
+
+def exec_cinder(
+    source: str | bytes | ast.Module | ast.Expression | ast.Interactive | CodeType,
+    locals: dict[str, Any],
+    globals: dict[str, Any],
+    modname: str = "<module>",
+) -> None:
+    if isinstance(source, CodeType):
+        code = source
+    else:
+        code = compile_code(
+            source, "<module>", "exec", compiler=CinderCodeGenerator, modname=modname
+        )
+
+    exec(code, locals, globals)
+
+
+__all__ = ("compile", "compile_code", "compileFile", "exec_cinder")

cinderx/compiler/__main__.py

@@ -0,0 +1,137 @@
+# Portions copyright (c) Meta Platforms, Inc. and affiliates.
+# pyre-strict
+
+import argparse
+import builtins
+import importlib.util
+import marshal
+import os
+import re
+import sys
+from dis import dis
+from types import CodeType
+from typing import Pattern, TextIO
+
+from .pycodegen import CinderCodeGenerator, CodeGenerator312, compile_code, make_header
+from .static import FIXED_MODULES, StaticCodeGenerator
+from .strict import StrictCodeGenerator
+
+try:
+    from cinder import StrictModule
+except ImportError:
+    StrictModule = None
+
+# https://www.python.org/dev/peps/pep-0263/
+coding_re: Pattern = re.compile(rb"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)")
+
+
+def open_with_coding(fname: str) -> TextIO:
+    with open(fname, "rb") as f:
+        line = f.readline()
+        m = coding_re.match(line)
+        if not m:
+            line = f.readline()
+            m = coding_re.match(line)
+        encoding = "utf-8"
+        if m:
+            encoding = m.group(1).decode()
+    return open(fname, encoding=encoding)
+
+
+def main() -> None:
+    argparser = argparse.ArgumentParser(
+        prog="compiler",
+        description="Compile/execute a Python3 source file",
+        epilog="""\
+    By default, compile source code into in-memory code object and execute it.
+    If -c is specified, instead of executing write .pyc file.
+    """,
+    )
+    argparser.add_argument(
+        "-c", action="store_true", help="compile into .pyc file instead of executing"
+    )
+    argparser.add_argument(
+        "--dis", action="store_true", help="disassemble compiled code"
+    )
+    argparser.add_argument("--output", help="path to the output .pyc file")
+    argparser.add_argument(
+        "--modname",
+        help="module name to compile as (default __main__)",
+        default="__main__",
+    )
+    argparser.add_argument("input", help="source .py file")
+    group = argparser.add_mutually_exclusive_group()
+    group.add_argument(
+        "--static", action="store_true", help="compile using static compiler"
+    )
+    group.add_argument(
+        "--builtin", action="store_true", help="compile using built-in C compiler"
+    )
+    argparser.add_argument(
+        "--opt",
+        action="store",
+        type=int,
+        default=-1,
+        help="set optimization level to compile with",
+    )
+    argparser.add_argument("--strict", action="store_true", help="run in strict module")
+    args = argparser.parse_args()
+
+    with open_with_coding(args.input) as f:
+        fileinfo = os.stat(args.input)
+        source = f.read()
+
+    if args.builtin:
+        codeobj = compile(source, args.input, "exec")
+        assert isinstance(codeobj, CodeType)
+    else:
+        if args.static and args.strict:
+            raise ValueError("Cannot specify both --static and --strict options.")
+
+        compiler = (
+            StaticCodeGenerator
+            if args.static
+            else StrictCodeGenerator
+            if args.strict
+            else CinderCodeGenerator
+        )
+
+        codeobj = compile_code(
+            source,
+            args.input,
+            "exec",
+            optimize=args.opt,
+            compiler=compiler,
+            modname=args.modname,
+        )
+
+    if args.dis:
+        dis(codeobj)
+
+    if args.c:
+        if args.output:
+            name = args.output
+        else:
+            name = args.input.rsplit(".", 1)[0] + ".pyc"
+        with open(name, "wb") as f:
+            hdr = make_header(int(fileinfo.st_mtime), fileinfo.st_size)
+            f.write(importlib.util.MAGIC_NUMBER)
+            f.write(hdr)
+            marshal.dump(codeobj, f)
+    else:
+        if args.strict and StrictModule is not None:
+            d: dict[str, object] = {"__name__": "__main__"}
+            mod = StrictModule(d, False)
+        else:
+            mod = type(sys)("__main__")
+            d = mod.__dict__
+        if args.static or args.strict:
+            d["<fixed-modules>"] = FIXED_MODULES
+        d["<builtins>"] = builtins.__dict__
+        sys.modules["__main__"] = mod
+        # don't confuse the script with args meant for us
+        sys.argv = sys.argv[:1]
+        exec(codeobj, d, d)
+
+
+main()

cinderx/compiler/config.py

@@ -0,0 +1,7 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# Compile comprehensions as using their own inner scope
+# (i.e. as lambdas).
+
+# pyre-strict
+COMPREHENSION_SCOPE = True

cinderx/compiler/consts.py

@@ -0,0 +1,72 @@
+# Portions copyright (c) Meta Platforms, Inc. and affiliates.
+
+# pyre-strict
+
+import sys
+
+# operation flags
+OP_ASSIGN = "OP_ASSIGN"
+OP_DELETE = "OP_DELETE"
+OP_APPLY = "OP_APPLY"
+
+SC_LOCAL = 1
+SC_GLOBAL_IMPLICIT = 2
+SC_GLOBAL_EXPLICIT = 3
+SC_FREE = 4
+SC_CELL = 5
+SC_TYPE_PARAM = 9
+SC_UNKNOWN = 6
+
+
+CO_OPTIMIZED = 0x0001
+CO_NEWLOCALS = 0x0002
+CO_VARARGS = 0x0004
+CO_VARKEYWORDS = 0x0008
+CO_NESTED = 0x0010
+CO_GENERATOR = 0x0020
+CO_NOFREE = 0x0040
+CO_COROUTINE = 0x0080
+CO_GENERATOR_ALLOWED = 0
+CO_ITERABLE_COROUTINE = 0x0100
+CO_ASYNC_GENERATOR = 0x0200
+CO_FUTURE_DIVISION = 0x20000
+CO_FUTURE_ABSOLUTE_IMPORT = 0x40000
+CO_FUTURE_WITH_STATEMENT = 0x80000
+CO_FUTURE_PRINT_FUNCTION = 0x100000
+CO_FUTURE_UNICODE_LITERALS = 0x200000
+CO_FUTURE_BARRY_AS_BDFL = 0x400000
+CO_FUTURE_GENERATOR_STOP = 0x800000
+CO_FUTURE_ANNOTATIONS = 0x1000000
+CO_HAS_DOCSTRING = 0x4000000
+CO_METHOD = 0x8000000
+if sys.version_info >= (3, 14):
+    CI_CO_STATICALLY_COMPILED = 0x4000
+else:
+    CI_CO_STATICALLY_COMPILED = 0x4000000
+CO_SUPPRESS_JIT = 0x40000000
+
+PyCF_MASK_OBSOLETE: int = CO_NESTED
+PyCF_SOURCE_IS_UTF8 = 0x0100
+PyCF_DONT_IMPLY_DEDENT = 0x0200
+PyCF_ONLY_AST = 0x0400
+PyCF_IGNORE_COOKIE = 0x0800
+PyCF_TYPE_COMMENTS = 0x1000
+PyCF_ALLOW_TOP_LEVEL_AWAIT = 0x2000
+PyCF_COMPILE_MASK: int = (
+    PyCF_ONLY_AST
+    | PyCF_ALLOW_TOP_LEVEL_AWAIT
+    | PyCF_TYPE_COMMENTS
+    | PyCF_DONT_IMPLY_DEDENT
+)
+
+PyCF_MASK: int = (
+    CO_FUTURE_DIVISION
+    | CO_FUTURE_ABSOLUTE_IMPORT
+    | CO_FUTURE_WITH_STATEMENT
+    | CO_FUTURE_PRINT_FUNCTION
+    | CO_FUTURE_UNICODE_LITERALS
+    | CO_FUTURE_BARRY_AS_BDFL
+    | CO_FUTURE_GENERATOR_STOP
+    | CO_FUTURE_ANNOTATIONS
+    | CI_CO_STATICALLY_COMPILED
+)

cinderx/compiler/debug.py

@@ -0,0 +1,70 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# pyre-strict
+"""Debugging output for various internal datatypes."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .opcodes import opcode
+
+if TYPE_CHECKING:
+    from .pyassem import Block, Instruction, PyFlowGraph
+
+
+def str_of_oparg(instr: Instruction) -> str:
+    if instr.target is None:
+        # This is not a block
+        return str(instr.oparg)
+    elif instr.target.label:
+        # This is a labelled block
+        return f"{instr.target.bid} ({instr.target.label})"
+    else:
+        # This is an unlabelled block
+        return str(instr.target.bid)
+
+
+def str_of_instr(instr: Instruction) -> str:
+    oparg = str_of_oparg(instr)
+    opname = instr.opname
+    return f"{instr.lineno} {opname} {oparg}"
+
+
+def str_of_block_header(block: Block) -> str:
+    return repr(block)
+
+
+def str_of_stack_effect(instr: Instruction) -> str:
+    d1 = opcode.stack_effect_raw(instr.opname, instr.oparg, False)
+    d2 = opcode.stack_effect_raw(instr.opname, instr.oparg, True)
+    if d1 != d2:
+        delta = f"{d1}/{d2}"
+    else:
+        delta = str(d1)
+    return delta
+
+
+def str_of_block_instr(
+    instr: Instruction, pc: int = 0, stack_effect: bool = False
+) -> str:
+    delta = str_of_stack_effect(instr) if stack_effect else ""
+    eh = f" EH: {instr.exc_handler.bid}" if instr.exc_handler is not None else ""
+
+    return f"{delta:>6} | {pc:3} {str_of_instr(instr)}{eh}"
+
+
+def dump_block(
+    graph: PyFlowGraph, block: Block, pc: int = 0, stack_effect: bool = False
+) -> int:
+    print(str_of_block_header(block))
+    for instr in block.getInstructions():
+        print("    ", str_of_block_instr(instr, pc, stack_effect))
+        pc += graph.instrsize(instr, instr.ioparg) * opcode.CODEUNIT_SIZE
+    return pc
+
+
+def dump_graph(graph: PyFlowGraph, stack_effect: bool = False) -> None:
+    pc = 0
+    for block in graph.getBlocks():
+        pc = dump_block(graph, block, pc, stack_effect)

cinderx/compiler/dis_stable.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python3
+# Portions copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Dissassemble code objects:
+# a) recursively (like dis.dis() in CPython behaves);
+# b) providing stable references to internal code objects (by replacing
+#    memory address with incrementing number);
+# c) besides disassembly, also dump other fields of code objects.
+# Useful for comparing disassembly outputs from different runs.
+#
+
+# pyre-strict
+
+import dis as _dis
+import re
+import sys
+from collections.abc import Generator, Iterable
+from pprint import pformat
+from re import Pattern
+from types import CodeType
+from typing import Optional, TextIO
+
+
+def _make_stable(
+    gen: Iterable[_dis.Instruction],
+) -> Generator[_dis.Instruction, None, None]:
+    for instr in gen:
+        if sys.version_info >= (3, 14):
+            yield _dis.Instruction.make(
+                instr.opname,
+                instr.arg,
+                instr.argval,
+                _stable_repr(instr.argval),
+                instr.offset,
+                instr.start_offset,
+                instr.starts_line,
+                instr.line_number,
+                instr.label,
+                instr.positions,
+                instr.cache_info,
+            )
+        else:
+            yield _dis.Instruction(
+                instr.opname,
+                instr.opcode,
+                instr.arg,
+                instr.argval,
+                _stable_repr(instr.argval),
+                instr.offset,
+                instr.starts_line,
+                instr.is_jump_target,
+            )
+
+
+def _stable_repr(obj: object) -> str:
+    if isinstance(obj, frozenset):
+        obj = frozenset(sorted(obj, key=repr))
+    return repr(obj)
+
+
+def _disassemble_bytes(
+    co: CodeType,
+    code: bytes,
+    lasti: int = -1,
+    varnames: Optional[tuple[str]] = None,
+    names: Optional[tuple[str]] = None,
+    constants: Optional[tuple[object]] = None,
+    cells: Optional[tuple[object]] = None,
+    linestarts: Optional[dict[int, int]] = None,
+    *,
+    file: Optional[TextIO] = None,
+    line_offset: int = 0,
+    localsplusnames: Optional[tuple[str]] = None,
+) -> None:
+    # Omit the line number column entirely if we have no line number info
+    show_lineno = linestarts is not None
+    if show_lineno:
+        # pyre-fixme [16]: `Optional` has no attribute `values`.
+        maxlineno = max(linestarts.values()) + line_offset
+        if maxlineno >= 1000:
+            lineno_width = len(str(maxlineno))
+        else:
+            lineno_width = 3
+    else:
+        lineno_width = 0
+    maxoffset = len(code) - 2
+    if maxoffset >= 10000:
+        offset_width = len(str(maxoffset))
+    else:
+        offset_width = 4
+    if sys.version_info >= (3, 14):
+        if co is not None:
+            exception_entries = _dis._parse_exception_table(co)
+            labels_map = _dis._make_labels_map(
+                code, exception_entries=exception_entries
+            )
+        else:
+            labels_map = None
+        instr_bytes = _dis._get_instructions_bytes(
+            code,
+            arg_resolver=_dis.ArgResolver(
+                constants, names, lambda oparg: localsplusnames[oparg], labels_map
+            ),
+            linestarts=linestarts,
+            line_offset=line_offset,
+        )
+    elif sys.version_info >= (3, 12):
+        # pyre-fixme[16]: Module `dis` has no attribute `_get_instructions_bytes`.
+        instr_bytes = _dis._get_instructions_bytes(
+            code,
+            # pyre-fixme[16]: `Optional` has no attribute `__getitem__`.
+            lambda oparg: localsplusnames[oparg],
+            names,
+            constants,
+            linestarts,
+            line_offset=line_offset,
+        )
+    else:
+        # pyre-fixme [16]: Module `dis` has no attribute `_get_instructions_bytes`
+        instr_bytes = _dis._get_instructions_bytes(
+            code, varnames, names, constants, cells, linestarts, line_offset=line_offset
+        )
+    for instr in _make_stable(instr_bytes):
+        new_source_line = (
+            show_lineno and instr.starts_line is not None and instr.offset > 0
+        )
+        if new_source_line:
+            print(file=file)
+        is_current_instr = instr.offset == lasti
+
+        if sys.version_info >= (3, 14):
+            _dis.Formatter(
+                file=file, lineno_width=lineno_width, offset_width=offset_width
+            ).print_instruction(instr, is_current_instr)
+        else:
+            print(
+                instr._disassemble(lineno_width, is_current_instr, offset_width),
+                file=file,
+            )
+
+
+def disassemble(
+    co: CodeType,
+    lasti: int = -1,
+    *,
+    file: Optional[TextIO] = None,
+    skip_line_nos: bool = False,
+) -> None:
+    cell_names = co.co_cellvars + co.co_freevars
+    if skip_line_nos:
+        linestarts = None
+    else:
+        linestarts = dict(_dis.findlinestarts(co))
+    localsplusnames = (
+        co.co_varnames
+        if sys.version_info < (3, 12)
+        else (co.co_varnames + co.co_cellvars + co.co_freevars)
+    )
+    _disassemble_bytes(
+        co,
+        co.co_code,
+        lasti,
+        co.co_varnames,
+        co.co_names,
+        co.co_consts,
+        cell_names,
+        linestarts,
+        file=file,
+        localsplusnames=localsplusnames,
+    )
+
+
+class Disassembler:
+    def __init__(self) -> None:
+        self.id_map: dict[int, int] = {}
+        self.id_cnt: int = 0
+
+    def get_co_id(self, co: CodeType) -> int:
+        addr = id(co)
+        if addr in self.id_map:
+            return self.id_map[addr]
+        self.id_map[addr] = self.id_cnt
+        self.id_cnt += 1
+        return self.id_cnt - 1
+
+    def co_repr(self, co: CodeType) -> str:
+        return '<code object %s at #%d, file "%s", line %d>' % (
+            co.co_name,
+            self.get_co_id(co),
+            co.co_filename,
+            co.co_firstlineno,
+        )
+
+    def disassemble(
+        self,
+        co: CodeType,
+        lasti: int = -1,
+        file: Optional[TextIO] = None,
+        skip_line_nos: bool = False,
+    ) -> None:
+        """Disassemble a code object."""
+        consts = tuple(
+            [self.co_repr(x) if hasattr(x, "co_code") else x for x in co.co_consts]
+        )
+        codeobj = co.replace(co_consts=consts)
+        disassemble(codeobj, file=file, skip_line_nos=skip_line_nos)
+
+    def dump_code(self, co: CodeType, file: Optional[TextIO] = None) -> None:
+        if not file:
+            file = sys.stdout
+        print(self.co_repr(co), file=file)
+        self.disassemble(co, file=file, skip_line_nos=True)
+        print("co_argcount:", co.co_argcount, file=file)
+        print("co_kwonlyargcount:", co.co_kwonlyargcount, file=file)
+        print("co_stacksize:", co.co_stacksize, file=file)
+        flags = []
+        co_flags = co.co_flags
+        for val, name in _dis.COMPILER_FLAG_NAMES.items():
+            if co_flags & val:
+                flags.append(name)
+                co_flags &= ~val
+        if co_flags:
+            flags.append(hex(co_flags))
+        print("co_flags:", hex(co.co_flags), "(" + " | ".join(flags) + ")", file=file)
+        print(
+            "co_consts:",
+            pformat(
+                tuple(
+                    [
+                        self.co_repr(x) if hasattr(x, "co_code") else _stable_repr(x)
+                        for x in co.co_consts
+                    ]
+                )
+            ),
+            file=file,
+        )
+        print("co_firstlineno:", co.co_firstlineno, file=file)
+        print("co_names:", co.co_names, file=file)
+        print("co_varnames:", co.co_varnames, file=file)
+        print("co_cellvars:", co.co_cellvars, file=file)
+        print("co_freevars:", co.co_freevars, file=file)
+        print("co_lines:", pformat(list(co.co_lines())), file=file)
+        if sys.version_info >= (3, 12):
+            print("co_positions:", file=file)
+            for i, position in enumerate(co.co_positions()):
+                print(f"Offset {i * 2}: {position}", file=file)
+            print("co_exceptiontable:", co.co_exceptiontable, file=file)
+            print("exception table: ", file=file)
+            print(
+                # pyre-fixme[16]: Module `dis` has no attribute
+                #  `_parse_exception_table`.
+                "\n".join("    " + str(x) for x in _dis._parse_exception_table(co)),
+                file=file,
+            )
+        print(file=file)
+        for c in co.co_consts:
+            if hasattr(c, "co_code"):
+                self.dump_code(c, file)
+
+
+# https://www.python.org/dev/peps/pep-0263/
+coding_re: Pattern[bytes] = re.compile(
+    rb"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)"
+)
+
+
+def open_with_coding(fname: str) -> TextIO:
+    with open(fname, "rb") as f:
+        line = f.readline()
+        m = coding_re.match(line)
+        if not m:
+            line = f.readline()
+            m = coding_re.match(line)
+        encoding = "utf-8"
+        if m:
+            encoding = m.group(1).decode()
+    return open(fname, encoding=encoding)
+
+
+if __name__ == "__main__":
+    with open_with_coding(sys.argv[1]) as f:
+        co: CodeType = compile(f.read(), sys.argv[1], "exec")
+    Disassembler().dump_code(co, file=sys.stdout)