mypy-dev 1.9.0.dev0__py3-none-any.whl

mypyc/codegen/literals.py

@@ -0,0 +1,302 @@
+from __future__ import annotations
+
+from typing import Final, FrozenSet, Tuple, Union
+from typing_extensions import TypeGuard
+
+# Supported Python literal types. All tuple / frozenset items must have supported
+# literal types as well, but we can't represent the type precisely.
+LiteralValue = Union[
+    str, bytes, int, bool, float, complex, Tuple[object, ...], FrozenSet[object], None
+]
+
+
+def _is_literal_value(obj: object) -> TypeGuard[LiteralValue]:
+    return isinstance(obj, (str, bytes, int, float, complex, tuple, frozenset, type(None)))
+
+
+# Some literals are singletons and handled specially (None, False and True)
+NUM_SINGLETONS: Final = 3
+
+
+class Literals:
+    """Collection of literal values used in a compilation group and related helpers."""
+
+    def __init__(self) -> None:
+        # Each dict maps value to literal index (0, 1, ...)
+        self.str_literals: dict[str, int] = {}
+        self.bytes_literals: dict[bytes, int] = {}
+        self.int_literals: dict[int, int] = {}
+        self.float_literals: dict[float, int] = {}
+        self.complex_literals: dict[complex, int] = {}
+        self.tuple_literals: dict[tuple[object, ...], int] = {}
+        self.frozenset_literals: dict[frozenset[object], int] = {}
+
+    def record_literal(self, value: LiteralValue) -> None:
+        """Ensure that the literal value is available in generated code."""
+        if value is None or value is True or value is False:
+            # These are special cased and always present
+            return
+        if isinstance(value, str):
+            str_literals = self.str_literals
+            if value not in str_literals:
+                str_literals[value] = len(str_literals)
+        elif isinstance(value, bytes):
+            bytes_literals = self.bytes_literals
+            if value not in bytes_literals:
+                bytes_literals[value] = len(bytes_literals)
+        elif isinstance(value, int):
+            int_literals = self.int_literals
+            if value not in int_literals:
+                int_literals[value] = len(int_literals)
+        elif isinstance(value, float):
+            float_literals = self.float_literals
+            if value not in float_literals:
+                float_literals[value] = len(float_literals)
+        elif isinstance(value, complex):
+            complex_literals = self.complex_literals
+            if value not in complex_literals:
+                complex_literals[value] = len(complex_literals)
+        elif isinstance(value, tuple):
+            tuple_literals = self.tuple_literals
+            if value not in tuple_literals:
+                for item in value:
+                    assert _is_literal_value(item)
+                    self.record_literal(item)
+                tuple_literals[value] = len(tuple_literals)
+        elif isinstance(value, frozenset):
+            frozenset_literals = self.frozenset_literals
+            if value not in frozenset_literals:
+                for item in value:
+                    assert _is_literal_value(item)
+                    self.record_literal(item)
+                frozenset_literals[value] = len(frozenset_literals)
+        else:
+            assert False, "invalid literal: %r" % value
+
+    def literal_index(self, value: LiteralValue) -> int:
+        """Return the index to the literals array for given value."""
+        # The array contains first None and booleans, followed by all str values,
+        # followed by bytes values, etc.
+        if value is None:
+            return 0
+        elif value is False:
+            return 1
+        elif value is True:
+            return 2
+        n = NUM_SINGLETONS
+        if isinstance(value, str):
+            return n + self.str_literals[value]
+        n += len(self.str_literals)
+        if isinstance(value, bytes):
+            return n + self.bytes_literals[value]
+        n += len(self.bytes_literals)
+        if isinstance(value, int):
+            return n + self.int_literals[value]
+        n += len(self.int_literals)
+        if isinstance(value, float):
+            return n + self.float_literals[value]
+        n += len(self.float_literals)
+        if isinstance(value, complex):
+            return n + self.complex_literals[value]
+        n += len(self.complex_literals)
+        if isinstance(value, tuple):
+            return n + self.tuple_literals[value]
+        n += len(self.tuple_literals)
+        if isinstance(value, frozenset):
+            return n + self.frozenset_literals[value]
+        assert False, "invalid literal: %r" % value
+
+    def num_literals(self) -> int:
+        # The first three are for None, True and False
+        return (
+            NUM_SINGLETONS
+            + len(self.str_literals)
+            + len(self.bytes_literals)
+            + len(self.int_literals)
+            + len(self.float_literals)
+            + len(self.complex_literals)
+            + len(self.tuple_literals)
+            + len(self.frozenset_literals)
+        )
+
+    # The following methods return the C encodings of literal values
+    # of different types
+
+    def encoded_str_values(self) -> list[bytes]:
+        return _encode_str_values(self.str_literals)
+
+    def encoded_int_values(self) -> list[bytes]:
+        return _encode_int_values(self.int_literals)
+
+    def encoded_bytes_values(self) -> list[bytes]:
+        return _encode_bytes_values(self.bytes_literals)
+
+    def encoded_float_values(self) -> list[str]:
+        return _encode_float_values(self.float_literals)
+
+    def encoded_complex_values(self) -> list[str]:
+        return _encode_complex_values(self.complex_literals)
+
+    def encoded_tuple_values(self) -> list[str]:
+        return self._encode_collection_values(self.tuple_literals)
+
+    def encoded_frozenset_values(self) -> list[str]:
+        return self._encode_collection_values(self.frozenset_literals)
+
+    def _encode_collection_values(
+        self, values: dict[tuple[object, ...], int] | dict[frozenset[object], int]
+    ) -> list[str]:
+        """Encode tuple/frozenset values into a C array.
+
+        The format of the result is like this:
+
+           <number of collections>
+           <length of the first collection>
+           <literal index of first item>
+           ...
+           <literal index of last item>
+           <length of the second collection>
+           ...
+        """
+        value_by_index = {index: value for value, index in values.items()}
+        result = []
+        count = len(values)
+        result.append(str(count))
+        for i in range(count):
+            value = value_by_index[i]
+            result.append(str(len(value)))
+            for item in value:
+                assert _is_literal_value(item)
+                index = self.literal_index(item)
+                result.append(str(index))
+        return result
+
+
+def _encode_str_values(values: dict[str, int]) -> list[bytes]:
+    value_by_index = {index: value for value, index in values.items()}
+    result = []
+    line: list[bytes] = []
+    line_len = 0
+    for i in range(len(values)):
+        value = value_by_index[i]
+        c_literal = format_str_literal(value)
+        c_len = len(c_literal)
+        if line_len > 0 and line_len + c_len > 70:
+            result.append(format_int(len(line)) + b"".join(line))
+            line = []
+            line_len = 0
+        line.append(c_literal)
+        line_len += c_len
+    if line:
+        result.append(format_int(len(line)) + b"".join(line))
+    result.append(b"")
+    return result
+
+
+def _encode_bytes_values(values: dict[bytes, int]) -> list[bytes]:
+    value_by_index = {index: value for value, index in values.items()}
+    result = []
+    line: list[bytes] = []
+    line_len = 0
+    for i in range(len(values)):
+        value = value_by_index[i]
+        c_init = format_int(len(value))
+        c_len = len(c_init) + len(value)
+        if line_len > 0 and line_len + c_len > 70:
+            result.append(format_int(len(line)) + b"".join(line))
+            line = []
+            line_len = 0
+        line.append(c_init + value)
+        line_len += c_len
+    if line:
+        result.append(format_int(len(line)) + b"".join(line))
+    result.append(b"")
+    return result
+
+
+def format_int(n: int) -> bytes:
+    """Format an integer using a variable-length binary encoding."""
+    if n < 128:
+        a = [n]
+    else:
+        a = []
+        while n > 0:
+            a.insert(0, n & 0x7F)
+            n >>= 7
+        for i in range(len(a) - 1):
+            # If the highest bit is set, more 7-bit digits follow
+            a[i] |= 0x80
+    return bytes(a)
+
+
+def format_str_literal(s: str) -> bytes:
+    utf8 = s.encode("utf-8")
+    return format_int(len(utf8)) + utf8
+
+
+def _encode_int_values(values: dict[int, int]) -> list[bytes]:
+    """Encode int values into C strings.
+
+    Values are stored in base 10 and separated by 0 bytes.
+    """
+    value_by_index = {index: value for value, index in values.items()}
+    result = []
+    line: list[bytes] = []
+    line_len = 0
+    for i in range(len(values)):
+        value = value_by_index[i]
+        encoded = b"%d" % value
+        if line_len > 0 and line_len + len(encoded) > 70:
+            result.append(format_int(len(line)) + b"\0".join(line))
+            line = []
+            line_len = 0
+        line.append(encoded)
+        line_len += len(encoded)
+    if line:
+        result.append(format_int(len(line)) + b"\0".join(line))
+    result.append(b"")
+    return result
+
+
+def float_to_c(x: float) -> str:
+    """Return C literal representation of a float value."""
+    s = str(x)
+    if s == "inf":
+        return "INFINITY"
+    elif s == "-inf":
+        return "-INFINITY"
+    elif s == "nan":
+        return "NAN"
+    return s
+
+
+def _encode_float_values(values: dict[float, int]) -> list[str]:
+    """Encode float values into a C array values.
+
+    The result contains the number of values followed by individual values.
+    """
+    value_by_index = {index: value for value, index in values.items()}
+    result = []
+    num = len(values)
+    result.append(str(num))
+    for i in range(num):
+        value = value_by_index[i]
+        result.append(float_to_c(value))
+    return result
+
+
+def _encode_complex_values(values: dict[complex, int]) -> list[str]:
+    """Encode float values into a C array values.
+
+    The result contains the number of values followed by pairs of doubles
+    representing complex numbers.
+    """
+    value_by_index = {index: value for value, index in values.items()}
+    result = []
+    num = len(values)
+    result.append(str(num))
+    for i in range(num):
+        value = value_by_index[i]
+        result.append(float_to_c(value.real))
+        result.append(float_to_c(value.imag))
+    return result

mypyc/common.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import sys
+import sysconfig
+from typing import Any, Dict, Final
+
+from mypy.util import unnamed_function
+
+PREFIX: Final = "CPyPy_"  # Python wrappers
+NATIVE_PREFIX: Final = "CPyDef_"  # Native functions etc.
+DUNDER_PREFIX: Final = "CPyDunder_"  # Wrappers for exposing dunder methods to the API
+REG_PREFIX: Final = "cpy_r_"  # Registers
+STATIC_PREFIX: Final = "CPyStatic_"  # Static variables (for literals etc.)
+TYPE_PREFIX: Final = "CPyType_"  # Type object struct
+MODULE_PREFIX: Final = "CPyModule_"  # Cached modules
+ATTR_PREFIX: Final = "_"  # Attributes
+
+ENV_ATTR_NAME: Final = "__mypyc_env__"
+NEXT_LABEL_ATTR_NAME: Final = "__mypyc_next_label__"
+TEMP_ATTR_NAME: Final = "__mypyc_temp__"
+LAMBDA_NAME: Final = "__mypyc_lambda__"
+PROPSET_PREFIX: Final = "__mypyc_setter__"
+SELF_NAME: Final = "__mypyc_self__"
+
+# Max short int we accept as a literal is based on 32-bit platforms,
+# so that we can just always emit the same code.
+
+TOP_LEVEL_NAME: Final = "__top_level__"  # Special function representing module top level
+
+# Maximal number of subclasses for a class to trigger fast path in isinstance() checks.
+FAST_ISINSTANCE_MAX_SUBCLASSES: Final = 2
+
+# Size of size_t, if configured.
+SIZEOF_SIZE_T_SYSCONFIG: Final = sysconfig.get_config_var("SIZEOF_SIZE_T")
+
+SIZEOF_SIZE_T: Final = (
+    int(SIZEOF_SIZE_T_SYSCONFIG)
+    if SIZEOF_SIZE_T_SYSCONFIG is not None
+    else (sys.maxsize + 1).bit_length() // 8
+)
+
+IS_32_BIT_PLATFORM: Final = int(SIZEOF_SIZE_T) == 4
+
+PLATFORM_SIZE = 4 if IS_32_BIT_PLATFORM else 8
+
+# Maximum value for a short tagged integer.
+MAX_SHORT_INT: Final = 2 ** (8 * int(SIZEOF_SIZE_T) - 2) - 1
+
+# Minimum value for a short tagged integer.
+MIN_SHORT_INT: Final = -(MAX_SHORT_INT) - 1
+
+# Maximum value for a short tagged integer represented as a C integer literal.
+#
+# Note: Assume that the compiled code uses the same bit width as mypyc
+MAX_LITERAL_SHORT_INT: Final = MAX_SHORT_INT
+MIN_LITERAL_SHORT_INT: Final = -MAX_LITERAL_SHORT_INT - 1
+
+# Description of the C type used to track the definedness of attributes and
+# the presence of argument default values that have types with overlapping
+# error values. Each tracked attribute/argument has a dedicated bit in the
+# relevant bitmap.
+BITMAP_TYPE: Final = "uint32_t"
+BITMAP_BITS: Final = 32
+
+# Runtime C library files
+RUNTIME_C_FILES: Final = [
+    "init.c",
+    "getargs.c",
+    "getargsfast.c",
+    "int_ops.c",
+    "float_ops.c",
+    "str_ops.c",
+    "bytes_ops.c",
+    "list_ops.c",
+    "dict_ops.c",
+    "set_ops.c",
+    "tuple_ops.c",
+    "exc_ops.c",
+    "misc_ops.c",
+    "generic_ops.c",
+]
+
+
+JsonDict = Dict[str, Any]
+
+
+def shared_lib_name(group_name: str) -> str:
+    """Given a group name, return the actual name of its extension module.
+
+    (This just adds a suffix to the final component.)
+    """
+    return f"{group_name}__mypyc"
+
+
+def short_name(name: str) -> str:
+    if name.startswith("builtins."):
+        return name[9:]
+    return name
+
+
+def use_vectorcall(capi_version: tuple[int, int]) -> bool:
+    # We can use vectorcalls to make calls on Python 3.8+ (PEP 590).
+    return capi_version >= (3, 8)
+
+
+def use_method_vectorcall(capi_version: tuple[int, int]) -> bool:
+    # We can use a dedicated vectorcall API to call methods on Python 3.9+.
+    return capi_version >= (3, 9)
+
+
+def get_id_from_name(name: str, fullname: str, line: int) -> str:
+    """Create a unique id for a function.
+
+    This creates an id that is unique for any given function definition, so that it can be used as
+    a dictionary key. This is usually the fullname of the function, but this is different in that
+    it handles the case where the function is named '_', in which case multiple different functions
+    could have the same name."""
+    if unnamed_function(name):
+        return f"{fullname}.{line}"
+    else:
+        return fullname
+
+
+def short_id_from_name(func_name: str, shortname: str, line: int | None) -> str:
+    if unnamed_function(func_name):
+        assert line is not None
+        partial_name = f"{shortname}.{line}"
+    else:
+        partial_name = shortname
+    return partial_name
+
+
+def bitmap_name(index: int) -> str:
+    if index == 0:
+        return "__bitmap"
+    return f"__bitmap{index + 1}"

mypyc/crash.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import sys
+import traceback
+from contextlib import contextmanager
+from typing import Iterator, NoReturn
+
+
+@contextmanager
+def catch_errors(module_path: str, line: int) -> Iterator[None]:
+    try:
+        yield
+    except Exception:
+        crash_report(module_path, line)
+
+
+def crash_report(module_path: str, line: int) -> NoReturn:
+    # Adapted from report_internal_error in mypy
+    err = sys.exc_info()[1]
+    tb = traceback.extract_stack()[:-4]
+    # Excise all the traceback from the test runner
+    for i, x in enumerate(tb):
+        if x.name == "pytest_runtest_call":
+            tb = tb[i + 1 :]
+            break
+    tb2 = traceback.extract_tb(sys.exc_info()[2])[1:]
+    print("Traceback (most recent call last):")
+    for s in traceback.format_list(tb + tb2):
+        print(s.rstrip("\n"))
+    print(f"{module_path}:{line}: {type(err).__name__}: {err}")
+    raise SystemExit(2)

mypyc/doc/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

mypyc/doc/bool_operations.rst

@@ -0,0 +1,27 @@
+.. _bool-ops:
+
+Native boolean operations
+=========================
+
+Operations on ``bool`` values that are listed here have fast,
+optimized implementations.
+
+Construction
+------------
+
+* ``True``
+* ``False``
+* ``bool(obj)``
+
+Operators
+---------
+
+* ``b1 and b2``
+* ``b1 or b2``
+* ``not b``
+
+Functions
+---------
+
+* ``any(expr for ... in ...)``
+* ``all(expr for ... in ...)``

mypyc/doc/compilation_units.rst

@@ -0,0 +1,20 @@
+.. _compilation-units:
+
+Compilation units
+=================
+
+When you run mypyc to compile a set of modules, these modules form a
+*compilation unit*. Mypyc will use early binding for references within
+the compilation unit.
+
+If you run mypyc multiple times to compile multiple sets of modules,
+each invocation will result in a new compilation unit. References
+between separate compilation units will fall back to late binding,
+i.e. looking up names using Python namespace dictionaries. Also, all
+calls will use the slower Python calling convention, where arguments
+and the return value will be boxed (and potentially unboxed again in
+the called function).
+
+For maximal performance, minimize interactions across compilation
+units. The simplest way to achieve this is to compile your entire
+program as a single compilation unit.

mypyc/doc/conf.py

@@ -0,0 +1,59 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+from __future__ import annotations
+
+import os
+import sys
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath("../.."))
+
+from mypy.version import __version__ as mypy_version
+
+# -- Project information -----------------------------------------------------
+
+project = "mypyc"
+copyright = "2020-2022, mypyc team"
+author = "mypyc team"
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = mypy_version.split("-")[0]
+# The full version, including alpha/beta/rc tags.
+release = mypy_version
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []  # type: ignore[var-annotated]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = "furo"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]

mypyc/doc/cpython-timings.md

@@ -0,0 +1,25 @@
+# Timings of CPython Operations
+
+Here are some *very rough* approximate timings of CPython interpreter
+operations:
+
+* `f(1)` (empty function body): 70-90ns
+* `f(n=1)` (empty function body): 90-110ns
+* `o.x`: 30-40ns
+* `o.f(1)` (empty method body): 80-160ns
+* `Cls(1)` (initialize attribute in `__init__`): 290-330ns
+* `x + y` (integers): 20-35ns
+* `a[i]` (list) : 20-40ns
+* `[i]` (also dealloc): 35-55ns
+* `a.append(i)` (list, average over 5 appends): 70ns
+* `d[s]` (dict, shared str key): 20ns
+* `d[s] = i` (dict, shared str key): 40ns
+* `isinstance(x, A)`: 100ns
+* `(x, y)`: 20-35ns
+* `x, y = t` (tuple expand): 10ns
+
+Note that these results are very imprecise due to many factors, but
+these should give a rough idea of the relative costs of various
+operations.
+
+Details: CPython 3.6.2, Macbook Pro 15" (Mid 2015), macOS Sierra