pythonfaster 1.8.0__py3-none-any.whl

pythonfaster/compiler.py

@@ -0,0 +1,425 @@
+"""Compilation pipeline: Python source -> Cython -> C -> shared object.
+
+The original source is first checked for known CPython/Cython semantic
+mismatches.  At compilation levels 2 and 3, the AST transform pipeline is
+attempted before Cython builds the extension.  If transformation or the
+transformed build fails, the pipeline retries with a verbatim ``.py`` copy;
+all failures ultimately return control to the standard Python importer.
+
+Intermediate files (.pyx/.c) live in the cache build area outside the
+project and are deleted after a successful build, keeping only the .so.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import time
+from pathlib import Path
+
+from . import cache
+
+_VERBOSE = os.environ.get("PYTHONFASTER_VERBOSE") == "1"
+_KEEP_INTERMEDIATE = os.environ.get("PYTHONFASTER_KEEP_INTERMEDIATE") == "1"
+
+
+class CompileError(RuntimeError):
+    """Raised when a module cannot be compiled. Caller falls back.
+
+    ``permanent`` marks failures that will reproduce for the same source
+    (cythonize / C build errors) -- only these belong on the skip list.
+    Transient failures (lock timeout, disk IO) must not be persisted.
+    """
+
+    def __init__(self, message: str, *, permanent: bool = True) -> None:
+        super().__init__(message)
+        self.permanent = permanent
+
+
+def _directives(level: int) -> dict:
+    # Global directives deliberately remain conservative.  Transform-specific
+    # directives are injected only when their local preconditions are proven.
+    d = {"language_level": "3"}
+    # Ensure source lines in tracebacks are str, not bytes
+    d["c_string_type"] = "unicode"
+    d["c_string_encoding"] = "utf-8"
+    return d
+
+
+# ---------------------------------------------------------------------------
+# Cython compatibility pre-check
+# ---------------------------------------------------------------------------
+
+def _cython_compat_check(source: str) -> str | None:
+    """Scan *source* for patterns known to cause runtime behaviour
+    differences between CPython and Cython-compiled code.
+
+    Returns a human-readable reason string when an incompatible pattern
+    is found, or ``None`` when the source appears safe to compile.
+
+    This check runs on the **original** user source, before any AST
+    transforms, and applies to *all* compilation levels (plain Cython
+    and transformed).  When a pattern is detected the caller should
+    skip Cython compilation entirely and fall back to the ``.py``
+    import, because the issue is a Cython limitation — not a transform
+    artefact.
+    """
+    import ast as _ast
+
+    try:
+        tree = _ast.parse(source)
+    except SyntaxError:
+        return None  # let Cython handle syntax issues
+
+    # Collect names of ALL class definitions (including nested/local classes)
+    # — only ``del ClassName.method`` where ClassName refers to a class is
+    # incompatible.  ``del self.attr`` or ``del local_var.attr`` are fine.
+    class_names: set[str] = set()
+    for node in _ast.walk(tree):
+        if isinstance(node, _ast.ClassDef):
+            class_names.add(node.name)
+
+    # --- Pattern 1: ``del ClassName.method`` ---
+    # Cython does not support runtime deletion of class-level attributes
+    # / methods.  In CPython this raises TypeError on the next access;
+    # in Cython it raises AttributeError instead.
+    for node in _ast.walk(tree):
+        if isinstance(node, _ast.Delete):
+            for target in node.targets:
+                if isinstance(target, _ast.Attribute) \
+                        and isinstance(target.value, _ast.Name) \
+                        and target.value.id in class_names \
+                        and target.value.id != "self":
+                    return (f"del {target.value.id}.{target.attr} — "
+                            "Cython does not support runtime method deletion")
+
+    # --- Pattern 2: dict mutation during iteration ---
+    # CPython raises RuntimeError when a dict changes size during
+    # iteration; Cython does not detect this and silently continues.
+    for node in _ast.walk(tree):
+        if not isinstance(node, _ast.For):
+            continue
+        if not isinstance(node.iter, _ast.Name):
+            continue
+        iter_name = node.iter.id
+        for child in _ast.walk(node):
+            if isinstance(child, _ast.Delete):
+                for target in child.targets:
+                    if isinstance(target, _ast.Subscript) \
+                            and isinstance(target.value, _ast.Name) \
+                            and target.value.id == iter_name:
+                        return (f"dict mutation during iteration — "
+                                "Cython does not detect dict size changes "
+                                "in for-loops")
+
+    # --- Pattern 3: load_tests + DocFileSuite ---
+    # DocFileSuite resolves file paths relative to the module's __file__,
+    # which changes from .py to .so/.pyx after compilation.
+    has_load_tests = False
+    has_docfilesuite = False
+    for node in _ast.walk(tree):
+        if isinstance(node, _ast.FunctionDef) and node.name == "load_tests":
+            has_load_tests = True
+        if isinstance(node, _ast.Call):
+            fname = None
+            if isinstance(node.func, _ast.Attribute):
+                fname = node.func.attr
+            elif isinstance(node.func, _ast.Name):
+                fname = node.func.id
+            if fname == "DocFileSuite":
+                has_docfilesuite = True
+    if has_load_tests and has_docfilesuite:
+        return ("load_tests + DocFileSuite — file path resolution "
+                "differs between .py and .so")
+
+    # ============================================================
+    # Patterns 4-10: silent-wrong-result regressions measured on
+    # the CPython official test suite (3.2.4 AND 3.2.9 — Cython
+    # does not fix these).  Each rule below maps to a concrete
+    # badcase; blocking here trades acceleration for correctness:
+    # the module falls back to the plain .py import.
+    #
+    # All 10 patterns are enabled by default (strict mode is the
+    # default since v1.8.1: compatibility wins over coverage).
+    # Set PYTHONFASTER_STRICT=0 to fall back to the lenient profile
+    # (patterns 4/7/9/10 disabled) if maximum acceleration is needed.
+    # ============================================================
+    strict = os.environ.get("PYTHONFASTER_STRICT", "1") != "0"
+
+    # --- Pattern 4: ``yield from`` (PEP 380 generator delegation) ---
+    # Cython-compiled generators propagate close()/throw() through
+    # the delegation chain differently (test_yield_from: the
+    # delegated exception never reaches the catcher).
+    if strict:
+        for node in _ast.walk(tree):
+            if isinstance(node, _ast.YieldFrom):
+                return ("yield from — Cython generator delegation changes "
+                        "close()/throw() semantics")
+
+    # --- Pattern 5: ``import cmath`` ---
+    # cmath.phase() loses the sign of negative-zero arguments under
+    # Cython numeric conversion (test_cmath: phase(complex(0,-0.0))
+    # returns 0.0 instead of pi).
+    imported = set()
+    for node in _ast.walk(tree):
+        if isinstance(node, _ast.Import):
+            imported.update(a.name for a in node.names)
+        elif isinstance(node, _ast.ImportFrom) and node.module:
+            imported.add(node.module)
+    if "cmath" in imported:
+        return ("import cmath — negative-zero sign lost in Cython "
+                "complex conversion (cmath.phase wrong)")
+
+    # --- Pattern 6: operator.countOf / operator.indexOf ---
+    # Cython replaces these with C implementations whose semantics
+    # differ (test_operator: countOf returns 0 instead of 2).
+    for node in _ast.walk(tree):
+        if isinstance(node, _ast.Attribute) and node.attr in (
+                "countOf", "indexOf"):
+            value = node.value
+            if isinstance(value, _ast.Name) and value.id == "operator":
+                return (f"operator.{node.attr} — Cython C implementation "
+                        "returns wrong results")
+        if isinstance(node, _ast.ImportFrom):
+            if node.module == "operator" and any(
+                    a.name in ("countOf", "indexOf") for a in node.names):
+                return ("operator.countOf/indexOf — Cython C "
+                        "implementation returns wrong results")
+
+    # --- Pattern 7: Fraction subclass / fractions import ---
+    # Mixed-arithmetic type dispatch is reordered under Cython
+    # (test_fractions: 10.0 != 9.0, DummyFloat isinstance fails,
+    # format() raises SystemError).
+    if strict:
+        for node in _ast.walk(tree):
+            if isinstance(node, _ast.ClassDef):
+                for base in node.bases:
+                    if isinstance(base, _ast.Name) and base.id == "Fraction":
+                        return ("class Fraction subclass — Cython numeric "
+                                "protocol dispatch differs from CPython")
+                    if isinstance(base, _ast.Attribute) and base.attr == "Fraction":
+                        return ("class Fraction subclass — Cython numeric "
+                                "protocol dispatch differs from CPython")
+        if "fractions" in imported:
+            return ("import fractions — Cython numeric protocol dispatch "
+                    "differs from CPython")
+
+    # --- Pattern 8: ``__class_getitem__`` definition ---
+    # Metaclass __class_getitem__ dispatch breaks under Cython class
+    # restructuring (test_genericclass: TypeError 'NoneType' object
+    # is not callable).
+    for node in _ast.walk(tree):
+        if isinstance(node, _ast.FunctionDef) \
+                and node.name == "__class_getitem__":
+            return ("__class_getitem__ — Cython class restructuring "
+                    "breaks metaclass dispatch")
+
+    # --- Pattern 9: ``__del__`` finalizer methods ---
+    # Cython cdef-style classes cannot accept runtime attribute
+    # assignment during finalization (test_finalization: cannot set
+    # the '_cleaning' attribute of immutable type).
+    if strict:
+        for node in _ast.walk(tree):
+            if isinstance(node, _ast.FunctionDef) and node.name == "__del__":
+                return ("__del__ — Cython immutable classes reject runtime "
+                        "attribute assignment in finalizers")
+
+    # --- Pattern 10: ``__file__``-relative file access ---
+    # Compiled modules get a .so __file__; relative data files
+    # (certs, fixtures, source lookups) resolve differently
+    # (test_baseexception/test_urllib2_localnet/test_zipimport_support:
+    # FileNotFoundError).
+    if strict:
+        uses_dunder_file = any(
+            isinstance(n, _ast.Name) and n.id == "__file__"
+            for n in _ast.walk(tree)
+        )
+        opens_files = any(
+            isinstance(n, _ast.Call) and isinstance(n.func, _ast.Name)
+            and n.func.id in ("open", "load")
+            for n in _ast.walk(tree)
+        )
+        if uses_dunder_file and opens_files:
+            return ("__file__ + open() — data-file resolution differs "
+                    "between .py and .so")
+
+    return None
+
+
+def _read_source(path: Path) -> str:
+    """Read a Python source file with PEP 263 encoding detection.
+
+    Uses :func:`tokenize.open` which honours ``# -*- coding: xxx -*-``
+    declarations and BOM markers.  Falls back to UTF-8 on any error.
+    """
+    try:
+        import tokenize as _tokenize
+        with _tokenize.open(str(path)) as fh:
+            return fh.read()
+    except Exception:
+        return path.read_text(encoding="utf-8", errors="replace")
+
+
+def compile_module(
+    *,
+    module: str,
+    source_path: Path,
+    key: str,
+    level: int,
+    aggressive: bool = False,
+) -> Path:
+    """Compile *module* and return the cached .so path.
+
+    Raises :class:`CompileError` on any failure (caller records the skip
+    and falls back to the normal .py import).
+    """
+    target = cache.lib_dir(key) / cache.so_rel_path(module)
+    if target.exists():
+        return target
+
+    # --- Cython compatibility pre-check ---
+    # Scan the original source for patterns known to cause runtime
+    # behaviour differences between CPython and Cython.  If found,
+    # skip compilation entirely and fall back to the .py import.
+    # This protects against silent semantic mismatches that the
+    # compile-time degradation ladder cannot detect.
+    source_text = _read_source(source_path)
+    compat_reason = _cython_compat_check(source_text)
+    if compat_reason is not None:
+        if _VERBOSE:
+            print(f"[pythonfaster] skipping {module}: {compat_reason}")
+        raise CompileError(
+            f"cython compatibility check failed: {compat_reason}",
+            permanent=True,
+        )
+
+    bdir = cache.build_dir(key)
+    lock = cache.FileLock(bdir)
+    try:
+        with lock:
+            if target.exists():  # another process won the race
+                return target
+            started = time.monotonic()
+            bdir.mkdir(parents=True, exist_ok=True)
+            applied: list[str] = []
+            try:
+                pyx_path, used_transform, applied = _prepare_pyx(
+                    module, source_path, bdir, level, aggressive
+                )
+                try:
+                    _cython_build(
+                        module=module,
+                        pyx_path=pyx_path,
+                        out_dir=cache.lib_dir(key),
+                        build_tmp=bdir / "tmp",
+                        level=level,
+                    )
+                except CompileError:
+                    if not used_transform:
+                        raise
+                    # degradation ladder ②: strip injections, compile plain
+                    applied = []
+                    pyx_path = bdir / (module.rsplit(".", 1)[-1] + ".pyx")
+                    shutil.copyfile(source_path, pyx_path)
+                    _cython_build(
+                        module=module,
+                        pyx_path=pyx_path,
+                        out_dir=cache.lib_dir(key),
+                        build_tmp=bdir / "tmp",
+                        level=level,
+                    )
+                # keep the exact compiled source for traceback line fidelity
+                final_pyx = cache.lib_dir(key) / cache.pyx_rel_path(module)
+                final_pyx.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copyfile(pyx_path, final_pyx)
+            finally:
+                if not _KEEP_INTERMEDIATE:
+                    shutil.rmtree(bdir, ignore_errors=True)
+            if not target.exists():
+                raise CompileError(f"build finished but artifact missing: {target}")
+            cache.CacheIndex().record_success(
+                key, module, time.monotonic() - started, strategies=applied
+            )
+            return target
+    except CompileError:
+        raise
+    except TimeoutError as exc:
+        raise CompileError(str(exc), permanent=False) from exc
+    except Exception as exc:  # cython/distutils raise many types
+        raise CompileError(f"{type(exc).__name__}: {exc}") from exc
+
+
+def _prepare_pyx(module: str, source_path: Path, bdir: Path,
+                  level: int, aggressive: bool = False):
+    """Produce the .pyx for *module* inside the build dir.
+
+    Runs the AST transform engine in memory (level >= 2) and writes the
+    transformed source; falls back to a verbatim copy when the transform
+    declines or fails. Returns (pyx_path, used_transform, applied_strategies).
+    The user project is never touched either way.
+    """
+    pyx_path = bdir / (module.rsplit(".", 1)[-1] + ".pyx")
+    if level >= 2:
+        try:
+            from .transform import transform_source
+
+            result = transform_source(
+                _read_source(source_path), level=level,
+                aggressive=aggressive,
+            )
+            if result.changed:
+                pyx_path.write_text(result.source, encoding="utf-8")
+                if _VERBOSE:
+                    print(f"[pythonfaster] transform {module}: "
+                          f"{', '.join(result.applied)}")
+                return pyx_path, True, list(result.applied)
+        except Exception as exc:
+            if _VERBOSE:
+                print(f"[pythonfaster] transform failed for {module}, "
+                      f"plain compile: {exc!r}")
+    shutil.copyfile(source_path, pyx_path)
+    return pyx_path, False, []
+
+
+def _cython_build(
+    *,
+    module: str,
+    pyx_path: Path,
+    out_dir: Path,
+    build_tmp: Path,
+    level: int,
+) -> None:
+    from Cython.Build import cythonize
+    from setuptools import Distribution, Extension
+    from setuptools.command.build_ext import build_ext
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+    build_tmp.mkdir(parents=True, exist_ok=True)
+
+    if not _VERBOSE:
+        from distutils import log as distutils_log
+
+        distutils_log.set_threshold(distutils_log.ERROR)
+
+    extension = Extension(
+        module,
+        sources=[str(pyx_path)],
+        extra_compile_args=list(cache.COMPILE_FLAGS),
+    )
+    try:
+        extensions = cythonize(
+            [extension],
+            compiler_directives=_directives(level),
+            quiet=not _VERBOSE,
+        )
+    except Exception as exc:
+        raise CompileError(f"cythonize failed: {exc}") from exc
+
+    distribution = Distribution({"ext_modules": extensions})
+    command = build_ext(distribution)
+    command.ensure_finalized()
+    command.build_lib = str(out_dir)
+    command.build_temp = str(build_tmp)
+    try:
+        command.run()
+    except Exception as exc:
+        raise CompileError(f"C build failed: {exc}") from exc

pythonfaster/config.py

@@ -0,0 +1,165 @@
+"""Project root detection and pythonfaster configuration loading.
+
+Configuration sources (later overrides earlier):
+1. ``pythonfaster.toml`` (``[pythonfaster]`` table) or ``pyproject.toml`` (``[tool.pythonfaster]``)
+   at the project root.
+2. Environment variables: ``PYTHONFASTER_DISABLE``, ``PYTHONFASTER_LEVEL``,
+   ``PYTHONFASTER_VERBOSE``, ``PYTHONFASTER_CACHE_DIR``.
+
+Global activation state lives in ``~/.config/pythonfaster/config.json``.
+When the file is absent (fresh install), pythonfaster defaults to **enabled**.
+Use ``pythonfaster enable`` to also install the ``.pth`` file that triggers
+auto-activation on every Python startup.
+"""
+from __future__ import annotations
+
+import fnmatch
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+ROOT_MARKERS = ("pythonfaster.toml", "pyproject.toml", ".git")
+
+#: Modules matching any of these (relative to project root) are never compiled.
+DEFAULT_EXCLUDES = (
+    "tests/*", "test/*", "*/tests/*", "*/test/*",
+    "conftest.py", "setup.py",
+    ".venv/*", "venv/*", "*/.venv/*", "*/venv/*",
+    "*/site-packages/*", "*/node_modules/*",
+)
+
+#: pythonfaster must never compile itself (it may live inside the project root
+#: when running from a source checkout).
+SELF_PACKAGE = "pythonfaster"
+
+
+@dataclass
+class Config:
+    """Effective configuration for one project."""
+
+    enabled: bool = True
+    level: int = 2
+    exclude: list[str] = field(default_factory=lambda: list(DEFAULT_EXCLUDES))
+    root: Path | None = None
+    #: High-risk template rewrites (S2 fannkuch, C2 nbody flat-buffer)
+    #: are disabled by default.  Enable via env PYTHONFASTER_AGGRESSIVE=1
+    #: or [pythonfaster] aggressive = true in config.
+    aggressive: bool = False
+
+
+# ---------------------------------------------------------------------------
+# project root detection
+# ---------------------------------------------------------------------------
+
+def find_project_root(start: Path) -> Path | None:
+    """Walk upwards from *start* looking for a project marker."""
+    p = start.resolve()
+    if p.is_file():
+        p = p.parent
+    for directory in (p, *p.parents):
+        for marker in ROOT_MARKERS:
+            if (directory / marker).exists():
+                return directory
+    return None
+
+
+# ---------------------------------------------------------------------------
+# per-project config
+# ---------------------------------------------------------------------------
+
+def _read_toml(path: Path) -> dict:
+    try:
+        import tomllib  # Python 3.11+
+    except ImportError:  # pragma: no cover - Python 3.9/3.10
+        try:
+            import tomli as tomllib
+        except ImportError:
+            return {}
+    try:
+        with open(path, "rb") as fh:
+            return tomllib.load(fh)
+    except Exception:
+        return {}
+
+
+def load_config(root: Path) -> Config:
+    """Load the effective config for the project at *root*."""
+    cfg = Config(root=root)
+
+    data = _read_toml(root / "pythonfaster.toml").get("pythonfaster", {})
+    if not data:
+        data = (
+            _read_toml(root / "pyproject.toml")
+            .get("tool", {})
+            .get("pythonfaster", {})
+        )
+
+    if isinstance(data.get("enable"), bool):
+        cfg.enabled = data["enable"]
+    if isinstance(data.get("level"), int) and data["level"] in (1, 2, 3):
+        cfg.level = data["level"]
+    if isinstance(data.get("exclude"), list):
+        cfg.exclude = list(DEFAULT_EXCLUDES) + [
+            str(p) for p in data["exclude"] if isinstance(p, str)
+        ]
+    if isinstance(data.get("aggressive"), bool):
+        cfg.aggressive = data["aggressive"]
+
+    # environment overrides
+    if os.environ.get("PYTHONFASTER_DISABLE") == "1":
+        cfg.enabled = False
+    level_env = os.environ.get("PYTHONFASTER_LEVEL")
+    if level_env in ("1", "2", "3"):
+        cfg.level = int(level_env)
+    if os.environ.get("PYTHONFASTER_AGGRESSIVE") == "1":
+        cfg.aggressive = True
+
+    return cfg
+
+
+def is_excluded(rel_posix_path: str, patterns: list[str] | tuple[str, ...]) -> bool:
+    """Return True if *rel_posix_path* matches any exclude pattern."""
+    for pattern in patterns:
+        if fnmatch.fnmatch(rel_posix_path, pattern):
+            return True
+    return False
+
+
+# ---------------------------------------------------------------------------
+# global activation state (written by `pythonfaster enable`)
+# ---------------------------------------------------------------------------
+
+def state_dir() -> Path:
+    xdg = os.environ.get("XDG_CONFIG_HOME")
+    base = Path(xdg) if xdg else Path.home() / ".config"
+    return base / "pythonfaster"
+
+
+def state_path() -> Path:
+    return state_dir() / "config.json"
+
+
+def load_state() -> dict:
+    """Load global activation state.
+
+    Defaults to **enabled** when no state file exists yet (fresh install).
+    ``pythonfaster disable`` writes ``{"enabled": false}`` to explicitly turn off.
+    """
+    try:
+        with open(state_path(), "r", encoding="utf-8") as fh:
+            state = json.load(fh)
+        if isinstance(state, dict):
+            return state
+    except Exception:
+        pass
+    return {"enabled": True, "mode": "all"}
+
+
+def save_state(state: dict) -> None:
+    d = state_dir()
+    d.mkdir(parents=True, exist_ok=True)
+    tmp = d / "config.json.tmp"
+    with open(tmp, "w", encoding="utf-8") as fh:
+        json.dump(state, fh, indent=2)
+    os.replace(tmp, state_path())