pythonfaster 1.8.0__py3-none-any.whl

pythonfaster/hook.py

@@ -0,0 +1,307 @@
+"""Import hook: transparently load compiled extensions for project modules.
+
+The finder sits at the front of ``sys.meta_path``. For every import it:
+
+1. Lets the standard PathFinder resolve the module's *source* location.
+2. Intercepts only ``.py`` files inside the project root (never stdlib,
+   site-packages, excluded paths, or pythonfaster itself).
+3. Serves the cached compiled artifact when available; otherwise compiles
+   (cross-process locked) and serves the fresh artifact.
+4. On ANY failure: records the skip list and returns ``None`` so the
+   default loaders import the original ``.py`` -- user code never breaks.
+
+Traceback/debugging fidelity: the original source is primed into
+``linecache`` under the original filename, so tracebacks and
+``inspect.getsource`` keep working on compiled modules.
+"""
+from __future__ import annotations
+
+import importlib.abc
+import importlib.machinery
+import importlib.util
+import linecache
+import os
+import sys
+from importlib.machinery import PathFinder
+from pathlib import Path
+
+from . import cache, compiler
+from .config import SELF_PACKAGE, Config, is_excluded
+
+_VERBOSE = os.environ.get("PYTHONFASTER_VERBOSE") == "1"
+
+
+def _log(msg: str) -> None:
+    if _VERBOSE:
+        print(f"[pythonfaster] {msg}", file=sys.stderr)
+
+
+class _PythonFasterLoader(importlib.machinery.ExtensionFileLoader):
+    """ExtensionFileLoader that primes linecache after module exec.
+
+    Cython embeds the build-area .pyx path (possibly *relative* to the
+    compile-time cwd) into every code object, and the build area is removed
+    after compilation. After exec we scan the module's code objects and
+    prime linecache for the exact co_filename values found -- bulletproof
+    against any path spelling Cython chose.
+
+    Runtime fallback: if the .so crashes during module initialisation
+    (e.g. a Cython semantic mismatch that static analysis could not catch),
+    we transparently re-exec the original .py source into the same module
+    object, record the module on the skip list, and delete the broken .so.
+    The user program continues running with pure-Python semantics — zero
+    breakage, zero manual intervention.
+    """
+
+    def __init__(self, name, path, source_path, source_bytes, key=None,
+                 source_text=None):
+        super().__init__(name, path)
+        self._pb_source_path = source_path
+        self._pb_source_bytes = source_bytes
+        self._pb_key = key
+        if source_text is None:
+            source_text = source_bytes.decode("utf-8", errors="replace")
+        self._pb_entry = (
+            len(source_text),
+            None,
+            source_text.splitlines(keepends=True),
+            str(source_path),
+        )
+
+    def exec_module(self, module):
+        try:
+            super().exec_module(module)
+        except Exception as exc:
+            # --- Runtime fallback ---
+            # The .so module init crashed. Re-exec the original .py source
+            # so the import succeeds with pure-Python semantics.
+            _log(
+                f"runtime error in compiled {module.__name__}, "
+                f"falling back to .py: {exc!r}"
+            )
+            self._fallback_to_py(module, exc)
+            return
+
+        # --- linecache priming (only when .so loaded successfully) ---
+        entry = self._pb_entry
+        seen = set()
+        stack = list(vars(module).values())
+        while stack:
+            obj = stack.pop()
+            code = getattr(obj, "__code__", None)
+            if code is None:
+                if isinstance(obj, type):
+                    stack.extend(vars(obj).values())
+                continue
+            fname = code.co_filename
+            if fname in seen:
+                continue
+            seen.add(fname)
+            # Prime ONLY phantom files (the deleted build-area .pyx) that no
+            # real on-disk source backs. Always overwrite entries that contain
+            # bytes lines (Cython sometimes injects bytes source lines into
+            # linecache, which crashes the traceback formatter).
+            if not os.path.exists(fname):
+                existing = linecache.cache.get(fname)
+                if existing is None or (
+                    existing[2]
+                    and isinstance(existing[2][0], bytes)
+                ):
+                    linecache.cache[fname] = entry
+            if isinstance(obj, type):
+                stack.extend(vars(obj).values())
+
+    def _fallback_to_py(self, module, exc):
+        """Re-exec original .py source after .so runtime failure."""
+        # Record skip so future imports go straight to .py
+        if self._pb_key is not None:
+            try:
+                cache.CacheIndex().record_skip(
+                    self._pb_key, module.__name__,
+                    f"runtime fallback: {exc!r}"[:500],
+                )
+            except Exception:
+                pass
+        # Delete the broken .so from cache to prevent re-use
+        try:
+            Path(self.path).unlink(missing_ok=True)
+        except Exception:
+            pass
+        # Also delete the stored .pyx copy
+        if self._pb_key is not None:
+            try:
+                stored_pyx = cache.pyx_lookup(self._pb_key, module.__name__)
+                if stored_pyx is not None:
+                    stored_pyx.unlink(missing_ok=True)
+            except Exception:
+                pass
+        # Preserve import-machinery attributes, clear partial .so state
+        _preserve = {
+            "__name__", "__spec__", "__loader__", "__package__",
+            "__path__", "__builtins__",
+        }
+        saved = {k: v for k, v in module.__dict__.items() if k in _preserve}
+        module.__dict__.clear()
+        module.__dict__.update(saved)
+        module.__file__ = str(self._pb_source_path)
+        # Execute the original Python source
+        code = compile(
+            self._pb_source_bytes, str(self._pb_source_path), "exec",
+        )
+        exec(code, module.__dict__)
+
+
+class PythonFasterFinder(importlib.abc.MetaPathFinder):
+    def __init__(self, config: Config) -> None:
+        self.config = config
+        self.root: Path = config.root  # type: ignore[assignment]
+        self.index = cache.CacheIndex()
+
+    def _matches(self, source_path: Path) -> bool:
+        """Check if *source_path* falls under this finder's project root."""
+        try:
+            source_path.resolve().relative_to(self.root.resolve())
+            return True
+        except (ValueError, OSError):
+            return False
+
+    # -- MetaPathFinder protocol -------------------------------------------
+
+    def find_spec(self, fullname, path=None, target=None):
+        try:
+            return self._find_spec(fullname, path)
+        except Exception as exc:  # absolute last line of defense
+            _log(f"hook error for {fullname!r}, falling back: {exc!r}")
+            return None
+
+    # -- internals ----------------------------------------------------------
+
+    def _find_spec(self, fullname: str, path):
+        if fullname == SELF_PACKAGE or fullname.startswith(SELF_PACKAGE + "."):
+            return None
+
+        # Resolve the original source location via the standard machinery.
+        spec = PathFinder.find_spec(fullname, path)
+        if spec is None or not spec.origin:
+            return None
+        origin = spec.origin
+        if not origin.endswith(".py"):
+            return None
+
+        source_path = Path(origin)
+        try:
+            rel = source_path.resolve().relative_to(self.root.resolve())
+        except (ValueError, OSError):
+            return None  # outside the project root
+        if is_excluded(rel.as_posix(), self.config.exclude):
+            return None
+
+        try:
+            source_bytes = source_path.read_bytes()
+        except OSError:
+            return None
+
+        # Decode using PEP 263 encoding detection (honours
+        # # -*- coding: xxx -*- declarations and BOM).
+        try:
+            import tokenize as _tokenize
+            with _tokenize.open(str(source_path)) as fh:
+                source_text = fh.read()
+        except Exception:
+            source_text = source_bytes.decode("utf-8", errors="replace")
+
+        key = cache.cache_key(
+            source=source_bytes, module=fullname, level=self.config.level
+        )
+
+        self.index.refresh()
+        if self.index.is_skipped(key):
+            return None  # known uncompilable -> default import
+
+        so_path = cache.so_lookup(key, fullname)
+        if so_path is None:
+            try:
+                so_path = compiler.compile_module(
+                    module=fullname,
+                    source_path=source_path,
+                    key=key,
+                    level=self.config.level,
+                    aggressive=self.config.aggressive,
+                )
+                _log(f"compiled {fullname} -> {so_path.name}")
+            except compiler.CompileError as exc:
+                _log(f"compile failed for {fullname}, falling back: {exc}")
+                if exc.permanent:
+                    # Source genuinely cannot compile -> remember it.
+                    # Transient errors (lock/IO) are retried next time.
+                    self.index.record_skip(key, fullname, str(exc))
+                return None
+            except Exception as exc:
+                _log(f"unexpected error for {fullname}, falling back: {exc!r}")
+                return None
+        else:
+            _log(f"cache hit {fullname}")
+
+        return self._compiled_spec(
+            fullname, spec, so_path, source_path, source_bytes, key,
+            source_text
+        )
+
+    @staticmethod
+    def _compiled_spec(fullname, original_spec, so_path, source_path,
+                       source_bytes, key, source_text=None):
+        # Prime linecache so tracebacks and inspect.getsource() keep working.
+        # The .pyx alias must map to the *actually compiled* source (which may
+        # be transform output with shifted line numbers); the original .py
+        # path always maps to the user's own source text.
+        if source_text is None:
+            source_text = source_bytes.decode("utf-8", errors="replace")
+        lines = source_text.splitlines(keepends=True)
+        entry = (len(source_text), None, lines, str(source_path))
+        linecache.cache[str(source_path)] = entry
+
+        compiled_bytes = source_bytes
+        stored_pyx = cache.pyx_lookup(key, fullname)
+        if stored_pyx is not None:
+            try:
+                compiled_bytes = stored_pyx.read_bytes()
+            except OSError:
+                pass
+        pyx_text = compiled_bytes.decode("utf-8", errors="replace")
+        pyx_lines = pyx_text.splitlines(keepends=True)
+        pyx_path = cache.build_dir(key) / (fullname.rsplit(".", 1)[-1] + ".pyx")
+        linecache.cache[str(pyx_path)] = (
+            len(pyx_text), None, pyx_lines, str(pyx_path)
+        )
+
+        loader = _PythonFasterLoader(fullname, str(so_path), source_path,
+                                compiled_bytes, key=key, source_text=source_text)
+        kwargs = {}
+        if original_spec.submodule_search_locations is not None:
+            # Package __init__: keep the original source dir on __path__ so
+            # submodules, data files and relative imports behave identically.
+            kwargs["submodule_search_locations"] = list(
+                original_spec.submodule_search_locations
+            )
+        return importlib.util.spec_from_file_location(
+            fullname, str(so_path), loader=loader, **kwargs
+        )
+
+
+def install(config: Config) -> PythonFasterFinder:
+    """Install the finder at the front of sys.meta_path.
+
+    Multiple finders can coexist for different project roots.
+    If a finder for the same root already exists, its config is updated
+    in place (so changed level/exclude/aggressive take effect).
+    """
+    for finder in sys.meta_path:
+        if isinstance(finder, PythonFasterFinder):
+            if finder.root == config.root:
+                finder.config = config
+                _log(f"import hook updated for project root: {config.root}")
+                return finder
+    finder = PythonFasterFinder(config)
+    sys.meta_path.insert(0, finder)
+    _log(f"import hook installed for project root: {config.root}")
+    return finder

pythonfaster/transform/__init__.py

@@ -0,0 +1,10 @@
+"""AST transformation pipeline used before Cython compilation.
+
+The engine applies conservative value, object-layout, container, call-overhead
+and recursion/loop-invariant optimizations.  A transform is optional: callers
+must preserve Python semantics by falling back when an analysis cannot prove
+its preconditions.
+"""
+from .engine import TransformResult, transform_source
+
+__all__ = ["TransformResult", "transform_source"]