codeterrain 0.3.0__py3-none-any.whl

codeterrain/__init__.py

@@ -0,0 +1,3 @@
+"""codeterrain: answers about your code in under a second, from a local cache (Python first)."""
+
+__version__ = "0.3.0"

codeterrain/_guard.py

@@ -0,0 +1,243 @@
+"""Best-effort network guard for code under analysis: loopback-only sockets.
+
+Installed inside the sandboxed subprocess by ``sandbox.BOOTSTRAP`` (and by the ``sitecustomize`` the
+sandbox writes, so child interpreters inherit it). It is a guard against accidental network calls
+from the target's code, not a security boundary against hostile code: references to socket
+functions taken before ``install()`` ran, C extensions that open sockets themselves, uvloop and
+aiodns are not covered. Stdlib only; importing this module changes nothing until ``install()``.
+
+``install()`` also replaces ``_socket.socket`` (and the ``SocketType`` aliases) with a guarded
+subclass, so ``isinstance(sock, _socket.socket)`` is False for sockets made by ``socket.socket``.
+"""
+
+from __future__ import annotations
+
+import _socket
+import ipaddress
+import socket
+import sys
+import warnings
+
+_LOOPBACK_NAMES = {"localhost", "localhost.localdomain", "ip6-localhost", ""}
+_INET_FAMILIES = {socket.AF_INET, socket.AF_INET6}
+
+#: socket.socket methods that can originate traffic to an address; each is replaced.
+GUARDED_METHODS = ("connect", "connect_ex", "sendto", "sendmsg")
+#: module-level functions in socket (and _socket) that resolve or reach a name; each is replaced.
+LOOKUP_FUNCTIONS = (
+    "getaddrinfo",
+    "gethostbyname",
+    "gethostbyname_ex",
+    "gethostbyaddr",
+    "getnameinfo",
+)
+GUARDED_FUNCTIONS = (*LOOKUP_FUNCTIONS, "create_connection")
+
+
+class SandboxNetworkBlocked(OSError):
+    pass
+
+
+def _is_loopback(host):
+    if host is None:
+        return True
+    if isinstance(host, (bytes, bytearray)):
+        host = bytes(host).decode("ascii", "replace")
+    host = str(host)
+    if host.lower() in _LOOPBACK_NAMES:
+        return True
+    try:
+        addr = ipaddress.ip_address(host.split("%", 1)[0])
+    except ValueError:
+        return False
+    mapped = getattr(addr, "ipv4_mapped", None)
+    return (mapped or addr).is_loopback
+
+
+def _refuse(host, port):
+    raise SandboxNetworkBlocked(
+        f"codeterrain sandbox: non-loopback network access is blocked ({host}:{port})"
+    )
+
+
+def _check_host(host, port=""):
+    if not _is_loopback(host):
+        _refuse(host, port)
+
+
+def _check_address(family, address):
+    if family in _INET_FAMILIES and isinstance(address, tuple) and address:
+        _check_host(address[0], address[1] if len(address) > 1 else "")
+        return
+    _refuse(address, "")
+
+
+def _name_of(sockaddr):
+    return sockaddr[0] if isinstance(sockaddr, tuple) and sockaddr else sockaddr
+
+
+def _build_methods():
+    connect_o, connect_ex_o = socket.socket.connect, socket.socket.connect_ex
+    sendto_o, sendmsg_o = socket.socket.sendto, getattr(socket.socket, "sendmsg", None)
+
+    def connect(self, address):
+        _check_address(self.family, address)
+        return connect_o(self, address)
+
+    def connect_ex(self, address):
+        _check_address(self.family, address)
+        return connect_ex_o(self, address)
+
+    def sendto(self, data, *args):
+        _check_address(self.family, args[-1])
+        return sendto_o(self, data, *args)
+
+    def sendmsg(self, buffers, *args, **kwargs):
+        address = args[2] if len(args) > 2 else kwargs.get("address")
+        if address is not None:
+            _check_address(self.family, address)
+        return sendmsg_o(self, buffers, *args, **kwargs)
+
+    built = {"connect": connect, "connect_ex": connect_ex, "sendto": sendto}
+    if sendmsg_o is not None:
+        built["sendmsg"] = sendmsg
+    return built
+
+
+def _build_lookups(module):
+    """Guarded wrappers for the name-resolution functions of ``module`` (socket or _socket)."""
+    found = {name: getattr(module, name) for name in LOOKUP_FUNCTIONS if hasattr(module, name)}
+
+    def getaddrinfo(host, *args, **kwargs):
+        _check_host(host, args[0] if args else kwargs.get("port", ""))
+        return found["getaddrinfo"](host, *args, **kwargs)
+
+    def gethostbyname(host):
+        _check_host(host)
+        return found["gethostbyname"](host)
+
+    def gethostbyname_ex(host):
+        _check_host(host)
+        return found["gethostbyname_ex"](host)
+
+    def gethostbyaddr(host):
+        _check_host(host)
+        return found["gethostbyaddr"](host)
+
+    def getnameinfo(sockaddr, flags):
+        _check_host(_name_of(sockaddr))
+        return found["getnameinfo"](sockaddr, flags)
+
+    wrappers = {
+        "getaddrinfo": getaddrinfo,
+        "gethostbyname": gethostbyname,
+        "gethostbyname_ex": gethostbyname_ex,
+        "gethostbyaddr": gethostbyaddr,
+        "getnameinfo": getnameinfo,
+    }
+    return {name: wrappers[name] for name in found}
+
+
+def _build_create_connection():
+    original = socket.create_connection
+
+    def create_connection(address, *args, **kwargs):
+        _check_host(_name_of(address), address[1] if len(address) > 1 else "")
+        return original(address, *args, **kwargs)
+
+    return create_connection
+
+
+_methods = _build_methods()
+_lookups = _build_lookups(socket)
+_raw_lookups = _build_lookups(_socket)
+create_connection = _build_create_connection()
+
+connect = _methods["connect"]
+connect_ex = _methods["connect_ex"]
+sendto = _methods["sendto"]
+sendmsg = _methods.get("sendmsg")
+getaddrinfo = _lookups["getaddrinfo"]
+
+
+def _guarded_raw_socket():
+    base = _socket.socket
+    return type("socket", (base,), dict(_methods))
+
+
+def _wrap_proactor():
+    """Windows' proactor loop connects and sends through overlapped I/O, past ``socket.connect``."""
+    if sys.platform != "win32":
+        return
+    try:
+        from asyncio import windows_events
+    except ImportError:
+        return
+    proactor = windows_events.IocpProactor
+    if getattr(proactor.connect, "_codeterrain_guard", False):
+        return
+    connect_o, sendto_o = proactor.connect, proactor.sendto
+    sendto_addr_o = getattr(proactor, "sendto_addr", None)
+
+    def connect(self, conn, address):
+        _check_address(conn.family, address)
+        return connect_o(self, conn, address)
+
+    def sendto(self, conn, buf, flags=0, addr=None):
+        if addr is not None:
+            _check_address(conn.family, addr)
+        return sendto_o(self, conn, buf, flags, addr)
+
+    def sendto_addr(self, conn, buf, addr, *args, **kwargs):
+        _check_address(conn.family, addr)
+        return sendto_addr_o(self, conn, buf, addr, *args, **kwargs)
+
+    wrapped = {"connect": connect, "sendto": sendto}
+    if sendto_addr_o is not None:
+        wrapped["sendto_addr"] = sendto_addr
+    for name, fn in wrapped.items():
+        fn._codeterrain_guard = True
+        setattr(proactor, name, fn)
+
+
+def proactor_guarded():
+    """True when the asyncio proactor's connect and sendto are wrapped (always True off Windows)."""
+    if sys.platform != "win32":
+        return True
+    try:
+        from asyncio import windows_events
+    except ImportError:
+        return True
+    proactor = windows_events.IocpProactor
+    return all(
+        getattr(getattr(proactor, name), "_codeterrain_guard", False)
+        for name in ("connect", "sendto")
+    )
+
+
+def install():
+    """Replace the socket entry points. Idempotent; call once, before the target is imported."""
+    if getattr(socket.socket, "_codeterrain_guard", False):
+        return
+    for name, fn in _methods.items():
+        setattr(socket.socket, name, fn)
+    socket.socket._codeterrain_guard = True
+    for name, fn in _lookups.items():
+        setattr(socket, name, fn)
+    for name, fn in _raw_lookups.items():
+        setattr(_socket, name, fn)
+    socket.create_connection = create_connection
+    guarded = _guarded_raw_socket()
+    _socket.socket = _socket.SocketType = socket.SocketType = guarded
+    _wrap_proactor()
+
+
+def use_selector_event_loop():
+    """Prefer the selector loop on Windows; the proactor is guarded too, this is a nicety."""
+    import asyncio
+
+    policy = getattr(asyncio, "WindowsSelectorEventLoopPolicy", None)
+    if policy is not None:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            asyncio.set_event_loop_policy(policy())

codeterrain/_pytest_dump_plugin.py

@@ -0,0 +1,253 @@
+"""pytest plugin, loaded into the TARGET's pytest run: dumps each test's fixture setup order.
+
+Run as ``python -m pytest --collect-only -q -p codeterrain._pytest_dump_plugin``; it writes a
+``codeterrain.order/1`` document (docs/extractor-protocol.md) to ``$CODETERRAIN_RESULT``. Items that
+share a fixture closure share one entry; ``nodes`` maps every node id to its closure.
+
+Only stdlib and pytest are imported, so any pytest 7+ environment can host it.
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+import os
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+SCHEMA = "codeterrain.order/1"
+
+
+def closure_of(item: Any) -> tuple[list[str], dict[str, Any]]:
+    """(names_closure, name -> tuple of FixtureDef) for ``item``.
+
+    Private API, isolated here: ``item._fixtureinfo``. Public fallback when it is absent:
+    ``item.fixturenames`` plus the fixture manager's ``getfixturedefs``.
+    """
+    info = getattr(item, "_fixtureinfo", None)
+    if info is not None and hasattr(info, "names_closure") and hasattr(info, "name2fixturedefs"):
+        return list(info.names_closure), dict(info.name2fixturedefs)
+    names = list(item.fixturenames)
+    manager = item.session._fixturemanager
+    defs: dict[str, Any] = {}
+    for name in names:
+        found = manager.getfixturedefs(name, item)
+        if found:
+            defs[name] = tuple(found)
+    return names, defs
+
+
+def lookup_fixturedefs(item: Any, name: str) -> tuple[Any, ...]:
+    """The fixture definitions visible to ``item`` under ``name`` (public manager API)."""
+    manager = getattr(getattr(item, "session", None), "_fixturemanager", None)
+    if manager is None:
+        return ()
+    try:
+        found = manager.getfixturedefs(name, item)
+    except (TypeError, AttributeError):
+        found = manager.getfixturedefs(name, item.nodeid)
+    return tuple(found or ())
+
+
+def setup_order(
+    names: list[str],
+    defs: dict[str, Any],
+    resolve: Callable[[str], tuple[Any, ...]] | None = None,
+) -> list[Any]:
+    """FixtureDefs in the order pytest sets them up: each requested name in closure order,
+    dependencies first.
+
+    Overrides resolve as pytest does: a request for ``name`` takes the definition one level up for
+    every fixture of that name still being set up in the chain of requests, however many other
+    fixtures sit between (an override requesting a helper that requests the same name gets the
+    parent). A name whose fixture already finished is not resolved again.
+
+    ``resolve`` finds a name the closure does not list: pytest 8 leaves out what only an
+    overridden parent definition requests, and resolves it lazily when the parent is set up."""
+    order: list[Any] = []
+    finished: set[str] = set()
+    chain: list[str] = []
+
+    def visit(name: str) -> None:
+        if name in finished:
+            return
+        candidates = defs.get(name)
+        if not candidates and resolve is not None:
+            candidates = defs[name] = resolve(name)
+        index = -1 - chain.count(name)
+        if not candidates or -index > len(candidates):
+            return
+        fixturedef = candidates[index]
+        chain.append(name)
+        for dep in fixturedef.argnames:
+            if dep != "request":
+                visit(dep)
+        chain.pop()
+        order.append(fixturedef)
+        finished.add(name)
+
+    for name in names:
+        visit(name)
+    return order
+
+
+def _location(func: Any, root: Path) -> tuple[str, int]:
+    try:
+        code = inspect.unwrap(func).__code__
+    except (AttributeError, ValueError):
+        return "", 0
+    real = Path(os.path.realpath(code.co_filename))
+    try:
+        return real.relative_to(root).as_posix(), code.co_firstlineno
+    except ValueError:
+        return real.as_posix(), code.co_firstlineno
+
+
+def param_scopes(item: Any) -> dict[str, str]:
+    """name -> the scope pytest runs a parametrized argument at, from the item's callspec.
+
+    ``parametrize(..., indirect=True, scope="module")`` runs a function-scoped fixture at module
+    scope. ``CallSpec2._arg2scope`` is private; without it the declared scope is reported.
+    """
+    raw = getattr(getattr(item, "callspec", None), "_arg2scope", None)
+    if not isinstance(raw, dict):
+        return {}
+    scopes = {name: getattr(scope, "value", scope) for name, scope in raw.items()}
+    return {name: scope for name, scope in scopes.items() if isinstance(scope, str)}
+
+
+def _entry(fixturedef: Any, root: Path, scope: str | None = None) -> dict[str, Any]:
+    file, line = _location(fixturedef.func, root)
+    return {
+        "name": fixturedef.argname,
+        "scope": scope or fixturedef.scope,
+        "file": file,
+        "line": line,
+        "params": bool(getattr(fixturedef, "params", None)),
+    }
+
+
+def build_document(
+    items: list[Any], root: Path, errors: int, pytest_version: str
+) -> dict[str, Any]:
+    closures: list[list[dict[str, Any]]] = []
+    index: dict[str, int] = {}
+    nodes: dict[str, int] = {}
+    for item in items:
+        names, defs = closure_of(item)
+        found = setup_order(names, defs, lambda name, item=item: lookup_fixturedefs(item, name))
+        # The parameter reaches the nearest definition of a name, the last one set up.
+        scopes = param_scopes(item)
+        last = {fd.argname: i for i, fd in enumerate(found)}
+        entries = [
+            _entry(fd, root, scopes.get(fd.argname) if last[fd.argname] == i else None)
+            for i, fd in enumerate(found)
+        ]
+        key = json.dumps(entries, sort_keys=True)
+        if key not in index:
+            index[key] = len(closures)
+            closures.append(entries)
+        nodes[item.nodeid] = index[key]
+    return {
+        "schema": SCHEMA,
+        "generator": {"name": "codeterrain-pytest-dump", "pytest": pytest_version},
+        "closures": closures,
+        "nodes": nodes,
+        "errors": errors,
+    }
+
+
+#: (pytest option name, flag) pairs: options whose value makes pytest write a file when set.
+REPORT_OPTIONS = (
+    ("xmlpath", "--junitxml"),
+    ("htmlpath", "--html"),
+    ("log_file", "--log-file"),
+    ("debug", "--debug"),
+    ("report_log", "--report-log"),
+    ("json_report", "--json-report"),
+    ("allure_report_dir", "--alluredir"),
+)
+
+
+def _writes_reports(config: Any) -> list[str]:
+    """Options in force that make pytest write report files (into the target, by default)."""
+    found: list[str] = [
+        flag for option, flag in REPORT_OPTIONS if config.getoption(option, None)
+    ]
+    cov = config.getoption("cov_report", None)
+    # A tuple, not ``list | tuple``: this module runs in the target's interpreter (3.9 and up).
+    names = cov.keys() if isinstance(cov, dict) else cov if isinstance(cov, (list, tuple)) else []  # noqa: UP038
+    if {str(n).split(":", 1)[0] for n in names} - {"term", "term-missing"}:
+        found.append("--cov-report")
+    return found
+
+
+class _Namespace:
+    """``getoption`` over the early argparse namespace, before the config is fully parsed."""
+
+    def __init__(self, namespace: Any, early_config: Any = None) -> None:
+        self.namespace = namespace
+        self.early_config = early_config
+
+    def getoption(self, name: str, default: Any = None) -> Any:
+        return getattr(self.namespace, name, default)
+
+    def getini(self, name: str) -> Any:
+        return self.early_config.getini(name)
+
+
+def _ini_log_file(config: Any) -> str:
+    """The ``log_file`` ini value after every ``-o`` override; empty when it cannot be read."""
+    try:
+        return str(config.getini("log_file") or "")
+    except (AttributeError, ValueError):
+        return ""
+
+
+def _refuse_report_options(config: Any) -> None:
+    """Refuse a run whose options make pytest write files, whichever source supplied them.
+
+    codeterrain blanks ``log_file`` and switches ``addopts`` off; what is left is ``pytest_args``
+    (user config only) and, under ``respect_addopts``, the project's own options.
+    """
+    import pytest
+
+    active = _writes_reports(config)
+    if _ini_log_file(config):
+        active.append("-o log_file=...")
+    if not active:
+        return
+    if os.environ.get("CODETERRAIN_RESPECT_ADDOPTS") == "1":
+        raise pytest.UsageError(
+            f"codeterrain: respect_addopts is on but the project's options ({', '.join(active)}) "
+            "make pytest write report files into the repository; remove them for the "
+            "extraction run or set respect_addopts = false"
+        )
+    raise pytest.UsageError(
+        f"codeterrain: the extraction options ({', '.join(active)}) make pytest write files "
+        "into the repository; remove them from pytest_args"
+    )
+
+
+def pytest_load_initial_conftests(early_config: Any, parser: Any = None, args: Any = None) -> None:
+    """Refuse before pytest opens ``--debug``'s log file, which happens right after parsing."""
+    namespace = getattr(early_config, "known_args_namespace", None)
+    if namespace is not None:
+        _refuse_report_options(_Namespace(namespace, early_config))
+
+
+def pytest_configure(config: Any) -> None:
+    _refuse_report_options(config)
+
+
+def pytest_collection_finish(session: Any) -> None:
+    import pytest
+
+    target = os.environ.get("CODETERRAIN_RESULT")
+    if not target:
+        return
+    root = Path(os.path.realpath(os.environ.get("CODETERRAIN_ROOT") or session.config.rootpath))
+    document = build_document(list(session.items), root, session.testsfailed, pytest.__version__)
+    # bytes, not write_text(newline=...), which needs 3.10 in the target's interpreter
+    Path(target).write_bytes(json.dumps(document).encode("utf-8"))

codeterrain/_routes_runner.py

@@ -0,0 +1,119 @@
+"""Generic app importer, run INSIDE the target interpreter (``-m codeterrain._routes_runner``).
+
+Imports each ``module:attr`` application, asks the pluggy ``iter_routes`` hook for its routes and
+writes a ``codeterrain.routes/1`` document (docs/extractor-protocol.md) to ``$CODETERRAIN_RESULT``.
+One app failing to import is recorded in ``errors`` and does not stop the others.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import importlib
+import json
+import os
+import sys
+import traceback
+from pathlib import Path
+from typing import Any
+
+import pluggy
+
+from codeterrain import hookspecs
+from codeterrain.plugins import fastapi_routes
+
+SCHEMA = "codeterrain.routes/1"
+
+
+def make_plugin_manager(extra: list[str]) -> pluggy.PluginManager:
+    manager = pluggy.PluginManager(hookspecs.PROJECT)
+    manager.add_hookspecs(hookspecs)
+    manager.register(fastapi_routes)
+    for module in extra:
+        manager.register(importlib.import_module(module))
+    return manager
+
+
+def relative_file(file: str, root: Path) -> str:
+    """Repo-relative POSIX path with symlinks and junctions resolved; absolute when outside."""
+    if not file:
+        return ""
+    real = Path(os.path.realpath(file))
+    try:
+        return real.relative_to(root).as_posix()
+    except ValueError:
+        return real.as_posix()
+
+
+def exception_chain(exc: BaseException | None) -> list[str]:
+    """``Type: message`` for the exception and each cause behind it, outermost first."""
+    chain: list[str] = []
+    while exc is not None and len(chain) < 12:
+        chain.append(f"{type(exc).__name__}: {exc}")
+        exc = exc.__cause__ or exc.__context__
+    return chain
+
+
+def collect(
+    apps: list[str], plugins: list[str], root: Path, moto_mock: bool
+) -> dict[str, Any]:
+    routes: list[dict[str, Any]] = []
+    errors: list[dict[str, str]] = []
+    notes: list[str] = []
+    sys.path.insert(0, str(root))
+    with contextlib.ExitStack() as stack:
+        if moto_mock:
+            try:
+                from moto import mock_aws
+            except ImportError:
+                notes.append("moto_mock is set but moto is not importable in the target")
+            else:
+                stack.enter_context(mock_aws())
+                notes.append("imports ran inside moto.mock_aws()")
+        manager = make_plugin_manager(plugins)
+        for spec in apps:
+            module_name, _, attr = spec.partition(":")
+            fastapi_routes.forget_host_routes()
+            try:
+                app = getattr(importlib.import_module(module_name), attr)
+                found = manager.hook.iter_routes(app=app)
+                if found is None:
+                    raise TypeError(f"no plugin recognises {type(app).__name__}")
+            except Exception as exc:
+                tail = traceback.format_exc().strip().splitlines()
+                errors.append(
+                    {"app": spec, "error": "\n".join(tail[-30:]), "chain": exception_chain(exc)}
+                )
+                continue
+            notes.extend(
+                f"warning: {spec}: Host route {host!r} is not walked, its routes are missing"
+                for host in fastapi_routes.host_routes_seen()
+            )
+            for record in found:
+                routes.append(
+                    {**record, "app": spec, "file": relative_file(record["file"], root)}
+                )
+    return {
+        "schema": SCHEMA,
+        "generator": {"name": "codeterrain-routes-runner", "python": sys.version.split()[0]},
+        "routes": routes,
+        "errors": errors,
+        "notes": notes,
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="codeterrain._routes_runner")
+    parser.add_argument("--apps", nargs="*", default=[])
+    parser.add_argument("--plugins", nargs="*", default=[])
+    parser.add_argument("--moto-mock", action="store_true")
+    args = parser.parse_args(argv)
+    root = Path(os.path.realpath(os.environ.get("CODETERRAIN_ROOT") or os.getcwd()))
+    document = collect(args.apps, args.plugins, root, args.moto_mock)
+    # bytes, not write_text(newline=...), which needs 3.10 in the target's interpreter
+    Path(os.environ["CODETERRAIN_RESULT"]).write_bytes(json.dumps(document).encode("utf-8"))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())