arborescent 0.2.0__py3-none-any.whl

arborescent/__init__.py

@@ -0,0 +1,13 @@
+"""Arborescent: one architecture policy that compiles to import-linter, tach and a type checker, and enforces more.
+
+- ``domain`` / ``facts`` / ``rules`` -- the policy, one-pass AST facts, and the pure checks (no I/O).
+- ``ports`` -- ``ToolRunner``, ``TypeChecker``, ``ModuleScanner``.
+- ``generate`` / ``engine`` / ``baseline`` / ``selftest`` / ``practices`` / ``bench`` -- the use cases.
+- ``adapters`` / ``typecheckers`` -- subprocesses, the file system + content-hash cache, pyright/basedpyright/ty/pyrefly.
+- ``testing`` -- fakes. ``cli`` -- the composition root.
+"""
+
+from .domain import Policy, PolicyError, Violation
+
+__all__ = ["Policy", "PolicyError", "Violation"]
+__version__ = "0.1.0"

arborescent/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

arborescent/adapters.py

@@ -0,0 +1,11 @@
+"""Compatibility exports for Arborescent's runtime adapters.
+
+New code should import adapters from their responsibility-specific modules.
+"""
+
+from .clock_adapter import SystemClock
+from .filesystem_adapter import LocalFileSystem
+from .scanner import FsScanner, ScanData
+from .tool_runner import SubprocessToolRunner, resolve_executable
+
+__all__ = ["FsScanner", "LocalFileSystem", "ScanData", "SubprocessToolRunner", "SystemClock", "resolve_executable"]

arborescent/baseline.py

@@ -0,0 +1,150 @@
+"""The shrink-only ratchet: a baseline file records today's violations; only NEW ones fail, and entries that no
+longer occur must be deleted (``arborescent baseline`` rewrites the file), so the debt can only go down."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+from .domain import PolicyError, Violation
+from .ports import FileSystem
+
+__all__ = ["Ratchet", "apply", "growth_consistent", "load", "record", "save"]
+
+VERSION = 2
+
+
+class BaselineError(PolicyError):
+    """A requested baseline change violates the ratchet contract."""
+
+
+@dataclass(frozen=True)
+class Ratchet:
+    new: list[Violation]  # not in the baseline: fail
+    stale: list[str]  # in the baseline but gone: fail (shrink-only)
+    tolerated: int  # in the baseline and still present
+
+    @property
+    def ok(self) -> bool:
+        return not self.new and not self.stale
+
+
+def load(fs: FileSystem, path: Path) -> set[str]:
+    violations = document(fs, path)["violations"]
+    if not isinstance(violations, list):
+        raise ValueError("baseline violations must be an array")
+    return {str(k) for k in violations}
+
+
+def document(fs: FileSystem, path: Path) -> dict[str, object]:
+    """Read a v1 or v2 baseline as a normalized document."""
+    if not fs.is_file(path):
+        return {"version": VERSION, "violations": [], "growth": []}
+    data = fs.read_json(path)
+    if not isinstance(data, dict):
+        raise ValueError("baseline must be a JSON object")
+    version = data.get("version", 1)
+    if version not in (1, VERSION):
+        raise ValueError(f"unsupported baseline version {version!r}")
+    violations = data.get("violations", [])
+    growth = data.get("growth", [])
+    if not isinstance(violations, list) or not isinstance(growth, list):
+        raise ValueError("baseline violations and growth must be arrays")
+    return {"version": VERSION, "violations": [str(key) for key in violations], "growth": growth}
+
+
+def save(fs: FileSystem, path: Path, violations: Iterable[Violation]) -> int:
+    keys = sorted({v.key for v in violations})
+    fs.write_json(path, {"version": VERSION, "violations": keys, "growth": []})
+    return len(keys)
+
+
+def record(
+    fs: FileSystem,
+    path: Path,
+    violations: Iterable[Violation],
+    *,
+    adopt: str | None = None,
+    allow_growth: bool = False,
+    reason: str | None = None,
+) -> int:
+    """Create or shrink a baseline; growth requires a one-time adoption or an explicit reason."""
+    if adopt is not None and allow_growth:
+        raise BaselineError("`--adopt` and `--allow-growth` cannot be used together")
+    if reason is not None and not allow_growth:
+        raise BaselineError("`--reason` requires `--allow-growth`")
+    if allow_growth and (reason is None or not reason.strip()):
+        raise BaselineError("`--allow-growth` requires a non-empty `--reason`")
+    current = {v.key for v in violations}
+    if not fs.is_file(path):
+        fs.write_json(path, {"version": VERSION, "violations": sorted(current), "growth": []})
+        return len(current)
+
+    old_doc = document(fs, path)
+    old_violations = old_doc["violations"]
+    existing_growth = old_doc["growth"]
+    if not isinstance(old_violations, list) or not isinstance(existing_growth, list):
+        raise ValueError("baseline violations and growth must be arrays")
+    old = {str(key) for key in old_violations}
+    growth = list(existing_growth)
+    added = current - old
+    if adopt is not None:
+        family_old = {key for key in old if _in_family(key, adopt)}
+        if family_old:
+            raise BaselineError(f"baseline already contains {adopt!r} family entries; adoption is allowed once per family")
+        other_new = {key for key in added if not _in_family(key, adopt)}
+        if other_new:
+            raise BaselineError(f"`--adopt {adopt}` cannot record findings outside that family: {', '.join(sorted(other_new))}")
+        added = current - old
+        if added:
+            growth.append({"reason": f"adopt {adopt} family", "added": sorted(added)})
+    elif added and not allow_growth:
+        raise BaselineError(f"baseline growth refused; {len(added)} new violation(s); use `--allow-growth --reason TEXT`")
+    if allow_growth and added:
+        growth.append({"reason": reason.strip() if reason is not None else "", "added": sorted(added)})
+    fs.write_json(path, {"version": VERSION, "violations": sorted(current), "growth": growth})
+    return len(current)
+
+
+def _in_family(key: str, family: str) -> bool:
+    rule = key.split("|", 1)[0]
+    if family == "coverage":
+        return rule in {"unclassified", "noise-dir-code"}
+    if family == "effects":
+        return rule.startswith(("effect-", "effects-")) or rule == "dynamic-import"
+    if family == "suppressions":
+        return rule == "suppression"
+    return rule == family or rule.startswith(f"{family}-")
+
+
+def growth_consistent(current: dict[str, object], at_ref: dict[str, object]) -> bool:
+    """Check that every newly added key is listed in growth entries appended since ``at_ref``."""
+    current_violations = current.get("violations", [])
+    ref_violations = at_ref.get("violations", [])
+    if not isinstance(current_violations, list) or not isinstance(ref_violations, list):
+        return False
+    current_keys = {str(key) for key in current_violations}
+    ref_keys = {str(key) for key in ref_violations}
+    current_log = current.get("growth", [])
+    ref_log = at_ref.get("growth", [])
+    if not isinstance(current_log, list) or not isinstance(ref_log, list) or current_log[: len(ref_log)] != ref_log:
+        return False
+    logged: set[str] = set()
+    for entry in current_log[len(ref_log) :]:
+        if not isinstance(entry, dict) or not isinstance(entry.get("reason"), str) or not entry["reason"].strip():
+            return False
+        keys = entry.get("added")
+        if not isinstance(keys, list) or not all(isinstance(key, str) for key in keys):
+            return False
+        logged.update(keys)
+    return current_keys - ref_keys == logged
+
+
+def apply(violations: Iterable[Violation], baseline: set[str], complete: bool = True) -> Ratchet:
+    """``complete`` is False for a partial (``--changed``) run, where an absent entry proves nothing."""
+    current = list(violations)
+    keys = {v.key for v in current}
+    new = [v for v in current if v.key not in baseline]
+    stale = sorted(baseline - keys) if complete else []
+    return Ratchet(new, stale, len(keys & baseline))

arborescent/bench.py

@@ -0,0 +1,93 @@
+"""``arborescent bench``: measure, don't assert. Cold and warm wall-clock for every engine and for the whole ``check``
+on codebases you prepared (each directory holds an ``arborescent.toml``). Prints a markdown table.
+
+"cold" = no Arborescent cache (and the first run of an external tool); "warm" = the immediately following run.
+External tools keep their own caches or the OS file cache; the numbers say what a developer would see, not what
+the algorithms cost.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import sys
+from collections.abc import Callable, Sequence
+from pathlib import Path
+
+from .config import load_policy
+from .engine import Options, Services, pick_checker, run_check, run_own, write_generated
+from .generate import generate
+from .ports import FileSystem
+
+__all__ = ["run_bench"]
+
+
+def _clear(fs: FileSystem, root: Path) -> None:
+    fs.remove_tree(root / ".arborescent")
+
+
+def bench_one(root: Path, names: Sequence[str], make: Callable[[Path], Services]) -> list[str]:
+    svc = make(root)
+    fs = svc.fs
+    policy, _ = load_policy(fs, root / "arborescent.toml")
+    policy = dataclasses.replace(policy, type_checkers=tuple(names))
+    _ = pick_checker(policy, svc)  # an unknown checker name is a PolicyError before anything is timed
+    rows: list[tuple[str, str, str, str]] = []  # engine, cold, warm, note
+
+    def twice(label: str, fn: Callable[[], str], reset: Callable[[], None] | None = None) -> None:
+        if reset:
+            reset()
+        t0 = svc.clock.now()
+        note = fn()
+        cold = svc.clock.now() - t0
+        t0 = svc.clock.now()
+        fn()
+        warm = svc.clock.now() - t0
+        rows.append((label, f"{cold:.2f}", f"{warm:.2f}", note))
+        print(f"  [{root.name}] {label}: cold {cold:.2f}s warm {warm:.2f}s {note}", file=sys.stderr, flush=True)
+
+    scan = svc.scanner.scan(policy, root)
+    n_files = len(scan.known)
+    _clear(fs, root)
+
+    def own() -> str:
+        found, s = run_own(policy, root, svc, practices=False)
+        return f"{len(found)} violations, {s.parsed} parsed"
+
+    twice("arborescent own checks (parse+rules)", own, lambda: _clear(fs, root))
+    chosen = [c for c in svc.checkers if c.name in names]
+    twice("generate (in memory)", lambda: f"{len(generate(policy, sorted(scan.known), chosen))} files")
+    _ = write_generated(policy, root, svc)
+
+    def tool(argv: list[str]) -> Callable[[], str]:
+        def go() -> str:
+            if not svc.runner.available(argv[0]):
+                return "not installed"
+            r = svc.runner.run(argv, root)
+            return f"rc={r.returncode}"
+
+        return go
+
+    for e in svc.engines:
+        twice(e.name, tool(e.command([])))
+    for c in chosen:
+        fs.write_text(root / c.config_file, c.render_config(policy, sorted(set(policy.sources))))
+        twice(f"type checker: {c.name}", tool(c.command([])))
+    for c in chosen[:1]:
+        fs.write_text(root / c.config_file, c.render_config(policy, sorted(set(policy.sources))))
+    first = names[0]
+
+    def whole() -> str:
+        rep = run_check(policy, root, svc, Options(type_checker=first, use_baseline=False))
+        slowest = max((e for e in rep.engines), key=lambda e: e.seconds)
+        return f"slowest engine: {slowest.name} {slowest.seconds:.2f}s; sum of engines {sum(e.seconds for e in rep.engines):.2f}s"
+
+    twice(f"WHOLE check (parallel, checker={first})", whole, lambda: _clear(fs, root))
+    out = [f"### {root.name}: {n_files} modules", "", "| engine | cold s | warm s | note |", "|---|---:|---:|---|"]
+    out += [f"| {a} | {b} | {c} | {d} |" for a, b, c, d in rows]
+    return out
+
+
+def run_bench(dirs: Sequence[Path], names: Sequence[str], make: Callable[[Path], Services]) -> int:
+    for d in dirs:
+        print("\n".join(bench_one(d.resolve(), list(names), make)) + "\n")
+    return 0

arborescent/certification.py

@@ -0,0 +1,251 @@
+"""Behavioral checks for the versioned Arborescent minimum profile."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .domain import Policy
+    from .facts import ModuleInfo
+    from .ports import FileSystem, ScanResult, ToolRunner
+
+from .domain import NOISE_DIRS, ToolResult
+
+PROFILE = "v1"
+RUFF_VERSION = "0.17.2"
+TACH_VERSION = "0.35.3"
+REQUIRED_RUFF = (
+    "BLE001", "E722", "F401", "B006", "C901", "PLR0912", "PLR0913", "PLR0915", "TID252",
+    "PLW1514", "DTZ001", "T201", "PLC0415",
+)
+_PROBES = {
+    "BLE001": "try:\n pass\nexcept Exception:\n pass\n",
+    "E722": "try:\n pass\nexcept:\n pass\n",
+    "F401": "import os\n",
+    "B006": "def f(value=[]):\n return value\n",
+    "C901": "def f(x):\n" + "".join(f" if x == {i}:\n  x += 1\n" for i in range(12)) + " return x\n",
+    "PLR0912": "def f(x):\n" + "".join(f" if x == {i}:\n  x += 1\n" for i in range(13)) + " return x\n",
+    "PLR0913": "def f(a,b,c,d,e,f):\n return a\n",
+    "PLR0915": "def f():\n" + "".join(f" x{i} = {i}\n" for i in range(60)),
+    "TID252": "from .. import sibling\n",
+    "PLW1514": "open('file.txt')\n",
+    "DTZ001": "from datetime import datetime\ndatetime(2024, 1, 1)\n",
+    "T201": "print('hello')\n",
+    "PLC0415": "def f():\n import os\n return os.name\n",
+}
+
+
+def certify(root: Path, policy: Policy, scan: ScanResult, runner: ToolRunner, fs: FileSystem) -> tuple[str, ...]:
+    """Verify exact tool versions and Ruff rules using probes beneath a configured source root."""
+    failures = _native_minimum(policy, scan)
+    failures.extend(_tool_versions(root, runner, policy.disabled_tools))
+    if "ruff" not in policy.disabled_tools and runner.available("ruff"):
+        failures.extend(_ruff_probes(root, policy.source_roots, runner, fs))
+    if "tach" not in policy.disabled_tools:
+        failures.extend(_tach_probes(root, policy, scan, runner, fs))
+    return tuple(failures)
+
+
+def _native_minimum(policy: Policy, scan: object) -> list[str]:
+    failures: list[str] = []
+    if not policy.forbid_circular_imports:
+        failures.append("minimum profile requires `forbid_circular_imports = true`")
+    failures.extend(
+        f"required profile tool {tool} is disabled by policy"
+        for tool in ("ruff", "tach")
+        if tool in policy.disabled_tools
+    )
+    outer = policy.rings[-1].name
+    for rule in policy.rings:
+        if rule.name != outer and "*" in rule.external:
+            failures.append(f"ring {rule.name} has wildcard external access")
+        if rule.name != outer and "*" in rule.may_import:
+            failures.append(f"ring {rule.name} has wildcard internal imports")
+    unclassified = getattr(scan, "unclassified", ())
+    if unclassified:
+        failures.append(f"{len(unclassified)} first-party module(s) are unclassified")
+    for rule in policy.rings[:-1]:
+        required = policy.forbidden_effects(rule.name)
+        if rule.forbid is not None and not required <= rule.forbid:
+            failures.append(f"ring {rule.name} weakens positional effect defaults")
+    return failures
+
+
+def _tool_versions(root: Path, runner: ToolRunner, disabled_tools: tuple[str, ...] = ()) -> list[str]:
+    failures: list[str] = []
+    for executable, expected in (("ruff", RUFF_VERSION), ("tach", TACH_VERSION)):
+        if executable in disabled_tools:
+            continue
+        if not runner.available(executable):
+            failures.append(f"{executable} is not installed")
+            continue
+        result = runner.run([executable, "--version"], root)
+        actual = result.stdout.strip().split()[-1] if result.returncode == 0 and result.stdout.strip() else "unknown"
+        if actual != expected:
+            failures.append(f"{executable} version {actual}; profile {PROFILE} requires {expected}")
+    return failures
+
+
+def _ruff_probes(root: Path, sources: tuple[str, ...], runner: ToolRunner, fs: FileSystem) -> list[str]:
+    failures: list[str] = []
+    source_root = next((root / source for source in sources if fs.is_dir(root / source)), root)
+    try:
+        with fs.temporary_directory("arborescent-profile-", source_root) as temp:
+            package = temp / "probe_package"
+            nested = package / "nested"
+            fs.mkdir(nested)
+            fs.write_text(package / "__init__.py", "")
+            fs.write_text(nested / "__init__.py", "")
+            filename = nested / "probe.py"
+            fs.write_text(filename, "\n")
+            for code, snippet in _PROBES.items():
+                fs.write_text(filename, snippet)
+                result = runner.run(
+                    ["ruff", "check", "--ignore-noqa", "--exit-zero", "--output-format", "json", str(filename)], root
+                )
+                if result.returncode != 0:
+                    failures.append(f"Ruff {code} probe exited with status {result.returncode}")
+                    continue
+                try:
+                    diagnostics = json.loads(result.stdout)
+                except json.JSONDecodeError:
+                    failures.append(f"Ruff {code} probe returned malformed JSON")
+                    continue
+                if not isinstance(diagnostics, list) or not any(
+                    isinstance(item, dict) and item.get("code") == code for item in diagnostics
+                ):
+                    failures.append(f"Ruff required rule {code} is not effective for {filename.relative_to(root)}")
+    except OSError as exc:
+        failures.append(f"could not create profile canaries: {exc}")
+    return failures
+
+
+def _tach_probes(root: Path, policy: Policy, scan: ScanResult, runner: ToolRunner, fs: FileSystem) -> list[str]:
+    """Prove active Tach config rejects a forbidden ring edge and undeclared external import."""
+    if not runner.available("tach"):
+        return ["Tach is not installed; profile behavior canaries cannot run"]
+    failures: list[str] = []
+    if not fs.is_file(root / "tach.toml"):
+        return ["Tach profile canaries require the active tach.toml at the project root"]
+    source_module, outer_modules = _tach_canary_modules(policy, scan)
+    if source_module is None or not outer_modules:
+        return ["Tach profile canaries need at least one classified innermost and outermost module"]
+    try:
+        with fs.temporary_directory("arborescent-tach-profile-") as temp:
+            _copy_tach_project(root, policy.source_roots, temp, fs)
+            source_path = temp / source_module.path
+            if not fs.is_file(source_path):
+                return [f"Tach canary source is unavailable in the isolated project: {source_module.path}"]
+            failures.extend(_tach_baseline(temp, runner))
+            if failures:
+                return failures
+            if not _tach_layer_probe(temp, source_module, outer_modules, runner, fs):
+                failures.append(
+                    f"Tach layer canary is not effective: {source_module.name} may import every outermost module"
+                )
+            if not _tach_external_probe(temp, source_path, runner, fs):
+                failures.append("Tach external-dependency canary is not effective for undeclared packages")
+    except (OSError, UnicodeDecodeError) as exc:
+        failures.append(f"could not create Tach profile canaries: {exc}")
+    return failures
+
+
+def _tach_baseline(temp: Path, runner: ToolRunner) -> list[str]:
+    failures: list[str] = []
+    for command, label in ((["tach", "check", "--dependencies"], "layer"), (["tach", "check-external"], "external-dependency")):
+        result = runner.run(command, temp)
+        if result.returncode != 0:
+            failures.append(f"Tach {label} baseline failed in isolated project: {_result_detail(result)}")
+    return failures
+
+
+def _tach_layer_probe(
+    temp: Path, source_module: ModuleInfo, outer_modules: tuple[str, ...], runner: ToolRunner, fs: FileSystem
+) -> bool:
+    source_path = temp / source_module.path
+    original = fs.read_bytes(source_path)
+    accepted_command = ["tach", "check", "--dependencies"]
+    for target in outer_modules:
+        _append_import(fs, source_path, original, target)
+        result = runner.run(accepted_command, temp)
+        fs.write_bytes(source_path, original)
+        if result.returncode != 0:
+            return True
+    return False
+
+
+def _tach_external_probe(temp: Path, source_path: Path, runner: ToolRunner, fs: FileSystem) -> bool:
+    original = fs.read_bytes(source_path)
+    _append_import(fs, source_path, original, "__arborescent_profile_missing_external_7f0f6c19__")
+    result = runner.run(["tach", "check-external"], temp)
+    fs.write_bytes(source_path, original)
+    return result.returncode != 0
+
+
+def _append_import(fs: FileSystem, path: Path, original: bytes, module: str) -> None:
+    source = original.decode("utf-8-sig")
+    fs.write_text(path, f"{source.rstrip()}\nimport {module}\n")
+
+
+def _tach_canary_modules(policy: Policy, scan: ScanResult) -> tuple[ModuleInfo | None, tuple[str, ...]]:
+    modules = tuple(scan.modules.values())
+    leaves = tuple(
+        module for module in modules if not any(other.name.startswith(module.name + ".") for other in modules)
+    )
+    inner = policy.rings[0].name
+    outer = policy.rings[-1].name
+    sources = sorted((module for module in leaves if policy.ring_of(module.name) == inner), key=lambda item: item.name)
+    targets = tuple(sorted(module.name for module in leaves if policy.ring_of(module.name) == outer))
+    if not sources:
+        return None, ()
+    return sources[0], tuple(target for target in targets if target != sources[0].name)
+
+
+def _copy_tach_project(root: Path, source_roots: tuple[str, ...], temp: Path, fs: FileSystem) -> None:
+    """Clone only source roots and dependency/tool config into an isolated relative-path-preserving project."""
+    roots = sorted({source.strip("/") or "." for source in source_roots}, key=lambda value: (value != ".", value.count("/")))
+    if "." in roots:
+        _copy_project_root(root, temp, fs)
+    else:
+        _copy_source_roots(root, roots, temp, fs)
+    _copy_project_configs(root, temp, fs)
+
+
+def _copy_project_root(root: Path, temp: Path, fs: FileSystem) -> None:
+    ignored = {*NOISE_DIRS, "dist", "build"}
+    for source in [*fs.glob(root, "*"), *fs.glob(root, ".*")]:
+        if source.name in ignored:
+            continue
+        target = temp / source.name
+        if fs.is_dir(source):
+            fs.copy_tree(source, target, ignore=[*ignored, "__pycache__"])
+        elif fs.is_file(source):
+            fs.write_bytes(target, fs.read_bytes(source))
+
+
+def _copy_source_roots(root: Path, roots: list[str], temp: Path, fs: FileSystem) -> None:
+    copied: list[str] = []
+    for source in roots:
+        if any(source == parent or source.startswith(parent + "/") for parent in copied):
+            continue
+        source_path = root / source
+        if fs.is_dir(source_path):
+            fs.copy_tree(source_path, temp / source, ignore=[*NOISE_DIRS, ".git", "node_modules"])
+            copied.append(source)
+
+
+def _copy_project_configs(root: Path, temp: Path, fs: FileSystem) -> None:
+    for relative in ("tach.toml", "pyproject.toml", "poetry.lock", "uv.lock", "requirements.txt", "setup.cfg"):
+        source = root / relative
+        target = temp / relative
+        if fs.is_file(source) and not fs.exists(target):
+            fs.mkdir(target.parent)
+            fs.write_bytes(target, fs.read_bytes(source))
+
+
+def _result_detail(result: ToolResult) -> str:
+    stdout = result.stdout.strip()
+    stderr = result.stderr.strip()
+    return stderr or stdout or "Tach exited unsuccessfully"