graphite-code 0.3.0__py3-none-any.whl

graphite/freshness.py

@@ -0,0 +1,127 @@
+"""Graph artifact freshness checks shared by CLI and diagnostics."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from .config import Config
+from .engine_identity import EngineIdentityError, engine_identity
+from .ingest import collect_files
+
+_DOCTOR_FILE_LIMIT = 10_000
+
+
+class FreshnessLimitError(RuntimeError):
+    """A stable, path-free freshness resource-limit failure."""
+
+
+def _manifest_size(path: Path) -> int:
+    return path.stat().st_size
+
+
+def _read_manifest_bounded(path: Path, limit: int) -> str:
+    try:
+        with path.open("rb") as handle:
+            data = handle.read(limit + 1)
+    except OSError as exc:
+        raise FreshnessLimitError("freshness manifest limit check failed") from exc
+    if len(data) > limit:
+        raise FreshnessLimitError("freshness manifest limit exceeded")
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise FreshnessLimitError("freshness manifest is not valid UTF-8") from exc
+
+
+def manifest_map(manifest: dict[str, Any]) -> dict[str, str]:
+    return {f["rel_path"]: f.get("hash", "") for f in manifest.get("files", []) if "rel_path" in f}
+
+
+def graph_engine_changed(cfg: Config, current_engine: dict[str, Any]) -> bool:
+    """True when a built graph records an engine identity other than the current one.
+
+    Narrow by design, for the daemon (#18): it reads the manifest and nothing
+    else. `check_graph_freshness` re-scans the whole repo, which is far too
+    expensive to run per supervised project on every discovery cycle.
+
+    A missing, unreadable, or engine-less manifest is deliberately NOT an engine
+    change. Those are the initial-build path's job; reporting them here would
+    queue a rebuild every cycle, forever, for any project without a graph.
+
+    The comparison is on the WHOLE identity, not the fingerprint alone, so a
+    `version` or `cache_version` bump counts as a change even when the packaged
+    engine files are byte-identical. That is deliberate: it is exactly what
+    `check_graph_freshness` treats as `engine_changed`, and the daemon reporting
+    a different notion of staleness than `graphite check` is how the two drift
+    apart.
+    """
+    manifest_path = cfg.output_dir / ".graphite_manifest.json"
+    try:
+        previous = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError, ValueError):
+        return False
+    if not isinstance(previous, dict):
+        return False
+    recorded = previous.get("engine")
+    if not recorded:
+        return False
+    return bool(recorded != current_engine)
+
+
+def check_graph_freshness(
+    root: Path,
+    cfg: Config,
+    *,
+    max_manifest_bytes: int | None = None,
+    ignore_engine: bool = False,
+) -> dict[str, Any]:
+    manifest_path = cfg.output_dir / ".graphite_manifest.json"
+    if not manifest_path.exists():
+        return {"stale": True, "reason": "missing manifest", "manifest": manifest_path.as_posix(), "added": [], "changed": [], "removed": []}
+    try:
+        if max_manifest_bytes is None:
+            manifest_text = manifest_path.read_text(encoding="utf-8")
+        else:
+            if max_manifest_bytes <= 0:
+                raise FreshnessLimitError("freshness manifest limit exceeded")
+            try:
+                size = _manifest_size(manifest_path)
+            except OSError as exc:
+                raise FreshnessLimitError("freshness manifest limit check failed") from exc
+            if size > max_manifest_bytes:
+                raise FreshnessLimitError("freshness manifest limit exceeded")
+            manifest_text = _read_manifest_bounded(manifest_path, max_manifest_bytes)
+        previous = json.loads(manifest_text)
+    except (OSError, json.JSONDecodeError) as exc:
+        return {"stale": True, "reason": f"unreadable manifest: {exc}", "added": [], "changed": [], "removed": []}
+    old = manifest_map(previous)
+    if max_manifest_bytes is not None and len(old) > _DOCTOR_FILE_LIMIT:
+        raise FreshnessLimitError("freshness file limit exceeded")
+    entries = collect_files(root, cfg)
+    if max_manifest_bytes is not None and len(entries) > _DOCTOR_FILE_LIMIT:
+        raise FreshnessLimitError("freshness file limit exceeded")
+    if not ignore_engine:
+        try:
+            current_engine = engine_identity(cfg.cache_version)
+        except EngineIdentityError:
+            return {
+                "stale": True,
+                "reason": "engine_identity_unavailable",
+                "added": [],
+                "changed": [],
+                "removed": [],
+            }
+        if previous.get("engine") != current_engine:
+            return {
+                "stale": True,
+                "reason": "engine_changed",
+                "added": [],
+                "changed": [],
+                "removed": [],
+            }
+    current = {e.rel_path: e.content_hash for e in entries}
+    added = sorted(set(current) - set(old))
+    removed = sorted(set(old) - set(current))
+    changed = sorted(p for p in set(current).intersection(old) if current[p] != old[p])
+    return {"stale": bool(added or removed or changed), "file_count": len(current), "manifest_file_count": len(old), "added": added, "changed": changed, "removed": removed}

graphite/git.py

@@ -0,0 +1,406 @@
+"""Hardened Git process execution for Graphite."""
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Sequence
+
+
+DEFAULT_GIT_STDOUT_MAX_BYTES = 16 * 1024 * 1024
+_MINIMUM_GIT_VERSION = (2, 38, 0)
+_GIT_VERSION_TIMEOUT_SECONDS = 2.0
+_GIT_VERSION_STDOUT_MAX_BYTES = 256
+_READ_CHUNK_BYTES = 64 * 1024
+_PROCESS_CLEANUP_TIMEOUT_SECONDS = 0.1
+
+#: Sanitized operator-facing message per version-check failure `reason`.
+#:
+#: Every value is a module constant and none is derived from Git's output, so
+#: looking a message up here can never leak a checkout path -- which is what
+#: lets `review` surface an accurate cause without abandoning the sanitising
+#: boundary it deliberately keeps between Git's text and a user's terminal.
+#:
+#: The table exists because there was only ever ONE message for three unrelated
+#: conditions, and it asserted a version requirement in the two cases where no
+#: version had been read. An operator acting on "Git 2.38 or newer is required"
+#: after a probe timeout goes and upgrades a working Git.
+_GIT_VERSION_FAILURE_MESSAGES = {
+    "too_old": "Git 2.38 or newer is required",
+    "unreadable": "Git version output could not be read",
+    "probe_GitTimeoutError": "Git version check timed out",
+    "probe_GitLaunchError": "Git version check could not be launched",
+    "probe_GitOutputLimitError": "Git version check output limit exceeded",
+}
+#: For a reason this table does not know. Deliberately admits ignorance rather
+#: than falling back to the version sentence, which is the failure being fixed:
+#: a new probe failure must not silently inherit "upgrade your Git".
+_GIT_VERSION_FAILURE_FALLBACK = "Git version could not be verified"
+
+
+def git_version_failure_message(reason: str | None) -> str:
+    """Map a version-check `reason` to its sanitized operator-facing message."""
+    return _GIT_VERSION_FAILURE_MESSAGES.get(reason, _GIT_VERSION_FAILURE_FALLBACK)
+
+
+@dataclass(frozen=True)
+class GitResult:
+    """Bounded output from one Git process."""
+
+    returncode: int
+    stdout: bytes
+
+
+class GitError(RuntimeError):
+    """Base class for sanitized Git execution failures.
+
+    The MESSAGE is a path-free constant, deliberately: Git's own text routinely
+    embeds a checkout path, which is why every raise site here is re-raised
+    `from None` upstream. That sanitising works, and it also left `git_unavailable`
+    carrying one bit of information -- the class name -- for faults with entirely
+    different fixes.
+
+    `diagnostic()` is the seam. It carries the step that failed and the OS error
+    number beside the message rather than inside it, so `str(exc)` stays exactly
+    the constant its tests assert. The vocabulary is fixed tokens and integers;
+    an `OSError`'s `.filename` is a path and must never reach it.
+    """
+
+    reason: str | None = None
+    os_error: int | None = None
+
+    def diagnostic(self) -> str:
+        """Path-free identity: which class, which step, which errno."""
+        parts = [type(self).__name__]
+        if self.reason:
+            parts.append(self.reason)
+        if self.os_error is not None:
+            parts.append(f"os={self.os_error}")
+        return ":".join(parts)
+
+
+class _StepGitError(GitError):
+    """A `GitError` that knows which step raised it and what the OS said."""
+
+    def __init__(
+        self, message: str, *, reason: str | None = None, os_error: int | None = None
+    ) -> None:
+        super().__init__(message)
+        self.reason = reason
+        # `None` stays `None`. Four launch sites hold no `OSError` at all, and an
+        # `os=None` would read as an errno that simply is not one.
+        self.os_error = os_error
+
+
+class GitUnavailableError(GitError):
+    """Raised when no trusted external Git executable is available."""
+
+
+class GitTimeoutError(GitError):
+    """Raised when a Git command exceeds its deadline."""
+
+
+class GitLaunchError(_StepGitError):
+    """Raised when the trusted Git process cannot be launched.
+
+    Seven distinct sites raise this inside `_run_bounded` alone -- root
+    resolution, spawn, wait, a missing stdout pipe, a reader thread that would
+    not join, a read that failed, and a close that failed. They are the reason
+    graphite#37's single sighting could not be taken past the bucket name, so
+    each one passes its own `reason`.
+    """
+
+
+class GitOutputLimitError(GitError):
+    """Raised when Git stdout exceeds the configured bound."""
+
+
+class GitUnsupportedVersionError(_StepGitError):
+    """Raised when Git's protected command configuration is unavailable.
+
+    Three unrelated conditions land here, and the failure this class is NAMED
+    for is the rarest of them. Git really being older than 2.38 is a permanent,
+    obvious, one-machine fact. The other two are transient: the `--version`
+    probe timed out inside `_GIT_VERSION_TIMEOUT_SECONDS`, or it could not be
+    launched at all. Both then widen into `git_unavailable` upstream, which is
+    how graphite#37 got a CI log saying an installed, working Git was
+    unavailable.
+
+    The probe's budget is two seconds for one process spawn, and `_version` is
+    cached per RUNNER -- `collect_diff_evidence`, `create_task_worktree` and
+    `accept` each build their own -- so a single routing operation pays that
+    race several times over. On a loaded Windows runner that is a plausible
+    intermittent failure, and `reason` is what lets the next sighting say so
+    instead of blaming the Git version.
+    """
+
+
+class GitRunner:
+    """Run Git from one canonical external executable in an isolated environment."""
+
+    def __init__(self, project_root: Path, *, platform_name: str | None = None) -> None:
+        try:
+            self.root = project_root.resolve()
+        except OSError as exc:
+            raise GitLaunchError(
+                "unable to run Git command", reason="resolve", os_error=exc.errno
+            ) from exc
+        self.executable = _resolve_git_executable(
+            self.root, platform_name=platform_name
+        )
+        self._environment = _isolated_environment()
+        self._version: tuple[int, int, int] | None = None
+        self._version_lock = threading.Lock()
+
+    def run(
+        self,
+        arguments: Sequence[str],
+        *,
+        timeout_seconds: float,
+        max_stdout_bytes: int = DEFAULT_GIT_STDOUT_MAX_BYTES,
+    ) -> GitResult:
+        """Run one bounded Git command with fixed process security controls."""
+        if max_stdout_bytes <= 0:
+            raise GitOutputLimitError("Git command output limit exceeded")
+        self._ensure_supported_version()
+        command = [
+            str(self.executable),
+            "--no-optional-locks",
+            "-c",
+            "core.fsmonitor=false",
+            "-c",
+            f"safe.directory={self.root}",
+            *arguments,
+        ]
+        return self._run_bounded(
+            command,
+            timeout_seconds=timeout_seconds,
+            max_stdout_bytes=max_stdout_bytes,
+        )
+
+    def _ensure_supported_version(self) -> None:
+        if self._version is not None:
+            return
+        with self._version_lock:
+            if self._version is not None:
+                return
+            try:
+                result = self._run_bounded(
+                    [str(self.executable), "--version"],
+                    timeout_seconds=_GIT_VERSION_TIMEOUT_SECONDS,
+                    max_stdout_bytes=_GIT_VERSION_STDOUT_MAX_BYTES,
+                )
+            except (GitTimeoutError, GitLaunchError, GitOutputLimitError) as exc:
+                # The probe never returned a version, so nothing here is
+                # evidence about Git's version -- carry what actually happened.
+                # `os_error` rides along when the inner failure had one; the
+                # class name alone cannot separate "spawn took longer than two
+                # seconds" from "spawn was refused".
+                probe_reason = f"probe_{type(exc).__name__}"
+                raise GitUnsupportedVersionError(
+                    git_version_failure_message(probe_reason),
+                    reason=probe_reason,
+                    os_error=getattr(exc, "os_error", None),
+                ) from exc
+            version = _parse_git_version(result.stdout) if result.returncode == 0 else None
+            if version is None:
+                # Ran, and said something this parser does not recognise. Not the
+                # same finding as a version that parsed and was too low.
+                raise GitUnsupportedVersionError(
+                    git_version_failure_message("unreadable"), reason="unreadable"
+                )
+            if version < _MINIMUM_GIT_VERSION:
+                raise GitUnsupportedVersionError(
+                    git_version_failure_message("too_old"), reason="too_old"
+                )
+            self._version = version
+
+    def _run_bounded(
+        self,
+        command: list[str],
+        *,
+        timeout_seconds: float,
+        max_stdout_bytes: int,
+    ) -> GitResult:
+        try:
+            process = subprocess.Popen(
+                command,
+                cwd=self.root,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.DEVNULL,
+                shell=False,
+                env=self._environment,
+            )
+        except FileNotFoundError as exc:
+            raise GitUnavailableError("Git executable was not found") from exc
+        except subprocess.TimeoutExpired as exc:
+            raise GitTimeoutError("Git command timeout") from exc
+        except OSError as exc:
+            raise GitLaunchError(
+                "unable to run Git command", reason="popen", os_error=exc.errno
+            ) from exc
+
+        if process.stdout is None:
+            _bounded_cleanup(process)
+            raise GitLaunchError("unable to run Git command", reason="no_stdout")
+
+        stdout = bytearray()
+        overflow = threading.Event()
+        read_failed = threading.Event()
+        # The reader runs on its own thread, so its exception cannot propagate --
+        # it can only leave a trace here. An `Event` alone records THAT the read
+        # failed and never why, which is how a bounded read fault and a pipe torn
+        # down by the peer became the same log line. One slot, written once,
+        # before the flag any waiter keys on.
+        read_error: list[int | None] = []
+
+        def read_stdout() -> None:
+            try:
+                while True:
+                    chunk = process.stdout.read(_READ_CHUNK_BYTES)
+                    if not chunk:
+                        return
+                    remaining = max_stdout_bytes - len(stdout)
+                    if len(chunk) > remaining:
+                        stdout.extend(chunk[:remaining])
+                        overflow.set()
+                        try:
+                            process.kill()
+                        except OSError:
+                            pass
+                        return
+                    stdout.extend(chunk)
+            except OSError as exc:
+                read_error.append(exc.errno)
+                read_failed.set()
+
+        reader = threading.Thread(target=read_stdout, daemon=True)
+        reader.start()
+        try:
+            returncode = process.wait(timeout=timeout_seconds)
+        except subprocess.TimeoutExpired as exc:
+            _bounded_cleanup(process, reader)
+            raise GitTimeoutError("Git command timeout") from exc
+        except OSError as exc:
+            _bounded_cleanup(process, reader)
+            raise GitLaunchError(
+                "unable to run Git command", reason="wait", os_error=exc.errno
+            ) from exc
+
+        reader.join(timeout=_PROCESS_CLEANUP_TIMEOUT_SECONDS)
+        if reader.is_alive():
+            _bounded_cleanup(process, reader)
+            raise GitLaunchError("unable to run Git command", reason="reader_hang")
+        if overflow.is_set():
+            _bounded_cleanup(process, reader)
+            raise GitOutputLimitError("Git command output limit exceeded")
+        if read_failed.is_set():
+            _close_stdout(process)
+            raise GitLaunchError(
+                "unable to run Git command",
+                reason="read",
+                os_error=read_error[0] if read_error else None,
+            )
+        if not _close_stdout(process):
+            raise GitLaunchError("unable to run Git command", reason="close")
+        return GitResult(returncode=returncode, stdout=bytes(stdout))
+
+
+def _parse_git_version(output: bytes) -> tuple[int, int, int] | None:
+    match = re.fullmatch(
+        rb"git version ([0-9]+)\.([0-9]+)(?:\.([0-9]+))?"
+        rb"(?:\.windows\.[0-9]+| \(Apple Git-[0-9A-Za-z][0-9A-Za-z.-]*\))?"
+        rb"\r?\n?",
+        output,
+    )
+    if match is None:
+        return None
+    try:
+        return (
+            int(match.group(1)),
+            int(match.group(2)),
+            int(match.group(3) or b"0"),
+        )
+    except ValueError:
+        return None
+
+
+def _bounded_cleanup(
+    process: subprocess.Popen[bytes], reader: threading.Thread | None = None
+) -> bool:
+    """Best-effort cleanup that never waits beyond the fixed cleanup deadline."""
+    try:
+        process.kill()
+    except OSError:
+        pass
+    wait_confirmed = True
+    try:
+        process.wait(timeout=_PROCESS_CLEANUP_TIMEOUT_SECONDS)
+    except (subprocess.TimeoutExpired, OSError):
+        wait_confirmed = False
+    reader_stopped = True
+    if reader is not None:
+        reader.join(timeout=_PROCESS_CLEANUP_TIMEOUT_SECONDS)
+        reader_stopped = not reader.is_alive()
+    if not reader_stopped:
+        return False
+    return wait_confirmed and _close_stdout(process)
+
+
+def _close_stdout(process: subprocess.Popen[bytes]) -> bool:
+    if process.stdout is None:
+        return True
+    try:
+        process.stdout.close()
+    except OSError:
+        return False
+    return True
+
+
+def _resolve_git_executable(
+    resolved_project_root: Path, *, platform_name: str | None = None
+) -> Path:
+    selected_platform = os.name if platform_name is None else platform_name
+    executable_name = "git.exe" if selected_platform == "nt" else "git"
+
+    for raw_directory in os.environ.get("PATH", "").split(os.pathsep):
+        if not raw_directory:
+            continue
+        directory = Path(raw_directory)
+        if not directory.is_absolute():
+            continue
+        candidate = directory / executable_name
+        try:
+            if not candidate.is_file():
+                continue
+            resolved_candidate = candidate.resolve(strict=True)
+            if not resolved_candidate.is_file():
+                continue
+            if selected_platform != "nt" and not os.access(resolved_candidate, os.X_OK):
+                continue
+            try:
+                resolved_candidate.relative_to(resolved_project_root)
+            except ValueError:
+                pass
+            else:
+                continue
+        except OSError:
+            continue
+        return resolved_candidate
+
+    raise GitUnavailableError("Git executable was not found")
+
+
+def _isolated_environment() -> dict[str, str]:
+    environment = {
+        name: value
+        for name, value in os.environ.items()
+        if not name.casefold().startswith("git_")
+    }
+    environment["GIT_OPTIONAL_LOCKS"] = "0"
+    environment["GIT_TERMINAL_PROMPT"] = "0"
+    return environment

graphite/graph.py

@@ -0,0 +1,117 @@
+"""Graph construction and normalization."""
+from __future__ import annotations
+
+import unicodedata
+from typing import Any
+
+import networkx as nx
+
+
+def normalize_id(value: str) -> str:
+    """Stable, portable node ID. Same rules as extract.ast._make_id."""
+    text = unicodedata.normalize("NFKC", value)
+    cleaned = "".join(c if c.isalnum() or c in ("_", ".") else "_" for c in text)
+    cleaned = cleaned.strip("_.")
+    cleaned = cleaned.replace(".", "_")
+    while "__" in cleaned:
+        cleaned = cleaned.replace("__", "_")
+    return cleaned.casefold()
+
+
+def edge_relations(data: dict[str, Any]) -> tuple[str, ...]:
+    """Every relation an edge carries.
+
+    A DiGraph holds one edge per node pair, so when edges with different
+    relations share a pair they are merged and `relations` lists all of them.
+    Read this rather than `relation` alone whenever the question is "does this
+    pair have relation X" -- `relation` is only the first-sorted value (#1).
+    """
+    relations = data.get("relations")
+    if relations:
+        return tuple(r for r in relations if r)
+    relation = data.get("relation")
+    return (relation,) if relation else ()
+
+
+def build_graph(nodes: list[dict[str, Any]], edges: list[dict[str, Any]]) -> nx.DiGraph:
+    """Build a normalized directed graph from extraction results."""
+    g = nx.DiGraph()
+
+    # Deterministic order.
+    sorted_nodes = sorted(nodes, key=lambda n: n.get("id", ""))
+    for n in sorted_nodes:
+        nid = normalize_id(n["id"])
+        attrs = dict(n)
+        attrs["id"] = nid
+        attrs["label"] = attrs.get("name", nid)
+        g.add_node(nid, **attrs)
+
+    sorted_edges = sorted(
+        edges,
+        key=lambda e: (e.get("source", ""), e.get("target", ""), e.get("relation", "")),
+    )
+    for e in sorted_edges:
+        src = normalize_id(e["source"])
+        tgt = normalize_id(e["target"])
+        if not src or not tgt or src == tgt:
+            continue
+        if not g.has_node(src):
+            g.add_node(src, id=src, kind="unknown", name=src, label=src)
+        if not g.has_node(tgt):
+            g.add_node(tgt, id=tgt, kind="unknown", name=tgt, label=tgt)
+        if g.has_edge(src, tgt):
+            data = g[src][tgt]
+            # Fold the incoming edge's own weight; a flat +1.0 discarded it
+            # (extraction-level _merge emits weights >= 1.0).
+            data["weight"] = data.get("weight", 1.0) + float(e.get("weight", 1.0) or 1.0)
+            # A DiGraph cannot hold parallel edges, so a second edge between the
+            # same pair with a DIFFERENT relation used to be dropped outright,
+            # losing its relation entirely (#1). Record it instead: `relation`
+            # keeps the first-sorted value for display, `relations` carries every
+            # relation this pair actually has.
+            relation = e.get("relation")
+            if relation and relation != data.get("relation"):
+                relations = data.setdefault("relations", [data.get("relation")])
+                if relation not in relations:
+                    relations.append(relation)
+        else:
+            attrs = dict(e)
+            attrs["source"] = src
+            attrs["target"] = tgt
+            attrs["weight"] = attrs.get("weight", 1.0)
+            g.add_edge(src, tgt, **attrs)
+
+    return g
+
+
+def graph_to_json(g: nx.DiGraph) -> dict[str, Any]:
+    """Convert networkx graph to graphify-compatible JSON."""
+    return {
+        "nodes": [
+            {"id": n, **{k: v for k, v in data.items() if k != "id"}}
+            for n, data in g.nodes(data=True)
+        ],
+        "edges": [
+            {
+                "source": u,
+                "target": v,
+                **{k: val for k, val in data.items() if k not in ("source", "target")},
+            }
+            for u, v, data in g.edges(data=True)
+        ],
+        "metadata": {
+            "node_count": g.number_of_nodes(),
+            "edge_count": g.number_of_edges(),
+        },
+    }
+
+
+def graph_from_json(data: dict[str, Any]) -> nx.DiGraph:
+    """Load graph from JSON."""
+    g = nx.DiGraph()
+    for n in data.get("nodes", []):
+        nid = n.get("id")
+        g.add_node(nid, **n)
+    for e in data.get("edges", []):
+        g.add_edge(e["source"], e["target"], **e)
+    return g