ref-agents 1.0.0__py3-none-any.whl

ref_agents/tools/full_suite_runner.py

@@ -0,0 +1,740 @@
+"""Full-suite test execution gate (STORY-TQ-002).
+
+Server-side subprocess driver. Detects each project's test runner from
+manifest files, executes them against the entire project tree, writes a
+tamper-resistant signed JSON report under ``ref-reports/platform_engineer/``,
+and powers the PHASE_4 → PHASE_5 ``full_suite_passed`` artifact.
+
+The detector, driver, report builder, and marker signer are private to this
+module. The public surface is one function: ``run_full_suite(directory)``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import secrets
+import shutil
+import subprocess
+import threading
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+# --- public surface -----------------------------------------------------------
+
+
+REPORT_DIR_RELATIVE: str = "ref-reports/platform_engineer"
+REPORT_FILENAME: str = "test_suite_report.json"
+REPORT_VERSION: int = 1
+DEFAULT_TIMEOUT_SECONDS: int = 600
+OUTPUT_CAP_BYTES: int = 1 * 1024 * 1024  # 1 MiB
+FAILURE_TAIL_LINES: int = 200
+ENV_ALLOWLIST: frozenset[str] = frozenset(
+    {"PATH", "HOME", "LANG", "CI", "LC_ALL", "TZ"}
+)
+MANIFEST_WALK_DEPTH: int = 3
+
+
+@dataclass(frozen=True)
+class RunnerSpec:
+    """A detected runner ready for execution."""
+
+    id: str
+    language: str
+    command: tuple[str, ...]
+    cwd: Path
+    manifest_path: Path
+
+
+@dataclass
+class RunnerResult:
+    """Outcome of one runner invocation."""
+
+    id: str
+    exit_code: int
+    runtime_seconds: float
+    status: str  # passed | failed | timeout | error | no_tests | skipped
+    failure_tail: str = ""
+    skipped_reason: str = ""
+
+
+def run_full_suite(
+    directory: str | Path,
+    *,
+    timeout_seconds: int | None = None,
+) -> str:
+    """Detect runners, execute each, write the signed report, return summary.
+
+    Args:
+        directory: Project root.
+        timeout_seconds: Optional per-runner timeout override; ``None`` uses default.
+
+    Returns:
+        Markdown summary string for the LLM agent. The gate consults the
+        on-disk report — not this string.
+    """
+    project_root = Path(directory).resolve()
+    timeout = timeout_seconds or _resolve_timeout(project_root)
+
+    detector = _ManifestDetector(project_root)
+    raw_runners = detector.detect()
+    config = _load_project_config(project_root)
+    runners = _RunnerRegistry(raw_runners, config).build()
+
+    if not runners:
+        report_dict = _build_report(
+            runners=[],
+            results=[],
+            project_root=project_root,
+            status_override="no_runners",
+        )
+        _write_report(project_root, report_dict)
+        return _format_summary(report_dict)
+
+    driver = _SubprocessDriver(timeout=timeout)
+    results: list[RunnerResult] = []
+    for spec in runners:
+        if spec.id in config.get("disabled_runners", ()):
+            results.append(
+                RunnerResult(
+                    id=spec.id,
+                    exit_code=0,
+                    runtime_seconds=0.0,
+                    status="skipped",
+                    skipped_reason="disabled_by_config",
+                )
+            )
+            continue
+        result = driver.execute(spec)
+        results.append(result)
+
+    report_dict = _build_report(
+        runners=runners,
+        results=results,
+        project_root=project_root,
+    )
+    _write_report(project_root, report_dict)
+    return _format_summary(report_dict)
+
+
+def validate_report_marker(report_path: Path) -> bool:
+    """True iff the report's marker was issued by this MCP server process."""
+    try:
+        with report_path.open("r", encoding="utf-8") as fh:
+            data = json.load(fh)
+    except (OSError, json.JSONDecodeError):
+        return False
+    marker = data.get("marker")
+    if not isinstance(marker, str):
+        return False
+    return _marker_signer().validate(marker)
+
+
+# --- manifest detection -------------------------------------------------------
+
+
+_MANIFEST_PRIORITY: tuple[tuple[str, str], ...] = (
+    # (manifest filename, language)
+    ("pyproject.toml", "python"),
+    ("setup.py", "python"),
+    ("pytest.ini", "python"),
+    ("tox.ini", "python"),
+    ("package.json", "javascript"),
+    ("go.mod", "go"),
+    ("Cargo.toml", "rust"),
+    ("pom.xml", "java"),
+    ("build.gradle", "java"),
+    ("build.gradle.kts", "java"),
+)
+_LANGUAGE_ORDER: tuple[str, ...] = (
+    "python",
+    "javascript",
+    "typescript",
+    "go",
+    "rust",
+    "java",
+)
+
+
+class _ManifestDetector:
+    """Walks the project tree and yields candidate runner specs."""
+
+    def __init__(self, project_root: Path) -> None:
+        self.project_root = project_root
+
+    def detect(self) -> list[RunnerSpec]:
+        per_dir: dict[Path, list[tuple[str, str]]] = {}
+        for path in self._walk():
+            if path.name not in {name for name, _ in _MANIFEST_PRIORITY}:
+                continue
+            language = next(
+                (lang for name, lang in _MANIFEST_PRIORITY if name == path.name),
+                None,
+            )
+            if language is None:
+                continue
+            per_dir.setdefault(path.parent, []).append((path.name, language))
+
+        specs: list[RunnerSpec] = []
+        for cwd, hits in per_dir.items():
+            specs.extend(self._build_specs(cwd, hits))
+
+        # Sort by language priority order; within a language, by cwd depth.
+        order_map = {lang: idx for idx, lang in enumerate(_LANGUAGE_ORDER)}
+        specs.sort(key=lambda s: (order_map.get(s.language, 99), len(s.cwd.parts)))
+        return specs
+
+    def _walk(self):
+        for root, dirs, files in os.walk(self.project_root):
+            depth = len(Path(root).relative_to(self.project_root).parts)
+            if depth > MANIFEST_WALK_DEPTH:
+                dirs.clear()
+                continue
+            # Prune common noise directories to keep detection fast.
+            dirs[:] = [
+                d
+                for d in dirs
+                if d
+                not in {
+                    "node_modules",
+                    ".git",
+                    "__pycache__",
+                    ".venv",
+                    "venv",
+                    "dist",
+                    "build",
+                    "target",
+                    ".tox",
+                    ".mypy_cache",
+                    ".pytest_cache",
+                }
+            ]
+            for filename in files:
+                yield Path(root) / filename
+
+    def _build_specs(
+        self,
+        cwd: Path,
+        hits: list[tuple[str, str]],
+    ) -> list[RunnerSpec]:
+        languages_seen: set[str] = set()
+        result: list[RunnerSpec] = []
+        # Honour priority: first match per language within the same directory.
+        for name, language in _MANIFEST_PRIORITY:
+            if (name, language) not in hits:
+                continue
+            if language in languages_seen:
+                continue
+            manifest_path = cwd / name
+            spec = _resolve_runner(name, language, cwd, manifest_path)
+            if spec is None:
+                continue
+            languages_seen.add(spec.language)
+            result.append(spec)
+        return result
+
+
+def _resolve_runner(
+    manifest_name: str,
+    language: str,
+    cwd: Path,
+    manifest_path: Path,
+) -> RunnerSpec | None:
+    if manifest_name == "package.json":
+        return _resolve_javascript_runner(cwd, manifest_path)
+    if manifest_name in ("build.gradle", "build.gradle.kts"):
+        return RunnerSpec(
+            id="gradle_test",
+            language="java",
+            command=("./gradlew", "test", "-q"),
+            cwd=cwd,
+            manifest_path=manifest_path,
+        )
+    table = {
+        "pyproject.toml": ("pytest", "python", ("pytest", "--tb=short", "-q")),
+        "setup.py": ("pytest", "python", ("pytest", "--tb=short", "-q")),
+        "pytest.ini": ("pytest", "python", ("pytest", "--tb=short", "-q")),
+        "tox.ini": ("pytest", "python", ("pytest", "--tb=short", "-q")),
+        "go.mod": ("go_test", "go", ("go", "test", "./...")),
+        "Cargo.toml": ("cargo_test", "rust", ("cargo", "test")),
+        "pom.xml": ("mvn_test", "java", ("mvn", "test", "-q")),
+    }
+    entry = table.get(manifest_name)
+    if entry is None:
+        return None
+    runner_id, lang, command = entry
+    return RunnerSpec(
+        id=runner_id,
+        language=lang,
+        command=command,
+        cwd=cwd,
+        manifest_path=manifest_path,
+    )
+
+
+def _resolve_javascript_runner(
+    cwd: Path,
+    manifest_path: Path,
+) -> RunnerSpec | None:
+    try:
+        data = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+    deps: dict[str, Any] = {}
+    for key in ("dependencies", "devDependencies"):
+        candidate = data.get(key)
+        if isinstance(candidate, dict):
+            deps.update(candidate)
+    if "vitest" in deps:
+        runner_id, language, command = "vitest", "typescript", ("npx", "vitest", "run")
+    elif "jest" in deps:
+        runner_id, language, command = "jest", "javascript", ("npx", "jest", "--ci")
+    else:
+        return None
+    scripts = data.get("scripts", {})
+    if isinstance(scripts, dict) and isinstance(scripts.get("test"), str):
+        command = ("npm", "test", "--silent")
+    return RunnerSpec(
+        id=runner_id,
+        language=language,
+        command=command,
+        cwd=cwd,
+        manifest_path=manifest_path,
+    )
+
+
+# --- registry + config --------------------------------------------------------
+
+
+class _RunnerRegistry:
+    """Filters detected specs by ref-config.yaml overrides."""
+
+    def __init__(self, runners: list[RunnerSpec], config: dict[str, Any]) -> None:
+        self.runners = runners
+        self.config = config
+
+    def build(self) -> list[RunnerSpec]:
+        extra_args = self.config.get("extra_args", {})
+        if not isinstance(extra_args, dict):
+            extra_args = {}
+        result: list[RunnerSpec] = []
+        for spec in self.runners:
+            extras = extra_args.get(spec.id, ())
+            if extras and isinstance(extras, (list, tuple)):
+                command = tuple(spec.command) + tuple(str(a) for a in extras)
+                result.append(
+                    RunnerSpec(
+                        id=spec.id,
+                        language=spec.language,
+                        command=command,
+                        cwd=spec.cwd,
+                        manifest_path=spec.manifest_path,
+                    )
+                )
+            else:
+                result.append(spec)
+        return result
+
+
+def _load_project_config(project_root: Path) -> dict[str, Any]:
+    config_path = project_root / "ref-config.yaml"
+    if not config_path.exists():
+        return {}
+    try:
+        import yaml
+
+        data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
+    except ImportError:
+        logger.warning("ref_config_yaml_unavailable")
+        return {}
+    except (OSError, yaml.YAMLError) as exc:  # type: ignore[attr-defined]
+        logger.warning("ref_config_malformed", error=str(exc))
+        return {}
+    block = data.get("full_suite") if isinstance(data, dict) else None
+    return block if isinstance(block, dict) else {}
+
+
+def _resolve_timeout(project_root: Path) -> int:
+    config = _load_project_config(project_root)
+    raw = config.get("timeout_seconds", DEFAULT_TIMEOUT_SECONDS)
+    try:
+        value = int(raw)
+        if value <= 0:
+            return DEFAULT_TIMEOUT_SECONDS
+        return value
+    except (TypeError, ValueError):
+        return DEFAULT_TIMEOUT_SECONDS
+
+
+# --- subprocess driver --------------------------------------------------------
+
+
+class _SubprocessDriver:
+    """Spawns each runner with env scrub, timeout, output cap."""
+
+    def __init__(self, timeout: int) -> None:
+        self.timeout = timeout
+
+    def execute(self, spec: RunnerSpec) -> RunnerResult:
+        # Honour PATH lookup; fall back to "error" status when binary missing.
+        binary = spec.command[0]
+        if not shutil.which(binary):
+            logger.warning("runner_binary_missing", runner=spec.id, binary=binary)
+            return RunnerResult(
+                id=spec.id,
+                exit_code=127,
+                runtime_seconds=0.0,
+                status="error",
+                failure_tail=f"binary `{binary}` not found on PATH",
+            )
+
+        env = self._scrub_env()
+        start = time.monotonic()
+        try:
+            proc = subprocess.Popen(
+                list(spec.command),
+                cwd=str(spec.cwd),
+                env=env,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+            )
+        except OSError as exc:
+            return RunnerResult(
+                id=spec.id,
+                exit_code=126,
+                runtime_seconds=time.monotonic() - start,
+                status="error",
+                failure_tail=str(exc),
+            )
+
+        timed_out, output = self._pump_with_timeout(proc, self.timeout)
+        elapsed = time.monotonic() - start
+        return self._classify_result(spec, proc, timed_out, output, elapsed)
+
+    @staticmethod
+    def _scrub_env() -> dict[str, str]:
+        return {key: os.environ[key] for key in ENV_ALLOWLIST if key in os.environ}
+
+    @staticmethod
+    def _pump_with_timeout(
+        proc: subprocess.Popen[bytes],
+        timeout: int,
+    ) -> tuple[bool, bytes]:
+        """Drain stdout into a 1 MiB ring buffer while honouring a wall-clock deadline.
+
+        Uses ``select.select`` so a subprocess that produces no output before the
+        deadline (e.g. a hanging ``sleep``) is still killed on time.
+        """
+        import select
+
+        buf = bytearray()
+        truncated_bytes = 0
+        timed_out = False
+        deadline = time.monotonic() + timeout
+        stdout = proc.stdout
+        if stdout is None:
+            try:
+                proc.wait(timeout=timeout)
+            except subprocess.TimeoutExpired:
+                timed_out = True
+                _terminate_process_tree(proc)
+            return timed_out, bytes(buf)
+
+        fd = stdout.fileno()
+        while True:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                timed_out = True
+                _terminate_process_tree(proc)
+                break
+            try:
+                readable, _, _ = select.select([fd], [], [], min(remaining, 1.0))
+            except (OSError, ValueError):
+                break
+            if readable:
+                try:
+                    chunk = os.read(fd, 64 * 1024)
+                except OSError:
+                    break
+                if not chunk:
+                    break
+                if len(buf) + len(chunk) <= OUTPUT_CAP_BYTES:
+                    buf.extend(chunk)
+                elif len(buf) < OUTPUT_CAP_BYTES:
+                    room = OUTPUT_CAP_BYTES - len(buf)
+                    buf.extend(chunk[:room])
+                    truncated_bytes += len(chunk) - room
+                else:
+                    truncated_bytes += len(chunk)
+            elif proc.poll() is not None:
+                break
+        if truncated_bytes:
+            buf.extend(f"\n[...truncated {truncated_bytes} bytes]\n".encode())
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            _terminate_process_tree(proc)
+            proc.wait()
+        return timed_out, bytes(buf)
+
+    @staticmethod
+    def _classify_result(  # noqa: PLR0913  # TECH_DEBT: refactor to TypedDict — STORY-046
+        spec: RunnerSpec,
+        proc: subprocess.Popen[bytes],
+        timed_out: bool,
+        output: bytes,
+        elapsed: float,
+    ) -> RunnerResult:
+        tail = _tail_lines(output.decode("utf-8", errors="replace"), FAILURE_TAIL_LINES)
+        if timed_out:
+            return RunnerResult(
+                id=spec.id,
+                exit_code=124,
+                runtime_seconds=elapsed,
+                status="timeout",
+                failure_tail=tail,
+            )
+        exit_code = proc.returncode if proc.returncode is not None else -1
+        if exit_code == 0:
+            status = "passed"
+        elif spec.id == "pytest" and exit_code == 5:  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+            status = "no_tests"
+        else:
+            status = "failed"
+        return RunnerResult(
+            id=spec.id,
+            exit_code=exit_code,
+            runtime_seconds=elapsed,
+            status=status,
+            failure_tail=tail if status != "passed" else "",
+        )
+
+
+def _terminate_process_tree(proc: subprocess.Popen[bytes]) -> None:
+    try:
+        proc.terminate()
+    except OSError:
+        return
+    try:
+        proc.wait(timeout=5)
+        return
+    except subprocess.TimeoutExpired:
+        pass
+    try:
+        proc.kill()
+    except OSError:
+        return
+
+
+def _tail_lines(text: str, max_lines: int) -> str:
+    lines = text.splitlines()
+    if len(lines) <= max_lines:
+        return text
+    return "\n".join(lines[-max_lines:])
+
+
+# --- report builder + writer --------------------------------------------------
+
+
+def _build_report(
+    runners: list[RunnerSpec],
+    results: list[RunnerResult],
+    project_root: Path,
+    *,
+    status_override: str | None = None,
+) -> dict[str, Any]:
+    exit_codes = {r.id: r.exit_code for r in results}
+    runtime_seconds = {r.id: r.runtime_seconds for r in results}
+    failure_tails = {r.id: r.failure_tail for r in results if r.failure_tail}
+    skipped = [
+        {"runner_id": r.id, "reason": r.skipped_reason}
+        for r in results
+        if r.status == "skipped"
+    ]
+
+    overall_status: str
+    if status_override is not None:
+        overall_status = status_override
+    elif any(r.status == "timeout" for r in results):
+        overall_status = "timeout"
+    elif all(r.status == "skipped" for r in results) and results:
+        overall_status = "no_runners"
+    elif all(r.status in ("passed", "skipped") for r in results):
+        overall_status = "passed"
+    elif all(r.status in ("no_tests", "skipped") for r in results):
+        overall_status = "no_tests"
+    elif any(r.status == "error" for r in results):
+        overall_status = "error"
+    else:
+        overall_status = "failed"
+
+    report: dict[str, Any] = {
+        "report_version": REPORT_VERSION,
+        "runners": [s.id for s in runners],
+        "executed_at": datetime.now(timezone.utc).isoformat(),
+        "exit_codes": exit_codes,
+        "status": overall_status,
+        "pass_summary": {
+            "total": 0,
+            "passed": 0,
+            "failed": 0,
+            "skipped": skipped,
+        },
+        "failure_tail": failure_tails,
+        "runtime_seconds": runtime_seconds,
+        "marker": _marker_signer().sign(),
+    }
+    logger.info(
+        "full_suite_report_built",
+        project_root=str(project_root),
+        status=overall_status,
+        runner_count=len(runners),
+    )
+    return report
+
+
+def _write_report(project_root: Path, report: dict[str, Any]) -> None:
+    report_dir = project_root / REPORT_DIR_RELATIVE
+    report_dir.mkdir(parents=True, exist_ok=True)
+    final_path = report_dir / REPORT_FILENAME
+    tmp_path = report_dir / f"{REPORT_FILENAME}.tmp"
+    tmp_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
+    os.replace(tmp_path, final_path)
+
+
+def _format_summary(report: dict[str, Any]) -> str:
+    status = report["status"]
+    runners = report["runners"]
+    exit_codes = report["exit_codes"]
+    lines = [
+        f"## Full-Suite Test Report — status: **{status}**",
+        f"Runners: {', '.join(runners) if runners else '(none detected)'}",
+    ]
+    if exit_codes:
+        for runner_id, code in exit_codes.items():
+            lines.append(f"- `{runner_id}` → exit {code}")
+    return "\n".join(lines)
+
+
+# --- marker signer (process-local, tamper-resistant) --------------------------
+
+
+class _MarkerSigner:
+    """Issues + validates per-process markers (FR-17, FR-18)."""
+
+    def __init__(self) -> None:
+        self._secret = secrets.token_bytes(32)
+        self._counter = 0
+        self._issued: set[str] = set()
+        self._lock = threading.Lock()
+
+    def sign(self) -> str:
+        with self._lock:
+            self._counter += 1
+            payload = (
+                str(os.getpid()).encode()
+                + str(time.monotonic_ns()).encode()
+                + str(self._counter).encode()
+                + self._secret
+            )
+            marker = hashlib.sha256(payload).hexdigest()[:32]
+            self._issued.add(marker)
+            return marker
+
+    def validate(self, marker: str) -> bool:
+        with self._lock:
+            return marker in self._issued
+
+    def reset_for_tests(self) -> None:
+        """Reset state — only intended for the pytest fixture."""
+        with self._lock:
+            self._secret = secrets.token_bytes(32)
+            self._counter = 0
+            self._issued.clear()
+
+
+_SIGNER: _MarkerSigner | None = None
+_SIGNER_LOCK = threading.Lock()
+
+
+def _marker_signer() -> _MarkerSigner:
+    global _SIGNER
+    if _SIGNER is None:
+        with _SIGNER_LOCK:
+            if _SIGNER is None:
+                _SIGNER = _MarkerSigner()
+    return _SIGNER
+
+
+def reset_marker_signer_for_tests() -> None:
+    """Reset the singleton — test-only entry point."""
+    signer = _marker_signer()
+    signer.reset_for_tests()
+
+
+# --- gate helper consumed by transitions.py ----------------------------------
+
+
+_REPORT_MAX_AGE_SECONDS: int = 7200  # 2 hours
+
+
+@dataclass
+class GateDecision:
+    passed: bool
+    reason: str
+    detail: dict[str, Any] = field(default_factory=dict)
+
+
+def evaluate_gate(project_root: Path) -> GateDecision:
+    """Return a GateDecision based on the on-disk report (FR-20).
+
+    Honours marker validation, freshness, and exit-code semantics. The caller
+    layers on top of this any deferral / greenfield artifact bypass logic.
+    """
+    report_path = project_root / REPORT_DIR_RELATIVE / REPORT_FILENAME
+    if not report_path.exists():
+        return GateDecision(passed=False, reason="report_missing")
+    try:
+        age = time.time() - report_path.stat().st_mtime
+    except OSError as exc:
+        return GateDecision(
+            passed=False, reason="stat_failed", detail={"err": str(exc)}
+        )
+    if age > _REPORT_MAX_AGE_SECONDS:
+        return GateDecision(passed=False, reason="report_stale", detail={"age": age})
+    try:
+        with report_path.open("r", encoding="utf-8") as fh:
+            data = json.load(fh)
+    except (OSError, json.JSONDecodeError) as exc:
+        return GateDecision(
+            passed=False, reason="report_unreadable", detail={"err": str(exc)}
+        )
+
+    marker = data.get("marker")
+    if not isinstance(marker, str) or not _marker_signer().validate(marker):
+        return GateDecision(passed=False, reason="signature_invalid")
+
+    status = data.get("status")
+    exit_codes = data.get("exit_codes", {}) or {}
+    if status != "passed":
+        return GateDecision(
+            passed=False,
+            reason="status_not_passed",
+            detail={"status": status, "exit_codes": exit_codes},
+        )
+    if any(code != 0 for code in exit_codes.values()):
+        return GateDecision(
+            passed=False,
+            reason="non_zero_exit",
+            detail={"exit_codes": exit_codes},
+        )
+    return GateDecision(passed=True, reason="ok")