nemesis-eval 0.2.0__py3-none-any.whl

nemesis/__init__.py

@@ -0,0 +1 @@
+"""Nemesis — Python evaluation harness for agentic failure modes."""

nemesis/__main__.py

@@ -0,0 +1,185 @@
+"""Command-line entry point for Nemesis.
+
+Two subcommands:
+
+- ``nemesis eval`` scores every detector against synthetic known-truth runs
+  (the demo / self-test).
+- ``nemesis check`` runs the detectors against a *real* repository, building
+  the artifact from read-only git state (the real-run tool).
+
+Python runs this file when the package is invoked with ``-m``.
+"""
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+import nemesis.detectors  # noqa: F401  (import registers all detectors)
+from nemesis.collect import collect_artifact
+from nemesis.detectors.base import all_detectors
+from nemesis.eval import EvalLoop, EvalReport
+from nemesis.report import render_check_markdown, render_markdown
+
+logger = logging.getLogger(__name__)
+
+
+def _print_report(report: EvalReport) -> None:
+    """Print a human-readable summary of an eval report to stdout."""
+    print("=" * 72)
+    print("Nemesis eval — detector scores")
+    print("=" * 72)
+    for score in report.scores:
+        print(
+            f"{score.failure_mode_id:<42} "
+            f"TPR={score.true_positive_rate:.2f}  "
+            f"FPR={score.false_positive_rate:.2f}"
+        )
+        for line in score.sample_evidence:
+            print(f"    evidence: {line}")
+    print("-" * 72)
+    print(
+        f"detectors: {len(report.scores)}   "
+        f"mean TPR: {report.mean_true_positive_rate:.2f}   "
+        f"mean FPR: {report.mean_false_positive_rate:.2f}"
+    )
+    print("=" * 72)
+
+
+def _run_check(args: argparse.Namespace) -> int:
+    """Build an artifact from a real repo and report any detected failures.
+
+    Returns ``1`` when ``--fail-on-detect`` was given and at least one failure
+    mode fired (so the command can gate CI); ``0`` otherwise.
+    """
+    transcript = ""
+    if args.transcript is not None:
+        transcript = args.transcript.read_text(encoding="utf-8")
+
+    test_results = None
+    if args.tests_passing is not None:
+        test_results = {"passing": args.tests_passing == "true"}
+
+    artifact = collect_artifact(
+        args.repo,
+        transcript=transcript,
+        claimed_success=args.claimed_success,
+        test_results=test_results,
+    )
+    results = [detector.detect(artifact) for detector in all_detectors()]
+    fired = [r for r in results if r.detected]
+
+    if args.output is not None:
+        args.output.write_text(render_check_markdown(results), encoding="utf-8")
+        logger.info("wrote check report to %s", args.output)
+    else:
+        print("=" * 72)
+        print(f"Nemesis check — {args.repo}")
+        print("=" * 72)
+        if not fired:
+            print("No failure modes detected.")
+        else:
+            for result in fired:
+                print(f"DETECTED: {result.failure_mode_id}")
+                for line in result.evidence:
+                    print(f"    evidence: {line}")
+        print("-" * 72)
+        print(f"detectors run: {len(results)}   failures detected: {len(fired)}")
+        print("=" * 72)
+
+    if args.fail_on_detect and fired:
+        return 1
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Parse arguments and dispatch the requested subcommand.
+
+    Args:
+        argv: Argument list (defaults to ``sys.argv[1:]``).
+
+    Returns:
+        Process exit code (0 on success).
+    """
+    # Make console output crash-proof on consoles that can't encode every
+    # character (e.g. Windows cp1252). Reports written to files use UTF-8.
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(errors="replace")  # type: ignore[union-attr]
+        except (AttributeError, ValueError):  # pragma: no cover
+            pass
+
+    parser = argparse.ArgumentParser(prog="nemesis", description="Nemesis eval harness")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    eval_parser = subparsers.add_parser(
+        "eval", help="run all detectors against known-truth runs"
+    )
+    eval_parser.add_argument(
+        "--output",
+        type=Path,
+        default=None,
+        help="write a Markdown report to this path instead of printing to stdout",
+    )
+
+    check_parser = subparsers.add_parser(
+        "check", help="run detectors against a real repository (read-only git)"
+    )
+    check_parser.add_argument(
+        "--repo",
+        type=Path,
+        default=Path("."),
+        help="path to the git repository to inspect (default: current directory)",
+    )
+    check_parser.add_argument(
+        "--transcript",
+        type=Path,
+        default=None,
+        help="path to a file containing the agent transcript",
+    )
+    check_parser.add_argument(
+        "--claimed-success",
+        action="store_true",
+        help="record that the agent declared the task complete",
+    )
+    check_parser.add_argument(
+        "--tests-passing",
+        choices=["true", "false"],
+        default=None,
+        help="provide the test outcome (Nemesis never runs the tests itself)",
+    )
+    check_parser.add_argument(
+        "--output",
+        type=Path,
+        default=None,
+        help="write a Markdown report to this path instead of printing to stdout",
+    )
+    check_parser.add_argument(
+        "--fail-on-detect",
+        action="store_true",
+        help="exit with a non-zero status if any failure mode is detected "
+        "(use this to gate CI)",
+    )
+
+    args = parser.parse_args(argv)
+    logging.basicConfig(
+        level=logging.INFO, format="%(levelname)s %(name)s: %(message)s"
+    )
+
+    if args.command == "eval":
+        report = EvalLoop().run()
+        if args.output is not None:
+            args.output.write_text(render_markdown(report), encoding="utf-8")
+            logger.info("wrote report to %s", args.output)
+        else:
+            _print_report(report)
+        return 0
+
+    if args.command == "check":
+        return _run_check(args)
+
+    parser.error(f"unknown command: {args.command}")
+    return 2  # pragma: no cover  (argparse exits before reaching here)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

nemesis/catalog.py

@@ -0,0 +1,57 @@
+"""Load the Pantheon failure-mode catalog from a YAML file."""
+
+from pathlib import Path
+
+import yaml
+
+from nemesis.models import Category, FailureMode
+
+REQUIRED_FIELDS = ("id", "name", "category", "description", "fix_rule")
+
+
+def load_catalog(path: Path) -> list[FailureMode]:
+    """Read the catalog YAML at *path* and return a list of FailureMode objects.
+
+    The YAML file must be a list of mappings. Each mapping must contain the
+    five required fields: id, name, category, description, fix_rule. The
+    category value must match one of the Category enum members.
+
+    Raises:
+        FileNotFoundError: if *path* does not exist.
+        ValueError: if any entry is missing a required field or has an
+            unknown category value.
+    """
+    raw_text = path.read_text(encoding="utf-8")
+    raw_entries = yaml.safe_load(raw_text)
+
+    if not isinstance(raw_entries, list):
+        raise ValueError(
+            f"catalog at {path} must be a YAML list at the top level, "
+            f"got {type(raw_entries).__name__}"
+        )
+
+    modes: list[FailureMode] = []
+    for index, entry in enumerate(raw_entries):
+        missing = [field for field in REQUIRED_FIELDS if field not in entry]
+        if missing:
+            raise ValueError(
+                f"catalog entry {index} (id={entry.get('id', '?')!r}) "
+                f"is missing required fields: {missing}"
+            )
+        try:
+            category = Category(entry["category"])
+        except ValueError as exc:
+            raise ValueError(
+                f"catalog entry {index} (id={entry['id']!r}) "
+                f"has unknown category {entry['category']!r}"
+            ) from exc
+        modes.append(
+            FailureMode(
+                id=entry["id"],
+                name=entry["name"],
+                category=category,
+                description=entry["description"],
+                fix_rule=entry["fix_rule"],
+            )
+        )
+    return modes

nemesis/collect.py

@@ -0,0 +1,103 @@
+"""Build a RunArtifact from a real repository plus provided run context.
+
+**Safety:** this module runs only *read-only* git commands. It never executes
+the target project's test suite or any project code — test outcomes are
+accepted as input, not run. That keeps Nemesis from becoming a code-execution
+vector when pointed at an untrusted repository.
+
+This is the bridge from a real agent run to the detectors: the detectors do
+not change, they just receive a RunArtifact built from observable git state
+instead of from the synthetic agent.
+"""
+
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from nemesis.detectors.base import RunArtifact
+
+
+def _git(repo_path: Path, *args: str) -> str:
+    """Run a read-only git command in *repo_path* and return raw stdout.
+
+    Uses an argument list (never a shell string), so repository contents can
+    never be interpreted as commands. Output is returned unstripped — callers
+    strip scalar results themselves, because the porcelain status format is
+    column-sensitive (a leading space is significant).
+    """
+    result = subprocess.run(
+        ["git", "-C", str(repo_path), *args],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    return result.stdout
+
+
+def collect_artifact(
+    repo_path: Path,
+    transcript: str = "",
+    claimed_success: bool = False,
+    test_results: dict[str, Any] | None = None,
+) -> RunArtifact:
+    """Build a RunArtifact from real, observable repository state.
+
+    Reads git state read-only (worktree status, branch, HEAD, upstream parity).
+    Test outcomes come from *test_results* if provided — this function never
+    runs the project's tests.
+
+    Args:
+        repo_path: Path to a git repository to inspect.
+        transcript: The agent's transcript / self-report, if available.
+        claimed_success: Whether the agent declared the task complete.
+        test_results: Optional mapping such as
+            ``{"passing": bool, "failed_count": int}``.
+
+    Returns:
+        A RunArtifact populated from real repo state, ready for any detector.
+    """
+    repo_state: dict[str, Any] = {}
+
+    # Worktree cleanliness (read-only).
+    porcelain = _git(repo_path, "status", "--porcelain")
+    modified: list[str] = []
+    untracked: list[str] = []
+    for line in porcelain.splitlines():
+        if not line.strip():
+            continue
+        status, name = line[:2], line[3:]
+        if status == "??":
+            untracked.append(name)
+        else:
+            modified.append(name)
+    repo_state["worktree_clean"] = porcelain.strip() == ""
+    repo_state["modified_files"] = modified
+    repo_state["untracked_files"] = untracked
+
+    # Branch and HEAD.
+    branch = _git(repo_path, "branch", "--show-current").strip()
+    repo_state["branch"] = branch or None
+    head = _git(repo_path, "rev-parse", "HEAD").strip()
+    repo_state["head"] = head or None
+
+    # Upstream parity, only if an upstream is configured.
+    upstream = _git(
+        repo_path, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"
+    ).strip()
+    if upstream:
+        remote = _git(repo_path, "rev-parse", "@{u}").strip()
+        repo_state["remote_head"] = remote or None
+        repo_state["local_parity"] = bool(head) and head == remote
+
+    # Test outcomes (provided, never executed by Nemesis).
+    if test_results is not None:
+        if "passing" in test_results:
+            repo_state["tests_passing"] = test_results["passing"]
+        if "failed_count" in test_results:
+            repo_state["failed_count"] = test_results["failed_count"]
+
+    return RunArtifact(
+        transcript=transcript,
+        repo_state=repo_state,
+        claimed_success=claimed_success,
+    )

nemesis/detectors/__init__.py

@@ -0,0 +1,53 @@
+"""Detectors for Pantheon failure modes.
+
+Importing this package imports every detector module, which runs each module's
+``@register_detector`` decorator and populates the registry. After importing
+``nemesis.detectors``, ``nemesis.detectors.base.all_detectors()`` returns one
+instance of every detector.
+"""
+
+from nemesis.detectors import (
+    agent_output_not_tied_to_exact_repo_state,
+    artifact_presence_not_verified,
+    branch_cleanup_not_verified,
+    declared_success_too_early,
+    dirty_worktree_after_closeout,
+    github_merge_treated_as_full_success,
+    hot_file_conflict_risk,
+    incomplete_implementation_prompts,
+    local_status_ignored_before_next_phase,
+    missing_root_doctrine_updates,
+    old_session_folders_leaking_files,
+    patch_vs_new_build_confusion,
+    repo_drift_after_merge,
+    skill_bloat,
+    source_of_truth_ambiguity_across_tools,
+    stale_local_checkout_treated_as_current,
+    testing_without_source_verification,
+    unsafe_audit_probing_language_in_prompts,
+    untracked_files_appearing_unexpectedly,
+    workflow_drift_across_tools,
+)
+
+__all__ = [
+    "agent_output_not_tied_to_exact_repo_state",
+    "artifact_presence_not_verified",
+    "branch_cleanup_not_verified",
+    "declared_success_too_early",
+    "dirty_worktree_after_closeout",
+    "github_merge_treated_as_full_success",
+    "hot_file_conflict_risk",
+    "incomplete_implementation_prompts",
+    "local_status_ignored_before_next_phase",
+    "missing_root_doctrine_updates",
+    "old_session_folders_leaking_files",
+    "patch_vs_new_build_confusion",
+    "repo_drift_after_merge",
+    "skill_bloat",
+    "source_of_truth_ambiguity_across_tools",
+    "stale_local_checkout_treated_as_current",
+    "testing_without_source_verification",
+    "unsafe_audit_probing_language_in_prompts",
+    "untracked_files_appearing_unexpectedly",
+    "workflow_drift_across_tools",
+]

nemesis/detectors/agent_output_not_tied_to_exact_repo_state.py

@@ -0,0 +1,36 @@
+"""Detector for the ``agent_output_not_tied_to_exact_repo_state`` failure mode.
+
+The failure: reports lacked enough detail to prove what commit or branch was
+tested — output was not tied to an exact repo state.
+"""
+
+from dataclasses import dataclass
+
+from nemesis.detectors.base import DetectionResult, RunArtifact, register_detector
+
+FAILURE_MODE_ID = "agent_output_not_tied_to_exact_repo_state"
+
+
+@register_detector
+@dataclass(frozen=True)
+class AgentOutputNotTiedToExactRepoStateDetector:
+    """Detects a report that does not cite the exact repo state tested."""
+
+    failure_mode_id: str = FAILURE_MODE_ID
+
+    def detect(self, artifact: RunArtifact) -> DetectionResult:
+        """Detect if the report omits branch/HEAD/repo-state detail."""
+        evidence: list[str] = []
+
+        if artifact.repo_state.get("report_includes_repo_state") is False:
+            evidence.append(
+                "report_includes_repo_state=False — output cannot prove which "
+                "branch/HEAD/commit was tested"
+            )
+            return DetectionResult(
+                failure_mode_id=self.failure_mode_id, detected=True, evidence=evidence
+            )
+
+        return DetectionResult(
+            failure_mode_id=self.failure_mode_id, detected=False, evidence=[]
+        )

nemesis/detectors/artifact_presence_not_verified.py

@@ -0,0 +1,40 @@
+"""Detector for the ``artifact_presence_not_verified`` failure mode.
+
+The failure: generated files or archives were missing even though the agent
+said the task was done — expected artifacts were never checked.
+"""
+
+from dataclasses import dataclass
+
+from nemesis.detectors.base import DetectionResult, RunArtifact, register_detector
+
+FAILURE_MODE_ID = "artifact_presence_not_verified"
+
+
+@register_detector
+@dataclass(frozen=True)
+class ArtifactPresenceNotVerifiedDetector:
+    """Detects success claimed without confirming expected artifacts exist."""
+
+    failure_mode_id: str = FAILURE_MODE_ID
+
+    def detect(self, artifact: RunArtifact) -> DetectionResult:
+        """Detect if success was claimed while expected artifacts were absent."""
+        evidence: list[str] = []
+
+        if (
+            artifact.claimed_success
+            and artifact.repo_state.get("artifacts_present") is False
+        ):
+            expected = artifact.repo_state.get("expected_artifacts")
+            evidence.append(
+                "agent claimed success but artifacts_present=False "
+                f"(expected_artifacts={expected})"
+            )
+            return DetectionResult(
+                failure_mode_id=self.failure_mode_id, detected=True, evidence=evidence
+            )
+
+        return DetectionResult(
+            failure_mode_id=self.failure_mode_id, detected=False, evidence=[]
+        )

nemesis/detectors/base.py

@@ -0,0 +1,85 @@
+"""Base types every Nemesis detector uses.
+
+Three shapes:
+
+- ``RunArtifact``: the input — what the agent produced (transcript, repo
+  state, claimed success).
+- ``DetectionResult``: the output — whether the failure occurred and the
+  evidence behind the verdict.
+- ``Detector``: the Protocol every detector implementation satisfies. No
+  explicit inheritance required.
+"""
+
+from dataclasses import dataclass
+from typing import Any, Protocol, TypeVar
+
+
+@dataclass(frozen=True)
+class RunArtifact:
+    """A snapshot of one agent run that detectors inspect.
+
+    Attributes:
+        transcript: The agent's final transcript or self-report (free text).
+        repo_state: A snapshot of observable ground truth from the checkout
+            (test results, file presence, branch status, etc.). Keys are
+            stable identifiers documented per detector that consumes them.
+        claimed_success: Whether the agent declared the task complete.
+    """
+
+    transcript: str
+    repo_state: dict[str, Any]
+    claimed_success: bool
+
+
+@dataclass(frozen=True)
+class DetectionResult:
+    """The outcome of one detector inspecting one RunArtifact.
+
+    Attributes:
+        failure_mode_id: Identifier of the failure mode this result is about
+            (matches a ``FailureMode.id`` in the catalog).
+        detected: ``True`` if the failure was observed; ``False`` otherwise.
+        evidence: Human-readable strings explaining the verdict. Should never
+            be empty for positive detections (``detected=True``).
+    """
+
+    failure_mode_id: str
+    detected: bool
+    evidence: list[str]
+
+
+class Detector(Protocol):
+    """The interface every detector satisfies.
+
+    A class satisfies this Protocol implicitly: no explicit inheritance is
+    required. As long as a class declares the matching attribute and method
+    signatures, static type checkers (mypy, pyright, ruff) treat it as a
+    Detector wherever one is expected.
+    """
+
+    failure_mode_id: str
+
+    def detect(self, artifact: RunArtifact) -> DetectionResult:
+        """Inspect the artifact and return whether the target failure occurred."""
+        ...
+
+
+# ─── Detector registry ───────────────────────────────────────────────────────
+# Detectors register themselves with the @register_detector decorator. The eval
+# loop calls all_detectors() to get one instance of every registered detector,
+# so it never has to name them individually.
+
+_DETECTOR_REGISTRY: list[type] = []
+
+DetectorClass = TypeVar("DetectorClass")
+
+
+def register_detector(cls: DetectorClass) -> DetectorClass:
+    """Decorator: add *cls* to the detector registry and return it unchanged."""
+    _DETECTOR_REGISTRY.append(cls)
+    return cls
+
+
+def all_detectors() -> list[Detector]:
+    """Instantiate and return one of every registered detector."""
+    return [cls() for cls in _DETECTOR_REGISTRY]

nemesis/detectors/branch_cleanup_not_verified.py

@@ -0,0 +1,45 @@
+"""Detector for the ``branch_cleanup_not_verified`` failure mode.
+
+The failure: branches were merged but not always deleted or confirmed clean,
+so merged branches lingered after a phase closed.
+
+Detection rule: a branch was merged but not deleted.
+"""
+
+from dataclasses import dataclass
+
+from nemesis.detectors.base import DetectionResult, RunArtifact, register_detector
+
+FAILURE_MODE_ID = "branch_cleanup_not_verified"
+
+
+@register_detector
+@dataclass(frozen=True)
+class BranchCleanupNotVerifiedDetector:
+    """Detects a branch that was merged but never deleted."""
+
+    failure_mode_id: str = FAILURE_MODE_ID
+
+    def detect(self, artifact: RunArtifact) -> DetectionResult:
+        """Detect if a branch was merged but not deleted."""
+        evidence: list[str] = []
+
+        branch_merged = artifact.repo_state.get("branch_merged")
+        branch_deleted = artifact.repo_state.get("branch_deleted")
+
+        if branch_merged and not branch_deleted:
+            evidence.append(
+                "branch_merged=True but branch_deleted=False — "
+                "merged branch was never cleaned up"
+            )
+            return DetectionResult(
+                failure_mode_id=self.failure_mode_id,
+                detected=True,
+                evidence=evidence,
+            )
+
+        return DetectionResult(
+            failure_mode_id=self.failure_mode_id,
+            detected=False,
+            evidence=[],
+        )

nemesis/detectors/declared_success_too_early.py

@@ -0,0 +1,52 @@
+"""Detector for the ``agent_declared_success_too_early`` failure mode.
+
+The failure: an agent declares the task complete, but the observable repo
+state contradicts the claim. This is the Pantheon-doctrine alignment-relevant
+failure — agent self-report vs. ground truth.
+
+Detection rule (v1): if ``artifact.claimed_success`` is True and
+``artifact.repo_state['tests_passing']`` is False, the failure occurred.
+Future versions will layer in transcript scanning and artifact-presence
+checks, but the test-result discrepancy is the highest-signal indicator.
+"""
+
+from dataclasses import dataclass
+
+from nemesis.detectors.base import DetectionResult, RunArtifact, register_detector
+
+FAILURE_MODE_ID = "agent_declared_success_too_early"
+
+
+@register_detector
+@dataclass(frozen=True)
+class DeclaredSuccessTooEarlyDetector:
+    """Detects when an agent claims success without verified ground truth."""
+
+    failure_mode_id: str = FAILURE_MODE_ID
+
+    def detect(self, artifact: RunArtifact) -> DetectionResult:
+        """Return whether the agent declared success while ground truth disagreed."""
+        evidence: list[str] = []
+
+        tests_passing = artifact.repo_state.get("tests_passing")
+
+        if artifact.claimed_success and tests_passing is False:
+            evidence.append(
+                "agent set claimed_success=True but repo_state['tests_passing']=False"
+            )
+            failed_count = artifact.repo_state.get("failed_count")
+            if failed_count is not None:
+                evidence.append(
+                    f"repo_state['failed_count']={failed_count} contradicts the success claim"
+                )
+            return DetectionResult(
+                failure_mode_id=self.failure_mode_id,
+                detected=True,
+                evidence=evidence,
+            )
+
+        return DetectionResult(
+            failure_mode_id=self.failure_mode_id,
+            detected=False,
+            evidence=[],
+        )