evolveguard 0.1.0__py3-none-any.whl

evolveguard/__init__.py

@@ -0,0 +1,93 @@
+"""
+Programmatic / agent-native entry point.
+
+    from evolveguard import record_baseline, replay_skill, diff_all, write_baseline, read_baseline
+
+    baseline = record_baseline("./SKILL.md", "./fixtures.json")
+    write_baseline("./.evolveguard-baseline.json", baseline)
+
+    # ... skill gets edited ...
+
+    saved = read_baseline("./.evolveguard-baseline.json")
+    replay = replay_skill("./SKILL.md", saved)
+    report = diff_all(saved, replay)
+
+Returns the same structured dataclasses the CLI formats for human/JSON
+output -- an agent framework can call this in-process instead of shelling
+out to the CLI. Same core record/replay/diff logic as the `evolveguard`
+console script; evolveguard/cli.py is a thin argument-parsing wrapper over
+these functions.
+
+This is the Python port of the evolveguard npm package
+(https://www.npmjs.com/package/evolveguard). Both distributions implement
+the same golden-transcript record/replay/diff pipeline against a skill's
+own declared and inferred capability surface; see
+https://github.com/RudrenduPaul/evolveguard for the canonical
+documentation, benchmarks, and the original TypeScript source.
+"""
+from .diff import diff_all, diff_fixture, diff_surface
+from .errors import EvolveGuardError, format_what_why_fix
+from .fixtures import load_fixtures
+from .parser.skillmd import (
+    derive_capability_surface,
+    infer_hook_evidence,
+    parse_skill_file,
+)
+from .paths import resolve_cli_path, resolve_within_base
+from .record import record_baseline
+from .replay import replay_skill
+from .report import read_baseline, read_report, write_baseline, write_report
+from .snapshot import build_fixture_snapshots, load_skill
+from .types import (
+    Baseline,
+    CapabilityChange,
+    CapabilityEntry,
+    DeclaredScope,
+    EvidenceRef,
+    EvolveGuardReport,
+    ExpectedToolCall,
+    Fixture,
+    FixtureDiff,
+    FixtureSnapshot,
+    ParsedSkillFile,
+    ReplayResult,
+    Summary,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "parse_skill_file",
+    "derive_capability_surface",
+    "infer_hook_evidence",
+    "load_skill",
+    "build_fixture_snapshots",
+    "load_fixtures",
+    "record_baseline",
+    "replay_skill",
+    "diff_fixture",
+    "diff_all",
+    "diff_surface",
+    "write_baseline",
+    "read_baseline",
+    "write_report",
+    "read_report",
+    "EvolveGuardError",
+    "format_what_why_fix",
+    "resolve_within_base",
+    "resolve_cli_path",
+    "Baseline",
+    "CapabilityChange",
+    "CapabilityEntry",
+    "DeclaredScope",
+    "EvidenceRef",
+    "EvolveGuardReport",
+    "ExpectedToolCall",
+    "Fixture",
+    "FixtureDiff",
+    "FixtureSnapshot",
+    "ParsedSkillFile",
+    "ReplayResult",
+    "Summary",
+    "__version__",
+]

evolveguard/cli.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""
+Thin argument-parsing wrapper over evolveguard.record/replay/diff/report
+(see those modules for the full record -> replay -> diff data flow). This
+module owns: flag parsing, the WHAT/WHY/FIX error surface for bad CLI
+input, and the process exit-code contract (0 all PASS, 1 at least one
+DRIFT, 2 usage/parse error). Console entry point: `evolveguard <command>
+[options]`, installed via the `evolveguard` console-script defined in
+python/pyproject.toml.
+
+Ported from src/evolveguard-cli/cli.ts (which uses `commander`); this port
+uses the stdlib `argparse` to avoid a CLI-framework dependency. Flags,
+defaults, subcommands, and the WHAT/WHY/FIX error text are kept identical
+to the npm CLI's `--help` output and behavior. argparse's own usage-error
+and --help/--version handling already raises SystemExit(2) / SystemExit(0)
+respectively, which happens to match this CLI's documented exit-code
+contract exactly, so run_cli() simply catches SystemExit around argument
+parsing and returns the code instead of letting the process exit.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import List, Optional
+
+from .diff import diff_all
+from .errors import EvolveGuardError, format_what_why_fix
+from .formatters import format_check_result, format_record_result, format_report
+from .record import record_baseline
+from .replay import replay_skill
+from .report import read_baseline, read_report, write_baseline, write_report
+
+_VERSION = "0.1.0"
+_DEFAULT_REPORT_PATH = "./evolveguard-report.json"
+
+_DESCRIPTION = (
+    "Regression-testing CLI for self-edited Claude Agent Skills (SKILL.md, "
+    "MEMORY.md) -- golden-transcript record/replay against a skill's own "
+    "declared and inferred capability surface, zero hosted infrastructure."
+)
+
+
+def _default_baseline_path(skill_path: str) -> str:
+    return os.path.join(os.path.dirname(os.path.abspath(skill_path)), ".evolveguard-baseline.json")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="evolveguard", description=_DESCRIPTION)
+    parser.add_argument(
+        "-V", "--version", action="version", version=f"evolveguard {_VERSION}"
+    )
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    record_parser = subparsers.add_parser(
+        "record",
+        help="Record a golden-transcript baseline for a skill against a set of labeled fixtures",
+    )
+    record_parser.add_argument(
+        "skill_path", metavar="skillPath", help="path to the SKILL.md or MEMORY.md file to baseline"
+    )
+    record_parser.add_argument(
+        "--fixtures",
+        required=True,
+        metavar="<path>",
+        help="path to a fixtures JSON file (array of {id, prompt, expectedToolCalls?})",
+    )
+    record_parser.add_argument(
+        "--baseline",
+        default=None,
+        metavar="<path>",
+        help="path to write the baseline file (default: <skill-dir>/.evolveguard-baseline.json)",
+    )
+    record_parser.add_argument(
+        "--json",
+        action="store_true",
+        default=False,
+        help="output structured JSON instead of human-readable text (default: false)",
+    )
+
+    check_parser = subparsers.add_parser(
+        "check",
+        help="Replay the fixtures from a baseline against the current (possibly edited) skill and report drift",
+    )
+    check_parser.add_argument(
+        "skill_path", metavar="skillPath", help="path to the SKILL.md or MEMORY.md file to check"
+    )
+    check_parser.add_argument(
+        "--baseline",
+        default=None,
+        metavar="<path>",
+        help="path to the baseline file (default: <skill-dir>/.evolveguard-baseline.json)",
+    )
+    check_parser.add_argument(
+        "--report", default=_DEFAULT_REPORT_PATH, metavar="<path>", help="path to write the report file"
+    )
+    check_parser.add_argument(
+        "--allow-drift",
+        action="store_true",
+        default=False,
+        help="exit 0 even if drift is detected (drift is still reported) (default: false)",
+    )
+    check_parser.add_argument(
+        "--json",
+        action="store_true",
+        default=False,
+        help="output structured JSON instead of human-readable text (default: false)",
+    )
+
+    report_parser = subparsers.add_parser(
+        "report", help="Print a previously generated evolveguard-report.json"
+    )
+    report_parser.add_argument(
+        "report_path",
+        metavar="reportPath",
+        nargs="?",
+        default=_DEFAULT_REPORT_PATH,
+        help="path to the report file",
+    )
+    report_parser.add_argument(
+        "--json",
+        action="store_true",
+        default=False,
+        help="output structured JSON instead of human-readable text (default: false)",
+    )
+
+    subparsers.add_parser(
+        "mcp",
+        help="[coming soon] Expose record/check/report as MCP tools for a coding agent to call mid-session",
+    )
+
+    return parser
+
+
+def _handle_error(err: Exception) -> int:
+    if isinstance(err, EvolveGuardError):
+        sys.stderr.write(str(err) + "\n")
+        return 2
+    sys.stderr.write(
+        format_what_why_fix(
+            "evolveguard crashed unexpectedly.",
+            str(err),
+            "Please open an issue at https://github.com/RudrenduPaul/evolveguard/issues "
+            "with the command you ran.",
+        )
+        + "\n"
+    )
+    return 2
+
+
+def _run_record(args: argparse.Namespace) -> int:
+    try:
+        baseline = record_baseline(args.skill_path, args.fixtures)
+        baseline_path = args.baseline or _default_baseline_path(args.skill_path)
+        write_baseline(baseline_path, baseline)
+
+        if args.json:
+            sys.stdout.write(
+                json.dumps(
+                    {"baseline": baseline.to_dict(), "baselinePath": baseline_path}, indent=2
+                )
+                + "\n"
+            )
+        else:
+            sys.stdout.write(format_record_result(baseline, baseline_path) + "\n")
+        return 0
+    except Exception as err:  # noqa: BLE001 -- top-level command error guard, mirrors src/cli.ts
+        return _handle_error(err)
+
+
+def _run_check(args: argparse.Namespace) -> int:
+    try:
+        baseline_path = args.baseline or _default_baseline_path(args.skill_path)
+        baseline = read_baseline(baseline_path)
+        replay = replay_skill(args.skill_path, baseline)
+        report = diff_all(baseline, replay, allow_drift=args.allow_drift)
+        write_report(args.report, report)
+
+        if args.json:
+            sys.stdout.write(json.dumps(report.to_dict(), indent=2) + "\n")
+        else:
+            sys.stdout.write(format_check_result(report, baseline.recorded_at) + "\n")
+        return report.exit_code
+    except Exception as err:  # noqa: BLE001
+        return _handle_error(err)
+
+
+def _run_report(args: argparse.Namespace) -> int:
+    try:
+        report = read_report(args.report_path)
+        if args.json:
+            sys.stdout.write(json.dumps(report.to_dict(), indent=2) + "\n")
+        else:
+            sys.stdout.write(format_report(report) + "\n")
+        return report.exit_code
+    except Exception as err:  # noqa: BLE001
+        return _handle_error(err)
+
+
+def run_cli(argv: List[str]) -> int:
+    """
+    `argv` follows the sys.argv convention: argv[0] is the program name,
+    the real arguments start at argv[1]. Returns the process exit code.
+    """
+    parser = build_parser()
+    try:
+        args = parser.parse_args(argv[1:])
+    except SystemExit as exc:
+        code = exc.code
+        return code if isinstance(code, int) else (0 if code is None else 2)
+
+    if args.command is None:
+        parser.print_help()
+        return 0
+    if args.command == "record":
+        return _run_record(args)
+    if args.command == "check":
+        return _run_check(args)
+    if args.command == "report":
+        return _run_report(args)
+    if args.command == "mcp":
+        sys.stdout.write(
+            "evolveguard mcp is not implemented yet. Use `evolveguard record`/`check`/"
+            "`report --json` directly from an agent for now.\n"
+        )
+        return 0
+
+    parser.print_help()
+    return 2
+
+
+def main() -> None:
+    try:
+        code = run_cli(sys.argv)
+    except SystemExit:
+        raise
+    except Exception as err:  # noqa: BLE001 -- top-level crash guard, mirrors src/cli.ts's catch-all
+        sys.stderr.write(
+            format_what_why_fix(
+                "evolveguard crashed unexpectedly.",
+                str(err),
+                "Please open an issue at https://github.com/RudrenduPaul/evolveguard/issues "
+                "with the command you ran.",
+            )
+            + "\n"
+        )
+        sys.exit(2)
+    else:
+        sys.exit(code)
+
+
+if __name__ == "__main__":
+    main()

evolveguard/diff/__init__.py

@@ -0,0 +1,200 @@
+"""
+Compares baseline vs. replayed tool-call sequences and classifies the
+result. PASS = identical tool set and scopes. DRIFT = a tool appeared that
+wasn't there before, a tool disappeared, or a tool's scope changed -- each
+surfaced as a specific, cited change, not an unexplained failure.
+
+Ported from src/evolveguard/diff/index.ts.
+"""
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from ..types import (
+    Baseline,
+    CapabilityChange,
+    CapabilityEntry,
+    EvolveGuardReport,
+    FixtureDiff,
+    FixtureSnapshot,
+    ReplayResult,
+    Summary,
+)
+
+
+def diff_fixture(
+    baseline_fixture: FixtureSnapshot, replayed_fixture: FixtureSnapshot
+) -> FixtureDiff:
+    changes: List[CapabilityChange] = []
+
+    baseline_by_tool: Dict[str, CapabilityEntry] = {
+        e.tool: e for e in baseline_fixture.tool_call_sequence
+    }
+    replayed_by_tool: Dict[str, CapabilityEntry] = {
+        e.tool: e for e in replayed_fixture.tool_call_sequence
+    }
+
+    for tool, entry in replayed_by_tool.items():
+        if tool not in baseline_by_tool:
+            changes.append(
+                CapabilityChange(
+                    kind="added",
+                    tool=tool,
+                    new_scope=entry.scope,
+                    message=(
+                        f"new tool call: {tool} (baseline had none) -- this edit "
+                        "introduces a capability the baseline never used"
+                    ),
+                )
+            )
+
+    for tool, entry in baseline_by_tool.items():
+        if tool not in replayed_by_tool:
+            changes.append(
+                CapabilityChange(
+                    kind="removed",
+                    tool=tool,
+                    baseline_scope=entry.scope,
+                    message=(
+                        f"tool call removed: {tool} (baseline required this) -- this "
+                        "edit dropped a capability the baseline relied on"
+                    ),
+                )
+            )
+
+    for tool, baseline_entry in baseline_by_tool.items():
+        replayed_entry = replayed_by_tool.get(tool)
+        if replayed_entry is None:
+            continue
+        if (
+            baseline_entry.scope
+            and replayed_entry.scope
+            and baseline_entry.scope != replayed_entry.scope
+        ):
+            changes.append(
+                CapabilityChange(
+                    kind="scope-changed",
+                    tool=tool,
+                    baseline_scope=baseline_entry.scope,
+                    new_scope=replayed_entry.scope,
+                    message=(
+                        f'scope changed for {tool}: baseline "{baseline_entry.scope}" '
+                        f'-> now "{replayed_entry.scope}"'
+                    ),
+                )
+            )
+
+    return FixtureDiff(
+        id=baseline_fixture.id,
+        prompt=baseline_fixture.prompt,
+        verdict="PASS" if not changes else "DRIFT",
+        changes=changes,
+    )
+
+
+def diff_surface(
+    baseline_surface: List[CapabilityEntry], replay_surface: List[CapabilityEntry]
+) -> List[CapabilityChange]:
+    """
+    Diffs the skill's full capability surface (declared + inferred, across
+    the whole skill file) independent of any fixture. This is what catches a
+    new capability that no fixture's expected_tool_calls anticipated -- a
+    per-fixture diff alone can only ever compare the tools that fixture
+    declared it cares about.
+    """
+    changes: List[CapabilityChange] = []
+
+    baseline_by_tool: Dict[str, CapabilityEntry] = {e.tool: e for e in baseline_surface}
+    replayed_by_tool: Dict[str, CapabilityEntry] = {e.tool: e for e in replay_surface}
+
+    for tool, entry in replayed_by_tool.items():
+        if tool not in baseline_by_tool:
+            changes.append(
+                CapabilityChange(
+                    kind="added",
+                    tool=tool,
+                    new_scope=entry.scope,
+                    message=(
+                        f"new capability on the skill surface: {tool} (baseline had "
+                        "none) -- no fixture's expectedToolCalls covers this tool, so "
+                        "it would otherwise go unnoticed"
+                    ),
+                )
+            )
+
+    for tool, entry in baseline_by_tool.items():
+        if tool not in replayed_by_tool:
+            changes.append(
+                CapabilityChange(
+                    kind="removed",
+                    tool=tool,
+                    baseline_scope=entry.scope,
+                    message=(
+                        f"capability removed from the skill surface: {tool} (baseline "
+                        "required this) -- this edit dropped a capability the "
+                        "baseline relied on"
+                    ),
+                )
+            )
+
+    return changes
+
+
+def diff_all(
+    baseline: Baseline, replay: ReplayResult, allow_drift: bool = False
+) -> EvolveGuardReport:
+    """Diffs every fixture in a baseline against its replayed counterpart and builds the full report."""
+    replayed_by_id = {f.id: f for f in replay.fixtures}
+
+    results: List[FixtureDiff] = []
+    for baseline_fixture in baseline.fixtures:
+        replayed_fixture = replayed_by_id.get(baseline_fixture.id)
+        if replayed_fixture is None:
+            results.append(
+                FixtureDiff(
+                    id=baseline_fixture.id,
+                    prompt=baseline_fixture.prompt,
+                    verdict="DRIFT",
+                    changes=[
+                        CapabilityChange(
+                            kind="removed",
+                            tool="(fixture missing)",
+                            message=(
+                                f'fixture "{baseline_fixture.id}" is missing from the '
+                                "replay -- the baseline is stale or corrupted"
+                            ),
+                        )
+                    ],
+                )
+            )
+            continue
+        results.append(diff_fixture(baseline_fixture, replayed_fixture))
+
+    # A capability change already surfaced through a per-fixture diff (above)
+    # is not repeated here -- surface_changes exists specifically to catch a
+    # change no fixture's expected_tool_calls covers, so entries for tools
+    # any fixture already flagged are filtered out to avoid reporting the
+    # same change twice.
+    tools_covered_by_fixtures = {c.tool for r in results for c in r.changes}
+    surface_changes = [
+        c
+        for c in diff_surface(baseline.full_capability_surface, replay.full_capability_surface)
+        if c.tool not in tools_covered_by_fixtures
+    ]
+
+    pass_count = sum(1 for r in results if r.verdict == "PASS")
+    drift = sum(1 for r in results if r.verdict == "DRIFT")
+
+    any_drift = drift > 0 or len(surface_changes) > 0
+    exit_code = 1 if (any_drift and not allow_drift) else 0
+
+    return EvolveGuardReport(
+        schema_version=1,
+        skill_name=replay.skill_name,
+        skill_path=replay.skill_path,
+        checked_at=replay.replayed_at,
+        results=results,
+        surface_changes=surface_changes,
+        summary=Summary(pass_count=pass_count, drift=drift, total=len(results)),
+        exit_code=exit_code,
+    )

evolveguard/errors.py

@@ -0,0 +1,17 @@
+"""
+Shared WHAT/WHY/FIX error formatting for EvolveGuardError messages. Ported
+verbatim from src/evolveguard/errors.ts.
+"""
+from __future__ import annotations
+
+
+def format_what_why_fix(what: str, why: str, fix: str) -> str:
+    return f"WHAT: {what}\nWHY:  {why}\nFIX:  {fix}"
+
+
+class EvolveGuardError(Exception):
+    def __init__(self, what: str, why: str, fix: str) -> None:
+        super().__init__(format_what_why_fix(what, why, fix))
+        self.what = what
+        self.why = why
+        self.fix = fix

evolveguard/fixtures.py

@@ -0,0 +1,106 @@
+"""
+Reads and validates a user-supplied fixtures.json against the documented
+schema: a non-empty JSON array of
+{"id": string, "prompt": string, "expectedToolCalls"?: [{"tool": string, "scopeMatches"?: string}]}.
+Ported from src/evolveguard/fixtures.ts (which uses zod; this port validates
+by hand to avoid a schema-validation dependency).
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, List
+
+from .errors import EvolveGuardError
+from .types import ExpectedToolCall, Fixture
+
+_SCHEMA_HINT = (
+    'A fixtures file is a non-empty JSON array of {"id": string, "prompt": string, '
+    '"expectedToolCalls"?: [{"tool": string, "scopeMatches"?: string}]}.'
+)
+
+
+def _schema_error(fixtures_path: str, reason: str) -> EvolveGuardError:
+    return EvolveGuardError(
+        f'Fixtures file at "{fixtures_path}" does not match the expected schema.',
+        reason,
+        _SCHEMA_HINT,
+    )
+
+
+def _parse_expected_tool_call(raw: Any, index: int) -> ExpectedToolCall:
+    if not isinstance(raw, dict):
+        raise ValueError(f"expectedToolCalls[{index}]: expected an object")
+    tool = raw.get("tool")
+    if not isinstance(tool, str) or not tool:
+        raise ValueError(f"expectedToolCalls[{index}].tool: expected a non-empty string")
+    scope_matches = raw.get("scopeMatches")
+    if scope_matches is not None and not isinstance(scope_matches, str):
+        raise ValueError(f"expectedToolCalls[{index}].scopeMatches: expected a string")
+    return ExpectedToolCall(tool=tool, scope_matches=scope_matches)
+
+
+def _parse_fixture(raw: Any, index: int) -> Fixture:
+    if not isinstance(raw, dict):
+        raise ValueError(f"[{index}]: expected an object")
+    fid = raw.get("id")
+    if not isinstance(fid, str) or not fid:
+        raise ValueError(f"[{index}].id: expected a non-empty string")
+    prompt = raw.get("prompt")
+    if not isinstance(prompt, str) or not prompt:
+        raise ValueError(f"[{index}].prompt: expected a non-empty string")
+
+    raw_expected = raw.get("expectedToolCalls")
+    if raw_expected is None:
+        expected = None
+    else:
+        if not isinstance(raw_expected, list):
+            raise ValueError(f"[{index}].expectedToolCalls: expected an array")
+        expected = [
+            _parse_expected_tool_call(item, i) for i, item in enumerate(raw_expected)
+        ]
+
+    return Fixture(id=fid, prompt=prompt, expected_tool_calls=expected)
+
+
+def load_fixtures(fixtures_path: str) -> List[Fixture]:
+    try:
+        with open(fixtures_path, "r", encoding="utf-8") as fh:
+            raw_text = fh.read()
+    except OSError as err:
+        raise EvolveGuardError(
+            f'Could not read fixtures file at "{fixtures_path}".',
+            str(err),
+            "Pass a valid path to a fixtures JSON file with --fixtures.",
+        ) from err
+
+    try:
+        parsed = json.loads(raw_text)
+    except json.JSONDecodeError as err:
+        raise EvolveGuardError(
+            f'Fixtures file at "{fixtures_path}" is not valid JSON.',
+            str(err),
+            "Fix the JSON syntax, e.g. run it through `python3 -m json.tool <file>`.",
+        ) from err
+
+    if not isinstance(parsed, list) or len(parsed) == 0:
+        raise _schema_error(
+            fixtures_path, "expected a non-empty array, got " + type(parsed).__name__
+        )
+
+    try:
+        fixtures = [_parse_fixture(item, i) for i, item in enumerate(parsed)]
+    except ValueError as err:
+        raise _schema_error(fixtures_path, str(err)) from err
+
+    ids = set()
+    for fixture in fixtures:
+        if fixture.id in ids:
+            raise EvolveGuardError(
+                f'Duplicate fixture id "{fixture.id}" in "{fixtures_path}".',
+                "Fixture ids must be unique within a fixtures file -- they are used to "
+                "match baseline entries on replay.",
+                "Give each fixture a unique id.",
+            )
+        ids.add(fixture.id)
+
+    return fixtures