evolveguard 0.1.0__py3-none-any.whl

evolveguard/formatters.py

@@ -0,0 +1,90 @@
+"""
+Human-readable output formatting for the CLI. Ported from
+src/evolveguard-cli/formatters.ts.
+"""
+from __future__ import annotations
+
+from typing import List
+
+from .types import Baseline, CapabilityChange, EvolveGuardReport
+
+_VERSION = "0.1.0"
+
+
+def format_record_result(baseline: Baseline, baseline_path: str) -> str:
+    lines: List[str] = []
+    lines.append(f"EvolveGuard v{_VERSION} -- Baseline Recorded")
+    lines.append(f"skill: {baseline.skill_name}  fixtures: {len(baseline.fixtures)}")
+    lines.append("")
+    for fixture in baseline.fixtures:
+        tool_names = ", ".join(e.tool for e in fixture.tool_call_sequence) or (
+            "(no capabilities detected)"
+        )
+        lines.append(f'  recorded  fixture: "{fixture.prompt}"  tools: {tool_names}')
+    lines.append("")
+    lines.append(f"baseline written to {baseline_path}")
+    return "\n".join(lines)
+
+
+def _summarize_change(change: CapabilityChange) -> str:
+    if change.kind == "added":
+        return f"new tool call: {change.tool} (baseline had none)"
+    if change.kind == "removed":
+        return f"tool call removed: {change.tool}"
+    return f"scope changed: {change.tool}"
+
+
+def format_check_result(report: EvolveGuardReport, baseline_recorded_at: str) -> str:
+    lines: List[str] = []
+    date = baseline_recorded_at[:10]
+    lines.append(f"EvolveGuard v{_VERSION} -- Regression Check")
+    lines.append(
+        f"skill: {report.skill_name}  baseline: {date}  fixtures: {report.summary.total}"
+    )
+    lines.append("")
+
+    for result in report.results:
+        tag = "[PASS] " if result.verdict == "PASS" else "[DRIFT]"
+        if result.verdict == "PASS":
+            lines.append(f'{tag} fixture: "{result.prompt}"  tool-call sequence unchanged')
+        else:
+            first = result.changes[0] if result.changes else None
+            summary = _summarize_change(first) if first else "behavior changed"
+            lines.append(f'{tag} fixture: "{result.prompt}"  {summary}')
+            for change in result.changes:
+                lines.append(f"         -> {change.message}")
+
+    if report.surface_changes:
+        lines.append("")
+        lines.append("skill-level surface changes (not tied to a specific fixture):")
+        for change in report.surface_changes:
+            lines.append(f"  -> {change.message}")
+
+    lines.append("")
+    lines.append(f"{report.summary.pass_count} PASS, {report.summary.drift} DRIFT, 0 FAIL")
+    if report.exit_code == 1:
+        lines.append("exit code 1 (DRIFT blocks merge by default; override with --allow-drift)")
+    else:
+        lines.append("exit code 0")
+
+    return "\n".join(lines)
+
+
+def format_report(report: EvolveGuardReport) -> str:
+    lines: List[str] = []
+    lines.append(f"EvolveGuard v{_VERSION} -- Report")
+    lines.append(f"skill: {report.skill_name}  checked: {report.checked_at}")
+    lines.append("")
+    for result in report.results:
+        lines.append(f"[{result.verdict}] {result.id}: {result.prompt}")
+        for change in result.changes:
+            lines.append(f"  -> {change.message}")
+    if report.surface_changes:
+        lines.append("")
+        lines.append("skill-level surface changes:")
+        for change in report.surface_changes:
+            lines.append(f"  -> {change.message}")
+    lines.append("")
+    lines.append(f"{report.summary.pass_count} PASS, {report.summary.drift} DRIFT, 0 FAIL")
+    lines.append(f"exit code {report.exit_code}")
+    return "\n".join(lines)

evolveguard/parser/skillmd.py

@@ -0,0 +1,249 @@
+"""
+Parses a Claude Agent Skill file (SKILL.md) or an auto-memory file
+(MEMORY.md). SKILL.md files carry YAML frontmatter declaring the skill's
+name and scope; MEMORY.md files typically do not, so a file with no
+frontmatter is treated as declaring an empty scope, and its capability
+surface is derived entirely from static evidence found in its own body
+text (see derive_capability_surface below).
+
+Frontmatter schema this parser understands:
+  ---
+  name: my-skill
+  description: ...
+  tools: [fs.read, fs.write]     # declared tool names, optional
+  network: false                  # boolean, optional (default false)
+  filesystem: read-only           # "none" | "read-only" | "read-write", optional (default none)
+  scope: "./workspace/**"         # glob the filesystem tools are scoped to, optional (default "./**")
+  hooks: ["scripts/pre-run.sh"]   # bundled hook scripts, paths relative to the skill file, optional
+  ---
+
+This schema is a superset compatible with SkillGuard's own DeclaredScope
+(network: boolean, filesystem: none|read-only|read-write) -- EvolveGuard
+reuses the same field names and parsing pattern deliberately so a skill
+author only has to think about one frontmatter shape across both tools.
+
+Ported from src/evolveguard/parser/skillmd.ts.
+"""
+from __future__ import annotations
+
+import os
+import re
+from typing import Dict, List, Optional, Tuple
+
+import yaml
+
+from ..paths import resolve_within_base
+from ..types import CapabilityEntry, DeclaredScope, EvidenceRef, ParsedSkillFile
+
+_FRONTMATTER_RE = re.compile(r"^---\r?\n([\s\S]*?)\r?\n---")
+
+_KNOWN_FILESYSTEM_VALUES = {"none", "read-only", "read-write"}
+
+
+def parse_skill_file(content: str, file_base_name: str) -> ParsedSkillFile:
+    match = _FRONTMATTER_RE.match(content)
+
+    def base_name() -> str:
+        return os.path.splitext(os.path.basename(file_base_name))[0]
+
+    if not match:
+        return ParsedSkillFile(
+            name=base_name(),
+            has_frontmatter=False,
+            declared_scope=DeclaredScope(
+                tools=[], network=False, filesystem="none", scope="./**", hooks=[]
+            ),
+            body=content,
+        )
+
+    try:
+        data = yaml.safe_load(match.group(1))
+    except yaml.YAMLError:
+        data = None
+
+    record = data if isinstance(data, dict) else {}
+
+    raw_name = record.get("name")
+    name = (
+        raw_name.strip()
+        if isinstance(raw_name, str) and raw_name.strip()
+        else base_name()
+    )
+
+    description = record.get("description")
+    if not isinstance(description, str):
+        description = None
+
+    raw_tools = record.get("tools")
+    tools = [t for t in raw_tools if isinstance(t, str)] if isinstance(raw_tools, list) else []
+
+    network = record.get("network") is True
+
+    filesystem_raw = record.get("filesystem")
+    filesystem = (
+        filesystem_raw
+        if isinstance(filesystem_raw, str) and filesystem_raw in _KNOWN_FILESYSTEM_VALUES
+        else "none"
+    )
+
+    raw_scope = record.get("scope")
+    scope = raw_scope if isinstance(raw_scope, str) and raw_scope.strip() else "./**"
+
+    raw_hooks = record.get("hooks")
+    hooks = [h for h in raw_hooks if isinstance(h, str)] if isinstance(raw_hooks, list) else []
+
+    body = content[match.end() :]
+
+    return ParsedSkillFile(
+        name=name,
+        description=description,
+        has_frontmatter=True,
+        declared_scope=DeclaredScope(
+            tools=tools, network=network, filesystem=filesystem, scope=scope, hooks=hooks
+        ),
+        body=body,
+    )
+
+
+# Note: `curl`/`wget` deliberately have no trailing `\s` baked into the
+# alternation -- `\b` after "curl" already requires a non-word character to
+# follow (a space, a flag like "-X", end of string, all count), so
+# consuming the whitespace into the match would break that boundary check
+# whenever the very next character after the space is itself non-word (e.g.
+# "curl -X ..."), silently missing real command lines.
+_NETWORK_EVIDENCE_RE = re.compile(
+    r"\b(fetch|axios|http\.request|https\.request|requests\.(get|post|put|delete)"
+    r"|urllib\.request|urlopen|socket\.socket|curl|wget)\b",
+    re.IGNORECASE,
+)
+
+_FS_WRITE_EVIDENCE_RE = re.compile(
+    r"""\b(fs\.writeFile|fs\.writeFileSync|fs\.appendFile|fs\.unlink|open\([^)]*['"]w"""
+    r"""|os\.remove|os\.rmdir|shutil\.rmtree)\b|>\s*/|rm\s+-rf""",
+    re.IGNORECASE,
+)
+
+
+def _first_match_line(content: str, pattern: re.Pattern) -> Optional[int]:
+    m = pattern.search(content)
+    if not m:
+        return None
+    return content.count("\n", 0, m.start()) + 1
+
+
+def infer_hook_evidence(
+    skill_dir: str, hooks: List[str]
+) -> Dict[str, List[Dict[str, object]]]:
+    """
+    Reads a skill's declared hook scripts (paths validated against the
+    skill's own directory -- see paths.py) and scans each for static
+    evidence of network or filesystem-write behavior. This is the same
+    read-only, regex-based evidence model SkillGuard uses; EvolveGuard
+    never executes anything from the skill or its hooks.
+    """
+    network_evidence: List[Dict[str, object]] = []
+    fs_write_evidence: List[Dict[str, object]] = []
+
+    for hook_path in hooks:
+        try:
+            abs_path = resolve_within_base(skill_dir, hook_path)
+        except Exception:
+            continue
+
+        try:
+            with open(abs_path, "r", encoding="utf-8") as fh:
+                content = fh.read()
+        except OSError:
+            continue
+
+        net_line = _first_match_line(content, _NETWORK_EVIDENCE_RE)
+        if net_line is not None:
+            network_evidence.append({"file": hook_path, "line": net_line})
+
+        fs_line = _first_match_line(content, _FS_WRITE_EVIDENCE_RE)
+        if fs_line is not None:
+            fs_write_evidence.append({"file": hook_path, "line": fs_line})
+
+    return {"networkEvidence": network_evidence, "fsWriteEvidence": fs_write_evidence}
+
+
+def derive_capability_surface(
+    parsed: ParsedSkillFile, skill_dir: str
+) -> List[CapabilityEntry]:
+    """
+    Derives the full capability surface for a parsed skill: declared entries
+    from frontmatter, plus inferred entries from static evidence in the
+    skill's own body text and any bundled hook scripts. This is the
+    deterministic "transcript baseline" EvolveGuard records and later
+    re-derives on replay -- see README "How it works" for why this is not a
+    live LLM run.
+    """
+    entries: List[CapabilityEntry] = []
+    seen = set()
+
+    def add_declared(tool: str, scope: Optional[str] = None) -> None:
+        key = f"declared:{tool}"
+        if key in seen:
+            return
+        seen.add(key)
+        entries.append(CapabilityEntry(tool=tool, source="declared", scope=scope))
+
+    for tool in parsed.declared_scope.tools:
+        add_declared(tool)
+
+    if parsed.declared_scope.network:
+        add_declared("network.fetch")
+
+    if parsed.declared_scope.filesystem in ("read-only", "read-write"):
+        add_declared("fs.read", parsed.declared_scope.scope)
+    if parsed.declared_scope.filesystem == "read-write":
+        add_declared("fs.write", parsed.declared_scope.scope)
+
+    # Inferred evidence from the skill body itself (covers MEMORY.md, which has no frontmatter).
+    body_net_line = _first_match_line(parsed.body, _NETWORK_EVIDENCE_RE)
+    body_fs_line = _first_match_line(parsed.body, _FS_WRITE_EVIDENCE_RE)
+
+    inferred_network_evidence: List[Dict[str, object]] = (
+        [{"file": "(body)", "line": body_net_line}] if body_net_line is not None else []
+    )
+    inferred_fs_evidence: List[Dict[str, object]] = (
+        [{"file": "(body)", "line": body_fs_line}] if body_fs_line is not None else []
+    )
+
+    # Inferred evidence from bundled hook scripts declared in frontmatter.
+    hook_evidence = infer_hook_evidence(skill_dir, parsed.declared_scope.hooks)
+    inferred_network_evidence.extend(hook_evidence["networkEvidence"])
+    inferred_fs_evidence.extend(hook_evidence["fsWriteEvidence"])
+
+    def add_inferred(tool: str, evidence: List[Dict[str, object]]) -> None:
+        if not evidence:
+            return
+        key = f"inferred:{tool}"
+        if key in seen:
+            existing = next(
+                (e for e in entries if e.source == "inferred" and e.tool == tool), None
+            )
+            if existing is not None:
+                existing.evidence = (existing.evidence or []) + [
+                    EvidenceRef(file=str(e["file"]), line=int(e["line"])) for e in evidence
+                ]
+            return
+        # If this tool is already declared, do not duplicate it as a separate inferred entry --
+        # declared coverage takes precedence and the inferred evidence is redundant confirmation.
+        if f"declared:{tool}" in seen:
+            return
+        seen.add(key)
+        entries.append(
+            CapabilityEntry(
+                tool=tool,
+                source="inferred",
+                evidence=[
+                    EvidenceRef(file=str(e["file"]), line=int(e["line"])) for e in evidence
+                ],
+            )
+        )
+
+    add_inferred("network.fetch", inferred_network_evidence)
+    add_inferred("fs.write", inferred_fs_evidence)
+
+    return entries

evolveguard/paths.py

@@ -0,0 +1,78 @@
+"""
+Resolves a user-supplied path (SKILL.md, fixtures.json, a hook script
+referenced from frontmatter) and rejects anything that escapes the
+expected base directory. This is the one place path-traversal protection
+lives -- every module that reads a file supplied indirectly through
+frontmatter (hook script paths) routes through this function rather than
+concatenating strings and calling open() directly.
+
+A skill file's own frontmatter is untrusted input (it may itself be the
+edit under test), so `hooks: ["../../../etc/passwd"]` must never resolve
+outside the skill's own directory.
+
+The lexical check alone is not enough: a path that looks safe
+("hooks/pre.sh") can itself be a symlink whose real target lives outside
+the skill directory, and open() follows symlinks transparently. So after
+the lexical check passes, this also resolves symlinks (via
+os.path.realpath) on both the base and the target and re-checks the same
+containment invariant against the real paths -- a symlink escape fails
+exactly like a literal "../" escape. A target that does not exist yet
+(e.g. a hook path being validated before the file is created) skips the
+realpath re-check rather than erroring; the caller's own file read will
+fail on it as before.
+
+Ported from src/evolveguard/paths.ts.
+"""
+from __future__ import annotations
+
+import os
+
+from .errors import EvolveGuardError
+
+
+def _resolve(base: str, target: str) -> str:
+    combined = target if os.path.isabs(target) else os.path.join(base, target)
+    return os.path.normpath(combined)
+
+
+def _escapes_base(resolved_base: str, resolved_target: str) -> bool:
+    relative = os.path.relpath(resolved_target, resolved_base)
+    return relative == ".." or relative.startswith(".." + os.sep)
+
+
+def _outside_base_error(user_path: str, resolved_base: str) -> EvolveGuardError:
+    return EvolveGuardError(
+        f'Path "{user_path}" resolves outside its skill directory.',
+        f"Hook and reference paths declared in a skill file must stay within "
+        f"the skill's own directory ({resolved_base}) -- this prevents a "
+        f"malicious or accidentally-broken skill file from making EvolveGuard "
+        f"read files outside its scope.",
+        "Use a path relative to the skill file that stays inside its own "
+        "directory, and make sure it is not a symlink pointing elsewhere.",
+    )
+
+
+def resolve_within_base(base_dir: str, user_path: str) -> str:
+    resolved_base = os.path.abspath(base_dir)
+    resolved_target = _resolve(resolved_base, user_path)
+
+    if _escapes_base(resolved_base, resolved_target):
+        raise _outside_base_error(user_path, resolved_base)
+
+    if not os.path.exists(resolved_base) or not os.path.exists(resolved_target):
+        # Base or target doesn't exist on disk (yet) -- nothing to follow, the
+        # lexical check above already stands as the answer for this call.
+        return resolved_target
+
+    real_base = os.path.realpath(resolved_base)
+    real_target = os.path.realpath(resolved_target)
+
+    if _escapes_base(real_base, real_target):
+        raise _outside_base_error(user_path, resolved_base)
+
+    return resolved_target
+
+
+def resolve_cli_path(cwd: str, user_path: str) -> str:
+    """Resolves and validates a top-level CLI-supplied path (SKILL.md, fixtures.json, baseline, report)."""
+    return _resolve(os.path.abspath(cwd), user_path)

evolveguard/record/__init__.py

@@ -0,0 +1,30 @@
+"""
+Captures a golden-transcript baseline for a skill: parses the skill file,
+derives its declared + inferred capability surface, and snapshots each
+fixture's expected tool-call sequence against that surface. This never
+invokes a live agent -- see README "How it works".
+
+Ported from src/evolveguard/record/index.ts.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from ..fixtures import load_fixtures
+from ..snapshot import build_fixture_snapshots, load_skill
+from ..types import Baseline
+
+
+def record_baseline(skill_path: str, fixtures_path: str) -> Baseline:
+    skill = load_skill(skill_path)
+    fixtures = load_fixtures(fixtures_path)
+    fixture_snapshots = build_fixture_snapshots(skill.capability_surface, fixtures)
+
+    return Baseline(
+        schema_version=1,
+        skill_name=skill.name,
+        skill_path=skill.skill_path,
+        recorded_at=datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+        full_capability_surface=skill.capability_surface,
+        fixtures=fixture_snapshots,
+    )

evolveguard/replay/__init__.py

@@ -0,0 +1,35 @@
+"""
+Re-parses the (possibly edited) skill file and re-derives its capability
+surface using the exact same deterministic logic `record` used, then
+re-snapshots each fixture from the baseline against the new surface. The
+fixture list is taken from the baseline itself (not re-read from an
+external fixtures.json) so `evolveguard check` only needs the skill path.
+
+Ported from src/evolveguard/replay/index.ts.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from ..snapshot import build_fixture_snapshots, load_skill
+from ..types import Baseline, Fixture, ReplayResult
+
+
+def replay_skill(skill_path: str, baseline: Baseline) -> ReplayResult:
+    skill = load_skill(skill_path)
+
+    fixtures = [
+        Fixture(id=f.id, prompt=f.prompt, expected_tool_calls=f.expected_tool_calls)
+        for f in baseline.fixtures
+    ]
+
+    fixture_snapshots = build_fixture_snapshots(skill.capability_surface, fixtures)
+
+    return ReplayResult(
+        schema_version=1,
+        skill_name=skill.name,
+        skill_path=skill.skill_path,
+        replayed_at=datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+        full_capability_surface=skill.capability_surface,
+        fixtures=fixture_snapshots,
+    )

evolveguard/report/__init__.py

@@ -0,0 +1,95 @@
+"""
+Reads and writes baseline (.evolveguard-baseline.json) and report
+(evolveguard-report.json) files as pretty-printed, deterministic JSON, with
+schema validation on read. Ported from src/evolveguard/report/index.ts
+(which uses zod; this port validates by hand to avoid a schema-validation
+dependency).
+"""
+from __future__ import annotations
+
+import json
+
+from ..errors import EvolveGuardError
+from ..types import Baseline, EvolveGuardReport
+
+
+def write_baseline(baseline_path: str, baseline: Baseline) -> None:
+    with open(baseline_path, "w", encoding="utf-8") as fh:
+        fh.write(json.dumps(baseline.to_dict(), indent=2) + "\n")
+
+
+def read_baseline(baseline_path: str) -> Baseline:
+    try:
+        with open(baseline_path, "r", encoding="utf-8") as fh:
+            raw_text = fh.read()
+    except OSError as err:
+        raise EvolveGuardError(
+            f'Could not read baseline file at "{baseline_path}".',
+            str(err),
+            "Run `evolveguard record <SKILL.md> --fixtures <fixtures.json>` first to "
+            "create a baseline.",
+        ) from err
+
+    try:
+        parsed = json.loads(raw_text)
+    except json.JSONDecodeError as err:
+        raise EvolveGuardError(
+            f'Baseline file at "{baseline_path}" is not valid JSON.',
+            str(err),
+            "Re-run `evolveguard record` to regenerate the baseline file.",
+        ) from err
+
+    try:
+        if not isinstance(parsed, dict) or parsed.get("schemaVersion") != 1:
+            raise ValueError("missing or unexpected schemaVersion")
+        if not parsed.get("fixtures"):
+            raise ValueError("fixtures: expected a non-empty array")
+        baseline = Baseline.from_dict(parsed)
+    except (KeyError, ValueError, TypeError) as err:
+        raise EvolveGuardError(
+            f'Baseline file at "{baseline_path}" does not match the expected schema.',
+            str(err),
+            "Re-run `evolveguard record` to regenerate the baseline file, or check "
+            "it was not hand-edited.",
+        ) from err
+
+    return baseline
+
+
+def write_report(report_path: str, report: EvolveGuardReport) -> None:
+    with open(report_path, "w", encoding="utf-8") as fh:
+        fh.write(json.dumps(report.to_dict(), indent=2) + "\n")
+
+
+def read_report(report_path: str) -> EvolveGuardReport:
+    try:
+        with open(report_path, "r", encoding="utf-8") as fh:
+            raw_text = fh.read()
+    except OSError as err:
+        raise EvolveGuardError(
+            f'Could not read report file at "{report_path}".',
+            str(err),
+            "Run `evolveguard check <SKILL.md>` first to generate a report.",
+        ) from err
+
+    try:
+        parsed = json.loads(raw_text)
+    except json.JSONDecodeError as err:
+        raise EvolveGuardError(
+            f'Report file at "{report_path}" is not valid JSON.',
+            str(err),
+            "Re-run `evolveguard check` to regenerate the report file.",
+        ) from err
+
+    try:
+        if not isinstance(parsed, dict) or parsed.get("schemaVersion") != 1:
+            raise ValueError("missing or unexpected schemaVersion")
+        report = EvolveGuardReport.from_dict(parsed)
+    except (KeyError, ValueError, TypeError) as err:
+        raise EvolveGuardError(
+            f'Report file at "{report_path}" does not match the expected schema.',
+            str(err),
+            "Re-run `evolveguard check` to regenerate the report file.",
+        ) from err
+
+    return report