skillproof 0.1.0__py3-none-any.whl

skillproof/__init__.py

@@ -0,0 +1,3 @@
+"""skillproof: pytest for agent skills."""
+
+__version__ = "0.1.0"

skillproof/__main__.py

@@ -0,0 +1,3 @@
+from skillproof.cli import main
+
+main()

skillproof/assertions/__init__.py

@@ -0,0 +1,62 @@
+"""Assertion types usable under `assert:` in evals.yaml.
+
+To add an assertion type: write a `BaseAssertion` subclass with a unique
+`type: Literal["..."]` field and an async `check()`, then add it to
+`Assertion` below. Validation and error messages come from pydantic.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, get_args
+
+from pydantic import Field
+
+from skillproof.assertions.base import AssertionResult, BaseAssertion, CheckContext
+from skillproof.assertions.builtin import (
+    Command,
+    Contains,
+    FileContains,
+    FileExists,
+    MaxTokens,
+    NotContains,
+    Regex,
+)
+from skillproof.assertions.llm_judge import LLMJudge
+from skillproof.judges.base import JudgeUnavailable
+
+Assertion = Annotated[
+    Contains | NotContains | Regex | FileExists | FileContains | Command | MaxTokens | LLMJudge,
+    Field(discriminator="type"),
+]
+
+# ["contains", "not_contains", ...], read from each class's `type` Literal.
+ASSERTION_TYPES: list[str] = [
+    get_args(cls.model_fields["type"].annotation)[0] for cls in get_args(get_args(Assertion)[0])
+]
+
+
+async def check_all(assertions: list[BaseAssertion], ctx: CheckContext) -> list[AssertionResult]:
+    """Run assertions in order. A crashing assertion counts as a failure, not a crash.
+
+    JudgeUnavailable is re-raised: the judge broke, not the agent, so the
+    engine marks the whole run as errored instead.
+    """
+    results = []
+    for assertion in assertions:
+        try:
+            results.append(await assertion.check(ctx))
+        except JudgeUnavailable:
+            raise
+        except Exception as exc:
+            results.append(assertion.result(False, f"assertion error: {exc}"))
+    return results
+
+
+__all__ = [
+    "ASSERTION_TYPES",
+    "Assertion",
+    "AssertionResult",
+    "BaseAssertion",
+    "CheckContext",
+    "check_all",
+]

skillproof/assertions/base.py

@@ -0,0 +1,44 @@
+"""Base class for assertions.
+
+An assertion is a pydantic model (so evals.yaml is validated for free) with an
+async `check()` that looks at one finished run and returns an AssertionResult.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from skillproof.judges.base import Judge
+from skillproof.runners.base import AgentRun
+
+
+@dataclass
+class CheckContext:
+    run: AgentRun
+    workdir: Path  # the sandbox the agent worked in
+    judge: Judge | None = None
+
+
+class AssertionResult(BaseModel):
+    type: str
+    passed: bool
+    message: str
+
+
+class BaseAssertion(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    type: str
+
+    def describe(self) -> str:
+        """Short human label, e.g. `contains "NullPointerException"`."""
+        return self.type
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        raise NotImplementedError
+
+    def result(self, passed: bool, message: str) -> AssertionResult:
+        return AssertionResult(type=self.type, passed=passed, message=message)

skillproof/assertions/builtin.py

@@ -0,0 +1,176 @@
+"""Deterministic assertions. These are cheap and reproducible: prefer them."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path, PurePosixPath
+from typing import Literal
+
+from pydantic import Field, field_validator
+
+from skillproof.assertions.base import AssertionResult, BaseAssertion, CheckContext
+from skillproof.sandbox import run_shell
+
+MAX_OUTPUT_IN_MESSAGE = 500
+
+
+def _sandbox_relative(path: str) -> str:
+    p = PurePosixPath(path)
+    if p.is_absolute() or ".." in p.parts:
+        raise ValueError("must be a relative path inside the sandbox (no '/' prefix, no '..')")
+    return path
+
+
+def _snippet(text: str, limit: int = 80) -> str:
+    text = " ".join(text.split())
+    return text if len(text) <= limit else text[: limit - 1] + "…"
+
+
+class Contains(BaseAssertion):
+    """The agent's final answer contains `value`."""
+
+    type: Literal["contains"]
+    value: str
+    case_sensitive: bool = False
+
+    def describe(self) -> str:
+        return f'contains "{self.value}"'
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        haystack, needle = ctx.run.final_text, self.value
+        if not self.case_sensitive:
+            haystack, needle = haystack.lower(), needle.lower()
+        if needle in haystack:
+            return self.result(True, f'answer contains "{self.value}"')
+        return self.result(False, f'"{self.value}" not in answer: "{_snippet(ctx.run.final_text)}"')
+
+
+class NotContains(BaseAssertion):
+    """The agent's final answer does NOT contain `value`."""
+
+    type: Literal["not_contains"]
+    value: str
+    case_sensitive: bool = False
+
+    def describe(self) -> str:
+        return f'not_contains "{self.value}"'
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        haystack, needle = ctx.run.final_text, self.value
+        if not self.case_sensitive:
+            haystack, needle = haystack.lower(), needle.lower()
+        if needle in haystack:
+            return self.result(False, f'answer contains forbidden "{self.value}"')
+        return self.result(True, f'answer does not contain "{self.value}"')
+
+
+class Regex(BaseAssertion):
+    """The agent's final answer matches a Python regular expression."""
+
+    type: Literal["regex"]
+    pattern: str
+
+    @field_validator("pattern")
+    @classmethod
+    def _compiles(cls, pattern: str) -> str:
+        try:
+            re.compile(pattern)
+        except re.error as exc:
+            raise ValueError(f"invalid regular expression: {exc}") from exc
+        return pattern
+
+    def describe(self) -> str:
+        return f"regex /{self.pattern}/"
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        if re.search(self.pattern, ctx.run.final_text, flags=re.MULTILINE):
+            return self.result(True, f"answer matches /{self.pattern}/")
+        return self.result(False, f"answer does not match /{self.pattern}/")
+
+
+class FileExists(BaseAssertion):
+    """A file exists in the sandbox after the run."""
+
+    type: Literal["file_exists"]
+    path: str
+
+    @field_validator("path")
+    @classmethod
+    def _inside_sandbox(cls, path: str) -> str:
+        return _sandbox_relative(path)
+
+    def describe(self) -> str:
+        return f"file_exists {self.path}"
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        if (ctx.workdir / self.path).exists():
+            return self.result(True, f"{self.path} exists")
+        return self.result(False, f"{self.path} was not created")
+
+
+class FileContains(BaseAssertion):
+    """A file in the sandbox contains `value`."""
+
+    type: Literal["file_contains"]
+    path: str
+    value: str
+    case_sensitive: bool = False
+
+    @field_validator("path")
+    @classmethod
+    def _inside_sandbox(cls, path: str) -> str:
+        return _sandbox_relative(path)
+
+    def describe(self) -> str:
+        return f'file_contains {self.path} "{self.value}"'
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        target: Path = ctx.workdir / self.path
+        if not target.is_file():
+            return self.result(False, f"{self.path} was not created")
+        content = target.read_text(encoding="utf-8", errors="replace")
+        needle = self.value
+        if not self.case_sensitive:
+            content, needle = content.lower(), needle.lower()
+        if needle in content:
+            return self.result(True, f'{self.path} contains "{self.value}"')
+        return self.result(False, f'{self.path} does not contain "{self.value}"')
+
+
+class Command(BaseAssertion):
+    """Run a shell command in the sandbox; passes if it exits 0.
+
+    API keys and tokens are removed from the command's environment.
+    """
+
+    type: Literal["command"]
+    run: str
+    timeout: float = Field(default=120, gt=0)
+
+    def describe(self) -> str:
+        return f"command `{self.run}`"
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        code, out = await run_shell(self.run, ctx.workdir, self.timeout)
+        if code is None:
+            return self.result(False, f"`{self.run}` timed out after {self.timeout:g}s")
+        if code == 0:
+            return self.result(True, f"`{self.run}` exited 0")
+        output = out.strip()[-MAX_OUTPUT_IN_MESSAGE:]
+        return self.result(False, f"`{self.run}` exited {code}: {output}")
+
+
+class MaxTokens(BaseAssertion):
+    """The run used at most `value` tokens (input + output + cache)."""
+
+    type: Literal["max_tokens"]
+    value: int = Field(gt=0)
+
+    def describe(self) -> str:
+        return f"max_tokens {self.value}"
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        used = ctx.run.usage.total_tokens
+        if used <= self.value:
+            return self.result(True, f"used {used} tokens (limit {self.value})")
+        return self.result(False, f"used {used} tokens, over the limit of {self.value}")

skillproof/assertions/llm_judge.py

@@ -0,0 +1,24 @@
+"""The optional LLM-graded assertion. Non-deterministic: use deterministic ones first."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from skillproof.assertions.base import AssertionResult, BaseAssertion, CheckContext
+
+
+class LLMJudge(BaseAssertion):
+    """A judge model decides whether the final answer satisfies `rubric`."""
+
+    type: Literal["llm_judge"]
+    rubric: str
+
+    def describe(self) -> str:
+        rubric = self.rubric if len(self.rubric) <= 40 else self.rubric[:39] + "…"
+        return f'llm_judge "{rubric}"'
+
+    async def check(self, ctx: CheckContext) -> AssertionResult:
+        if ctx.judge is None:
+            return self.result(False, "no judge configured")
+        verdict = await ctx.judge.grade(self.rubric, ctx.run.final_text)
+        return self.result(verdict.passed, f"judge ({ctx.judge.name}): {verdict.reason}")

skillproof/cli.py

@@ -0,0 +1,313 @@
+"""The `skillproof` command line."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn, TimeElapsedColumn
+
+from skillproof import __version__
+from skillproof.config import EvalConfig, load_config
+from skillproof.engine import Engine, RunOptions
+from skillproof.errors import RunAborted, SkillproofError
+from skillproof.estimate import CONFIRM_ABOVE_USD, Estimate, estimate
+from skillproof.report import FORMATS, Summary, build_summary, render_text, terminal, write_reports
+from skillproof.report import badge as badges
+from skillproof.results import SuiteResult, find_latest_run, load_suite
+from skillproof.scaffold import init_skill
+
+app = typer.Typer(
+    name="skillproof",
+    help="pytest for agent skills: prove your skill actually works.",
+    no_args_is_help=True,
+    add_completion=False,
+    pretty_exceptions_enable=False,
+)
+console = Console()
+err_console = Console(stderr=True)
+
+
+def fail(exc: SkillproofError) -> typer.Exit:
+    err_console.print(f"[bold red]error:[/] {exc.message}", highlight=False)
+    if exc.hint:
+        err_console.print(f"[bold]fix:[/] {exc.hint}", highlight=False)
+    return typer.Exit(code=2)
+
+
+def version_callback(value: bool) -> None:
+    if value:
+        console.print(f"skillproof {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def root(
+    version: Annotated[
+        bool,
+        typer.Option("--version", callback=version_callback, is_eager=True, help="Show version."),
+    ] = False,
+) -> None:
+    pass
+
+
+@app.command()
+def init(
+    path: Annotated[Path, typer.Argument(help="Skill folder (contains SKILL.md).")] = Path("."),
+    runner: Annotated[str, typer.Option(help="Agent to test with.")] = "claude-code",
+    force: Annotated[
+        bool, typer.Option("--force", help="Overwrite an existing evals.yaml.")
+    ] = False,
+) -> None:
+    """Scaffold an evals.yaml with example cases."""
+    try:
+        result = init_skill(path, runner=runner, force=force)
+    except SkillproofError as exc:
+        raise fail(exc) from exc
+    for created in result.created:
+        console.print(f"[green]created[/] {created}")
+    for warning in result.warnings:
+        console.print(f"[yellow]warning:[/] {warning}")
+    console.print(f"\nNext: edit {path / 'evals.yaml'}, then run [bold]skillproof run {path}[/]")
+
+
+@app.command()
+def run(
+    path: Annotated[Path, typer.Argument(help="Skill folder or evals.yaml file.")] = Path("."),
+    trials: Annotated[
+        int | None, typer.Option("--trials", "-n", min=1, help="Override trials per case.")
+    ] = None,
+    case: Annotated[
+        list[str] | None, typer.Option("--case", "-k", help="Only run this case (repeatable).")
+    ] = None,
+    parallel: Annotated[int, typer.Option("--parallel", "-p", min=1, help="Concurrent runs.")] = 4,
+    report_format: Annotated[
+        str, typer.Option("--format", "-f", help="terminal, md, html or json.")
+    ] = "terminal",
+    output: Annotated[
+        Path | None, typer.Option("--output", "-o", help="Write the md/html/json report here.")
+    ] = None,
+    out: Annotated[Path, typer.Option(help="Where transcripts and results are saved.")] = Path(
+        ".skillproof"
+    ),
+    keep_sandboxes: Annotated[
+        bool, typer.Option("--keep-sandboxes", help="Save each run's final working dir.")
+    ] = False,
+    yes: Annotated[
+        bool, typer.Option("--yes", "-y", help="Don't ask before an expensive run.")
+    ] = False,
+    confirm_above: Annotated[
+        float, typer.Option(help="Ask for confirmation above this estimated cost (USD).")
+    ] = CONFIRM_ABOVE_USD,
+    dry_run: Annotated[
+        bool, typer.Option("--dry-run", help="Show the plan and cost estimate, run nothing.")
+    ] = False,
+    min_lift: Annotated[
+        float | None,
+        typer.Option(help="Exit 1 if the lift is below this many percentage points (CI gate)."),
+    ] = None,
+    min_trigger_rate: Annotated[
+        float | None,
+        typer.Option(help="Exit 1 if the trigger rate is below this percentage (CI gate)."),
+    ] = None,
+) -> None:
+    """Run the evals with and without the skill, and print a report."""
+    check_format(report_format)
+    options = RunOptions(
+        parallel=parallel,
+        out_dir=out,
+        keep_sandboxes=keep_sandboxes,
+        trials=trials,
+        only_cases=case,
+    )
+    # Machine-readable output goes to stdout, so everything else goes to stderr.
+    info = console if report_format == "terminal" else err_console
+    try:
+        config = load_config(path)
+        check_placeholders(config)
+        engine = Engine(config, options)
+        plan = estimate(engine, out)
+        info.print(f"[bold]plan:[/] {plan.describe()}", highlight=False)
+        if dry_run:
+            return
+        confirm_cost(plan, confirm_above, yes)
+        with Progress(
+            TextColumn("[bold]running[/]"),
+            BarColumn(),
+            MofNCompleteColumn(),
+            TimeElapsedColumn(),
+            console=info,
+            transient=True,
+        ) as progress:
+            task = progress.add_task("runs", total=len(engine.jobs()))
+            engine.on_trial = lambda _trial: progress.advance(task)
+            suite = asyncio.run(engine.run())
+        summary = write_reports(suite)
+    except KeyboardInterrupt:
+        saved = save_partial_reports(out)
+        err_console.print(f"\n[yellow]interrupted.[/] Partial results: {saved}", highlight=False)
+        raise typer.Exit(code=130) from None
+    except RunAborted as exc:
+        save_partial_reports(out)
+        raise fail(exc) from exc
+    except SkillproofError as exc:
+        raise fail(exc) from exc
+    emit(report_format, summary, suite, output)
+    gate(summary, min_lift, min_trigger_rate)
+
+
+def check_placeholders(config: EvalConfig) -> None:
+    """Refuse to pay for running the TODO examples that `skillproof init` writes."""
+    for case in config.cases:
+        texts = [case.prompt, *(str(a.model_dump()) for a in case.assertions)]
+        if any("TODO" in text for text in texts):
+            raise SkillproofError(
+                f"Case {case.name!r} still has TODO placeholders from `skillproof init`.",
+                hint=f"Write a real prompt and assertions in {config.path}, then run again.",
+            )
+
+
+def confirm_cost(plan: Estimate, confirm_above: float, yes: bool) -> None:
+    total = plan.total_usd
+    if yes or total is None or total <= confirm_above:
+        return
+    if sys.stdin.isatty() and sys.stdout.isatty():
+        if typer.confirm(f"This may cost about ${total:.2f}. Continue?", default=False):
+            return
+        raise typer.Exit(code=1)
+    raise fail(
+        SkillproofError(
+            f"Estimated cost ~${total:.2f} is above ${confirm_above:.2f}.",
+            hint="Pass --yes to run anyway, lower --trials, or raise --confirm-above.",
+        )
+    )
+
+
+def save_partial_reports(out: Path) -> str:
+    """After an interruption, render reports for whatever finished."""
+    try:
+        suite = load_suite(find_latest_run(out))
+        write_reports(suite)
+        return f"{suite.run_dir}/report.html"
+    except (SkillproofError, OSError, ValueError):
+        return str(out / "runs")
+
+
+def gate(summary: Summary, min_lift: float | None, min_trigger_rate: float | None) -> None:
+    """CI gates: exit 1 when the skill doesn't meet the bar."""
+    failures = []
+    if min_lift is not None:
+        if summary.lift is None:
+            failures.append("lift could not be measured (no graded runs)")
+        elif summary.lift * 100 < min_lift:
+            failures.append(f"lift {summary.lift * 100:+.0f}% is below --min-lift {min_lift:+g}%")
+    if min_trigger_rate is not None:
+        rate = summary.trigger_rate
+        if rate is None:
+            failures.append("trigger rate could not be measured with this runner")
+        elif rate * 100 < min_trigger_rate:
+            failures.append(
+                f"trigger rate {rate * 100:.0f}% is below --min-trigger-rate {min_trigger_rate:g}%"
+            )
+    for failure in failures:
+        err_console.print(f"[bold red]gate failed:[/] {failure}", highlight=False)
+    if failures:
+        raise typer.Exit(code=1)
+
+
+@app.command()
+def report(
+    path: Annotated[
+        Path | None,
+        typer.Argument(help="A run directory or results.json. Default: the latest run."),
+    ] = None,
+    report_format: Annotated[
+        str, typer.Option("--format", "-f", help="terminal, md, html or json.")
+    ] = "terminal",
+    output: Annotated[
+        Path | None, typer.Option("--output", "-o", help="Write the md/html/json report here.")
+    ] = None,
+    out: Annotated[Path, typer.Option(help="Where runs are saved.")] = Path(".skillproof"),
+) -> None:
+    """Re-render the report for a saved run, without running anything."""
+    check_format(report_format)
+    try:
+        suite = load_suite(path if path is not None else find_latest_run(out))
+    except SkillproofError as exc:
+        raise fail(exc) from exc
+    summary = write_reports(suite)  # refresh the saved reports with this version
+    emit(report_format, summary, suite, output)
+
+
+@app.command()
+def badge(
+    path: Annotated[Path, typer.Argument(help="Skill folder (or its evals.yaml).")] = Path("."),
+    endpoint: Annotated[
+        bool, typer.Option("--json", help="Print shields.io endpoint JSON instead of markdown.")
+    ] = False,
+    out: Annotated[Path, typer.Option(help="Where runs are saved.")] = Path(".skillproof"),
+) -> None:
+    """Print a shields.io badge for the skill's latest run."""
+    try:
+        skill_path = load_config(path).skill
+    except SkillproofError:
+        skill_path = path.resolve()
+    try:
+        summary = build_summary(load_suite(find_latest_run(out, skill_path)))
+    except SkillproofError as exc:
+        raise fail(exc) from exc
+    print(badges.endpoint_json(summary) if endpoint else badges.markdown(summary))
+
+
+def check_format(report_format: str) -> None:
+    if report_format not in FORMATS:
+        raise fail(
+            SkillproofError(
+                f"Unknown report format {report_format!r}.",
+                hint=f"Use one of: {', '.join(FORMATS)}.",
+            )
+        )
+
+
+def emit(report_format: str, summary: Summary, suite: SuiteResult, output: Path | None) -> None:
+    """Show or write the report in the requested format."""
+    if report_format == "terminal":
+        terminal.render(summary, console)
+        return
+    text = render_text(report_format, summary, suite)
+    if output is None and report_format == "html":
+        output = Path(suite.run_dir) / "report.html"
+    if output is None:
+        sys.stdout.write(text)
+        return
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(text, encoding="utf-8")
+    err_console.print(f"wrote {output}", highlight=False)
+
+
+ISSUES_URL = "https://github.com/akhilgarg07/skillproof/issues"
+
+
+def main() -> None:
+    """Console entry point: user errors get a fix-it message, bugs get a bug report link."""
+    try:
+        app()
+    except Exception as exc:
+        if os.environ.get("SKILLPROOF_DEBUG"):
+            raise
+        err_console.print(
+            f"[bold red]skillproof crashed:[/] {type(exc).__name__}: {exc}\n"
+            f"This is a bug in skillproof, not in your skill. Re-run with SKILLPROOF_DEBUG=1 for "
+            f"the traceback and please report it at {ISSUES_URL}",
+            highlight=False,
+        )
+        sys.exit(70)
+
+
+if __name__ == "__main__":
+    main()