skillproof 0.1.0__py3-none-any.whl

skillproof/judges/anthropic_judge.py

@@ -0,0 +1,59 @@
+"""Claude as a judge (Anthropic Messages API, structured JSON output)."""
+
+from __future__ import annotations
+
+import os
+
+from skillproof.errors import JudgeNotReady
+from skillproof.judges.base import (
+    VERDICT_SCHEMA,
+    Judge,
+    JudgeUnavailable,
+    Verdict,
+    judge_prompt,
+    parse_verdict,
+)
+
+DEFAULT_MODEL = "claude-opus-5-5"
+
+
+class AnthropicJudge(Judge):
+    name = "anthropic"
+    vendor = "anthropic"
+
+    def __init__(
+        self, model: str | None = None, api_key_env: str | None = None, base_url: str | None = None
+    ) -> None:
+        import anthropic
+
+        self.model = model or DEFAULT_MODEL
+        api_key = None
+        if api_key_env:
+            api_key = os.environ.get(api_key_env)
+            if not api_key:
+                raise JudgeNotReady(
+                    f"Judge API key env var {api_key_env} is not set.",
+                    hint=f"export {api_key_env}=... or change `judge.api_key_env`.",
+                )
+        self.client = anthropic.AsyncAnthropic(api_key=api_key, base_url=base_url)
+
+    async def grade(self, rubric: str, response: str) -> Verdict:
+        import anthropic
+
+        try:
+            message = await self.client.messages.create(
+                model=self.model,
+                max_tokens=16000,
+                messages=[{"role": "user", "content": judge_prompt(rubric, response)}],
+                output_config={"format": {"type": "json_schema", "schema": VERDICT_SCHEMA}},
+            )
+        except anthropic.APIError as exc:
+            raise JudgeUnavailable(f"judge API error: {exc}") from exc
+        except TypeError as exc:  # raised by the SDK when no credentials are found
+            raise JudgeUnavailable(
+                "no Anthropic credentials for the judge: export ANTHROPIC_API_KEY"
+            ) from exc
+        if message.stop_reason == "refusal":
+            raise JudgeUnavailable("judge refused to grade this answer")
+        text = next((b.text for b in message.content if b.type == "text"), "")
+        return parse_verdict(text)

skillproof/judges/base.py

@@ -0,0 +1,69 @@
+"""The Judge interface used by `llm_judge` assertions."""
+
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel
+
+
+class Verdict(BaseModel):
+    passed: bool
+    reason: str
+
+
+class JudgeUnavailable(Exception):
+    """The judge itself failed (API error, refusal). The run is marked errored,
+    not failed, so judge outages don't skew pass rates."""
+
+
+class Judge(ABC):
+    name: str
+    vendor: str
+    model: str | None = None
+
+    @abstractmethod
+    async def grade(self, rubric: str, response: str) -> Verdict: ...
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}:{self.model}" if self.model else self.name
+
+
+JUDGE_PROMPT = """\
+You are grading an AI agent's final answer against a rubric.
+Be strict: the answer passes only if it clearly satisfies the rubric.
+Judge only what is in the answer; do not give credit for intent.
+
+<rubric>
+{rubric}
+</rubric>
+
+<answer>
+{response}
+</answer>
+
+Respond with JSON: {{"passed": true or false, "reason": "<one sentence>"}}
+"""
+
+VERDICT_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "passed": {"type": "boolean"},
+        "reason": {"type": "string"},
+    },
+    "required": ["passed", "reason"],
+    "additionalProperties": False,
+}
+
+
+def judge_prompt(rubric: str, response: str) -> str:
+    return JUDGE_PROMPT.format(rubric=rubric.strip(), response=response.strip() or "(empty)")
+
+
+def parse_verdict(text: str) -> Verdict:
+    try:
+        return Verdict.model_validate(json.loads(text))
+    except (json.JSONDecodeError, ValueError) as exc:
+        raise JudgeUnavailable(f"judge returned an unreadable verdict: {text[:200]!r}") from exc

skillproof/judges/mock.py

@@ -0,0 +1,21 @@
+"""A judge for tests: passes when every `pass_if` phrase appears in the answer."""
+
+from __future__ import annotations
+
+from skillproof.judges.base import Judge, Verdict
+
+
+class MockJudge(Judge):
+    name = "mock"
+    vendor = "mock"
+
+    def __init__(self, pass_if: list[str] | None = None) -> None:
+        self.pass_if = pass_if or []
+        self.calls: list[tuple[str, str]] = []
+
+    async def grade(self, rubric: str, response: str) -> Verdict:
+        self.calls.append((rubric, response))
+        missing = [p for p in self.pass_if if p.lower() not in response.lower()]
+        if missing:
+            return Verdict(passed=False, reason=f"missing: {', '.join(missing)}")
+        return Verdict(passed=True, reason="mock judge: all phrases present")

skillproof/judges/openai_judge.py

@@ -0,0 +1,58 @@
+"""An OpenAI model as a judge (Responses API, structured JSON output)."""
+
+from __future__ import annotations
+
+import os
+
+from skillproof.errors import JudgeNotReady
+from skillproof.judges.base import (
+    VERDICT_SCHEMA,
+    Judge,
+    JudgeUnavailable,
+    Verdict,
+    judge_prompt,
+    parse_verdict,
+)
+
+DEFAULT_MODEL = "gpt-6.1-sol"
+
+
+class OpenAIJudge(Judge):
+    name = "openai"
+    vendor = "openai"
+
+    def __init__(
+        self, model: str | None = None, api_key_env: str | None = None, base_url: str | None = None
+    ) -> None:
+        import openai
+
+        self.model = model or DEFAULT_MODEL
+        key_env = api_key_env or "OPENAI_API_KEY"
+        api_key = os.environ.get(key_env)
+        if not api_key:
+            raise JudgeNotReady(
+                f"{key_env} is not set, and llm_judge needs it.",
+                hint=f"export {key_env}=..., or set `judge: {{provider: anthropic}}`.",
+            )
+        self.client = openai.AsyncOpenAI(api_key=api_key, base_url=base_url)
+
+    async def grade(self, rubric: str, response: str) -> Verdict:
+        import openai
+
+        try:
+            result = await self.client.responses.create(
+                model=self.model,
+                input=judge_prompt(rubric, response),
+                text={
+                    "format": {
+                        "type": "json_schema",
+                        "name": "verdict",
+                        "strict": True,
+                        "schema": VERDICT_SCHEMA,
+                    }
+                },
+                store=False,
+            )
+        except openai.APIError as exc:
+            raise JudgeUnavailable(f"judge API error: {exc}") from exc
+        return parse_verdict(result.output_text)

skillproof/parsers/__init__.py

@@ -0,0 +1,6 @@
+"""Transcript parsers: turn an agent's raw event log into an AgentRun.
+
+Agents change their output formats often, so each parser is a set of small
+pure functions over plain dicts (the same dicts we save as the transcript).
+Tests feed them recorded transcripts; no API calls needed.
+"""

skillproof/parsers/claude_code.py

@@ -0,0 +1,144 @@
+"""Parse a Claude Agent SDK message stream (as dicts) into an AgentRun.
+
+Each message is saved as a dict with a `_type` key naming the SDK class
+(`SystemMessage`, `AssistantMessage`, `UserMessage`, `ResultMessage`, ...).
+Content blocks get the same treatment (`TextBlock`, `ToolUseBlock`, ...).
+
+A skill invocation shows up as a `Skill` tool call:
+    {"_type": "ToolUseBlock", "name": "Skill", "input": {"skill": "my-skill"}}
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any
+
+from skillproof.runners.base import AgentRun, ToolCall, Usage
+
+SKILL_TOOL = "Skill"
+DROPPED_FIELDS = {"signature"}  # opaque thinking signatures: large and useless for debugging
+
+
+def to_dict(obj: Any) -> Any:
+    """Convert SDK dataclasses (recursively) into JSON-friendly dicts."""
+    if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
+        out: dict[str, Any] = {"_type": type(obj).__name__}
+        for field in dataclasses.fields(obj):
+            if field.name not in DROPPED_FIELDS:
+                out[field.name] = to_dict(getattr(obj, field.name))
+        return out
+    if isinstance(obj, dict):
+        return {str(k): to_dict(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [to_dict(v) for v in obj]
+    if obj is None or isinstance(obj, (str, int, float, bool)):
+        return obj
+    return str(obj)
+
+
+def _blocks(message: dict[str, Any]) -> list[dict[str, Any]]:
+    content = message.get("content")
+    return content if isinstance(content, list) else []
+
+
+def _result_text(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(str(c.get("text", "")) for c in content if isinstance(c, dict))
+    return ""
+
+
+def tool_calls(transcript: list[dict[str, Any]]) -> list[ToolCall]:
+    calls: dict[str, ToolCall] = {}
+    for message in transcript:
+        for block in _blocks(message):
+            if block.get("_type") == "ToolUseBlock":
+                calls[block["id"]] = ToolCall(name=block["name"], input=block.get("input") or {})
+            elif block.get("_type") == "ToolResultBlock" and block.get("tool_use_id") in calls:
+                calls[block["tool_use_id"]].output = _result_text(block.get("content"))
+    return list(calls.values())
+
+
+def is_skill_call(call: ToolCall, skill_name: str) -> bool:
+    """A Skill tool call naming this skill, or a direct read of its SKILL.md."""
+    if call.name == SKILL_TOOL:
+        invoked = str(call.input.get("skill", ""))
+        return invoked == skill_name or invoked.endswith(f":{skill_name}")
+    marker = f"skills/{skill_name}/SKILL.md"
+    return any(isinstance(v, str) and marker in v for v in call.input.values())
+
+
+def init_data(transcript: list[dict[str, Any]]) -> dict[str, Any]:
+    for message in transcript:
+        if message.get("_type") == "SystemMessage" and message.get("subtype") == "init":
+            return message.get("data") or {}
+    return {}
+
+
+def final_text(transcript: list[dict[str, Any]], result: dict[str, Any] | None) -> str:
+    if result and result.get("result"):
+        return str(result["result"])
+    for message in reversed(transcript):  # fall back to the last assistant text
+        if message.get("_type") == "AssistantMessage" and not message.get("parent_tool_use_id"):
+            text = "".join(b.get("text", "") for b in _blocks(message) if b["_type"] == "TextBlock")
+            if text:
+                return text
+    return ""
+
+
+def parse_usage(result: dict[str, Any] | None) -> Usage:
+    usage = (result or {}).get("usage") or {}
+    return Usage(
+        input_tokens=usage.get("input_tokens") or 0,
+        output_tokens=usage.get("output_tokens") or 0,
+        cache_read_tokens=usage.get("cache_read_input_tokens") or 0,
+        cache_write_tokens=usage.get("cache_creation_input_tokens") or 0,
+        cost_usd=(result or {}).get("total_cost_usd"),
+    )
+
+
+def parse_error(result: dict[str, Any] | None) -> tuple[str | None, list[str]]:
+    """Split failures into infrastructure errors and agent-side warnings."""
+    if result is None:
+        return "Claude Code ended without a result message", []
+    if not result.get("is_error"):
+        return None, []
+    subtype = result.get("subtype", "")
+    details = "; ".join(result.get("errors") or []) or str(result.get("result") or "")
+    if result.get("api_error_status"):
+        return f"API error {result['api_error_status']}: {details}".strip(": "), []
+    if subtype in ("error_max_turns", "error_max_budget_usd"):
+        # The agent ran out of its allowance: a real outcome, not an infrastructure error.
+        return None, [f"stopped early: {subtype}"]
+    return f"Claude Code error ({subtype}): {details}".strip(": "), []
+
+
+def parse_transcript(transcript: list[dict[str, Any]], skill_name: str | None) -> AgentRun:
+    result = next((m for m in reversed(transcript) if m.get("_type") == "ResultMessage"), None)
+    calls = tool_calls(transcript)
+    error, warnings = parse_error(result)
+    init = init_data(transcript)
+
+    triggered: bool | None = None
+    if skill_name is not None:
+        triggered = any(is_skill_call(c, skill_name) for c in calls)
+        listed = init.get("skills")
+        if isinstance(listed, list) and skill_name not in listed:
+            warnings.append(
+                f"skill {skill_name!r} was not listed at startup; "
+                "check SKILL.md frontmatter (or `user-invocable: false`)"
+            )
+
+    return AgentRun(
+        final_text=final_text(transcript, result),
+        transcript=transcript,
+        tool_calls=calls,
+        usage=parse_usage(result),
+        duration_s=((result or {}).get("duration_ms") or 0) / 1000,
+        num_turns=(result or {}).get("num_turns"),
+        model=init.get("model"),
+        triggered=triggered,
+        error=error,
+        warnings=warnings,
+    )

skillproof/parsers/codex.py

@@ -0,0 +1,112 @@
+"""Parse `codex exec --json` events (JSONL, one dict per line) into an AgentRun.
+
+Event types: thread.started, turn.started, turn.completed {usage},
+turn.failed {error}, item.started/updated/completed {item}, error {message}.
+Item types we use: agent_message {text}, command_execution {command,
+aggregated_output, exit_code}, file_change {changes}, mcp_tool_call, error.
+
+Codex has no dedicated skill tool: the model reads SKILL.md (or runs a script
+from the skill folder) with a shell command, so a skill counts as triggered
+when a command touches `skills/<name>/`.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from skillproof.runners.base import AgentRun, ToolCall, Usage
+
+
+def parse_jsonl(text: str) -> list[dict[str, Any]]:
+    events = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue  # not an event (stray log line)
+        if isinstance(event, dict):
+            events.append(event)
+    return events
+
+
+def completed_items(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [e["item"] for e in events if e.get("type") == "item.completed" and "item" in e]
+
+
+def to_tool_call(item: dict[str, Any]) -> ToolCall | None:
+    kind = item.get("type")
+    if kind == "command_execution":
+        return ToolCall(
+            name="command_execution",
+            input={"command": item.get("command", ""), "exit_code": item.get("exit_code")},
+            output=item.get("aggregated_output"),
+        )
+    if kind == "file_change":
+        return ToolCall(name="file_change", input={"changes": item.get("changes", [])})
+    if kind == "mcp_tool_call":
+        return ToolCall(
+            name=f"mcp:{item.get('server')}/{item.get('tool')}",
+            input={"arguments": item.get("arguments")},
+            output=json.dumps(item.get("result")) if item.get("result") else None,
+        )
+    if kind == "web_search":
+        return ToolCall(name="web_search", input={"query": item.get("query")})
+    return None
+
+
+def is_skill_call(call: ToolCall, skill_name: str) -> bool:
+    return call.name == "command_execution" and f"skills/{skill_name}/" in str(
+        call.input.get("command", "")
+    )
+
+
+def parse_usage(events: list[dict[str, Any]]) -> Usage:
+    """Sum usage over all turns. Codex's input_tokens include cached tokens."""
+    usage = Usage()
+    for event in events:
+        if event.get("type") != "turn.completed":
+            continue
+        u = event.get("usage") or {}
+        cached = u.get("cached_input_tokens") or 0
+        written = u.get("cache_write_input_tokens") or 0
+        usage.input_tokens += max(0, (u.get("input_tokens") or 0) - cached - written)
+        usage.cache_read_tokens += cached
+        usage.cache_write_tokens += written
+        usage.output_tokens += u.get("output_tokens") or 0
+    return usage
+
+
+def parse_events(events: list[dict[str, Any]], skill_name: str | None) -> AgentRun:
+    items = completed_items(events)
+    calls = [c for c in (to_tool_call(i) for i in items) if c is not None]
+    messages = [i.get("text", "") for i in items if i.get("type") == "agent_message"]
+
+    error = None
+    failed = [e for e in events if e.get("type") == "turn.failed"]
+    if failed:
+        error = str((failed[-1].get("error") or {}).get("message") or "turn failed")
+    elif not any(e.get("type") == "turn.completed" for e in events):
+        top_errors = [e.get("message") for e in events if e.get("type") == "error"]
+        error = str(top_errors[-1]) if top_errors else "codex exited without completing a turn"
+
+    # Non-fatal problems Codex reports as error items (e.g. unknown model metadata).
+    warnings = [str(i.get("message")) for i in items if i.get("type") == "error"]
+
+    triggered = None
+    if skill_name is not None:
+        triggered = any(is_skill_call(c, skill_name) for c in calls)
+
+    return AgentRun(
+        final_text=messages[-1] if messages else "",
+        transcript=events,
+        tool_calls=calls,
+        usage=parse_usage(events),
+        num_turns=sum(1 for e in events if e.get("type") == "turn.completed") or None,
+        triggered=triggered,
+        error=error,
+        warnings=warnings,
+    )

skillproof/report/__init__.py

@@ -0,0 +1,34 @@
+"""Report renderers. Every format reads the same `Summary` (see summary.py)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from skillproof.report import html_report, json_report, markdown
+from skillproof.report.summary import Summary, build_summary
+from skillproof.results import SuiteResult
+
+FORMATS = ("terminal", "md", "html", "json")
+
+
+def render_text(fmt: str, summary: Summary, suite: SuiteResult) -> str:
+    """Render a file-style format (md, html, json) to a string."""
+    if fmt == "md":
+        return markdown.render(summary)
+    if fmt == "html":
+        return html_report.render(summary, suite)
+    if fmt == "json":
+        return json_report.render(summary)
+    raise ValueError(f"not a file format: {fmt}")
+
+
+def write_reports(suite: SuiteResult) -> Summary:
+    """Save summary.json, report.md and report.html next to the transcripts."""
+    summary = build_summary(suite)
+    run_dir = Path(suite.run_dir)
+    for name, fmt in (("summary.json", "json"), ("report.md", "md"), ("report.html", "html")):
+        (run_dir / name).write_text(render_text(fmt, summary, suite), encoding="utf-8")
+    return summary
+
+
+__all__ = ["FORMATS", "Summary", "build_summary", "render_text", "write_reports"]

skillproof/report/badge.py

@@ -0,0 +1,60 @@
+"""shields.io badges for a skill's README.
+
+Static badge (markdown):  [![skillproof](https://img.shields.io/badge/...)](link)
+Endpoint JSON (--json):   {"schemaVersion": 1, "label": ..., "message": ..., "color": ...}
+Host the JSON (e.g. a gist, updated by CI) and point shields.io/endpoint at it
+to get a badge that updates itself.
+"""
+
+from __future__ import annotations
+
+import json
+from urllib.parse import quote
+
+from skillproof.report.summary import Summary
+
+LABEL = "skillproof"
+HOMEPAGE = "https://github.com/akhilgarg07/skillproof"
+COLORS = {
+    "helps": "brightgreen",
+    "hurts": "red",
+    "no clear effect": "lightgrey",
+    "too few runs": "lightgrey",
+    "not measured": "lightgrey",
+}
+
+
+def message(s: Summary) -> str:
+    if s.lift is None:
+        text = "lift not measured"
+    else:
+        text = f"{s.lift * 100:+.0f}% lift"
+        if s.verdict == "no clear effect":
+            text += " (n.s.)"  # not significant: the 95% interval includes zero
+        elif s.verdict == "too few runs":
+            text += " (too few runs)"
+    if s.trigger_runs:
+        text += f" · {s.triggers}/{s.trigger_runs} triggered"
+    if s.approximate:
+        text += " · approx"
+    return text
+
+
+def _escape(text: str) -> str:
+    """shields.io path escaping: '-' -> '--', '_' -> '__', then URL-encode."""
+    return quote(text.replace("-", "--").replace("_", "__"), safe="")
+
+
+def badge_url(s: Summary) -> str:
+    return (
+        f"https://img.shields.io/badge/{_escape(LABEL)}-{_escape(message(s))}-{COLORS[s.verdict]}"
+    )
+
+
+def markdown(s: Summary, link: str = HOMEPAGE) -> str:
+    return f"[![{LABEL}: {message(s)}]({badge_url(s)})]({link})"
+
+
+def endpoint_json(s: Summary) -> str:
+    data = {"schemaVersion": 1, "label": LABEL, "message": message(s), "color": COLORS[s.verdict]}
+    return json.dumps(data, ensure_ascii=False)

skillproof/report/format.py

@@ -0,0 +1,75 @@
+"""Number formatting shared by all report renderers."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from skillproof.report.summary import Summary
+
+DASH = "–"
+
+
+def pct(value: float | None) -> str:
+    return DASH if value is None else f"{value * 100:.0f}%"
+
+
+def signed_pct(value: float | None) -> str:
+    """+23% / −12% (typographic minus) / – for unknown."""
+    if value is None:
+        return DASH
+    text = f"{value * 100:+.0f}%"
+    return "±0%" if text in ("+0%", "-0%") else text.replace("-", "−")
+
+
+def interval(ci: tuple[float, float] | None) -> str:
+    if ci is None:
+        return DASH
+    return f"{signed_pct(ci[0])} to {signed_pct(ci[1])}"
+
+
+def fraction(num: int, den: int) -> str:
+    return DASH if den == 0 else f"{num}/{den}"
+
+
+def seconds(value: float | None) -> str:
+    if value is None:
+        return DASH
+    return f"{value:.1f}s" if value < 100 else f"{value:.0f}s"
+
+
+def tokens(value: float | None) -> str:
+    if value is None:
+        return DASH
+    return f"{value / 1000:.1f}k" if value >= 1000 else f"{value:.0f}"
+
+
+def usd(value: float | None) -> str:
+    if value is None:
+        return DASH
+    return f"${value:.3f}" if value < 1 else f"${value:.2f}"
+
+
+def summary_line(s: Summary) -> str:
+    """skill lift: +23% (95% CI +5% to +41%)  |  triggers 9/10  |  tokens −12% ..."""
+    parts = [f"skill lift: {signed_pct(s.lift)}"]
+    if s.lift_ci is not None:
+        parts[0] += f" (95% CI {interval(s.lift_ci)})"
+    parts.append(f"triggers {fraction(s.triggers, s.trigger_runs)}")
+    if s.false_trigger_runs:
+        parts.append(f"false triggers {fraction(s.false_triggers, s.false_trigger_runs)}")
+    parts.append(f"tokens {signed_pct(s.token_delta)}")
+    if s.cost_delta is not None:
+        parts.append(f"cost {signed_pct(s.cost_delta)}")
+    w, wo = s.with_skill.mean_duration_s, s.without_skill.mean_duration_s
+    parts.append(f"latency {signed_pct(s.latency_delta)} ({seconds(w)} vs {seconds(wo)})")
+    return "  |  ".join(parts)
+
+
+VERDICT_TEXT = {
+    "helps": "the skill helps",
+    "hurts": "the skill hurts",
+    "no clear effect": "no clear effect",
+    "too few runs": "too few runs for a verdict",
+    "not measured": "lift not measured",
+}