skillbench-cli 0.1.0__py3-none-any.whl

skillbench/__init__.py

@@ -0,0 +1 @@
+"""skillbench: eval framework for agent skills."""

skillbench/assertions.py

@@ -0,0 +1,250 @@
+"""Built-in assertions: parsing a test's `assert:` list and checking it against a finished run."""
+
+import subprocess
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath, PureWindowsPath
+from typing import Any
+
+from skillbench.claude import SKILL_TOOL, final_text, walk_tool_calls
+
+# How much of a failed command's output its reason quotes: enough to see the error, not a log.
+_COMMAND_OUTPUT_TAIL_CHARS = 2000
+_TOOL_ORDER_SHOWN_CALLS = 20
+
+
+class InvalidAssertionError(Exception):
+    """An `assert:` entry is malformed; the message says what is wrong with it."""
+
+
+@dataclass(frozen=True)
+class Assertion:
+    type: str
+    expected: Any
+
+
+@dataclass(frozen=True)
+class RunFacts:
+    """What assertions are checked against."""
+
+    trace: dict[str, Any]
+    workspace: Path
+    prompt: str
+    loaded_skills: list[Any]
+
+    @property
+    def tool_names(self) -> list[str]:
+        """Every tool call's name in order, subagents included."""
+        return [call["name"] for call in walk_tool_calls(self.trace["items"])]
+
+    @property
+    def invoked_skills(self) -> list[Any]:
+        """The skill named by each Skill tool call, in order."""
+        return [
+            call["input"].get("skill") if isinstance(call["input"], dict) else None
+            for call in walk_tool_calls(self.trace["items"])
+            if call["name"] == SKILL_TOOL
+        ]
+
+    @property
+    def user_messages(self) -> list[str]:
+        """The prompt, then each answer sent."""
+        answers = [item["text"] for item in self.trace["items"] if item["kind"] == "user_message"]
+        return [self.prompt, *answers]
+
+
+@dataclass(frozen=True)
+class AssertionResult:
+    assertion: Assertion
+    passed: bool
+    reason: str
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "type": self.assertion.type,
+            "expected": self.assertion.expected,
+            "passed": self.passed,
+            "reason": self.reason,
+        }
+
+
+def parse_assertion(entry: Any) -> Assertion:
+    """One `assert:` list entry, e.g. `{"tool_not_called": "Write"}`."""
+    if not isinstance(entry, dict) or len(entry) != 1:
+        raise InvalidAssertionError(
+            f"must be a mapping with exactly one assertion type, got {entry!r}"
+        )
+    [(assertion_type, expected)] = entry.items()
+    if assertion_type not in _ASSERTION_TYPES:
+        known = ", ".join(_ASSERTION_TYPES)
+        raise InvalidAssertionError(
+            f"unknown assertion '{assertion_type}' (known assertions: {known})"
+        )
+    is_valid, description, _ = _ASSERTION_TYPES[assertion_type]
+    if not is_valid(expected):
+        raise InvalidAssertionError(f"'{assertion_type}' must be {description}, got {expected!r}")
+    return Assertion(type=assertion_type, expected=expected)
+
+
+def evaluate_assertions(assertions: list[Assertion], facts: RunFacts) -> list[AssertionResult]:
+    results = []
+    for assertion in assertions:
+        _, _, check = _ASSERTION_TYPES[assertion.type]
+        passed, reason = check(assertion.expected, facts)
+        results.append(AssertionResult(assertion=assertion, passed=passed, reason=reason))
+    return results
+
+
+# --- checks: each returns (passed, reason) -------------------------------------------------
+
+_Check = Callable[[Any, RunFacts], tuple[bool, str]]
+
+
+def _negated(check: _Check) -> _Check:
+    """The opposite check, with the same reason."""
+
+    def negated_check(expected: Any, facts: RunFacts) -> tuple[bool, str]:
+        passed, reason = check(expected, facts)
+        return not passed, reason
+
+    return negated_check
+
+
+def _check_skill_invoked(skill_name: str, facts: RunFacts) -> tuple[bool, str]:
+    """Through the Skill tool, or by a `/skill` prompt or answer when Claude loaded it."""
+    count = sum(1 for name in facts.invoked_skills if _skill_name_matches(name, skill_name))
+    if count:
+        return True, f"{skill_name} was invoked {_times(count)}"
+    if any(_skill_name_matches(name, skill_name) for name in facts.loaded_skills):
+        for index, message in enumerate(facts.user_messages):
+            if _is_slash_command_for(message, skill_name):
+                by = "the prompt" if index == 0 else "a user message"
+                return True, f"{skill_name} was invoked by {by}"
+    others = ", ".join(str(name) for name in facts.invoked_skills) or "none"
+    return False, f"{skill_name} was never invoked (skills invoked: {others})"
+
+
+def _skill_name_matches(actual: Any, expected: str) -> bool:
+    """`planner` matches both `planner` and a plugin's `my-plugin:planner`."""
+    if not isinstance(actual, str):
+        return False
+    return actual == expected or (":" not in expected and actual.rsplit(":", 1)[-1] == expected)
+
+
+def _is_slash_command_for(message: str, skill_name: str) -> bool:
+    command = message.split(maxsplit=1)[0] if message.strip() else ""
+    return command.startswith("/") and _skill_name_matches(command[1:], skill_name)
+
+
+def _check_tool_called(tool_name: str, facts: RunFacts) -> tuple[bool, str]:
+    count = facts.tool_names.count(tool_name)
+    if count == 0:
+        return False, f"{tool_name} was never called"
+    return True, f"{tool_name} was called {_times(count)}"
+
+
+def _check_tool_order(expected: list[str], facts: RunFacts) -> tuple[bool, str]:
+    """The tools were called in this order; other calls may come in between."""
+    called = facts.tool_names
+    matched = 0
+    for name in called:
+        if matched < len(expected) and name == expected[matched]:
+            matched += 1
+    if matched == len(expected):
+        return True, f"called in order: {' -> '.join(expected)}"
+    shown = ", ".join(called[:_TOOL_ORDER_SHOWN_CALLS]) or "none"
+    if len(called) > _TOOL_ORDER_SHOWN_CALLS:
+        shown += f", ... ({len(called)} calls)"
+    after = " -> ".join(expected[:matched]) or "nothing"
+    return False, f"{expected[matched]} was not called after {after} (calls in order: {shown})"
+
+
+def _check_file_contains(spec: dict[str, str], facts: RunFacts) -> tuple[bool, str]:
+    relative_path, text = spec["path"], spec["text"]
+    file_path = facts.workspace / relative_path
+    if not file_path.is_file():
+        return False, f"{relative_path} does not exist in the workspace"
+    if text in file_path.read_text(encoding="utf-8", errors="replace"):
+        return True, f"{relative_path} contains {text!r}"
+    return False, f"{relative_path} does not contain {text!r}"
+
+
+def _check_final_text_contains(text: str, facts: RunFacts) -> tuple[bool, str]:
+    agent_final_text = final_text(facts.trace["items"])
+    if agent_final_text is None:
+        return False, "the agent produced no text"
+    if text in agent_final_text:
+        return True, f"final text contains {text!r}"
+    return False, f"final text does not contain {text!r} (final text: {agent_final_text!r})"
+
+
+def _check_command_succeeds(command: str, facts: RunFacts) -> tuple[bool, str]:
+    """Runs through the platform shell: sh on macOS and Linux, cmd on Windows."""
+    completed = subprocess.run(
+        command,
+        shell=True,
+        cwd=facts.workspace,
+        stdin=subprocess.DEVNULL,
+        capture_output=True,
+        encoding="utf-8",
+        errors="replace",
+    )
+    reason = f"{command!r} exited with code {completed.returncode}"
+    if completed.returncode == 0:
+        return True, reason
+    output_tail = (completed.stdout + completed.stderr).strip()[-_COMMAND_OUTPUT_TAIL_CHARS:]
+    return False, f"{reason}; output: {output_tail!r}"
+
+
+def _times(count: int) -> str:
+    return "1 time" if count == 1 else f"{count} times"
+
+
+# --- validators for the expected value -----------------------------------------------------
+
+
+def _is_non_empty_string(value: Any) -> bool:
+    return isinstance(value, str) and value != ""
+
+
+def _is_non_empty_string_list(value: Any) -> bool:
+    return isinstance(value, list) and bool(value) and all(map(_is_non_empty_string, value))
+
+
+def _is_workspace_path(value: Any) -> bool:
+    """A relative path that stays inside the workspace (checked lexically, before any run)."""
+    if not _is_non_empty_string(value):
+        return False
+    if PurePosixPath(value).is_absolute() or PureWindowsPath(value).anchor:
+        return False
+    depth = 0
+    for part in PureWindowsPath(value).parts:
+        depth += -1 if part == ".." else 0 if part == "." else 1
+        if depth < 0:
+            return False
+    return True
+
+
+def _is_file_contains_spec(value: Any) -> bool:
+    return (
+        isinstance(value, dict)
+        and value.keys() == {"path", "text"}
+        and _is_workspace_path(value["path"])
+        and _is_non_empty_string(value["text"])
+    )
+
+
+# assertion type -> (validator of the expected value, its description, check)
+_ASSERTION_TYPES: dict[str, tuple[Callable[[Any], bool], str, _Check]] = {
+    "skill_invoked": (_is_non_empty_string, "a skill name", _check_skill_invoked),
+    "skill_not_invoked": (_is_non_empty_string, "a skill name", _negated(_check_skill_invoked)),
+    "tool_not_called": (_is_non_empty_string, "a tool name", _negated(_check_tool_called)),
+    "tool_order": (_is_non_empty_string_list, "a non-empty list of tool names", _check_tool_order),
+    "final_text_contains": (_is_non_empty_string, "a string", _check_final_text_contains),
+    "file_contains": (
+        _is_file_contains_spec,
+        "a mapping with 'path' (a relative path inside the workspace) and 'text'",
+        _check_file_contains,
+    ),
+    "command_succeeds": (_is_non_empty_string, "a command string", _check_command_succeeds),
+}

skillbench/claude.py

@@ -0,0 +1,277 @@
+"""Claude Code: finding the CLI, the command line for each turn, and reading its `stream-json`
+output into a trace.
+
+A trace is plain JSON data:
+    {"items": [...], "usage": {...}, "cost_usd": float | None, "num_turns": int | None}
+Each item is one of
+    {"kind": "assistant_text", "text": str}
+    {"kind": "tool_call", "id", "name", "input", "output", "ok", "children": [items]}
+    {"kind": "user_message", "turn": int, "text": str, "source": "scripted" | "live"}
+    {"kind": "unknown", "raw_type": str}
+Items emitted inside a subagent go into the `children` of the tool call that spawned it. A
+multi-turn run's trace is the turns' traces joined in order, each turn after the first preceded
+by the user message that started it.
+"""
+
+import json
+import os
+import shutil
+import subprocess
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+SKILL_TOOL = "Skill"
+_INSTALL_HINT = "install Claude Code: npm install -g @anthropic-ai/claude-code"
+# A healthy CLI prints its version in well under this; a hung stub must not stall the run.
+_VERSION_CHECK_TIMEOUT_S = 30
+# Asking the user mid-turn would block a headless run; skillbench answers between turns instead.
+_ALWAYS_DENIED_TOOLS = ("AskUserQuestion",)
+_EMPTY_MCP_CONFIG = json.dumps({"mcpServers": {}})
+
+
+class ClaudeNotFoundError(Exception):
+    """`claude` is not on PATH, or is installed but won't run."""
+
+
+@dataclass
+class TurnInfo:
+    """Everything recovered from one turn's output."""
+
+    trace: dict[str, Any]
+    session_id: str | None = None
+    reported_model: str | None = None
+    plugins: list[Any] = field(default_factory=list)
+    skills: list[Any] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class UserMessage:
+    """A message sent to the agent to start a follow-up turn."""
+
+    text: str
+    source: str  # "scripted" (from the test's answers:) or "live" (typed in the terminal)
+
+
+def find_claude() -> str:
+    """The path of a `claude` that answers `--version`. Raises ClaudeNotFoundError."""
+    executable = shutil.which("claude")
+    if executable is None:
+        raise ClaudeNotFoundError(f"claude is not on PATH ({_INSTALL_HINT})")
+    try:
+        check = subprocess.run(
+            [executable, "--version"],
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            timeout=_VERSION_CHECK_TIMEOUT_S,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired) as error:
+        raise ClaudeNotFoundError(f"{executable} does not run ({error})") from error
+    if check.returncode != 0:
+        raise ClaudeNotFoundError(f"{executable} --version failed ({_INSTALL_HINT})")
+    return executable
+
+
+def turn_command(
+    executable: str,
+    message: str,
+    session_id: str | None,
+    model: str | None,
+    plugin_dirs: list[Path],
+) -> list[str]:
+    """The full command for one turn. The message comes first: it is positional, and the
+    list-valued flags (--disallowedTools, ...) would otherwise swallow it."""
+    command = [executable, "-p", message]
+    if session_id is not None:
+        command += ["--resume", session_id]
+    command += ["--output-format", "stream-json", "--verbose"]
+    command += ["--permission-mode", "bypassPermissions"]
+    command += ["--disallowedTools", *_ALWAYS_DENIED_TOOLS]
+    if model is not None:
+        command += ["--model", model]
+    command += _ignore_user_config_args()
+    for plugin_dir in plugin_dirs:
+        command += ["--plugin-dir", str(plugin_dir)]
+    return command
+
+
+def _ignore_user_config_args() -> list[str]:
+    """--bare needs an API key; with a subscription login, keep only project settings and no
+    MCP servers instead."""
+    if os.environ.get("ANTHROPIC_API_KEY"):
+        return ["--bare"]
+    return [
+        "--setting-sources",
+        "project",
+        "--strict-mcp-config",
+        "--mcp-config",
+        _EMPTY_MCP_CONFIG,
+    ]
+
+
+def parse_stream(raw_stdout: bytes) -> TurnInfo:
+    """Parse raw stream-json stdout. Never raises on odd input; odd lines become `unknown`."""
+    builder = _TraceBuilder()
+    for line in raw_stdout.decode("utf-8", errors="replace").splitlines():
+        if line.strip():
+            builder.add_line(line)
+    return builder.info
+
+
+def empty_trace() -> dict[str, Any]:
+    return {"items": [], "usage": {}, "cost_usd": None, "num_turns": None}
+
+
+def walk_tool_calls(items: list[dict[str, Any]]) -> Iterator[dict[str, Any]]:
+    """Every tool call in order, including those made inside subagents."""
+    for item in items:
+        if item["kind"] == "tool_call":
+            yield item
+            yield from walk_tool_calls(item["children"])
+
+
+def final_text(items: list[dict[str, Any]]) -> str | None:
+    """The last top-level assistant text: what the agent ended on."""
+    texts = [item["text"] for item in items if item["kind"] == "assistant_text"]
+    return texts[-1] if texts else None
+
+
+def merge_turns(turns: list[tuple[UserMessage | None, dict[str, Any]]]) -> dict[str, Any]:
+    """One trace for the whole conversation from each turn's (starting message, trace): items in
+    order, usage, cost and turn counts summed."""
+    items: list[dict[str, Any]] = []
+    for turn_number, (message, trace) in enumerate(turns, start=1):
+        if message is not None:
+            items.append(
+                {"kind": "user_message", "turn": turn_number, **vars(message)},
+            )
+        items += trace["items"]
+    traces = [trace for _, trace in turns]
+    return {
+        "items": items,
+        "usage": _sum_usage([trace["usage"] for trace in traces]),
+        "cost_usd": _sum_known([trace["cost_usd"] for trace in traces]),
+        "num_turns": _sum_known([trace["num_turns"] for trace in traces]),
+    }
+
+
+def _sum_known(values: list[Any]) -> Any:
+    """Sum of the values that were reported; None if no turn reported one."""
+    known = [value for value in values if value is not None]
+    return sum(known) if known else None
+
+
+def _sum_usage(usages: list[dict[str, Any]]) -> dict[str, Any]:
+    """Numeric counters add up across turns; anything else keeps the latest turn's value."""
+    total: dict[str, Any] = {}
+    for usage in usages:
+        for key, value in usage.items():
+            is_counter = isinstance(value, int | float) and not isinstance(value, bool)
+            if is_counter and isinstance(total.get(key, 0), int | float):
+                total[key] = total.get(key, 0) + value
+            else:
+                total[key] = value
+    return total
+
+
+class _TraceBuilder:
+    """Builds a TurnInfo from stream-json events, nesting subagent events under their parent."""
+
+    def __init__(self) -> None:
+        self.info = TurnInfo(trace=empty_trace())
+        self._tool_calls_by_id: dict[str, dict[str, Any]] = {}
+
+    def add_line(self, line: str) -> None:
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            event = None
+        if isinstance(event, dict):
+            self._add_event(event)
+        else:
+            self._add_unknown(None, "unparseable")
+
+    def _add_event(self, event: dict[str, Any]) -> None:
+        event_type = event.get("type")
+        if event_type == "system" and event.get("subtype") == "init":
+            self._add_init(event)
+        elif event_type == "assistant":
+            self._add_assistant(event)
+        elif event_type == "user":
+            self._add_user(event)
+        elif event_type == "result":
+            self._add_result(event)
+        elif event_type == "system":
+            self._add_unknown(None, f"system.{event.get('subtype')}")
+        else:
+            self._add_unknown(None, str(event_type))
+
+    def _add_init(self, event: dict[str, Any]) -> None:
+        self.info.session_id = event.get("session_id")
+        self.info.reported_model = event.get("model")
+        self.info.plugins = event.get("plugins", [])
+        self.info.skills = event.get("skills", [])
+
+    def _add_assistant(self, event: dict[str, Any]) -> None:
+        parent_id = event.get("parent_tool_use_id")
+        for block in event.get("message", {}).get("content", []):
+            block_type = block.get("type")
+            if block_type == "text":
+                text_item = {"kind": "assistant_text", "text": block["text"]}
+                self._items_under(parent_id).append(text_item)
+            elif block_type == "tool_use":
+                self._add_tool_call(parent_id, block)
+            else:
+                self._add_unknown(parent_id, f"assistant.{block_type}")
+
+    def _add_tool_call(self, parent_id: str | None, block: dict[str, Any]) -> None:
+        tool_call = {
+            "kind": "tool_call",
+            "id": block["id"],
+            "name": block["name"],
+            "input": block.get("input", {}),
+            "output": None,
+            "ok": None,
+            "children": [],
+        }
+        self._tool_calls_by_id[block["id"]] = tool_call
+        self._items_under(parent_id).append(tool_call)
+
+    def _add_user(self, event: dict[str, Any]) -> None:
+        parent_id = event.get("parent_tool_use_id")
+        content = event.get("message", {}).get("content", [])
+        if isinstance(content, str):
+            self._add_unknown(parent_id, "user.text")
+            return
+        for block in content:
+            tool_call = self._tool_calls_by_id.get(block.get("tool_use_id", ""))
+            if block.get("type") == "tool_result" and tool_call is not None:
+                tool_call["output"] = _tool_result_text(block.get("content"))
+                tool_call["ok"] = not block.get("is_error", False)
+            else:
+                self._add_unknown(parent_id, f"user.{block.get('type')}")
+
+    def _add_result(self, event: dict[str, Any]) -> None:
+        self.info.session_id = event.get("session_id", self.info.session_id)
+        trace = self.info.trace
+        trace["usage"] = event.get("usage", {})
+        trace["cost_usd"] = event.get("total_cost_usd")
+        trace["num_turns"] = event.get("num_turns")
+
+    def _add_unknown(self, parent_id: str | None, raw_type: str) -> None:
+        self._items_under(parent_id).append({"kind": "unknown", "raw_type": raw_type})
+
+    def _items_under(self, parent_id: str | None) -> list[dict[str, Any]]:
+        parent = self._tool_calls_by_id.get(parent_id) if parent_id else None
+        return parent["children"] if parent is not None else self.info.trace["items"]
+
+
+def _tool_result_text(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    is_all_text = isinstance(content, list) and all(
+        isinstance(block, dict) and block.get("type") == "text" for block in content
+    )
+    return "".join(block["text"] for block in content) if is_all_text else json.dumps(content)