skilltest-sdk 0.3.0__tar.gz → 0.5.0__tar.gz

{skilltest_sdk-0.3.0 → skilltest_sdk-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skilltest-sdk
-Version: 0.3.0
+Version: 0.5.0
 Summary: Python SDK for the skilltest CLI: run AI-skill tests and natural-language evals from Python, with a typed report contract.
 Author: Nick DeRobertis
 License-Expression: MIT
@@ -17,17 +17,52 @@ integrations build on it — use [`skilltest-pytest`](../../plugins/pytest) if y
 want pytest collection; use this package directly from any other Python code.
 
 ```python
-from skilltest_sdk import run_skill, validate_skill
+from skilltest_sdk import run_skill, validate_skill, describe_failures, assistant_text
 
 report = run_skill("cases/greet.yaml")
-assert report.passed, report.describe_failures()
+assert report.passed, describe_failures(report)
 # Mix in deterministic checks on the transcript:
-assert "Dr. Smith" in report.runs[0].transcript.assistant_text()
+assert "Dr. Smith" in assistant_text(report.runs[0].transcript)
 
 result = validate_skill("skills/greeter")
 assert result.valid
 ```
 
+### Tool events
+
+Each assistant turn carries the normalized tool events the skill took (shell
+commands, file edits, tool uses), lifted from oneharness's `--events`. Assert on
+*what the skill did* with `tool_calls` (the `tool_call` events across a
+transcript, in order); each `ToolEvent` has `kind`, `name`, `input`, `output`,
+`index`:
+
+```python
+from skilltest_sdk import run_skill, tool_calls
+
+report = run_skill("cases/edit.skilltest.yaml")
+calls = tool_calls(report.runs[0].transcript)
+assert any("git commit" in str(c.input) for c in calls)
+assert not any("rm -rf" in str(c.input) for c in calls)   # never destructive
+```
+
+### Streaming (opt-in)
+
+`stream_skill` returns a `SkillStream` you iterate with `async for` to receive
+each event live, and `break` to **short-circuit** — closing the stream tears the
+harness down, so a bad turn is cut off instead of paid for in full. `.report`
+holds the final report once the stream runs to completion:
+
+```python
+from skilltest_sdk import stream_skill
+
+async def guard():
+    stream = stream_skill("cases/edit.skilltest.yaml")
+    async for ev in stream:  # ev: StreamEvent — .case/.platform/.model/.turn/.event
+        if ev.event.name == "bash" and "rm -rf" in str(ev.event.input):
+            break
+    return stream.report
+```
+
 The `skilltest` binary is resolved from the `bin=` argument, the
 `SKILLTEST_BIN` env var, or `PATH`; a provider override comes from `provider=`
 or `SKILLTEST_PROVIDER`. A failing eval is *reported* (`report.passed` is

skilltest_sdk-0.5.0/README.md

@@ -0,0 +1,65 @@
+# skilltest-sdk
+
+The Python SDK for the [`skilltest`](https://github.com/nickderobertis/skilltest)
+CLI. A thin, typed wrapper and nothing else: it runs the CLI as a subprocess and
+parses the stable `--format json` contract into Pydantic models. Test-framework
+integrations build on it — use [`skilltest-pytest`](../../plugins/pytest) if you
+want pytest collection; use this package directly from any other Python code.
+
+```python
+from skilltest_sdk import run_skill, validate_skill, describe_failures, assistant_text
+
+report = run_skill("cases/greet.yaml")
+assert report.passed, describe_failures(report)
+# Mix in deterministic checks on the transcript:
+assert "Dr. Smith" in assistant_text(report.runs[0].transcript)
+
+result = validate_skill("skills/greeter")
+assert result.valid
+```
+
+### Tool events
+
+Each assistant turn carries the normalized tool events the skill took (shell
+commands, file edits, tool uses), lifted from oneharness's `--events`. Assert on
+*what the skill did* with `tool_calls` (the `tool_call` events across a
+transcript, in order); each `ToolEvent` has `kind`, `name`, `input`, `output`,
+`index`:
+
+```python
+from skilltest_sdk import run_skill, tool_calls
+
+report = run_skill("cases/edit.skilltest.yaml")
+calls = tool_calls(report.runs[0].transcript)
+assert any("git commit" in str(c.input) for c in calls)
+assert not any("rm -rf" in str(c.input) for c in calls)   # never destructive
+```
+
+### Streaming (opt-in)
+
+`stream_skill` returns a `SkillStream` you iterate with `async for` to receive
+each event live, and `break` to **short-circuit** — closing the stream tears the
+harness down, so a bad turn is cut off instead of paid for in full. `.report`
+holds the final report once the stream runs to completion:
+
+```python
+from skilltest_sdk import stream_skill
+
+async def guard():
+    stream = stream_skill("cases/edit.skilltest.yaml")
+    async for ev in stream:  # ev: StreamEvent — .case/.platform/.model/.turn/.event
+        if ev.event.name == "bash" and "rm -rf" in str(ev.event.input):
+            break
+    return stream.report
+```
+
+The `skilltest` binary is resolved from the `bin=` argument, the
+`SKILLTEST_BIN` env var, or `PATH`; a provider override comes from `provider=`
+or `SKILLTEST_PROVIDER`. A failing eval is *reported* (`report.passed` is
+false), not raised; bad input raises `SkilltestUsageError` (CLI exit 2) and
+provider problems raise `SkilltestProviderError` (exit 3).
+
+The Pydantic models are **generated** from `schemas/report.schema.json` /
+`schemas/validation.schema.json` — themselves generated from the CLI's own
+types — via `just gen-contract`, and a drift gate in CI fails if anything is
+stale, so the models cannot diverge from the binary.

{skilltest_sdk-0.3.0 → skilltest_sdk-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "skilltest-sdk"
-version = "0.3.0"
+version = "0.5.0"
 description = "Python SDK for the skilltest CLI: run AI-skill tests and natural-language evals from Python, with a typed report contract."
 readme = "README.md"
 requires-python = ">=3.12"

{skilltest_sdk-0.3.0 → skilltest_sdk-0.5.0}/skilltest_sdk/__init__.py

@@ -16,14 +16,30 @@ so they cannot drift from the binary. Test frameworks build on this —
 from __future__ import annotations
 
 from .errors import SkilltestError, SkilltestProviderError, SkilltestUsageError
+from .mock import (
+    Matcher,
+    ToolCall,
+    ToolMock,
+    ToolSpy,
+    anything,
+    contains,
+    deny,
+    matching,
+    rewrite,
+    spy,
+    stub,
+)
 from .models import (
     BooleanDetail,
+    CallsDetail,
     CaseRun,
     EvalOutcome,
     Message,
+    MockCall,
     NumericDetail,
     Report,
     Summary,
+    ToolEvent,
     Transcript,
     Usage,
     ValidationFinding,
@@ -32,30 +48,50 @@ from .models import (
     describe_failures,
     failed_evals,
     failed_runs,
+    tool_calls,
 )
 from .runner import ENV_BIN, ENV_PROVIDER, run_skill, validate_skill
+from .stream import SkillStream, StreamEvent, stream_skill
 
 __all__ = [
     "ENV_BIN",
     "ENV_PROVIDER",
     "BooleanDetail",
+    "CallsDetail",
     "CaseRun",
     "EvalOutcome",
+    "Matcher",
     "Message",
+    "MockCall",
     "NumericDetail",
     "Report",
+    "SkillStream",
     "SkilltestError",
     "SkilltestProviderError",
     "SkilltestUsageError",
+    "StreamEvent",
     "Summary",
+    "ToolCall",
+    "ToolEvent",
+    "ToolMock",
+    "ToolSpy",
     "Transcript",
     "Usage",
     "ValidationFinding",
     "ValidationReport",
+    "anything",
     "assistant_text",
+    "contains",
+    "deny",
     "describe_failures",
     "failed_evals",
     "failed_runs",
+    "matching",
+    "rewrite",
     "run_skill",
+    "spy",
+    "stream_skill",
+    "stub",
+    "tool_calls",
     "validate_skill",
 ]

{skilltest_sdk-0.3.0 → skilltest_sdk-0.5.0}/skilltest_sdk/_report.py

@@ -2,7 +2,7 @@
 #   filename:  report.schema.json
 
 from __future__ import annotations
-from typing import Literal
+from typing import Any, Literal
 from pydantic import BaseModel, Field
 
 
@@ -35,17 +35,100 @@ class NumericDetail(BaseModel):
     value: float
 
 
+class CallsDetail(BaseModel):
+    """
+    The deterministic `called`/`not_called` verdict: how many observed calls
+    matched, against what expectation.
+    """
+
+    count: int = Field(..., description="Matching calls observed.", ge=0)
+    kind: Literal["calls"]
+    negated: bool | None = Field(
+        False, description="True for `not_called` (the eval required absence)."
+    )
+    times: int | None = Field(
+        None,
+        description='The exact count required (`times`); `null` means "at least one"\n(or, with `negated`, "none").',
+        ge=0,
+    )
+
+
 class EvalOutcome(BaseModel):
     """
     The result of running one eval against a transcript.
     """
 
-    detail: BooleanDetail | NumericDetail = Field(..., description="Kind-specific verdict detail.")
+    detail: BooleanDetail | NumericDetail | CallsDetail = Field(
+        ..., description="Kind-specific verdict detail."
+    )
     label: str = Field(..., description="The eval's label (name or criterion).")
     passed: bool = Field(..., description="Whether the eval passed.")
     reason: str = Field(..., description="The judge's stated reason.")
 
 
+class MockCall(BaseModel):
+    """
+    One observed tool call, as recorded by the mock/spy channel: the harness
+    hook's spy log for real runs, or the provider's `mock_calls` response for
+    the command protocol. Carries the **original, pre-rewrite** input — the
+    transcript's `events` show post-rewrite reality (the stub that actually
+    ran); this shows what the skill *attempted*.
+    """
+
+    action: str = Field(
+        ...,
+        description="The verdict applied: `allow` (fell through every rule), `deny`,\n`rewrite`, or `stub`.",
+    )
+    input: Any | None = Field(
+        None, description="The tool's original input arguments; `null` when the event carried none."
+    )
+    mock: str | None = Field(
+        None,
+        description="Name of the mock declaration the intercepting rule came from; `null`\nfor `allow`. Filled by the runner, so SDKs bind records to mock objects\nby name instead of re-deriving rule indices.",
+    )
+    rule: int | None = Field(
+        None, description="Index of the compiled rule that intercepted; `null` for `allow`.", ge=0
+    )
+    tool: str | None = Field(
+        None, description="Tool name as the harness reported it; `null` when the event named none."
+    )
+
+
+class ToolEvent(BaseModel):
+    """
+    One normalized tool-call / action event the skill took during a turn, lifted
+    from oneharness's `events` array (its `--events` output). Harness-agnostic, so
+    a consumer can inspect *what the skill did* — shell commands, file edits, tool
+    uses — across any harness, not just the final text. Mirrors the oneharness
+    action-event shape; `input` is the structured, tool-shaped args so a consumer
+    can match on the command string or file path without re-parsing.
+
+    `input` is a free-form JSON value, so `Message`/`Transcript` are `PartialEq`
+    but not `Eq`.
+    """
+
+    index: int | None = Field(
+        0,
+        description='Position within the turn, so ordering ("did X before Y") is expressible.',
+        ge=0,
+    )
+    input: Any | None = Field(
+        None,
+        description="Structured tool arguments (the command, the file path); `null` when none.",
+    )
+    kind: str = Field(
+        ...,
+        description="`tool_call` (the skill invoked a tool) or `tool_result` (the observation).",
+    )
+    name: str | None = Field(
+        None,
+        description="Normalized tool name where knowable (e.g. `bash`, `edit_file`); `null` for\na `tool_result` or when the harness did not name it.",
+    )
+    output: str | None = Field(
+        None, description="The result/observation text, when the transcript exposed it."
+    )
+
+
 class Usage(BaseModel):
     """
     Token / cost usage for one provider call.
@@ -67,6 +150,10 @@ class Message(BaseModel):
     """
 
     content: str
+    events: list[ToolEvent] | None = Field(
+        None,
+        description="The normalized tool events the skill took producing this turn (assistant\nturns only, and only when the harness exposed a tool transcript via\noneharness `--events`). Empty otherwise. Surfaced for post-hoc analysis\nand streamed live for short-circuiting.",
+    )
     role: Literal["user", "assistant", "system"] = Field(..., description="Who produced a message.")
 
 
@@ -101,6 +188,10 @@ class CaseRun(BaseModel):
 
     case: str = Field(..., description="The test case name.")
     evals: list[EvalOutcome] = Field(..., description="Per-eval outcomes, in declaration order.")
+    mock_calls: list[MockCall] | None = Field(
+        None,
+        description='Every tool call the mock/spy channel observed, in order, with the\noriginal (pre-rewrite) input and the verdict applied. `null` when the\nchannel was off for this run (no `mocks`, no `spy`); an empty array\nmeans the channel was on and the skill made no tool calls — SDKs use\nthat distinction so a spy on a channel-less run errs instead of reading\nas "zero calls".',
+    )
     model: str = Field(..., description="The model this run used.")
     passed: bool = Field(..., description="True iff every eval in this run passed.")
     platform: str = Field(..., description="The harness platform this run used.")