skilltest-sdk 0.2.2__tar.gz → 0.4.0__tar.gz

{skilltest_sdk-0.2.2 → skilltest_sdk-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skilltest-sdk
-Version: 0.2.2
+Version: 0.4.0
 Summary: Python SDK for the skilltest CLI: run AI-skill tests and natural-language evals from Python, with a typed report contract.
 Author: Nick DeRobertis
 License-Expression: MIT

{skilltest_sdk-0.2.2 → skilltest_sdk-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "skilltest-sdk"
-version = "0.2.2"
+version = "0.4.0"
 description = "Python SDK for the skilltest CLI: run AI-skill tests and natural-language evals from Python, with a typed report contract."
 readme = "README.md"
 requires-python = ">=3.12"

{skilltest_sdk-0.2.2 → skilltest_sdk-0.4.0}/skilltest_sdk/__init__.py

@@ -24,6 +24,7 @@ from .models import (
     NumericDetail,
     Report,
     Summary,
+    ToolEvent,
     Transcript,
     Usage,
     ValidationFinding,
@@ -32,8 +33,10 @@ from .models import (
     describe_failures,
     failed_evals,
     failed_runs,
+    tool_calls,
 )
 from .runner import ENV_BIN, ENV_PROVIDER, run_skill, validate_skill
+from .stream import SkillStream, StreamEvent, stream_skill
 
 __all__ = [
     "ENV_BIN",
@@ -44,10 +47,13 @@ __all__ = [
     "Message",
     "NumericDetail",
     "Report",
+    "SkillStream",
     "SkilltestError",
     "SkilltestProviderError",
     "SkilltestUsageError",
+    "StreamEvent",
     "Summary",
+    "ToolEvent",
     "Transcript",
     "Usage",
     "ValidationFinding",
@@ -57,5 +63,7 @@ __all__ = [
     "failed_evals",
     "failed_runs",
     "run_skill",
+    "stream_skill",
+    "tool_calls",
     "validate_skill",
 ]

{skilltest_sdk-0.2.2 → skilltest_sdk-0.4.0}/skilltest_sdk/_report.py

@@ -2,7 +2,7 @@
 #   filename:  report.schema.json
 
 from __future__ import annotations
-from typing import Literal
+from typing import Any, Literal
 from pydantic import BaseModel, Field
 
 
@@ -46,6 +46,41 @@ class EvalOutcome(BaseModel):
     reason: str = Field(..., description="The judge's stated reason.")
 
 
+class ToolEvent(BaseModel):
+    """
+    One normalized tool-call / action event the skill took during a turn, lifted
+    from oneharness's `events` array (its `--events` output). Harness-agnostic, so
+    a consumer can inspect *what the skill did* — shell commands, file edits, tool
+    uses — across any harness, not just the final text. Mirrors the oneharness
+    action-event shape; `input` is the structured, tool-shaped args so a consumer
+    can match on the command string or file path without re-parsing.
+
+    `input` is a free-form JSON value, so `Message`/`Transcript` are `PartialEq`
+    but not `Eq`.
+    """
+
+    index: int | None = Field(
+        0,
+        description='Position within the turn, so ordering ("did X before Y") is expressible.',
+        ge=0,
+    )
+    input: Any | None = Field(
+        None,
+        description="Structured tool arguments (the command, the file path); `null` when none.",
+    )
+    kind: str = Field(
+        ...,
+        description="`tool_call` (the skill invoked a tool) or `tool_result` (the observation).",
+    )
+    name: str | None = Field(
+        None,
+        description="Normalized tool name where knowable (e.g. `bash`, `edit_file`); `null` for\na `tool_result` or when the harness did not name it.",
+    )
+    output: str | None = Field(
+        None, description="The result/observation text, when the transcript exposed it."
+    )
+
+
 class Usage(BaseModel):
     """
     Token / cost usage for one provider call.
@@ -67,6 +102,10 @@ class Message(BaseModel):
     """
 
     content: str
+    events: list[ToolEvent] | None = Field(
+        None,
+        description="The normalized tool events the skill took producing this turn (assistant\nturns only, and only when the harness exposed a tool transcript via\noneharness `--events`). Empty otherwise. Surfaced for post-hoc analysis\nand streamed live for short-circuiting.",
+    )
     role: Literal["user", "assistant", "system"] = Field(..., description="Who produced a message.")
 
 

{skilltest_sdk-0.2.2 → skilltest_sdk-0.4.0}/skilltest_sdk/models.py

@@ -19,6 +19,7 @@ from ._report import (
     NumericDetail,
     Report,
     Summary,
+    ToolEvent,
     Transcript,
     Usage,
 )
@@ -32,6 +33,7 @@ __all__ = [
     "NumericDetail",
     "Report",
     "Summary",
+    "ToolEvent",
     "Transcript",
     "Usage",
     "ValidationFinding",
@@ -40,6 +42,7 @@ __all__ = [
     "describe_failures",
     "failed_evals",
     "failed_runs",
+    "tool_calls",
 ]
 
 
@@ -48,6 +51,21 @@ def assistant_text(transcript: Transcript) -> str:
     return "\n".join(m.content for m in transcript.messages if m.role == "assistant")
 
 
+def tool_calls(transcript: Transcript) -> list[ToolEvent]:
+    """Every ``tool_call`` event across the transcript's assistant turns, in order.
+
+    The normalized tool events the skill took (shell commands, file edits, tool
+    uses), lifted from oneharness ``--events`` — for asserting on *what the skill
+    did*, not just what it said. Empty for harnesses that expose no transcript.
+    """
+    return [
+        event
+        for message in transcript.messages
+        for event in (message.events or [])
+        if event.kind == "tool_call"
+    ]
+
+
 def failed_evals(run: CaseRun) -> list[EvalOutcome]:
     """The evals of a run that did not pass."""
     return [e for e in run.evals if not e.passed]

{skilltest_sdk-0.2.2 → skilltest_sdk-0.4.0}/skilltest_sdk/runner.py

@@ -104,10 +104,41 @@ def run_skill(
     the caller can assert and inspect. Only bad input ([`SkilltestUsageError`])
     and provider failures ([`SkilltestProviderError`]) raise.
     """
+    argv = build_run_argv(
+        case,
+        bin=bin,
+        provider=provider,
+        platforms=platforms,
+        models=models,
+        judge_model=judge_model,
+        max_turns=max_turns,
+        config=config,
+        fmt="json",
+    )
+    proc = _run(argv, cwd)
+    _raise_for_status(proc)
+    return _parse(Report, proc.stdout)
+
+
+def build_run_argv(
+    case: str | Path,
+    *,
+    bin: str | Path | None,
+    provider: str | Sequence[str] | None,
+    platforms: Sequence[str],
+    models: Sequence[str],
+    judge_model: str | None,
+    max_turns: int | None,
+    config: str | Path | None,
+    fmt: str,
+) -> list[str]:
+    """Build the ``skilltest run`` argv for output format ``fmt`` (``json`` for the
+    buffered API, ``json-stream`` for the streaming API). Internal, shared by
+    ``run_skill`` and the streaming API."""
     argv = [_resolve_bin(bin)]
     if config is not None:
         argv += ["--config", str(config)]
-    argv += ["run", str(case), "--format", "json"]
+    argv += ["run", str(case), "--format", fmt]
 
     resolved_provider = _resolve_provider(provider)
     if resolved_provider is not None:
@@ -120,10 +151,7 @@ def run_skill(
         argv += ["--judge-model", judge_model]
     if max_turns is not None:
         argv += ["--max-turns", str(max_turns)]
-
-    proc = _run(argv, cwd)
-    _raise_for_status(proc)
-    return _parse(Report, proc.stdout)
+    return argv
 
 
 def validate_skill(
@@ -140,14 +168,19 @@ def validate_skill(
 
 
 def _raise_for_status(proc: subprocess.CompletedProcess[str]) -> None:
-    if proc.returncode in _REPORTING_CODES:
+    raise_for_code(proc.returncode, proc.stderr.strip() or proc.stdout.strip())
+
+
+def raise_for_code(code: int | None, detail: str) -> None:
+    """Map a skilltest exit code to an exception (shared by the buffered and
+    streaming APIs). Codes 0/1 produce a report and never raise."""
+    if code in _REPORTING_CODES:
         return
-    detail = proc.stderr.strip() or proc.stdout.strip()
-    if proc.returncode == 2:
+    if code == 2:
         raise SkilltestUsageError(detail)
-    if proc.returncode == 3:
+    if code == 3:
         raise SkilltestProviderError(detail)
-    raise SkilltestError(f"skilltest exited {proc.returncode}: {detail}")
+    raise SkilltestError(f"skilltest exited {code}: {detail}")
 
 
 def _parse[T: BaseModel](model: type[T], stdout: str) -> T:

skilltest_sdk-0.4.0/skilltest_sdk/stream.py

@@ -0,0 +1,128 @@
+"""Stream a running skilltest case as an async iterator of tool events.
+
+Opt-in on top of the buffered [`run_skill`][skilltest_sdk.runner.run_skill]:
+iterate a [`SkillStream`][skilltest_sdk.stream.SkillStream] with ``async for`` to
+receive each normalized tool event the instant the skill takes it, and ``break``
+to **short-circuit** — the CLI subprocess is killed, which closes oneharness's
+stream and tears the harness down, so a bad turn is cut off instead of paid for
+in full. After the stream completes normally, ``.report`` holds the final
+[`Report`].
+
+```python
+stream = stream_skill("cases/edit.skilltest.yaml")
+async for ev in stream:
+    if ev.event.name == "bash" and "rm -rf" in str(ev.event.input):
+        break  # disallowed action — abort the run now
+report = stream.report  # the full Report, when the stream ran to completion
+```
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+from collections.abc import AsyncIterator, Sequence
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from ._report import ToolEvent
+from .errors import SkilltestProviderError
+from .models import Report
+from .runner import ENV_BIN, build_run_argv, raise_for_code
+
+
+class StreamEvent(BaseModel):
+    """One streamed tool event, tagged with the run it belongs to."""
+
+    case: str
+    platform: str
+    model: str
+    #: 1-based assistant-turn index within the run.
+    turn: int
+    event: ToolEvent
+
+
+class SkillStream:
+    """An async stream of [`StreamEvent`][skilltest_sdk.stream.StreamEvent]s from
+    a running case. Iterate with ``async for``; ``break`` to short-circuit. When
+    the stream runs to completion, ``report`` holds the final [`Report`]."""
+
+    def __init__(self, argv: list[str], cwd: str | None) -> None:
+        self._argv = argv
+        self._cwd = cwd
+        #: The final report, populated once the stream completes normally.
+        self.report: Report | None = None
+
+    def __aiter__(self) -> AsyncIterator[StreamEvent]:
+        return self._iterate()
+
+    async def _iterate(self) -> AsyncIterator[StreamEvent]:
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                *self._argv,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                cwd=self._cwd,
+            )
+        except FileNotFoundError as exc:
+            raise SkilltestProviderError(
+                f"could not run skilltest binary `{self._argv[0]}`: {exc}. "
+                f"Set {ENV_BIN} or pass bin=..."
+            ) from exc
+
+        assert proc.stdout is not None
+        try:
+            async for raw in proc.stdout:
+                line = raw.decode().strip()
+                if not line:
+                    continue
+                obj = json.loads(line)
+                kind = obj.get("type")
+                if kind == "event":
+                    yield StreamEvent.model_validate(obj)
+                elif kind == "result":
+                    self.report = Report.model_validate(obj["report"])
+            await proc.wait()
+            # A hard failure (bad input / provider error) once the stream ends.
+            detail = ""
+            if proc.stderr is not None:
+                detail = (await proc.stderr.read()).decode().strip()
+            raise_for_code(proc.returncode, detail)
+        finally:
+            # The consumer stopped early (break): kill the CLI so oneharness's
+            # stream closes and the harness is torn down.
+            if proc.returncode is None:
+                proc.kill()
+                with contextlib.suppress(ProcessLookupError):
+                    await proc.wait()
+
+
+def stream_skill(
+    case: str | Path,
+    *,
+    bin: str | Path | None = None,
+    provider: str | Sequence[str] | None = None,
+    platforms: Sequence[str] = (),
+    models: Sequence[str] = (),
+    judge_model: str | None = None,
+    max_turns: int | None = None,
+    config: str | Path | None = None,
+    cwd: str | Path | None = None,
+) -> SkillStream:
+    """Start a streaming run and return a [`SkillStream`] to iterate. Same
+    arguments as [`run_skill`][skilltest_sdk.runner.run_skill]; the run does not
+    begin until iteration starts."""
+    argv = build_run_argv(
+        case,
+        bin=bin,
+        provider=provider,
+        platforms=platforms,
+        models=models,
+        judge_model=judge_model,
+        max_turns=max_turns,
+        config=config,
+        fmt="json-stream",
+    )
+    return SkillStream(argv, str(cwd) if cwd is not None else None)

{skilltest_sdk-0.2.2 → skilltest_sdk-0.4.0}/tests/test_api.py

@@ -14,6 +14,7 @@ from skilltest_sdk import (
     describe_failures,
     failed_evals,
     run_skill,
+    tool_calls,
     validate_skill,
 )
 
@@ -26,6 +27,13 @@ def test_happy_path_passes_and_exposes_transcript(cases: Path) -> None:
     assert "Dr. Smith" in assistant_text(report.runs[0].transcript)
 
 
+def test_tool_calls_are_exposed_for_analysis(cases: Path) -> None:
+    report = run_skill(cases / "tool_events.yaml")
+    calls = tool_calls(report.runs[0].transcript)
+    assert [c.name for c in calls] == ["edit_file", "bash"]
+    assert calls[1].input == {"command": 'git commit -m "update config"'}
+
+
 def test_numeric_eval_detail_is_typed(cases: Path) -> None:
     report = run_skill(cases / "greet_numeric.yaml")
     assert report.passed

skilltest_sdk-0.4.0/tests/test_stream.py

@@ -0,0 +1,37 @@
+"""Streaming-API e2e tests against the built binary + fake provider."""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+from skilltest_sdk import Report, StreamEvent, stream_skill
+
+
+def test_stream_yields_events_then_exposes_report(cases: Path) -> None:
+    async def go() -> tuple[list[str | None], Report | None]:
+        stream = stream_skill(cases / "tool_events.yaml")
+        names: list[str | None] = []
+        async for ev in stream:
+            assert isinstance(ev, StreamEvent)
+            assert ev.case == "tool_events"
+            assert ev.turn == 1
+            names.append(ev.event.name)
+        return names, stream.report
+
+    names, report = asyncio.run(go())
+    assert names == ["edit_file", "bash"]
+    assert report is not None
+    assert report.passed
+
+
+def test_stream_short_circuits_on_break(cases: Path) -> None:
+    async def go() -> int:
+        stream = stream_skill(cases / "tool_events.yaml")
+        seen = 0
+        async for _ in stream:
+            seen += 1
+            break  # abort after the first event
+        return seen
+
+    assert asyncio.run(go()) == 1

{skilltest_sdk-0.2.2 → skilltest_sdk-0.4.0}/uv.lock

@@ -482,7 +482,7 @@ wheels = [
 
 [[package]]
 name = "skilltest-sdk"
-version = "0.2.2"
+version = "0.4.0"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },