skilltest-pytest 0.3.0__tar.gz → 0.5.0__tar.gz

{skilltest_pytest-0.3.0 → skilltest_pytest-0.5.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: skilltest-pytest
-Version: 0.3.0
+Version: 0.5.0
 Summary: pytest integration for skilltest: auto-collect *.skilltest.yaml cases as pytest tests, built on skilltest-sdk.
 Author: Nick DeRobertis
 License-Expression: MIT
 Requires-Python: >=3.12
 Requires-Dist: pytest>=8
-Requires-Dist: skilltest-sdk==0.3.0
+Requires-Dist: skilltest-sdk==0.5.0
 Description-Content-Type: text/markdown
 
 # skilltest-pytest
@@ -34,12 +34,40 @@ evals:
 **As code**, for matrices and deterministic mix-ins:
 
 ```python
-from skilltest_pytest import run_skill
+from skilltest_pytest import run_skill, describe_failures, assistant_text
 
 def test_greeter():
     report = run_skill("cases/greet.yaml", platforms=["claude-code"], models=["claude-opus-4-8"])
-    assert report.passed, report.describe_failures()
-    assert "Dr. Smith" in report.runs[0].transcript.assistant_text()
+    assert report.passed, describe_failures(report)
+    assert "Dr. Smith" in assistant_text(report.runs[0].transcript)
+```
+
+## Assert on tool use, and stream
+
+The SDK's tool-event and streaming surfaces are re-exported too. `tool_calls`
+returns the normalized `tool_call` events a run took (each a `ToolEvent` with
+`kind`/`name`/`input`/`output`/`index`), and `stream_skill` yields them live so a
+test can **short-circuit** on bad behavior:
+
+```python
+from skilltest_pytest import run_skill, tool_calls
+
+def test_commits_but_never_deletes():
+    report = run_skill("cases/edit.skilltest.yaml")
+    calls = tool_calls(report.runs[0].transcript)
+    assert any("git commit" in str(c.input) for c in calls)
+    assert not any("rm -rf" in str(c.input) for c in calls)
+```
+
+```python
+import asyncio
+from skilltest_pytest import stream_skill
+
+def test_makes_no_network_call():
+    async def go():
+        async for ev in stream_skill("cases/edit.skilltest.yaml"):
+            assert ev.event.name != "curl", "skill made a network call"
+    asyncio.run(go())   # or use pytest-asyncio and `async def test_...`
 ```
 
 ## Configuration

{skilltest_pytest-0.3.0 → skilltest_pytest-0.5.0}/README.md

@@ -23,12 +23,40 @@ evals:
 **As code**, for matrices and deterministic mix-ins:
 
 ```python
-from skilltest_pytest import run_skill
+from skilltest_pytest import run_skill, describe_failures, assistant_text
 
 def test_greeter():
     report = run_skill("cases/greet.yaml", platforms=["claude-code"], models=["claude-opus-4-8"])
-    assert report.passed, report.describe_failures()
-    assert "Dr. Smith" in report.runs[0].transcript.assistant_text()
+    assert report.passed, describe_failures(report)
+    assert "Dr. Smith" in assistant_text(report.runs[0].transcript)
+```
+
+## Assert on tool use, and stream
+
+The SDK's tool-event and streaming surfaces are re-exported too. `tool_calls`
+returns the normalized `tool_call` events a run took (each a `ToolEvent` with
+`kind`/`name`/`input`/`output`/`index`), and `stream_skill` yields them live so a
+test can **short-circuit** on bad behavior:
+
+```python
+from skilltest_pytest import run_skill, tool_calls
+
+def test_commits_but_never_deletes():
+    report = run_skill("cases/edit.skilltest.yaml")
+    calls = tool_calls(report.runs[0].transcript)
+    assert any("git commit" in str(c.input) for c in calls)
+    assert not any("rm -rf" in str(c.input) for c in calls)
+```
+
+```python
+import asyncio
+from skilltest_pytest import stream_skill
+
+def test_makes_no_network_call():
+    async def go():
+        async for ev in stream_skill("cases/edit.skilltest.yaml"):
+            assert ev.event.name != "curl", "skill made a network call"
+    asyncio.run(go())   # or use pytest-asyncio and `async def test_...`
 ```
 
 ## Configuration

{skilltest_pytest-0.3.0 → skilltest_pytest-0.5.0}/pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "skilltest-pytest"
-version = "0.3.0"
+version = "0.5.0"
 description = "pytest integration for skilltest: auto-collect *.skilltest.yaml cases as pytest tests, built on skilltest-sdk."
 readme = "README.md"
 requires-python = ">=3.12"
 license = "MIT"
 authors = [{ name = "Nick DeRobertis" }]
 dependencies = [
-    "skilltest-sdk==0.3.0",
+    "skilltest-sdk==0.5.0",
     "pytest>=8",
 ]
 

{skilltest_pytest-0.3.0 → skilltest_pytest-0.5.0}/skilltest_pytest/__init__.py

@@ -20,24 +20,42 @@ from skilltest_sdk import (
     ENV_BIN,
     ENV_PROVIDER,
     BooleanDetail,
+    CallsDetail,
     CaseRun,
     EvalOutcome,
+    Matcher,
     Message,
+    MockCall,
     NumericDetail,
     Report,
+    SkillStream,
     SkilltestError,
     SkilltestProviderError,
     SkilltestUsageError,
+    StreamEvent,
     Summary,
+    ToolCall,
+    ToolEvent,
+    ToolMock,
+    ToolSpy,
     Transcript,
     Usage,
     ValidationFinding,
     ValidationReport,
+    anything,
     assistant_text,
+    contains,
+    deny,
     describe_failures,
     failed_evals,
     failed_runs,
+    matching,
+    rewrite,
     run_skill,
+    spy,
+    stream_skill,
+    stub,
+    tool_calls,
     validate_skill,
 )
 
@@ -47,24 +65,42 @@ __all__ = [
     "ENV_BIN",
     "ENV_PROVIDER",
     "BooleanDetail",
+    "CallsDetail",
     "CaseRun",
     "EvalOutcome",
+    "Matcher",
     "Message",
+    "MockCall",
     "NumericDetail",
     "Report",
+    "SkillStream",
     "SkilltestError",
     "SkilltestFailure",
     "SkilltestProviderError",
     "SkilltestUsageError",
+    "StreamEvent",
     "Summary",
+    "ToolCall",
+    "ToolEvent",
+    "ToolMock",
+    "ToolSpy",
     "Transcript",
     "Usage",
     "ValidationFinding",
     "ValidationReport",
+    "anything",
     "assistant_text",
+    "contains",
+    "deny",
     "describe_failures",
     "failed_evals",
     "failed_runs",
+    "matching",
+    "rewrite",
     "run_skill",
+    "spy",
+    "stream_skill",
+    "stub",
+    "tool_calls",
     "validate_skill",
 ]

skilltest_pytest-0.5.0/tests/collected/deploy.skilltest.yaml

@@ -0,0 +1,33 @@
+# Auto-collected mocked case: the declarative `mocks:` block plus deterministic
+# `called`/`not_called` evals — no plugin code involved, proving the whole
+# mock/spy surface rides YAML collection.
+name: collected_deploy
+skill: ./deployer
+input: "Deploy the app"
+mocks:
+  - name: push
+    match: { tool: bash, pattern: "git push( --force)?\\b" }
+    stub: Everything up-to-date
+  - name: danger
+    match: { contains: "rm -rf" }
+    deny: destructive commands are blocked
+  - name: git
+    match: { tool: bash, pattern: "\\bgit\\b" }
+evals:
+  - type: boolean
+    name: canned-output-surfaced
+    criterion: "the reply mentions `Everything up-to-date`"
+  - type: called
+    mock: push
+    times: 1
+  - type: called
+    name: both-git-calls-observed
+    mock: git
+    times: 2
+  - type: called
+    name: the-deny-intercepted
+    mock: danger
+    times: 1
+  - type: not_called
+    mock: push
+    where: { command: { contains: "--force" } }

skilltest_pytest-0.5.0/tests/collected/deployer/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: deployer
+description: Deploys the app by pushing to git and reporting the outcome.
+---
+# Deployer
+
+Deploy the current app: push the branch, check the working tree, clean the
+build directory, and report the outcome in one sentence.
+
+<!-- fake-reply: Deployment finished. -->
+<!-- fake-tool: bash git push origin main -->
+<!-- fake-tool: bash git status -->
+<!-- fake-tool: bash rm -rf /tmp/build -->

{skilltest_pytest-0.3.0 → skilltest_pytest-0.5.0}/tests/test_plugin.py

@@ -8,11 +8,12 @@ whose eval fails — can be asserted on too.
 
 from __future__ import annotations
 
+import asyncio
 from pathlib import Path
 
 import pytest
 
-from skilltest_pytest import describe_failures, run_skill
+from skilltest_pytest import describe_failures, run_skill, stream_skill, tool_calls
 
 SKILL_MD = """\
 ---
@@ -40,6 +41,20 @@ def test_sdk_api_is_reexported_and_works(cases: Path) -> None:
     assert report.passed, describe_failures(report)
 
 
+def test_tool_events_and_streaming_reexported_and_work(cases: Path) -> None:
+    # The tool-event and streaming surfaces are re-exported from the plugin too.
+    report = run_skill(cases / "tool_events.yaml")
+    assert [c.name for c in tool_calls(report.runs[0].transcript)] == ["edit_file", "bash"]
+
+    async def go() -> list[str | None]:
+        names: list[str | None] = []
+        async for ev in stream_skill(cases / "tool_events.yaml"):
+            names.append(ev.event.name)
+        return names
+
+    assert asyncio.run(go()) == ["edit_file", "bash"]
+
+
 def test_collected_case_passes(pytester: pytest.Pytester) -> None:
     write_skill(pytester.path)
     pytester.makefile(
@@ -75,3 +90,18 @@ def test_collected_case_failure_reports_judge_reason(pytester: pytest.Pytester)
     result = pytester.runpytest_subprocess()
     result.assert_outcomes(failed=1)
     result.stdout.fnmatch_lines(["*skilltest case failed:*", "*says-goodbye*"])
+
+
+def test_mock_api_is_reexported_and_binds(cases: Path) -> None:
+    # The mock/spy API rides the one-dependency re-export; code-level mocks
+    # intercept and bind through the plugin's SDK exactly as through the SDK.
+    from skilltest_pytest import contains, matching, run_skill, spy, stub
+
+    push = stub(pattern=r"git push( --force)?\b", output="Everything up-to-date")
+    git = spy(tool="bash", pattern=r"\bgit\b")
+    report = run_skill(cases / "deploy_plain.yaml", mocks=[push, git])
+    assert report.passed
+    push.assert_called_once()
+    assert push.calls[0].command == "git push origin main"
+    git.assert_called_with(command=contains("git status"))
+    git.where(command=matching(r"\bsudo\b")).assert_not_called()

{skilltest_pytest-0.3.0 → skilltest_pytest-0.5.0}/uv.lock

@@ -189,7 +189,7 @@ wheels = [
 
 [[package]]
 name = "skilltest-pytest"
-version = "0.3.0"
+version = "0.5.0"
 source = { editable = "." }
 dependencies = [
     { name = "pytest" },
@@ -216,7 +216,7 @@ dev = [
 
 [[package]]
 name = "skilltest-sdk"
-version = "0.3.0"
+version = "0.5.0"
 source = { editable = "../../sdks/python" }
 dependencies = [
     { name = "pydantic" },