skilltest-pytest 0.2.0__py3-none-any.whl

skilltest_pytest/__init__.py

@@ -0,0 +1,70 @@
+"""skilltest-pytest: run AI-skill tests and natural-language evals as pytest.
+
+The pytest integration on top of [`skilltest-sdk`][skilltest_sdk]: drop a
+``*.skilltest.yaml`` next to your other tests and pytest collects it as a test
+item. The SDK's code-level API is re-exported here for convenience, so a pytest
+suite only needs one dependency:
+
+    from skilltest_pytest import run_skill, validate_skill
+
+    def test_greeter():
+        report = run_skill("cases/greet.yaml")
+        assert report.passed, describe_failures(report)
+        # Mix in a deterministic check on the transcript:
+        assert "Dr. Smith" in assistant_text(report.runs[0].transcript)
+"""
+
+from __future__ import annotations
+
+from skilltest_sdk import (
+    ENV_BIN,
+    ENV_PROVIDER,
+    BooleanDetail,
+    CaseRun,
+    EvalOutcome,
+    Message,
+    NumericDetail,
+    Report,
+    SkilltestError,
+    SkilltestProviderError,
+    SkilltestUsageError,
+    Summary,
+    Transcript,
+    Usage,
+    ValidationFinding,
+    ValidationReport,
+    assistant_text,
+    describe_failures,
+    failed_evals,
+    failed_runs,
+    run_skill,
+    validate_skill,
+)
+
+from .plugin import SkilltestFailure
+
+__all__ = [
+    "ENV_BIN",
+    "ENV_PROVIDER",
+    "BooleanDetail",
+    "CaseRun",
+    "EvalOutcome",
+    "Message",
+    "NumericDetail",
+    "Report",
+    "SkilltestError",
+    "SkilltestFailure",
+    "SkilltestProviderError",
+    "SkilltestUsageError",
+    "Summary",
+    "Transcript",
+    "Usage",
+    "ValidationFinding",
+    "ValidationReport",
+    "assistant_text",
+    "describe_failures",
+    "failed_evals",
+    "failed_runs",
+    "run_skill",
+    "validate_skill",
+]

skilltest_pytest/plugin.py

@@ -0,0 +1,86 @@
+"""pytest integration: collect ``*.skilltest.yaml`` files as test items.
+
+Drop a ``greets.skilltest.yaml`` next to your other tests and `pytest` will run
+it as a case, failing with the judge's reasons when an eval does not pass. For
+finer control — multiple platforms/models, or deterministic mix-in assertions on
+the transcript — call [`run_skill`][skilltest_sdk.runner.run_skill] from an
+ordinary test function instead.
+
+Settings come from ``pytest.ini``/``pyproject.toml`` (``skilltest_bin``,
+``skilltest_provider``, ``skilltest_platforms``, ``skilltest_models``,
+``skilltest_config``) or the ``SKILLTEST_BIN`` / ``SKILLTEST_PROVIDER`` env vars.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+from skilltest_sdk import Report, describe_failures, run_skill
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+_SUFFIXES = (".skilltest.yaml", ".skilltest.yml")
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addini("skilltest_bin", "Path to the skilltest binary", default=None)
+    parser.addini("skilltest_provider", "Provider command for skilltest", default=None)
+    parser.addini("skilltest_platforms", "Platforms to run cases on", type="args", default=[])
+    parser.addini("skilltest_models", "Models to run cases on", type="args", default=[])
+    parser.addini("skilltest_config", "Path to a skilltest config file", default=None)
+
+
+def pytest_collect_file(parent: pytest.Collector, file_path) -> SkilltestFile | None:
+    name = file_path.name
+    if any(name.endswith(suffix) for suffix in _SUFFIXES):
+        return SkilltestFile.from_parent(parent, path=file_path)
+    return None
+
+
+class _Settings:
+    """Resolved collector settings, read once from the pytest config."""
+
+    def __init__(self, config: pytest.Config) -> None:
+        self.bin: str | None = config.getini("skilltest_bin") or None
+        self.provider: str | None = config.getini("skilltest_provider") or None
+        self.platforms: Sequence[str] = config.getini("skilltest_platforms")
+        self.models: Sequence[str] = config.getini("skilltest_models")
+        self.config: str | None = config.getini("skilltest_config") or None
+
+
+class SkilltestFailure(Exception):
+    """Raised when a collected case fails, carrying the report for reporting."""
+
+    def __init__(self, report: Report) -> None:
+        super().__init__(describe_failures(report))
+        self.report = report
+
+
+class SkilltestFile(pytest.File):
+    def collect(self):  # type: ignore[override]
+        yield SkilltestItem.from_parent(self, name=self.path.stem)
+
+
+class SkilltestItem(pytest.Item):
+    def runtest(self) -> None:
+        settings = _Settings(self.config)
+        report = run_skill(
+            self.path,
+            bin=settings.bin,
+            provider=settings.provider,
+            platforms=settings.platforms,
+            models=settings.models,
+            config=settings.config,
+        )
+        if not report.passed:
+            raise SkilltestFailure(report)
+
+    def repr_failure(self, excinfo, style=None):  # type: ignore[override]
+        if isinstance(excinfo.value, SkilltestFailure):
+            return f"skilltest case failed:\n{excinfo.value}"
+        return super().repr_failure(excinfo, style=style)
+
+    def reportinfo(self):  # type: ignore[override]
+        return self.path, 0, f"skilltest: {self.name}"

skilltest_pytest-0.2.0.dist-info/METADATA

@@ -0,0 +1,58 @@
+Metadata-Version: 2.4
+Name: skilltest-pytest
+Version: 0.2.0
+Summary: pytest integration for skilltest: auto-collect *.skilltest.yaml cases as pytest tests, built on skilltest-sdk.
+Author: Nick DeRobertis
+License-Expression: MIT
+Requires-Python: >=3.12
+Requires-Dist: pytest>=8
+Requires-Dist: skilltest-sdk<0.2,>=0.1.0
+Description-Content-Type: text/markdown
+
+# skilltest-pytest
+
+A [pytest](https://pytest.org) plugin for [skilltest](../../README.md): run
+AI-skill tests and natural-language evals as ordinary pytest tests, and mix in
+your own deterministic checks. Built on
+[`skilltest-sdk`](../../sdks/python/README.md) — the SDK's code API is
+re-exported here, so a pytest suite needs only this one dependency.
+
+## Two ways to use it
+
+**Auto-collected case files.** Name a case `something.skilltest.yaml` and pytest
+runs it:
+
+```yaml
+# greet.skilltest.yaml
+skill: ./skills/greeter
+input: "Greet Dr. Smith."
+evals:
+  - type: boolean
+    criterion: "the reply greets Dr. Smith by name"
+```
+
+**As code**, for matrices and deterministic mix-ins:
+
+```python
+from skilltest_pytest import run_skill
+
+def test_greeter():
+    report = run_skill("cases/greet.yaml", platforms=["claude-code"], models=["claude-opus-4-8"])
+    assert report.passed, report.describe_failures()
+    assert "Dr. Smith" in report.runs[0].transcript.assistant_text()
+```
+
+## Configuration
+
+The plugin shells out to the `skilltest` binary. Point it at one with the
+`SKILLTEST_BIN` env var (or `bin=`), the provider with `SKILLTEST_PROVIDER` (or
+`provider=`), and set defaults in `pyproject.toml`:
+
+```toml
+[tool.pytest.ini_options]
+skilltest_provider = "oneharness"
+skilltest_platforms = ["claude-code"]
+skilltest_models = ["claude-opus-4-8"]
+```
+
+See the repository root for the provider protocol and the full schema.

skilltest_pytest-0.2.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+skilltest_pytest/__init__.py,sha256=zUtKpvSZqJlJvX3LicBIJ0xB6ni0vPrWyyQ86WSTPV4,1645
+skilltest_pytest/plugin.py,sha256=6MgeZLz7s_gfM0NodYxG-A_Qlm1RSD-mglt6Y7gE0i4,3374
+skilltest_pytest-0.2.0.dist-info/METADATA,sha256=DmT5vH94tbSGAuiyRLTC9Vl7xMqR3uKKjFbEC9kaO98,1788
+skilltest_pytest-0.2.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+skilltest_pytest-0.2.0.dist-info/entry_points.txt,sha256=r7Haj7qWnhqk2CyI8I1yttI8L7qXG9ssAZ5Nyccz4Ro,47
+skilltest_pytest-0.2.0.dist-info/RECORD,,

skilltest_pytest-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

skilltest_pytest-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+skilltest = skilltest_pytest.plugin