skilltest-sdk 0.12.1__py3-none-win_amd64.whl

skilltest_sdk/__init__.py

@@ -0,0 +1,150 @@
+"""skilltest-sdk: the Python SDK for the ``skilltest`` CLI.
+
+A thin, typed wrapper around the CLI and nothing else: run test cases, validate
+skills, and get back models mirroring the ``--format json`` contract. The
+models are generated from the CLI's own JSON Schemas (``just gen-contract``),
+so they cannot drift from the binary. Test frameworks build on this —
+``skilltest-pytest`` adds pytest collection on top.
+
+Define the whole case in code (the recommended form):
+
+    from skilltest_sdk import TestCase, run_skill, boolean, describe_failures
+
+    case = TestCase(
+        skill="skills/greeter",
+        input="Greet Dr. Smith, who has an appointment today.",
+        evals=[boolean("the reply greets Dr. Smith by name")],
+    )
+    report = run_skill(case)
+    assert report.passed, describe_failures(report)
+
+``run_skill`` also takes a path to an existing test-case YAML file (or a
+directory of them), and the pytest plugin auto-discovers ``*.skilltest.yaml``.
+"""
+
+from __future__ import annotations
+
+from .case import (
+    Eval,
+    MockRefEval,
+    SimulatedUser,
+    TestCase,
+    boolean,
+    called,
+    not_called,
+    numeric,
+    user,
+)
+from .errors import (
+    ProviderErrorKind,
+    SkilltestAuthError,
+    SkilltestError,
+    SkilltestModelNotFoundError,
+    SkilltestOverloadedError,
+    SkilltestProtocolError,
+    SkilltestProviderError,
+    SkilltestQuotaError,
+    SkilltestRateLimitError,
+    SkilltestSpawnError,
+    SkilltestTimeoutError,
+    SkilltestUsageError,
+)
+from .mock import (
+    Matcher,
+    ToolCall,
+    ToolMock,
+    ToolSpy,
+    anything,
+    contains,
+    deny,
+    matching,
+    rewrite,
+    spy,
+    stub,
+)
+from .models import (
+    BooleanDetail,
+    CallsDetail,
+    CaseRun,
+    EvalOutcome,
+    Message,
+    MockCall,
+    NumericDetail,
+    Report,
+    ReportError,
+    Summary,
+    ToolEvent,
+    Transcript,
+    Usage,
+    ValidationFinding,
+    ValidationReport,
+    assistant_text,
+    describe_failures,
+    failed_evals,
+    failed_runs,
+    tool_calls,
+)
+from .runner import ENV_BIN, ENV_PROVIDER, run_skill, validate_skill
+from .stream import SkillStream, StreamEvent, stream_skill
+
+__all__ = [
+    "ENV_BIN",
+    "ENV_PROVIDER",
+    "BooleanDetail",
+    "CallsDetail",
+    "CaseRun",
+    "Eval",
+    "EvalOutcome",
+    "Matcher",
+    "Message",
+    "MockCall",
+    "MockRefEval",
+    "NumericDetail",
+    "ProviderErrorKind",
+    "Report",
+    "ReportError",
+    "SimulatedUser",
+    "SkillStream",
+    "SkilltestAuthError",
+    "SkilltestError",
+    "SkilltestModelNotFoundError",
+    "SkilltestOverloadedError",
+    "SkilltestProtocolError",
+    "SkilltestProviderError",
+    "SkilltestQuotaError",
+    "SkilltestRateLimitError",
+    "SkilltestSpawnError",
+    "SkilltestTimeoutError",
+    "SkilltestUsageError",
+    "StreamEvent",
+    "Summary",
+    "TestCase",
+    "ToolCall",
+    "ToolEvent",
+    "ToolMock",
+    "ToolSpy",
+    "Transcript",
+    "Usage",
+    "ValidationFinding",
+    "ValidationReport",
+    "anything",
+    "assistant_text",
+    "boolean",
+    "called",
+    "contains",
+    "deny",
+    "describe_failures",
+    "failed_evals",
+    "failed_runs",
+    "matching",
+    "not_called",
+    "numeric",
+    "rewrite",
+    "run_skill",
+    "spy",
+    "stream_skill",
+    "stub",
+    "tool_calls",
+    "user",
+    "validate_skill",
+]

skilltest_sdk/_case.py

@@ -0,0 +1,251 @@
+# generated by datamodel-codegen:
+#   filename:  case.schema.json
+
+from __future__ import annotations
+from typing import Any, Literal
+from pydantic import BaseModel, ConfigDict, Field, RootModel
+
+
+class DenyMessage(BaseModel):
+    """
+    The explicit form of a [`DenySpec`]: the model-visible message.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    message: str
+
+
+class BooleanEval(BaseModel):
+    """
+    Assert a plain-English criterion holds (or, with `expected: false`, that
+    it does not).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    criterion: str = Field(
+        ..., description="The criterion the judge evaluates against the transcript."
+    )
+    expected: bool | None = Field(
+        True, description="What the judge's verdict must equal to pass. Defaults to `true`."
+    )
+    name: str | None = Field(None, description="Optional human label for reports.")
+    type: Literal["boolean"]
+
+
+class NumericEval(BaseModel):
+    """
+    Score a plain-English criterion on a numeric scale and compare it to a
+    threshold.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    comparator: Literal["gte", "gt", "lte", "lt"] | None = Field(
+        "gte", description="How the score is compared to `threshold`. Defaults to `>=`."
+    )
+    criterion: str = Field(..., description="The criterion the judge scores.")
+    max: float = Field(..., description="Inclusive upper bound of the scale.")
+    min: float = Field(..., description="Inclusive lower bound of the scale.")
+    name: str | None = Field(None, description="Optional human label for reports.")
+    threshold: float = Field(..., description="The passing threshold.")
+    type: Literal["numeric"]
+
+
+class FieldPredicateSpec(BaseModel):
+    """
+    The explicit form of a [`FieldPredicate`]; every given key must hold.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    contains: str | None = None
+    equals: str | None = None
+    pattern: str | None = Field(
+        None, description="Unanchored regex (linear-time; same engine oneharness uses)."
+    )
+
+
+class SimulatedUser(BaseModel):
+    """
+    The simulated-user block that turns a single-turn case into a multi-turn one.
+    When present, after each assistant turn the runner asks the provider to play
+    the user (guided by `persona`) until `done_when` holds or `max_turns` is hit.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    done_when: str | None = Field(
+        None,
+        description="A plain-English condition; when the judge decides it holds, the\nconversation ends. Optional — without it the run ends at `max_turns` or\nwhen the skill reports itself done.",
+    )
+    max_turns: int | None = Field(
+        None, description="Per-case override of the global assistant-turn cap.", ge=0
+    )
+    persona: str = Field(
+        ..., description="Instructions describing how the simulated user should behave."
+    )
+
+
+class StubOutput(BaseModel):
+    """
+    The explicit form of a [`StubSpec`]: the canned output plus an exit code.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    exit_code: int | None = Field(0, description="Non-zero fakes a failing command. Default 0.")
+    output: str
+
+
+class StubSpec1(RootModel[list[str | StubOutput]]):
+    root: list[str | StubOutput] = Field(
+        ...,
+        description="Ordered responses for successive intercepted calls; the last repeats.",
+        min_length=1,
+    )
+
+
+class MockMatch(BaseModel):
+    """
+    What a mock/spy declaration matches on. At least one criterion is required;
+    every given criterion must hold (AND).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    contains: str | None = Field(
+        None,
+        description="Substring match over the raw hook event JSON (the tool name and its\ninput always serialize into it). Note the haystack is JSON: quotes and\nbackslashes inside tool input appear escaped.",
+    )
+    input: dict[str, str | FieldPredicateSpec] | None = Field(
+        None,
+        description="Per-field predicates on the tool's input: the key is the argument name\n(`command`, `file_path`, …), and every listed field must match (a field\nabsent from the call fails the matcher). Non-string fields are compared\nagainst their compact JSON form.",
+    )
+    pattern: str | None = Field(
+        None,
+        description="Unanchored regex over the same haystack as `contains`, for non-exact\nneedles like `git push( --force)?`. Linear-time engine (no lookarounds).",
+    )
+    tool: str | None = Field(
+        None,
+        description="Case-insensitive exact match on the tool name. Tool names are\nper-harness (`Bash` on claude-code, `bash` on opencode/crush), so\ncross-harness matchers usually prefer `contains`/`pattern`/`input`.",
+    )
+
+
+class CalledEval(BaseModel):
+    """
+    Deterministic: assert the referenced mock/spy observed at least one
+    matching call (or exactly `times`). Evaluated against the mock channel's
+    records, not by a judge.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    mock: str = Field(
+        ..., description="The `mocks:` declaration (mock or spy) this asserts on, by name."
+    )
+    name: str | None = Field(None, description="Optional human label for reports.")
+    times: int | None = Field(
+        None, description='Exact required call count; absent means "at least once".', ge=0
+    )
+    type: Literal["called"]
+    where: dict[str, str | FieldPredicateSpec] | None = Field(
+        None, description="Optional per-field input predicates narrowing which calls count."
+    )
+
+
+class NotCalledEval(BaseModel):
+    """
+    Deterministic: assert the referenced mock/spy observed **no** matching
+    call.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    mock: str = Field(
+        ..., description="The `mocks:` declaration (mock or spy) this asserts on, by name."
+    )
+    name: str | None = Field(None, description="Optional human label for reports.")
+    type: Literal["not_called"]
+    where: dict[str, str | FieldPredicateSpec] | None = Field(
+        None, description="Optional per-field input predicates narrowing which calls count."
+    )
+
+
+class MockDecl(BaseModel):
+    """
+    One mock or spy declaration — an entry of a test case's `mocks:` block, of
+    the CLI's `--mocks` file, or synthesized by an SDK. Exactly one of `stub` /
+    `deny` / `rewrite` makes it a **mock** (the call is intercepted); none makes
+    it a **spy** (observed only, matched locally against the returned records).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    deny: str | DenyMessage | None = Field(
+        None, description="Block the call; the model reads the message as the tool's feedback."
+    )
+    match: MockMatch
+    name: str | None = Field(
+        None,
+        description="Name evals (`type: called` / `not_called`) reference this declaration\nby. Optional; unnamed declarations get a positional fallback\n(`mock_<i>` / `spy_<i>`) used in reports and error messages.",
+    )
+    rewrite: Any | None = Field(
+        None,
+        description="Substitute raw input fields (a JSON object) — the low-level escape\nhatch, and the way to mock file reads (rewrite `file_path` to a\nfixture).",
+    )
+    stub: str | StubOutput | StubSpec1 | None = Field(
+        None,
+        description="Fake a shell call's result: the real command never runs and the model\nreceives this output as the tool's genuine result.",
+    )
+
+
+class TestCase(BaseModel):
+    """
+    One test case.
+
+    This type is the source of truth for the **input contract**
+    (`schemas/case.schema.json`): the shape a `--case-json` payload — and the
+    SDKs' generated case models — must have. Serialization skips
+    absent/default fields so the canonical JSON form is minimal; the SDK case
+    builders emit that same form (pinned by the kitchen-sink golden in
+    `tests/fixtures/contract/`).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    evals: list[BooleanEval | NumericEval | CalledEval | NotCalledEval] = Field(
+        ..., description="The evals that decide whether this case passes. Must be non-empty."
+    )
+    input: str = Field(
+        ..., description="The initial data/prompt handed to the skill as the first user message."
+    )
+    mocks: list[MockDecl] | None = Field(
+        None,
+        description="Mock/spy declarations for this case: a declaration with a `stub`/`deny`/\n`rewrite` action intercepts matching tool calls; one without observes\nonly. `called`/`not_called` evals reference these by `name`.",
+    )
+    name: str | None = Field(
+        None, description="Human-readable name (defaults to the file stem when loaded from a file)."
+    )
+    skill: str = Field(
+        ..., description="Path to the skill directory under test, relative to the test-case file."
+    )
+    spy: bool | None = Field(
+        None,
+        description="Record every tool call through the mock/spy channel even with no\n`mocks` declared, so code-level consumers (the SDKs' spies) get records.\nImplied whenever `mocks` is non-empty.",
+    )
+    user: SimulatedUser | None = Field(
+        None, description="Present for multi-turn cases; absent for single-turn."
+    )

skilltest_sdk/_error.py

@@ -0,0 +1,50 @@
+# generated by datamodel-codegen:
+#   filename:  error.schema.json
+
+from __future__ import annotations
+from typing import Literal
+from pydantic import BaseModel, Field
+
+
+class ReportError(BaseModel):
+    """
+    A structured error, emitted as the `--format json` / `json-stream` output
+    when a `skilltest run` cannot produce a [`Report`].
+
+    This is the machine-readable counterpart to the human hint the CLI prints on
+    stderr: it rides on stdout so SDK/plugin consumers get the [`ProviderErrorKind`]
+    (and the `code`/`context`) for targeted handling — retry on
+    [`ProviderErrorKind::Timeout`], fail fast on [`ProviderErrorKind::Auth`] —
+    instead of matching substrings in the message. For `json` the object is
+    emitted bare; for `json-stream` it is the terminal
+    `{"type":"error","error":{…}}` line.
+    """
+
+    code: Literal["usage", "provider"] = Field(
+        ..., description="The coarse failure class, matching the process exit code."
+    )
+    context: str | None = Field(
+        None,
+        description="The provider context the failure came from (e.g. `oneharness:claude-code`,\n`api-judge`). Absent for usage errors.",
+    )
+    kind: (
+        Literal[
+            "auth",
+            "rate_limit",
+            "model_not_found",
+            "quota",
+            "overloaded",
+            "timeout",
+            "spawn",
+            "protocol",
+            "other",
+        ]
+        | None
+    ) = Field(
+        None,
+        description="The structured provider-failure category, when skilltest could classify\nit. Absent for usage errors and for unclassified provider failures.",
+    )
+    message: str = Field(
+        ...,
+        description="A human-readable description of what went wrong (the same text printed on\nstderr, minus the suggested-action hint).",
+    )

skilltest_sdk/_report.py

@@ -0,0 +1,220 @@
+# generated by datamodel-codegen:
+#   filename:  report.schema.json
+
+from __future__ import annotations
+from typing import Any, Literal
+from pydantic import BaseModel, Field
+
+
+class BooleanDetail(BaseModel):
+    """
+    The kind-specific detail of an eval outcome, for reporting.
+
+    The variant titles name the generated SDK model for each union arm, so keep
+    them stable: they are part of the SDK API surface.
+    """
+
+    expected: bool
+    kind: Literal["boolean"]
+    value: bool
+
+
+class NumericDetail(BaseModel):
+    """
+    The kind-specific detail of an eval outcome, for reporting.
+
+    The variant titles name the generated SDK model for each union arm, so keep
+    them stable: they are part of the SDK API surface.
+    """
+
+    comparator: Literal["gte", "gt", "lte", "lt"] = Field(
+        ..., description="How a numeric score is compared to its threshold."
+    )
+    kind: Literal["numeric"]
+    threshold: float
+    value: float
+
+
+class CallsDetail(BaseModel):
+    """
+    The deterministic `called`/`not_called` verdict: how many observed calls
+    matched, against what expectation.
+    """
+
+    count: int = Field(..., description="Matching calls observed.", ge=0)
+    kind: Literal["calls"]
+    negated: bool | None = Field(
+        False, description="True for `not_called` (the eval required absence)."
+    )
+    times: int | None = Field(
+        None,
+        description='The exact count required (`times`); `null` means "at least one"\n(or, with `negated`, "none").',
+        ge=0,
+    )
+
+
+class EvalOutcome(BaseModel):
+    """
+    The result of running one eval against a transcript.
+    """
+
+    detail: BooleanDetail | NumericDetail | CallsDetail = Field(
+        ..., description="Kind-specific verdict detail."
+    )
+    label: str = Field(..., description="The eval's label (name or criterion).")
+    passed: bool = Field(..., description="Whether the eval passed.")
+    reason: str = Field(..., description="The judge's stated reason.")
+
+
+class MockCall(BaseModel):
+    """
+    One observed tool call, as recorded by the mock/spy channel: the harness
+    hook's spy log for real runs, or the provider's `mock_calls` response for
+    the command protocol. Carries the **original, pre-rewrite** input — the
+    transcript's `events` show post-rewrite reality (the stub that actually
+    ran); this shows what the skill *attempted*.
+    """
+
+    action: str = Field(
+        ...,
+        description="The verdict applied: `allow` (fell through every rule), `deny`,\n`rewrite`, or `stub`.",
+    )
+    input: Any | None = Field(
+        None, description="The tool's original input arguments; `null` when the event carried none."
+    )
+    mock: str | None = Field(
+        None,
+        description="Name of the mock declaration the intercepting rule came from; `null`\nfor `allow`. Filled by the runner, so SDKs bind records to mock objects\nby name instead of re-deriving rule indices.",
+    )
+    rule: int | None = Field(
+        None, description="Index of the compiled rule that intercepted; `null` for `allow`.", ge=0
+    )
+    tool: str | None = Field(
+        None, description="Tool name as the harness reported it; `null` when the event named none."
+    )
+
+
+class ToolEvent(BaseModel):
+    """
+    One normalized tool-call / action event the skill took during a turn, lifted
+    from oneharness's `events` array (its `--events` output). Harness-agnostic, so
+    a consumer can inspect *what the skill did* — shell commands, file edits, tool
+    uses — across any harness, not just the final text. Mirrors the oneharness
+    action-event shape; `input` is the structured, tool-shaped args so a consumer
+    can match on the command string or file path without re-parsing.
+
+    `input` is a free-form JSON value, so `Message`/`Transcript` are `PartialEq`
+    but not `Eq`.
+    """
+
+    index: int | None = Field(
+        0,
+        description='Position within the turn, so ordering ("did X before Y") is expressible.',
+        ge=0,
+    )
+    input: Any | None = Field(
+        None,
+        description="Structured tool arguments (the command, the file path); `null` when none.",
+    )
+    kind: str = Field(
+        ...,
+        description="`tool_call` (the skill invoked a tool) or `tool_result` (the observation).",
+    )
+    name: str | None = Field(
+        None,
+        description="Normalized tool name where knowable (e.g. `bash`, `edit_file`); `null` for\na `tool_result` or when the harness did not name it.",
+    )
+    output: str | None = Field(
+        None, description="The result/observation text, when the transcript exposed it."
+    )
+
+
+class Usage(BaseModel):
+    """
+    Token / cost usage for one provider call.
+
+    Each field is independently optional because not every harness reports every
+    signal (cost is commonly absent on subscription auth; some harnesses report
+    no usage at all). The whole struct is `Option<Usage>` on a turn — `None`
+    means "no signal," not "zero."
+    """
+
+    cost_usd: float | None = None
+    input_tokens: int | None = Field(None, ge=0)
+    output_tokens: int | None = Field(None, ge=0)
+
+
+class Message(BaseModel):
+    """
+    A single turn in the conversation.
+    """
+
+    content: str
+    events: list[ToolEvent] | None = Field(
+        None,
+        description="The normalized tool events the skill took producing this turn (assistant\nturns only, and only when the harness exposed a tool transcript via\noneharness `--events`). Empty otherwise. Surfaced for post-hoc analysis\nand streamed live for short-circuiting.",
+    )
+    role: Literal["user", "assistant", "system"] = Field(..., description="Who produced a message.")
+
+
+class Summary(BaseModel):
+    """
+    Aggregate pass/fail counts for a report.
+    """
+
+    cases: int = Field(..., description="Distinct test cases represented.", ge=0)
+    failed: int = Field(..., description="Runs that failed.", ge=0)
+    passed: int = Field(..., description="Runs that passed.", ge=0)
+    runs: int = Field(..., description="Total (case × platform × model) runs.", ge=0)
+    usage: Usage | None = Field(
+        None,
+        description="Aggregated token/cost usage across every run in the report. Omitted\nwhen no run reported usage.",
+    )
+
+
+class Transcript(BaseModel):
+    """
+    An ordered list of messages. Thin wrapper so the type reads clearly at call
+    sites and so we can grow conversation-level helpers without churn.
+    """
+
+    messages: list[Message]
+
+
+class CaseRun(BaseModel):
+    """
+    The result of running one test case on one (platform, model) pair.
+    """
+
+    case: str = Field(..., description="The test case name.")
+    evals: list[EvalOutcome] = Field(..., description="Per-eval outcomes, in declaration order.")
+    history_command: str | None = Field(
+        None,
+        description="A ready-to-run command that replays this run's recorded transcript — e.g.\n`oneharness history show <name> --history-dir <dir>` — so a past run can\nbe reviewed after the fact. Present only when the run was recorded (the\noneharness provider with history enabled); `null` for providers/configs\nthat record no history.",
+    )
+    mock_calls: list[MockCall] | None = Field(
+        None,
+        description='Every tool call the mock/spy channel observed, in order, with the\noriginal (pre-rewrite) input and the verdict applied. `null` when the\nchannel was off for this run (no `mocks`, no `spy`); an empty array\nmeans the channel was on and the skill made no tool calls — SDKs use\nthat distinction so a spy on a channel-less run errs instead of reading\nas "zero calls".',
+    )
+    model: str = Field(..., description="The model this run used.")
+    passed: bool = Field(..., description="True iff every eval in this run passed.")
+    platform: str = Field(..., description="The harness platform this run used.")
+    skill: str = Field(..., description="Absolute-ish path to the skill that was exercised.")
+    transcript: Transcript = Field(
+        ..., description="The full conversation, for debugging and deterministic mix-in checks."
+    )
+    turns: int = Field(..., description="Number of assistant turns produced.", ge=0)
+    usage: Usage | None = Field(
+        None,
+        description="Aggregated token/cost usage across every provider call in this run\n(skill turns + simulated-user turns + judge calls). Omitted when no\nusage was reported (e.g. the fake provider or a harness that doesn't\nsurface usage).",
+    )
+
+
+class Report(BaseModel):
+    """
+    The top-level report for a `skilltest run` invocation.
+    """
+
+    passed: bool = Field(..., description="True iff every run passed.")
+    runs: list[CaseRun] = Field(..., description="Every individual run.")
+    summary: Summary = Field(..., description="Aggregate counts.")

skilltest_sdk/_validation.py

@@ -0,0 +1,24 @@
+# generated by datamodel-codegen:
+#   filename:  validation.schema.json
+
+from __future__ import annotations
+from pydantic import BaseModel, Field
+
+
+class ValidationFinding(BaseModel):
+    """
+    One problem found while validating a skill, as serialized in the
+    `skilltest validate --format json` output.
+    """
+
+    message: str = Field(..., description="What is wrong and how to fix it.")
+    skill: str = Field(..., description="The skill directory the finding is about.")
+
+
+class ValidationReport(BaseModel):
+    """
+    The top-level report for a `skilltest validate` invocation.
+    """
+
+    findings: list[ValidationFinding] = Field(..., description="Every finding, in discovery order.")
+    valid: bool = Field(..., description="True iff no findings were produced.")