workflowval 0.1.0__py3-none-any.whl

workflowval/schemas/__init__.py

@@ -0,0 +1,78 @@
+"""Pydantic models for every data format. Reviewed by the user before M1."""
+
+from workflowval.schemas._base import atomic_write_text, load_json, save_json
+from workflowval.schemas.auth import ApiKeyAuth, AuthConfig, BasicAuth, BearerAuth, LoginAuth
+from workflowval.schemas.labels import Labels, SeededBug
+from workflowval.schemas.results import (
+    Action,
+    ActionKind,
+    Confidence,
+    Evidence,
+    Finding,
+    HttpExchange,
+    Label,
+    RunResults,
+    TestResult,
+    TestStatus,
+    Verdict,
+)
+from workflowval.schemas.review import (
+    Answer,
+    Decision,
+    MissingJourney,
+    Review,
+    ReviewLog,
+    WorkflowFix,
+)
+from workflowval.schemas.state import RunState
+from workflowval.schemas.stats import CallCounts, RunStats, StatsTotal
+from workflowval.schemas.workflow import (
+    Basis,
+    CleanupStep,
+    Expect,
+    KnownIssue,
+    Source,
+    Step,
+    Workflow,
+)
+
+__all__ = [
+    "Action",
+    "ActionKind",
+    "Answer",
+    "ApiKeyAuth",
+    "AuthConfig",
+    "BasicAuth",
+    "Basis",
+    "BearerAuth",
+    "CallCounts",
+    "CleanupStep",
+    "Confidence",
+    "Decision",
+    "Evidence",
+    "Expect",
+    "Finding",
+    "HttpExchange",
+    "KnownIssue",
+    "Label",
+    "Labels",
+    "LoginAuth",
+    "MissingJourney",
+    "Review",
+    "ReviewLog",
+    "RunResults",
+    "RunState",
+    "RunStats",
+    "SeededBug",
+    "Source",
+    "StatsTotal",
+    "Step",
+    "TestResult",
+    "TestStatus",
+    "Verdict",
+    "Workflow",
+    "WorkflowFix",
+    "atomic_write_text",
+    "load_json",
+    "save_json",
+]

workflowval/schemas/_base.py

@@ -0,0 +1,43 @@
+"""Shared base model and JSON file helpers for every data format."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import TypeVar
+
+from pydantic import BaseModel, ConfigDict
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class Strict(BaseModel):
+    """Base for on-disk formats: unknown fields are errors, so typos fail loudly."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+def atomic_write_text(path: Path, text: str) -> None:
+    """Write text so that a killed process never leaves a half-written file behind."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            handle.write(text)
+        os.replace(tmp, path)
+    except BaseException:
+        Path(tmp).unlink(missing_ok=True)
+        raise
+
+
+def save_json(model: BaseModel, path: Path) -> None:
+    """Write a model as indented JSON, atomically."""
+    payload = json.loads(model.model_dump_json(exclude_none=True))
+    atomic_write_text(path, json.dumps(payload, indent=2) + "\n")
+
+
+def load_json(cls: type[T], path: Path) -> T:
+    """Read and validate a JSON file. Raises pydantic.ValidationError on bad data."""
+    return cls.model_validate_json(path.read_text(encoding="utf-8"))

workflowval/schemas/auth.py

@@ -0,0 +1,66 @@
+"""The auth config file (FR-2). It names env vars and never holds a secret value (NFR-7)."""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from pydantic import Field, StringConstraints
+
+from workflowval.schemas._base import Strict
+
+EnvVarName = Annotated[str, StringConstraints(pattern=r"^[A-Z_][A-Z0-9_]*$")]
+
+
+class ApiKeyAuth(Strict):
+    """An API key sent in a header or query parameter."""
+
+    type: Literal["api_key"]
+    name: str = Field(min_length=1, description="Header or query parameter name")
+    location: Literal["header", "query"]
+    value_env: EnvVarName
+
+
+class BearerAuth(Strict):
+    """A bearer token."""
+
+    type: Literal["bearer"]
+    token_env: EnvVarName
+
+
+class BasicAuth(Strict):
+    """HTTP basic auth."""
+
+    type: Literal["basic"]
+    username_env: EnvVarName
+    password_env: EnvVarName
+
+
+class LoginAuth(Strict):
+    """Call a login operation, extract a token, reuse it and refresh it on 401.
+
+    Leave `token_header` and `token_prefix` out and the agent works out how to send the token, from
+    the spec's security section, then the docs, then the login response and what the API accepts.
+    Values set here always win. The agent never sees the secret values.
+    """
+
+    type: Literal["login"]
+    op: str = Field(min_length=1, description="Login operationId")
+    body_env: dict[str, EnvVarName] = Field(
+        description="Request body field to env var name. `user.email` nests the field.",
+    )
+    token_path: str = Field(pattern=r"^\$", description="JSONPath of the token in the response")
+    token_header: str | None = Field(default=None, description="Override, e.g. Cookie")
+    token_prefix: str | None = Field(default=None, description="Override, e.g. token=")
+    refresh_on_401: bool = True
+
+
+AuthScheme = Annotated[
+    ApiKeyAuth | BearerAuth | BasicAuth | LoginAuth,
+    Field(discriminator="type"),
+]
+
+
+class AuthConfig(Strict):
+    """Top level of the auth config file."""
+
+    scheme: AuthScheme

workflowval/schemas/labels.py

@@ -0,0 +1,31 @@
+"""`benchmarks/seeded/labels.json`: ground truth for the seeded-bug app (Evaluation)."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import Field, model_validator
+
+from workflowval.schemas._base import Strict
+
+
+class SeededBug(Strict):
+    """One planted defect or conflict and the label it should get."""
+
+    id: str = Field(min_length=1)
+    description: str = Field(min_length=1)
+    expected_label: Literal["API_BUG", "SPEC_DRIFT", "CONFLICT"]
+    operations: list[str] = Field(min_length=1, description="operationIds where it can be seen")
+
+
+class Labels(Strict):
+    """The whole ground-truth file."""
+
+    bugs: list[SeededBug] = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def _ids_are_unique(self) -> Labels:
+        ids = [bug.id for bug in self.bugs]
+        if len(ids) != len(set(ids)):
+            raise ValueError("bug ids must be unique")
+        return self

workflowval/schemas/results.py

@@ -0,0 +1,222 @@
+"""`results.json`: per-test outcomes and failure verdicts (FR-6, FR-8, FR-11)."""
+
+from __future__ import annotations
+
+from collections import Counter
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import Field, computed_field, model_validator
+
+from workflowval.schemas._base import Strict
+from workflowval.schemas.review import Review
+from workflowval.schemas.workflow import Basis
+
+
+class Label(StrEnum):
+    """Every failure ends as exactly one label."""
+
+    TEST_BUG = "TEST_BUG"
+    API_BUG = "API_BUG"
+    SPEC_DRIFT = "SPEC_DRIFT"
+    UNRESOLVED = "UNRESOLVED"
+    CONFLICT = "CONFLICT"
+
+
+class Confidence(StrEnum):
+    """How sure the verdict is."""
+
+    LOW = "low"
+    MED = "med"
+    HIGH = "high"
+
+
+class TestStatus(StrEnum):
+    """What pytest reported."""
+
+    __test__ = False
+
+    PASSED = "passed"
+    FAILED = "failed"
+    ERROR = "error"
+    SKIPPED = "skipped"
+    XFAILED = "xfailed"
+    XPASSED = "xpassed"
+
+
+_OK_STATUSES = frozenset({TestStatus.PASSED, TestStatus.XFAILED, TestStatus.SKIPPED})
+
+
+class ActionKind(StrEnum):
+    """What was done about a failure."""
+
+    PATCHED_TEST = "patched_test"
+    BUG_REPORT = "bug_report"
+    SPEC_PATCH = "spec_patch"
+    NONE = "none"
+
+
+class HttpExchange(Strict):
+    """One request/response pair. Secrets must be redacted before this is built."""
+
+    method: str
+    url: str
+    request_headers: dict[str, str] = Field(default_factory=dict)
+    request_body: Any = None
+    status: int | None = Field(default=None, description="None when no response arrived")
+    response_headers: dict[str, str] = Field(default_factory=dict)
+    response_body: Any = None
+
+
+class Evidence(Strict):
+    """What a verdict rests on."""
+
+    exchanges: list[HttpExchange] = Field(default_factory=list)
+    spec_excerpt: str | None = None
+    docs_excerpt: str | None = None
+
+
+class Action(Strict):
+    """The outcome of a verdict, with the file it produced."""
+
+    kind: ActionKind
+    path: str | None = None
+
+
+class Verdict(Strict):
+    """One label for one failure, with confidence, evidence and an action."""
+
+    label: Label
+    confidence: Confidence
+    reason: str = Field(min_length=1)
+    evidence: Evidence = Field(default_factory=Evidence)
+    action: Action = Field(default_factory=lambda: Action(kind=ActionKind.NONE))
+    fix_target: Literal["test", "workflow"] | None = Field(
+        default=None, description="For a TEST_BUG: what's wrong, the test code or the workflow"
+    )
+
+
+class Finding(Strict):
+    """A response that breaks the spec, once however many tests saw it (FR-8, FR-10).
+
+    `label` is None until the Reviewer rules on it. `TEST_BUG` means it ruled this isn't a real
+    problem, for example a spec that's merely loose.
+    """
+
+    id: str = Field(min_length=1)
+    message: str = Field(min_length=1)
+    operations: list[str] = Field(default_factory=list)
+    tests: list[str] = Field(default_factory=list)
+    label: Label | None = None
+    confidence: Confidence | None = None
+    reason: str | None = None
+    action: Action = Field(default_factory=lambda: Action(kind=ActionKind.NONE))
+
+
+class TestResult(Strict):
+    """The result of one test, keeping the last 3 HTTP exchanges (FR-6)."""
+
+    __test__ = False
+
+    id: str = Field(min_length=1, description="pytest node id")
+    workflow_id: str
+    basis: Basis
+    status: TestStatus
+    duration_s: float = Field(ge=0)
+    failing_assertion: str | None = None
+    stdout: str = ""
+    exchanges: list[HttpExchange] = Field(default_factory=list, max_length=3)
+    fix_attempts: int = Field(default=0, ge=0)
+    verdict: Verdict | None = None
+
+    @model_validator(mode="after")
+    def _assumed_rules_cap_at_medium(self) -> TestResult:
+        if (
+            self.basis is Basis.ASSUMED
+            and self.verdict is not None
+            and self.verdict.confidence is Confidence.HIGH
+        ):
+            raise ValueError("a failure on an assumed workflow is capped at medium confidence")
+        return self
+
+
+class RunResults(Strict):
+    """Everything a run reports."""
+
+    stop_reason: str
+    stop_detail: str | None = None
+    tests: list[TestResult] = Field(default_factory=list)
+    findings: list[Finding] = Field(default_factory=list)
+    reviews: list[Review] = Field(default_factory=list)
+    tasks_stopped: list[str] = Field(
+        default_factory=list,
+        description="Tasks that hit their call limit, so may be incomplete",
+    )
+    uncovered_operations: list[str] = Field(
+        default_factory=list, description="Spec operations no workflow calls"
+    )
+    input_tokens: int = Field(default=0, ge=0)
+    output_tokens: int = Field(default=0, ge=0)
+    cost_usd: float = Field(default=0.0, ge=0, description="Sum over calls with a known price")
+    unpriced_calls: int = Field(default=0, ge=0, description="Calls missing from cost_usd")
+
+    @model_validator(mode="before")
+    @classmethod
+    def _ignore_computed_fields(cls, data: Any) -> Any:
+        """`label_counts` is written for readers and computed again on load."""
+        if isinstance(data, dict):
+            return {key: value for key, value in data.items() if key != "label_counts"}
+        return data
+
+    @property
+    def has_unexpected_failures(self) -> bool:
+        """True when a test failed in a way nobody has reviewed and marked as expected."""
+        return any(t.status not in _OK_STATUSES for t in self.tests)
+
+    def ci_failures(self, *, max_api_bugs: int, max_spec_drift: int) -> list[str]:
+        """Why `--ci` should fail the build. Expected failures still count toward the thresholds."""
+        reasons = []
+        unresolved = sum(
+            t.status not in _OK_STATUSES
+            or (t.verdict is not None and t.verdict.label is Label.UNRESOLVED)
+            for t in self.tests
+        )
+        if unresolved:
+            reasons.append(f"{unresolved} unresolved")
+        counts = self.label_counts
+        if counts.get("API_BUG", 0) > max_api_bugs:
+            reasons.append(f"{counts['API_BUG']} API_BUG, above the limit of {max_api_bugs}")
+        if counts.get("SPEC_DRIFT", 0) > max_spec_drift:
+            reasons.append(
+                f"{counts['SPEC_DRIFT']} SPEC_DRIFT, above the limit of {max_spec_drift}"
+            )
+        return reasons
+
+    def summary(self) -> str:
+        """One line of counts, keeping expected failures apart from passes."""
+        passed = sum(t.status is TestStatus.PASSED for t in self.tests)
+        parts = [f"{passed} passed"]
+        expected = Counter(
+            t.verdict.label.value
+            for t in self.tests
+            if t.status is TestStatus.XFAILED and t.verdict is not None
+        )
+        parts += [f"{n} expected to fail ({label})" for label, n in sorted(expected.items())]
+        skipped = sum(t.status is TestStatus.SKIPPED for t in self.tests)
+        if skipped:
+            parts.append(f"{skipped} skipped")
+        failed = len(self.tests) - passed - skipped - sum(expected.values())
+        if failed:
+            parts.append(f"{failed} failed")
+        confirmed = sum(f.label not in {None, Label.TEST_BUG} for f in self.findings)
+        if confirmed:
+            parts.append(f"{confirmed} spec mismatch{'es' if confirmed > 1 else ''}")
+        return ", ".join(parts)
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def label_counts(self) -> dict[str, int]:
+        """Number of tests per label."""
+        labels = [t.verdict.label.value for t in self.tests if t.verdict is not None]
+        labels += [f.label.value for f in self.findings if f.label not in {None, Label.TEST_BUG}]
+        return dict(Counter(labels))

workflowval/schemas/review.py

@@ -0,0 +1,94 @@
+"""`review.json`: the Reviewer's fixes and missing journeys, with the Planner's answers."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+from pydantic import Field, model_validator
+
+from workflowval.schemas._base import Strict
+from workflowval.schemas.workflow import Basis
+
+
+class Decision(StrEnum):
+    """How the Planner answered a review item."""
+
+    ACCEPTED = "accepted"
+    REJECTED = "rejected"
+
+
+class Answer(Strict):
+    """The Planner's answer. Nothing is dropped silently, so a rejection needs a reason."""
+
+    decision: Decision
+    reason: str | None = None
+    workflow_id: str | None = Field(default=None, description="File that carries out the answer")
+
+    @model_validator(mode="after")
+    def _check(self) -> Answer:
+        if self.decision is Decision.REJECTED and not self.reason:
+            raise ValueError("a rejection must give a reason")
+        if self.decision is Decision.ACCEPTED and not self.workflow_id:
+            raise ValueError("an accepted item must name its workflow_id")
+        return self
+
+
+class WorkflowFix(Strict):
+    """A workflow that is wrong or weak."""
+
+    id: str = Field(min_length=1)
+    workflow_id: str
+    reason: str = Field(min_length=1)
+    answer: Answer | None = None
+
+
+class MissingJourney(Strict):
+    """A user flow the workflows do not cover."""
+
+    id: str = Field(min_length=1)
+    user: str = Field(min_length=1, description="The user named in the docs, or 'API consumer'")
+    steps: list[str] = Field(min_length=1)
+    rule: str = Field(min_length=1)
+    basis: Basis
+    answer: Answer | None = None
+
+
+class Review(Strict):
+    """One review round's output."""
+
+    round: int = Field(ge=1)
+    after_tests: bool = Field(default=False, description="True if it followed a test run")
+    fixes: list[WorkflowFix] = Field(default_factory=list)
+    missing_journeys: list[MissingJourney] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def _ids_are_unique(self) -> Review:
+        ids = [item.id for item in self._items()]
+        if len(ids) != len(set(ids)):
+            raise ValueError("review item ids must be unique")
+        return self
+
+    def unanswered(self) -> list[str]:
+        """Ids of items the Planner has not answered yet."""
+        return [item.id for item in self._items() if item.answer is None]
+
+    def _items(self) -> list[WorkflowFix | MissingJourney]:
+        return [*self.fixes, *self.missing_journeys]
+
+    def items(self) -> list[WorkflowFix | MissingJourney]:
+        """Every fix and journey in this round."""
+        return self._items()
+
+
+class ReviewLog(Strict):
+    """`review.json`: every round, so nothing proposed is ever dropped."""
+
+    rounds: list[Review] = Field(default_factory=list)
+
+    def item(self, item_id: str) -> WorkflowFix | MissingJourney | None:
+        """Find an item in any round."""
+        return next((i for r in self.rounds for i in r.items() if i.id == item_id), None)
+
+    def unanswered(self) -> list[str]:
+        """Ids of items the Planner has not answered yet, in any round."""
+        return [item_id for review in self.rounds for item_id in review.unanswered()]

workflowval/schemas/state.py

@@ -0,0 +1,32 @@
+"""`.workflowval/state.json`: the resume point and spec hash (FR-13)."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import Field
+
+from workflowval.schemas._base import Strict
+
+
+class RunState(Strict):
+    """A checkpoint, rewritten atomically as the run advances."""
+
+    format_version: Literal[1] = 1
+    spec_hash: str = ""
+    completed_steps: list[str] = Field(default_factory=list)
+    review_rounds: int = Field(default=0, ge=0)
+    iterations: int = Field(default=0, ge=0)
+    steps_used: int = Field(default=0, ge=0)
+    fix_attempts: dict[str, int] = Field(default_factory=dict)
+    closed_tests: dict[str, str] = Field(
+        default_factory=dict, description="Test id to 'passed' or its final label"
+    )
+    open_tests: list[str] = Field(default_factory=list)
+    input_tokens: int = Field(default=0, ge=0)
+    output_tokens: int = Field(default=0, ge=0)
+    cost_usd: float = Field(default=0.0, ge=0)
+    unpriced_calls: int = Field(default=0, ge=0)
+    tasks_stopped: list[str] = Field(default_factory=list)
+    cost_tracked: bool = True
+    stop_reason: str | None = None

workflowval/schemas/stats.py

@@ -0,0 +1,36 @@
+"""`.workflowval/stats.json` and `stats_history.jsonl`: names and counts, never arguments."""
+
+from __future__ import annotations
+
+from pydantic import Field
+
+from workflowval.schemas._base import Strict
+
+
+class CallCounts(Strict):
+    """Model calls, and tool calls by tool name, including refused and invalid attempts."""
+
+    model_calls: int = Field(default=0, ge=0)
+    tool_calls: dict[str, int] = Field(default_factory=dict)
+    cost_usd: float = Field(default=0.0, ge=0, description="Sum over calls with a known price")
+
+
+class StatsTotal(Strict):
+    """Totals for a run. `model_calls` is what `--max-steps` limits."""
+
+    model_calls: int = Field(default=0, ge=0)
+    tool_calls: int = Field(default=0, ge=0)
+    cost_usd: float = Field(default=0.0, ge=0)
+    limit: int = Field(ge=1, description="The --max-steps value for this run")
+
+
+class RunStats(Strict):
+    """One run's calls. `run` holds work that belongs to no single workflow."""
+
+    run: CallCounts = Field(default_factory=CallCounts)
+    workflows: dict[str, CallCounts] = Field(default_factory=dict)
+    roles: dict[str, CallCounts] = Field(default_factory=dict)
+    total: StatsTotal
+    spec_hash: str = ""
+    stop_reason: str | None = None
+    finished_at: str | None = None