workflowval 0.1.0__py3-none-any.whl

workflowval/schemas/workflow.py

@@ -0,0 +1,148 @@
+"""`workflows/<id>.json`: one reviewable workflow, which becomes one test file (FR-4)."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import Field, field_validator, model_validator
+
+from workflowval.schemas._base import Strict
+
+_PLACEHOLDER = re.compile(r"\{([A-Za-z_]\w*)\}")
+_JSON_PATH = re.compile(r"\$(\.[A-Za-z_][\w-]*|\[\d+\])*")
+BUILTIN_VALUES = frozenset({"unique", "unique_int"})
+"""Placeholders always available: fresh values per test run, so runs don't collide (FR-5)."""
+
+
+class Source(StrEnum):
+    """Who added the workflow."""
+
+    PLANNER = "planner"
+    REVIEWER = "reviewer"
+    USER = "user"
+
+
+class Basis(StrEnum):
+    """Whether the rule is documented or was inferred by the Reviewer."""
+
+    DOCUMENTED = "documented"
+    ASSUMED = "assumed"
+
+
+class Expect(Strict):
+    """Expected response. `body` maps JSONPath expressions to expected values."""
+
+    status: int = Field(ge=100, le=599)
+    body: dict[str, Any] = Field(default_factory=dict)
+
+    @field_validator("body")
+    @classmethod
+    def _body_keys_are_jsonpath(cls, value: dict[str, Any]) -> dict[str, Any]:
+        for key in value:
+            if not _JSON_PATH.fullmatch(key):
+                raise ValueError(f"expect.body key must be a JSONPath like $.a.b[0]: {key!r}")
+        return value
+
+
+class CleanupStep(Strict):
+    """A request that removes data the workflow created (NFR-3)."""
+
+    op: str = Field(min_length=1, description="Spec operationId")
+    params: dict[str, Any] = Field(default_factory=dict)
+    body: Any = None
+
+
+class Step(CleanupStep):
+    """A request plus its expectation.
+
+    `body` and `auth` are additions to the doc's example, which has no way to send a request body
+    or to call a route unauthenticated. See docs/m0-review.md.
+    """
+
+    save: dict[str, str] = Field(default_factory=dict, description="Name to JSONPath")
+    expect: Expect
+    auth: bool = Field(default=True, description="False sends this request without credentials")
+
+    @field_validator("save")
+    @classmethod
+    def _save_values_are_jsonpath(cls, value: dict[str, str]) -> dict[str, str]:
+        for name, path in value.items():
+            if not _JSON_PATH.fullmatch(path):
+                raise ValueError(f"save[{name!r}] must be a JSONPath like $.a.b[0]: {path!r}")
+        return value
+
+
+class KnownIssue(Strict):
+    """A failure reviewed as real, which the test now expects. Written by code, never by a model.
+
+    It excuses only this check on this step: any other failure still fails the test.
+    """
+
+    label: Literal["API_BUG", "SPEC_DRIFT", "CONFLICT"]
+    confidence: Literal["low", "med", "high"]
+    reason: str = Field(min_length=1)
+    step: int = Field(ge=0, description="Index in `steps` of the step that fails")
+    check: Literal["status", "body"]
+    status: int | None = Field(default=None, description="The status seen, for a status check")
+    report: str | None = Field(default=None, description="The bug report or spec patch file")
+
+    @model_validator(mode="after")
+    def _status_check_names_a_status(self) -> KnownIssue:
+        if self.check == "status" and self.status is None:
+            raise ValueError("a status check must name the status it saw")
+        return self
+
+
+def _placeholders(value: Any) -> Iterator[str]:
+    if isinstance(value, str):
+        yield from _PLACEHOLDER.findall(value)
+    elif isinstance(value, dict):
+        for item in value.values():
+            yield from _placeholders(item)
+    elif isinstance(value, list):
+        for item in value:
+            yield from _placeholders(item)
+
+
+class Workflow(Strict):
+    """One user journey. A locked workflow belongs to the user; agents may never change it."""
+
+    id: str = Field(
+        pattern=r"^[a-z0-9_]+$",
+        description="Lowercase, digits and underscores: names workflows/<id>.json and test_<id>.py",
+    )
+    source: Source
+    basis: Basis
+    locked: bool = False
+    rule: str = Field(
+        pattern=r"\S",
+        description="The business rule this workflow checks, and where it comes from. Required.",
+    )
+    steps: list[Step] = Field(min_length=1)
+    cleanup: list[CleanupStep] = Field(default_factory=list)
+    known_issue: KnownIssue | None = None
+
+    @model_validator(mode="after")
+    def _check(self) -> Workflow:
+        if self.locked and self.source is not Source.USER:
+            raise ValueError("only workflows with source 'user' can be locked")
+        if self.known_issue is not None and self.known_issue.step >= len(self.steps):
+            raise ValueError("known_issue.step isn't one of this workflow's steps")
+        saved = set(BUILTIN_VALUES)
+        for index, step in enumerate(self.steps):
+            self._require_saved(_placeholders([step.params, step.body]), saved, f"steps[{index}]")
+            saved |= set(step.save)
+            self._require_saved(_placeholders(step.expect.body), saved, f"steps[{index}].expect")
+        for index, cleanup in enumerate(self.cleanup):
+            where = f"cleanup[{index}]"
+            self._require_saved(_placeholders([cleanup.params, cleanup.body]), saved, where)
+        return self
+
+    @staticmethod
+    def _require_saved(names: Iterator[str], saved: set[str], where: str) -> None:
+        for name in names:
+            if name not in saved:
+                raise ValueError(f"{where} uses {{{name}}}, which no earlier step saves")

workflowval/spec/__init__.py

@@ -0,0 +1,38 @@
+"""OpenAPI loading, the dependency graph and response validation."""
+
+from __future__ import annotations
+
+from workflowval.spec.graph import (
+    Link,
+    Operation,
+    Parameter,
+    SpecGraph,
+    build_graph,
+    resolve_base_url,
+)
+from workflowval.spec.loader import (
+    SpecError,
+    load_raw_spec,
+    load_spec,
+    parse_spec,
+    resolve_refs,
+    spec_hash,
+)
+from workflowval.spec.validation import ResponseChecker, SpecMismatch
+
+__all__ = [
+    "Link",
+    "Operation",
+    "Parameter",
+    "ResponseChecker",
+    "SpecError",
+    "SpecGraph",
+    "SpecMismatch",
+    "build_graph",
+    "load_raw_spec",
+    "load_spec",
+    "parse_spec",
+    "resolve_base_url",
+    "resolve_refs",
+    "spec_hash",
+]

workflowval/spec/graph.py

@@ -0,0 +1,196 @@
+"""The internal graph: resources, operations, parameters and producer-to-consumer links (FR-3)."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+from pydantic import BaseModel, Field
+
+_METHODS = ("get", "put", "post", "delete", "patch", "head", "options")
+_PATH_PARAM = re.compile(r"\{([^{}]+)\}")
+
+
+class Parameter(BaseModel):
+    """One operation parameter."""
+
+    name: str
+    location: str
+    required: bool = False
+    schema_: dict[str, Any] = Field(default_factory=dict, alias="schema")
+
+
+class Operation(BaseModel):
+    """One API call."""
+
+    operation_id: str
+    method: str
+    path: str
+    resource: str
+    summary: str = ""
+    description: str = ""
+    parameters: list[Parameter] = Field(default_factory=list)
+    request_schema: dict[str, Any] | None = None
+    responses: dict[str, dict[str, Any]] = Field(default_factory=dict)
+    security: list[str] = Field(default_factory=list)
+
+
+class Link(BaseModel):
+    """A value one operation returns that another operation consumes."""
+
+    producer: str
+    field: str
+    consumer: str
+    parameter: str
+
+
+class SpecGraph(BaseModel):
+    """Everything the agents and the test generator need to know about an API."""
+
+    title: str = ""
+    base_path: str = ""
+    resources: list[str] = Field(default_factory=list)
+    operations: list[Operation] = Field(default_factory=list)
+    links: list[Link] = Field(default_factory=list)
+    security_schemes: dict[str, dict[str, Any]] = Field(default_factory=dict)
+
+    def operation_at(self, method: str, path: str) -> Operation | None:
+        """Find an operation by its method and path template."""
+        wanted = method.upper()
+        return next(
+            (op for op in self.operations if op.path == path and op.method.upper() == wanted),
+            None,
+        )
+
+    def operation(self, operation_id: str) -> Operation | None:
+        """Find an operation by its id."""
+        return next((op for op in self.operations if op.operation_id == operation_id), None)
+
+
+def _synthetic_id(method: str, path: str) -> str:
+    return "_".join([method, *re.findall(r"[A-Za-z0-9]+", path)])
+
+
+def _json_schema(content: dict[str, Any] | None) -> dict[str, Any] | None:
+    for media_type, media in (content or {}).items():
+        if "json" in media_type:
+            schema = media.get("schema")
+            return schema if isinstance(schema, dict) else {}
+    return None
+
+
+def _build_operation(
+    path: str,
+    method: str,
+    raw: dict[str, Any],
+    inherited: list[Any],
+    default_security: list[dict[str, Any]],
+) -> Operation:
+    merged = {(p["name"], p["in"]): p for p in [*inherited, *raw.get("parameters", [])]}
+    segments = [part for part in path.split("/") if part and not part.startswith("{")]
+    request = raw.get("requestBody")
+    responses = {
+        str(status): {
+            "description": response.get("description", ""),
+            "schema": _json_schema(response.get("content")),
+        }
+        for status, response in raw.get("responses", {}).items()
+    }
+    return Operation(
+        operation_id=raw.get("operationId") or _synthetic_id(method, path),
+        method=method.upper(),
+        path=path,
+        resource=segments[0] if segments else "",
+        summary=raw.get("summary", ""),
+        description=raw.get("description", ""),
+        parameters=[
+            Parameter(
+                name=p["name"],
+                location=p["in"],
+                required=bool(p.get("required")),
+                schema=p.get("schema") or {},
+            )
+            for p in merged.values()
+        ],
+        request_schema=_json_schema(request.get("content")) if request else None,
+        responses=responses,
+        # An operation's own `security` replaces the spec's, and an empty list means no auth at all.
+        security=[name for option in raw.get("security", default_security) for name in option],
+    )
+
+
+def _is_success(status: str) -> bool:
+    return status == "default" or status.startswith("2")
+
+
+def _returned_fields(operation: Operation) -> set[str]:
+    fields: set[str] = set()
+    for status, response in operation.responses.items():
+        if _is_success(status):
+            fields |= set((response.get("schema") or {}).get("properties", {}))
+    return fields
+
+
+def _singular(word: str) -> str:
+    return word[:-1] if word.endswith("s") and not word.endswith("ss") else word
+
+
+def _matching_field(fields: set[str], parameter: str, collection: str) -> str | None:
+    """Find the returned field that feeds a path parameter, most specific first."""
+    resource = _singular(collection.rstrip("/").rsplit("/", 1)[-1]).lower()
+    wanted = (parameter.lower(), f"{resource}id", f"{resource}_id", "id")
+    by_lower = {field.lower(): field for field in fields}
+    return next((by_lower[name] for name in wanted if name in by_lower), None)
+
+
+def _build_links(operations: list[Operation]) -> list[Link]:
+    links = []
+    for consumer in operations:
+        for match in _PATH_PARAM.finditer(consumer.path):
+            collection = consumer.path[: match.start()].rstrip("/")
+            for producer in operations:
+                if producer.method != "POST" or producer.path.rstrip("/") != collection:
+                    continue
+                field = _matching_field(_returned_fields(producer), match.group(1), collection)
+                if field:
+                    links.append(
+                        Link(
+                            producer=producer.operation_id,
+                            field=field,
+                            consumer=consumer.operation_id,
+                            parameter=match.group(1),
+                        )
+                    )
+    return links
+
+
+def build_graph(spec: dict[str, Any]) -> SpecGraph:
+    """Build the graph from a spec whose `$ref`s are already resolved."""
+    operations = []
+    for path, item in spec.get("paths", {}).items():
+        for method in _METHODS:
+            if method in item:
+                inherited = item.get("parameters", [])
+                operations.append(
+                    _build_operation(
+                        path, method, item[method], inherited, spec.get("security", [])
+                    )
+                )
+    servers = spec.get("servers") or [{}]
+    return SpecGraph(
+        title=spec.get("info", {}).get("title", ""),
+        base_path=urlparse(servers[0].get("url", "")).path.rstrip("/"),
+        resources=sorted({op.resource for op in operations if op.resource}),
+        operations=operations,
+        links=_build_links(operations),
+        security_schemes=spec.get("components", {}).get("securitySchemes", {}),
+    )
+
+
+def resolve_base_url(base_url: str, graph: SpecGraph) -> str:
+    """Add the spec's server path to a base URL, unless the URL already ends with it."""
+    url = base_url.rstrip("/")
+    if graph.base_path and not urlparse(url).path.rstrip("/").endswith(graph.base_path):
+        return url + graph.base_path
+    return url

workflowval/spec/loader.py

@@ -0,0 +1,102 @@
+"""Load and validate an OpenAPI 3.x JSON spec from a file or URL, and resolve its `$ref`s (FR-1)."""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+from openapi_spec_validator import validate
+from openapi_spec_validator.validation.exceptions import OpenAPIValidationError
+
+
+class SpecError(Exception):
+    """A spec can't be used. The message says what's wrong and where."""
+
+
+def read_spec_text(source: str) -> str:
+    """Read a spec's text from a path or an http(s) URL."""
+    if urlparse(source).scheme in {"http", "https"}:
+        try:
+            response = httpx.get(source, timeout=30, follow_redirects=True)
+            response.raise_for_status()
+        except httpx.HTTPError as exc:
+            raise SpecError(f"can't fetch spec from {source}: {exc}") from exc
+        return response.text
+    path = Path(source)
+    if not path.is_file():
+        raise SpecError(f"spec file not found: {source}")
+    return path.read_text(encoding="utf-8")
+
+
+def parse_spec(text: str, source: str) -> dict[str, Any]:
+    """Parse and validate spec text. Only JSON is supported in v1."""
+    try:
+        spec = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise SpecError(
+            f"{source} isn't valid JSON ({exc}). Only JSON specs are supported: "
+            "convert YAML to JSON first."
+        ) from exc
+    if not isinstance(spec, dict):
+        raise SpecError(f"{source} must be a JSON object, not {type(spec).__name__}")
+    version = str(spec.get("openapi", ""))
+    if not version.startswith(("3.0", "3.1")):
+        found = version or "no 'openapi' field"
+        raise SpecError(f"{source}: only OpenAPI 3.0 and 3.1 are supported, found {found}")
+    try:
+        validate(spec)
+    except OpenAPIValidationError as exc:
+        where = "/".join(str(part) for part in exc.absolute_path) or "(top level)"
+        raise SpecError(f"{source}: invalid spec at {where}: {exc.message}") from exc
+    return spec
+
+
+def load_raw_spec(source: str) -> dict[str, Any]:
+    """Load a spec as written, with its `$ref`s. Spec patches apply to this form."""
+    return parse_spec(read_spec_text(source), source)
+
+
+def load_spec(source: str) -> dict[str, Any]:
+    """Load a spec from a path or URL, validated, with every `$ref` resolved."""
+    return resolve_refs(parse_spec(read_spec_text(source), source))
+
+
+def spec_hash(spec: dict[str, Any]) -> str:
+    """Return a stable hash of a spec, used to decide whether to re-plan."""
+    return hashlib.sha256(json.dumps(spec, sort_keys=True).encode()).hexdigest()
+
+
+def _lookup(root: dict[str, Any], ref: str) -> Any:
+    if not ref.startswith("#/"):
+        raise SpecError(f"only local $refs are supported, found {ref!r}")
+    node: Any = root
+    for part in ref[2:].split("/"):
+        key = part.replace("~1", "/").replace("~0", "~")
+        if not isinstance(node, dict) or key not in node:
+            raise SpecError(f"$ref points at nothing: {ref}")
+        node = node[key]
+    return node
+
+
+def resolve_refs(spec: dict[str, Any]) -> dict[str, Any]:
+    """Return a copy with `$ref`s replaced. A recursive reference stays as a `$ref`, marking it."""
+
+    def walk(node: Any, active: tuple[str, ...]) -> Any:
+        if isinstance(node, list):
+            return [walk(item, active) for item in node]
+        if not isinstance(node, dict):
+            return node
+        ref = node.get("$ref")
+        if isinstance(ref, str):
+            if ref in active:
+                return {"$ref": ref}
+            return walk(copy.deepcopy(_lookup(spec, ref)), (*active, ref))
+        return {key: walk(value, active) for key, value in node.items()}
+
+    resolved: dict[str, Any] = walk(spec, ())
+    return resolved

workflowval/spec/validation.py

@@ -0,0 +1,98 @@
+"""Checks live responses against a spec. The one boundary around openapi-core (FR-8)."""
+
+from __future__ import annotations
+
+import copy
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from openapi_core import Config, OpenAPI
+from openapi_core.exceptions import OpenAPIError
+from openapi_core.testing import MockRequest, MockResponse
+
+from workflowval.redact import MASK
+
+_SERVER = "http://workflowval.invalid"
+_MAX_MESSAGE = 300
+
+
+@dataclass(frozen=True)
+class SpecMismatch:
+    """One way a response breaks the spec. `key` is the same however the value differs."""
+
+    key: str
+    message: str
+
+
+def _without_servers(spec: dict[str, Any]) -> dict[str, Any]:
+    """Point the spec at one fixed server, since callers check by operation path, not by host."""
+    copied = copy.deepcopy(spec)
+    copied["servers"] = [{"url": _SERVER}]
+    for item in copied.get("paths", {}).values():
+        item.pop("servers", None)
+        for operation in item.values():
+            if isinstance(operation, dict):
+                operation.pop("servers", None)
+    return copied
+
+
+def _schema_errors(exc: BaseException) -> list[Any]:
+    seen: BaseException | None = exc
+    while seen is not None:
+        errors = getattr(seen, "schema_errors", None)
+        if errors:
+            return list(errors)
+        seen = seen.__cause__
+    return []
+
+
+def _from_schema_error(error: Any) -> SpecMismatch:
+    parts = list(error.absolute_path)
+    path = "$" + "".join("[]" if isinstance(part, int) else f".{part}" for part in parts)
+    message = str(error.message)
+    # The key must be the same wherever one schema shows up, in a list or under another status, and
+    # whatever the bad value was. So it uses the field's name and the rule broken, not the path.
+    if error.validator == "required":
+        return SpecMismatch(message, f"{path}: {message}"[:_MAX_MESSAGE])
+    names = [part for part in parts if not isinstance(part, int)]
+    field = names[-1] if names else "$"
+    shape = f"{field}: {error.validator} {error.validator_value!r}"[:_MAX_MESSAGE]
+    return SpecMismatch(shape, f"{path}: {message}"[:_MAX_MESSAGE])
+
+
+class ResponseChecker:
+    """Checks responses against one spec, whichever host they came from."""
+
+    def check_call(self, call: Mapping[str, Any]) -> list[SpecMismatch]:
+        """Check one recorded call. A 5xx is an API failure, and its body says nothing of drift."""
+        template, status = call.get("template"), call.get("status")
+        if not isinstance(template, str) or not isinstance(status, int) or status >= 500:
+            return []
+        body = str(call.get("raw_body") or "").encode()
+        found = self.check(str(call["method"]), template, status, call.get("content_type"), body)
+        # A value that was hidden can't be judged: the mask itself would break a format rule.
+        return [mismatch for mismatch in found if MASK not in mismatch.message]
+
+    def __init__(self, spec: dict[str, Any], *, validate_spec: bool = False) -> None:
+        config = None if validate_spec else Config(spec_validator_cls=None)
+        self._api = OpenAPI.from_dict(_without_servers(spec), config=config)
+
+    def check(
+        self, method: str, path: str, status: int, content_type: str | None, body: bytes
+    ) -> list[SpecMismatch]:
+        """Return the ways a response breaks the spec. `path` is the spec's path template."""
+        request = MockRequest(_SERVER, method.lower(), path)
+        response = MockResponse(body, status_code=status, content_type=content_type or "")
+        try:
+            self._api.validate_response(request, response)
+        except OpenAPIError as exc:
+            errors = _schema_errors(exc)
+            if errors:
+                return [_from_schema_error(error) for error in errors]
+            kind = type(exc).__name__
+            detail = str(exc.__cause__ or exc)[:_MAX_MESSAGE]
+            if kind == "ResponseNotFound":
+                return [SpecMismatch(f"status {status}", f"status {status} isn't in the spec")]
+            return [SpecMismatch(f"{kind} on status {status}", detail)]
+        return []