api-shakedown 0.1.0__py3-none-any.whl

api_shakedown/__init__.py

@@ -0,0 +1,3 @@
+"""API Shakedown: OpenAPI spec in, ready-to-run chaos test harness out."""
+
+__version__ = "0.1.0"

api_shakedown/__main__.py

@@ -0,0 +1,3 @@
+from api_shakedown.cli import app
+
+app()

api_shakedown/agents/__init__.py

@@ -0,0 +1 @@
+"""Specialist agents for API Shakedown."""

api_shakedown/agents/_common.py

@@ -0,0 +1,36 @@
+"""Shared plumbing for the shakedown specialists."""
+
+from collections.abc import Callable
+from typing import TypeVar
+
+from agentpanel.structured import StructuredOutputError, ainvoke_structured
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import HumanMessage, SystemMessage
+from pydantic import BaseModel
+
+T = TypeVar("T", bound=BaseModel)
+
+
+async def generate_checked(
+    model: BaseChatModel,
+    instructions: str,
+    prompt: str,
+    schema: type[T],
+    *,
+    check: Callable[[T], list[str]],
+    max_attempts: int,
+    notes: list[str],
+) -> T | None:
+    """Structured generation that never raises on bad output.
+
+    Returns the passing reply, or the last reply that parsed (so callers can keep its
+    valid parts), or None. Problems are recorded in `notes`.
+    """
+    messages = [SystemMessage(instructions), HumanMessage(prompt)]
+    try:
+        return await ainvoke_structured(
+            model, messages, schema, max_attempts=max_attempts, check=check
+        )
+    except StructuredOutputError as exc:
+        notes.append(f"Some LLM output was unusable: {exc}")
+        return exc.value if isinstance(exc.value, schema) else None

api_shakedown/agents/chaos.py

@@ -0,0 +1,146 @@
+"""Agent that invents failure scenarios: timeouts, malformed payloads, partial outages.
+
+Baseline scenarios and robustness probes come from code. The LLM adds scenarios aimed at
+this particular API (which calls are critical, which can be retried, where partial
+failure hurts most), expressed in the constrained `ScenarioProposal` format and checked
+before being compiled to WireMock mappings.
+"""
+
+from dataclasses import dataclass
+
+from agentpanel import AgentContext
+from agentpanel.llm_agents import render_feedback
+from langchain_core.language_models import BaseChatModel
+from pydantic import BaseModel
+
+from api_shakedown.agents._common import generate_checked
+from api_shakedown.chaos import (
+    ChaosSuite,
+    ScenarioProposal,
+    baseline_scenarios,
+    build_probes,
+    compile_scenario,
+    scenario_problems,
+)
+from api_shakedown.spec import ApiSpec
+from api_shakedown.testdata import Dataset
+from api_shakedown.wiremock import DEFAULT_STUB_URL, base_path
+
+MAX_LLM_SCENARIOS = 10
+
+INSTRUCTIONS = """\
+You are a chaos engineer. You design failure scenarios, injected into a mock of an API, \
+that test how CLIENTS of that API cope.
+
+Propose scenarios specific to this API, for example:
+- partial outages: writes fail while reads still work, or one critical endpoint is down;
+- a slow dependency on the most-used read;
+- a create call that fails twice then succeeds (tests retries and idempotency);
+- schema drift (wrong_types, missing_required) in an important response;
+- rate limiting on list endpoints.
+
+Rules:
+- Do not repeat the baseline scenarios listed in the prompt.
+- Each effect uses: delay_ms, fault, status (400-599), or malformation. fault cannot be \
+combined with status or malformation; status and malformation are exclusive.
+- operations must list the affected operation ids (never empty: the baseline already \
+covers every operation). A scenario about one endpoint lists only that endpoint.
+- Use "times" for failures that recover after N calls; null means always while active. \
+A flaky scenario must set times.
+- Match effect to category: latency/timeout need delay_ms; rate_limit uses status 429; \
+flaky, outage and partial_outage use a 5xx/429 status or a fault (a 4xx is not a \
+transient failure); a partial_outage leaves some operations working; malformed_response \
+needs a malformation.
+- expectation says concretely what a resilient client should do."""
+
+
+class ScenarioProposals(BaseModel):
+    scenarios: list[ScenarioProposal]
+
+
+@dataclass
+class ChaosAgent:
+    model: BaseChatModel | None = None
+    name: str = "chaos"
+    kind: str = "chaos scenarios and robustness probes"
+    max_attempts: int = 3
+
+    async def produce(self, ctx: AgentContext) -> ChaosSuite:
+        spec: ApiSpec = ctx.inputs["spec"]
+        dataset: Dataset | None = ctx.inputs.get("dataset")
+        notes: list[str] = []
+        baseline = baseline_scenarios(spec)
+        proposals: list[ScenarioProposal] = []
+        if self.model is not None:
+            feedback = render_feedback(ctx.feedback_for(self.name))
+            proposals = await self._generate(self.model, spec, baseline, feedback, notes)
+
+        scenarios = [compile_scenario(spec, p, "baseline", dataset) for p in baseline]
+        scenarios += [compile_scenario(spec, p, "llm", dataset) for p in proposals]
+        return ChaosSuite(
+            title=spec.title,
+            version=spec.version,
+            default_target=DEFAULT_STUB_URL + base_path(spec),
+            scenarios=scenarios,
+            probes=build_probes(spec, dataset),
+            notes=notes,
+        )
+
+    async def _generate(
+        self,
+        model: BaseChatModel,
+        spec: ApiSpec,
+        baseline: list[ScenarioProposal],
+        feedback: str,
+        notes: list[str],
+    ) -> list[ScenarioProposal]:
+        taken = {p.name for p in baseline}
+        output = await generate_checked(
+            model,
+            INSTRUCTIONS,
+            _prompt(spec, baseline, feedback),
+            ScenarioProposals,
+            check=lambda out: _problems(spec, taken, out),
+            max_attempts=self.max_attempts,
+            notes=notes,
+        )
+        usable: list[ScenarioProposal] = []
+        for proposal in output.scenarios if output else []:
+            if proposal.name in taken or scenario_problems(spec, proposal):
+                continue
+            taken.add(proposal.name)
+            usable.append(proposal)
+        return usable[:MAX_LLM_SCENARIOS]
+
+
+def _prompt(spec: ApiSpec, baseline: list[ScenarioProposal], feedback: str) -> str:
+    operations = "\n".join(
+        f"- {op.id}: {op.method.upper()} {op.path}"
+        + (f" ({op.summary})" if op.summary else "")
+        + f" statuses {[r.status for r in op.responses]}"
+        for op in spec.operations
+    )
+    existing = "\n".join(f"- {p.name}: {p.title}" for p in baseline)
+    prompt = (
+        f"API: {spec.title} {spec.version}\n\nOperations:\n{operations}\n\n"
+        f"Baseline scenarios (already included, do not repeat):\n{existing}\n\n"
+        "Propose 3 to 6 additional scenarios targeted at this API."
+    )
+    if feedback:
+        prompt += (
+            f"\n\nReviewers rejected the previous scenarios. Address this feedback:\n{feedback}"
+        )
+    return prompt
+
+
+def _problems(spec: ApiSpec, taken: set[str], output: ScenarioProposals) -> list[str]:
+    problems = []
+    seen = set(taken)
+    if not output.scenarios:
+        problems.append("Propose at least one scenario.")
+    for proposal in output.scenarios:
+        if proposal.name in seen:
+            problems.append(f"scenario {proposal.name}: duplicate or baseline name.")
+        seen.add(proposal.name)
+        problems += [f"scenario {proposal.name}: {p}" for p in scenario_problems(spec, proposal)]
+    return problems

api_shakedown/agents/contracts.py

@@ -0,0 +1,176 @@
+"""Agent that writes contract tests against the spec.
+
+The LLM proposes test cases as data; code validates each against the spec, feeds problems
+back, and guarantees a happy-path case per operation (falling back to a schema-derived
+case). The cases run under a fixed pytest runner, so no generated code is executed.
+Without a model, the agent emits only the deterministic happy-path cases.
+"""
+
+from dataclasses import dataclass
+
+from agentpanel import AgentContext
+from agentpanel.llm_agents import render_feedback
+from langchain_core.language_models import BaseChatModel
+from pydantic import BaseModel
+
+from api_shakedown.agents._common import generate_checked
+from api_shakedown.contracts import (
+    CaseProposal,
+    ContractSuite,
+    build_suite,
+    happy_proposal,
+    proposal_problems,
+)
+from api_shakedown.spec import ApiSpec, Operation, chunk_operations
+from api_shakedown.spec.chunking import DEFAULT_MAX_CHARS
+from api_shakedown.testdata import Dataset
+
+MAX_CASES_PER_OPERATION = 8
+
+INSTRUCTIONS = """\
+You design HTTP contract test cases from an OpenAPI description.
+
+For each operation, propose:
+- one happy-path case with realistic, valid inputs;
+- one case per documented error status where the inputs would plausibly cause it, \
+e.g. a well-formed but unknown id for 404, or a body missing a required field for 400;
+- boundary cases for constrained inputs (minimum/maximum values, enum members).
+
+Rules:
+- expected_status must be one of the operation's documented statuses.
+- Path parameters must always be valid values for their schema, even in error cases, \
+so the URL still routes.
+- Always include required query parameters.
+- Success cases must send inputs that satisfy every schema.
+- Never add authentication headers; they are added automatically.
+- Give each case a short snake_case name describing what it checks."""
+
+
+class CaseProposals(BaseModel):
+    cases: list[CaseProposal]
+
+
+@dataclass
+class ContractTestAgent:
+    model: BaseChatModel | None = None
+    name: str = "contracts"
+    kind: str = "contract test suite"
+    max_chars: int = DEFAULT_MAX_CHARS
+    max_attempts: int = 3
+
+    async def produce(self, ctx: AgentContext) -> ContractSuite:
+        spec: ApiSpec = ctx.inputs["spec"]
+        dataset: Dataset | None = ctx.inputs.get("dataset")
+        notes: list[str] = []
+        proposals: list[CaseProposal] = []
+        if self.model is not None:
+            feedback = render_feedback(ctx.feedback_for(self.name))
+            for chunk in chunk_operations(spec.operations, self.max_chars):
+                proposals += await self._generate(self.model, spec, dataset, chunk, feedback, notes)
+        cases = _with_happy_paths(spec, dataset, proposals, notes, self.model)
+        return build_suite(spec, cases, notes)
+
+    async def _generate(
+        self,
+        model: BaseChatModel,
+        spec: ApiSpec,
+        dataset: Dataset | None,
+        chunk: list[Operation],
+        feedback: str,
+        notes: list[str],
+    ) -> list[CaseProposal]:
+        ops = {op.id: op for op in chunk}
+        output = await generate_checked(
+            model,
+            INSTRUCTIONS,
+            _prompt(spec, dataset, chunk, feedback),
+            CaseProposals,
+            check=lambda out: _problems(spec, dataset, ops, out),
+            max_attempts=self.max_attempts,
+            notes=notes,
+        )
+        usable: list[CaseProposal] = []
+        per_op: dict[str, int] = {}
+        for proposal in output.cases if output else []:
+            op = ops.get(proposal.operation_id)
+            if op is None or proposal_problems(spec, op, proposal, dataset):
+                continue
+            if per_op.get(op.id, 0) < MAX_CASES_PER_OPERATION:
+                per_op[op.id] = per_op.get(op.id, 0) + 1
+                usable.append(proposal)
+        return usable
+
+
+def _with_happy_paths(
+    spec: ApiSpec,
+    dataset: Dataset | None,
+    proposals: list[CaseProposal],
+    notes: list[str],
+    model: BaseChatModel | None,
+) -> list[CaseProposal]:
+    covered = {p.operation_id for p in proposals if p.expected_status.startswith("2")}
+    result = list(proposals)
+    for op in spec.operations:
+        if op.id in covered:
+            continue
+        fallback = happy_proposal(op, dataset)
+        if fallback is not None:
+            result.append(fallback)
+            if model is not None:
+                notes.append(f"{op.id}: used a schema-generated happy-path case.")
+    order = {op.id: i for i, op in enumerate(spec.operations)}
+    return sorted(result, key=lambda p: order[p.operation_id])  # stable within an operation
+
+
+def _prompt(spec: ApiSpec, dataset: Dataset | None, chunk: list[Operation], feedback: str) -> str:
+    statuses = "\n".join(
+        f"- {op.id}: documented statuses {[r.status for r in op.responses]}" for op in chunk
+    )
+    prompt = (
+        f"API: {spec.title} {spec.version}\n\n"
+        "<operations>\n" + "\n".join(op.to_prompt() for op in chunk) + "\n</operations>\n\n"
+        f"Propose test cases for these operations:\n{statuses}"
+    )
+    known = _known_ids(dataset, chunk)
+    if known:
+        prompt += (
+            "\n\nThese ids exist in the test data. Success cases must use one of them; "
+            "404 cases must use a well-formed id that is NOT listed:\n" + "\n".join(known)
+        )
+    if feedback:
+        prompt += f"\n\nReviewers rejected the previous tests. Address this feedback:\n{feedback}"
+    return prompt
+
+
+def _known_ids(dataset: Dataset | None, chunk: list[Operation]) -> list[str]:
+    lines = []
+    for op in chunk:
+        for param in (p for p in op.parameters if p.location == "path"):
+            ids = dataset.known_ids(op.path, param.name) if dataset else None
+            if ids:
+                lines.append(f"- {op.id} {param.name}: {ids}")
+    return lines
+
+
+def _problems(
+    spec: ApiSpec, dataset: Dataset | None, ops: dict[str, Operation], output: CaseProposals
+) -> list[str]:
+    problems = []
+    for proposal in output.cases:
+        label = f"case {proposal.operation_id}/{proposal.name}"
+        op = ops.get(proposal.operation_id)
+        if op is None:
+            problems.append(f"{label}: unknown operation_id; use one of {sorted(ops)}.")
+            continue
+        problems += [f"{label}: {p}" for p in proposal_problems(spec, op, proposal, dataset)]
+    happy = {
+        p.operation_id
+        for p in output.cases
+        if p.expected_status.startswith("2")
+        and p.operation_id in ops
+        and not proposal_problems(spec, ops[p.operation_id], p, dataset)
+    }
+    problems += [
+        f"No valid happy-path (2xx) case for {op_id}." for op_id in ops if op_id not in happy
+    ]
+    return problems

api_shakedown/agents/coordinator.py

@@ -0,0 +1,211 @@
+"""Coordinator agent that assembles reviewed artifacts into the final harness.
+
+Assembly is deterministic: every file comes from an artifact or a fixed template, and the
+README is generated from what is actually in the harness. Review outcomes are reported,
+not hidden: anything still rejected when the revision budget ran out is listed.
+"""
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from agentpanel import AgentContext, Review
+from pydantic import BaseModel, Field
+
+from api_shakedown.chaos import CHAOS_CLI_FILE, SCENARIOS_FILE, ChaosSuite
+from api_shakedown.contracts import ContractSuite
+from api_shakedown.spec import ApiSpec
+from api_shakedown.testdata import Dataset
+from api_shakedown.wiremock import DEFAULT_STUB_URL, STATUS_HEADER, StubSet, base_path
+
+WIREMOCK_IMAGE = "wiremock/wiremock:3.13.1"
+TEST_REQUIREMENTS = ["pytest>=8", "httpx>=0.27", "openapi-schema-validator>=0.6"]
+
+
+class Harness(BaseModel):
+    title: str
+    version: str
+    files: dict[str, str]  # relative path -> content
+    summary: dict[str, int]
+    notes: list[str] = Field(default_factory=list)
+    reviews: list[str] = Field(default_factory=list)  # final-round outcome per artifact
+    unresolved: list[str] = Field(default_factory=list)
+
+
+@dataclass
+class HarnessCoordinator:
+    generated_at: str = ""  # shown in the README; empty keeps output reproducible
+
+    async def assemble(self, ctx: AgentContext) -> Harness:
+        spec: ApiSpec = ctx.inputs["spec"]
+        dataset: Dataset | None = ctx.inputs.get("dataset")
+        stubs: StubSet = ctx.artifacts["stubs"].content
+        contracts: ContractSuite = ctx.artifacts["contracts"].content
+        chaos: ChaosSuite = ctx.artifacts["chaos"].content
+        return build_harness(
+            spec, dataset, stubs, contracts, chaos, ctx.reviews, ctx.unresolved, self.generated_at
+        )
+
+
+def build_harness(
+    spec: ApiSpec,
+    dataset: Dataset | None,
+    stubs: StubSet,
+    contracts: ContractSuite,
+    chaos: ChaosSuite,
+    reviews: Sequence[Review] = (),
+    unresolved: list[str] | None = None,
+    generated_at: str = "",
+) -> Harness:
+    files: dict[str, str] = {
+        "docker-compose.yml": _compose(),
+        "wiremock/mappings/stubs.json": _json(stubs.to_wiremock()),
+        "requirements.txt": "\n".join(TEST_REQUIREMENTS) + "\n",
+        "tests/pytest.ini": "[pytest]\n",
+    }
+    files |= {f"tests/{name}": content for name, content in contracts.files().items()}
+    for name, content in chaos.files().items():
+        folder = "chaos" if name in (CHAOS_CLI_FILE, SCENARIOS_FILE) else "tests"
+        files[f"{folder}/{name}"] = content
+    if dataset is not None:
+        files["data/dataset.json"] = _json(dataset.model_dump(by_alias=True))
+
+    final_reviews = _final_reviews(reviews)
+    summary = {
+        "operations": len(spec.operations),
+        "stub_mappings": len(stubs.mappings),
+        "contract_cases": len(contracts.cases),
+        "chaos_scenarios": len(chaos.scenarios),
+        "robustness_probes": len(chaos.probes),
+        "records": sum(len(e.records) for e in dataset.entities.values()) if dataset else 0,
+    }
+    notes = [
+        *(f"testdata: {n}" for n in (dataset.notes if dataset else [])),
+        *(f"stubs: {n}" for n in stubs.notes),
+        *(f"contracts: {n}" for n in contracts.notes),
+        *(f"chaos: {n}" for n in chaos.notes),
+    ]
+    review_lines = [
+        f"{r.target}: {'approved' if r.approved else 'REJECTED'}"
+        + "".join(f"\n  - {c}" for c in r.comments)
+        for r in final_reviews
+    ]
+    harness = Harness(
+        title=spec.title,
+        version=spec.version,
+        files=files,
+        summary=summary,
+        notes=notes,
+        reviews=review_lines,
+        unresolved=list(unresolved or []),
+    )
+    harness.files["README.md"] = _readme(spec, dataset, chaos, harness, generated_at)
+    harness.files["shakedown.json"] = _json(
+        harness.model_dump(exclude={"files"}) | {"generated_at": generated_at or None}
+    )
+    return harness
+
+
+def _final_reviews(reviews: Sequence[Review]) -> list[Review]:
+    """Each artifact's latest review (merging comments from all its reviewers)."""
+    latest: dict[str, Review] = {}
+    for review in reviews:
+        current = latest.get(review.target)
+        if current is None or review.round > current.round:
+            latest[review.target] = review
+        elif review.round == current.round:
+            latest[review.target] = current.model_copy(
+                update={
+                    "approved": current.approved and review.approved,
+                    "comments": [*current.comments, *review.comments],
+                }
+            )
+    return [latest[k] for k in sorted(latest)]
+
+
+def _compose() -> str:
+    return f"""\
+# WireMock serving the generated stubs. Port: SHAKEDOWN_WIREMOCK_PORT (default 8080).
+services:
+  wiremock:
+    image: {WIREMOCK_IMAGE}
+    ports:
+      - "${{SHAKEDOWN_WIREMOCK_PORT:-8080}}:8080"
+    volumes:
+      - ./wiremock:/home/wiremock
+"""
+
+
+def _readme(
+    spec: ApiSpec,
+    dataset: Dataset | None,
+    chaos: ChaosSuite,
+    harness: Harness,
+    generated_at: str,
+) -> str:
+    target = DEFAULT_STUB_URL + base_path(spec)
+    s = harness.summary
+    when = f" on {generated_at}" if generated_at else ""
+    scenario_rows = "\n".join(
+        f"| `{sc.id}` | {sc.category} | {sc.title} | {sc.expectation} |" for sc in chaos.scenarios
+    )
+    sections = [
+        f"# Chaos test harness: {spec.title} {spec.version}",
+        f"Generated by API Shakedown{when} from the OpenAPI spec.",
+        "## Quick start",
+        "```bash\n"
+        "docker compose up -d                 # WireMock with the stubs on :8080\n"
+        "pip install -r requirements.txt\n\n"
+        "# Contract tests against the stubs (the header asks the stubs for each status)\n"
+        "SHAKEDOWN_SELECT_STATUS=1 pytest tests/test_contract.py\n\n"
+        "# Switch failure scenarios on and off while your client talks to the stubs\n"
+        "python chaos/chaos.py list\n"
+        "python chaos/chaos.py on flaky-start\n"
+        "python chaos/chaos.py off\n\n"
+        "# Against a real deployment: contract tests and robustness probes\n"
+        "SHAKEDOWN_TARGET_URL=https://api.example.com/v1 pytest tests\n"
+        "```",
+        'PowerShell: `$env:SHAKEDOWN_SELECT_STATUS = "1"; pytest tests/test_contract.py`.',
+        f"The stubs serve `{target}`. Use another port with `SHAKEDOWN_WIREMOCK_PORT`, then "
+        "point `SHAKEDOWN_TARGET_URL` and `chaos.py --wiremock` at it.",
+        "## What's inside",
+        "| | |\n|---|---|\n"
+        f"| `wiremock/mappings/` | {s['stub_mappings']} stub mappings for {s['operations']} "
+        "operations |\n"
+        f"| `tests/test_contract.py` | {s['contract_cases']} contract test cases |\n"
+        f"| `tests/test_robustness.py` | {s['robustness_probes']} malformed-request probes "
+        "(expect 4xx, never 5xx) |\n"
+        f"| `chaos/chaos.py` | {s['chaos_scenarios']} failure scenarios to inject into the "
+        "stubs |\n"
+        + (
+            f"| `data/dataset.json` | {s['records']} test records and edge-case values |\n"
+            if dataset
+            else ""
+        ),
+        "## How the stubs behave",
+        "- Each operation returns its success response; with test data, known ids return "
+        "their record and unknown ids return 404.\n"
+        f"- Any other documented response is returned when the request carries "
+        f"`{STATUS_HEADER}: <status>`, e.g. `{STATUS_HEADER}: 404`.\n"
+        "- An active chaos scenario overrides everything until `chaos.py off`.",
+        "## Chaos scenarios",
+        "| Scenario | Category | What happens | A resilient client should |\n"
+        "|---|---|---|---|\n" + scenario_rows,
+    ]
+    if harness.reviews or harness.unresolved:
+        review = "\n".join(f"- {line}" for line in harness.reviews)
+        if harness.unresolved:
+            review += (
+                "\n\nStill rejected when the revision budget ran out: "
+                + ", ".join(harness.unresolved)
+                + ". Check those parts before relying on them."
+            )
+        sections += ["## Review", review]
+    if harness.notes:
+        sections += ["## Generation notes", "\n".join(f"- {n}" for n in harness.notes)]
+    return "\n\n".join(sections) + "\n"
+
+
+def _json(value: Any) -> str:
+    return json.dumps(value, indent=2, ensure_ascii=False, default=str) + "\n"