api-shakedown 0.1.0__py3-none-any.whl

api_shakedown/agents/reviewer.py

@@ -0,0 +1,301 @@
+"""Agent that reviews the other specialists' output.
+
+Code already guarantees validity (schemas, routing, ids); the reviewer judges what code
+cannot: realism, coverage, and whether expectations make sense. Findings are either
+blockers (reject: the author revises) or suggestions (approve with comments), and every
+finding must cite a real item in the artifact, so hallucinated complaints are fed back
+and dropped rather than triggering pointless revisions.
+"""
+
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from agentpanel import AgentContext, Artifact, Verdict
+from langchain_core.language_models import BaseChatModel
+from pydantic import BaseModel, Field
+
+from api_shakedown.agents._common import generate_checked
+from api_shakedown.chaos import ChaosSuite
+from api_shakedown.contracts import ContractSuite
+from api_shakedown.spec import ApiSpec, Operation
+from api_shakedown.testdata import Dataset
+from api_shakedown.wiremock import RECORD_PRIORITY, StubSet
+
+GENERAL = "general"
+MAX_CHARS = 16_000
+MAX_VALUE_CHARS = 300
+
+INSTRUCTIONS = """\
+You review artifacts generated for an API test harness. Validity (schemas, routing, \
+ids) is already checked by code; you judge quality.
+
+Report only real problems, each about one specific item, with a concrete fix:
+- "blocker": the artifact would be wrong or misleading for its purpose; the author \
+must fix it.
+- "suggestion": an improvement that is not required.
+If the artifact is good, return an empty findings list. Do not invent problems.
+
+OpenAPI note: "default" and ranges such as "4XX" are valid response keys. "default" \
+means any status the spec does not list explicitly, so a case that expects "default" \
+passes for any such status (400, 500, ...)."""
+
+CHECKLISTS = {
+    "stubs": """\
+Checklist for WireMock stubs:
+- Bodies look like real data: no placeholders such as "string", "foo", or "test".
+- Ids and names are consistent across operations.
+- Error bodies match their status and plausibly describe the failure.
+- Response headers have plausible values.
+Request matching (paths, priorities, which stub answers which request) is generated by \
+code and is correct; judge only the response content.""",
+    "contracts": """\
+Checklist for contract test cases:
+- Each operation has a happy path, its documented error statuses where feasible, and \
+boundary cases for constrained inputs.
+- Each case's inputs would actually produce its expected status (a 400 case must send \
+invalid input; a 404 case an unknown id).
+- Inputs are realistic; there are no near-duplicate cases.""",
+    "chaos": """\
+Checklist for chaos scenarios:
+- Scenarios are realistic and valuable for this API (writes, most-used reads, \
+retries, partial failure).
+- Each expectation is concrete and matches its effect (e.g. retry advice only for \
+transient failures).
+- No two scenarios test the same thing.
+Baseline scenarios and probes are generated by code and cannot be changed by the \
+author: only suggest about them, never block.""",
+    "dataset": """\
+Checklist for test data:
+- Records are realistic and varied: no placeholders, enum values spread across records.
+- Data is internally consistent (names, dates, amounts, references).""",
+}
+
+
+class Finding(BaseModel):
+    severity: Literal["blocker", "suggestion"]
+    item: str = Field(description='Id of the stub/case/scenario/record, or "general".')
+    problem: str
+    fix: str = Field(description="The concrete change the author should make.")
+
+
+class ReviewFindings(BaseModel):
+    findings: list[Finding] = Field(default_factory=list)
+
+
+@dataclass
+class ArtifactReviewer:
+    model: BaseChatModel
+    name: str = "reviewer"
+    max_findings: int = 8
+    max_attempts: int = 3
+
+    async def review(self, artifact: Artifact, ctx: AgentContext) -> Verdict:
+        spec: ApiSpec = ctx.inputs["spec"]
+        reviewable = describe(artifact.content)
+        if reviewable is None:
+            return Verdict(approved=True, comments=["Not reviewed: unknown artifact type."])
+        kind, rendering, items, blockable = reviewable
+        notes: list[str] = []
+        output = await generate_checked(
+            self.model,
+            f"{INSTRUCTIONS}\n\n{CHECKLISTS[kind]}",
+            _prompt(spec, artifact, rendering, items),
+            ReviewFindings,
+            check=lambda out: _problems(out, items),
+            max_attempts=self.max_attempts,
+            notes=notes,
+        )
+        if output is None:
+            return Verdict(approved=True, comments=[f"Review unavailable: {' '.join(notes)}"])
+        findings = [
+            _downgrade(f, blockable) for f in output.findings if not _finding_problems(f, items)
+        ]
+        findings.sort(key=lambda f: f.severity != "blocker")
+        findings = findings[: self.max_findings]
+        return Verdict(
+            approved=not any(f.severity == "blocker" for f in findings),
+            comments=[f"[{f.severity}] {f.item}: {f.problem} Fix: {f.fix}" for f in findings],
+        )
+
+
+Rendered = tuple[list[str], set[str], set[str]]  # lines, citable ids, blockable ids
+
+
+def describe(content: Any) -> tuple[str, str, set[str], set[str]] | None:
+    """(checklist kind, rendering, citable ids, ids the author can fix) for an artifact."""
+    renderers: list[tuple[type, str, Callable[[Any], Rendered]]] = [
+        (StubSet, "stubs", _stubs),
+        (ContractSuite, "contracts", _contracts),
+        (ChaosSuite, "chaos", _chaos),
+        (Dataset, "dataset", _dataset),
+    ]
+    for model, kind, render in renderers:
+        if isinstance(content, model):
+            lines, items, blockable = render(content)
+            return kind, _fit(lines), items, blockable
+    return None
+
+
+def _stubs(stubs: StubSet) -> Rendered:
+    lines = [
+        "(Which stub answers which request is decided by code and is correct; the bracket "
+        "says when each one is used. The X-Shakedown-Status header is a deliberate test "
+        "hook, not part of the API: it lets tests request each documented response on "
+        "demand. Never report it as a problem.)"
+    ]
+    for m in stubs.mappings:
+        request, response = m["request"], m["response"]
+        meta = m["metadata"]["shakedown"]
+        body = response.get("jsonBody", response.get("body"))
+        lines.append(
+            f"- {m['name']}: {request['method']} [{_when(m, meta)}] "
+            f"-> {response.get('status')} {_short(body)}"
+        )
+    items = {m["name"] for m in stubs.mappings}
+    items |= {m["metadata"]["shakedown"]["operationId"] for m in stubs.mappings}
+    # Bodies taken from the test dataset are not the stub author's to change.
+    from_dataset = {
+        m["name"] for m in stubs.mappings if m["metadata"]["shakedown"].get("source") == "dataset"
+    }
+    return lines, items, items - from_dataset
+
+
+def _when(mapping: dict[str, Any], meta: dict[str, Any]) -> str:
+    if meta.get("selectedBy"):
+        header, value = next(iter(meta["selectedBy"].items()))
+        meaning = " (the spec's 'default' response)" if value == "default" else ""
+        return f"only when the client sends {header}: {value}{meaning}"
+    if mapping["priority"] == RECORD_PRIORITY:
+        return f"only for {mapping['name'].split(' ', 2)[2]} (a record in the test data)"
+    if mapping["name"].endswith("(unknown id)"):
+        return "for ids not in the test data"
+    return "normal response"
+
+
+def _contracts(suite: ContractSuite) -> Rendered:
+    lines = [
+        f"- {c.id}: {c.method.upper()} {c.path} path={_short(c.path_params)} "
+        f"query={_short(c.query)} body={_short(c.body)} -> expect {_status(c.expected_status)}"
+        for c in suite.cases
+    ]
+    items = {c.id for c in suite.cases} | {c.operation_id for c in suite.cases}
+    return lines, items, items
+
+
+def _status(label: str) -> str:
+    if label == "default":
+        return "default (any status the spec does not list explicitly)"
+    if not label.isdigit():
+        return f"{label} (any {label[0]}xx)"
+    return label
+
+
+def _chaos(suite: ChaosSuite) -> Rendered:
+    lines = ["Scenarios:"] + [
+        f"- {s.id} [{s.category}, {'baseline, fixed' if s.source == 'baseline' else 'llm'}] "
+        f"ops={s.operations or 'all'} "
+        f"effect={_short(s.effect.model_dump(exclude_none=True))} times={s.times} "
+        f"| expect: {s.expectation}"
+        for s in suite.scenarios
+    ]
+    per_op: dict[str, list[str]] = {}
+    for probe in suite.probes:
+        per_op.setdefault(probe.operation_id, []).append(probe.description)
+    lines.append("Robustness probes (malformed requests that must get a 4xx):")
+    lines += [f"- {op}: {len(d)} probes, e.g. {'; '.join(d[:4])}" for op, d in per_op.items()]
+    items = {s.id for s in suite.scenarios} | {p.id for p in suite.probes} | set(per_op)
+    blockable = {s.id for s in suite.scenarios if s.source == "llm"} | set(per_op)
+    return lines, items, blockable
+
+
+def _dataset(dataset: Dataset) -> Rendered:
+    lines: list[str] = []
+    items: set[str] = set()
+    for entity in dataset.entities.values():
+        items.add(entity.name)
+        for record in entity.records:
+            ref = f"{entity.name}#{record.get(entity.id_field or '', '?')}"
+            items.add(ref)
+            lines.append(f"- {ref}: {_short(record)}")
+    return lines, items, items
+
+
+def _prompt(spec: ApiSpec, artifact: Artifact, rendering: str, items: set[str]) -> str:
+    operations = "\n".join(_operation_line(op) for op in spec.operations)
+    return (
+        f"API: {spec.title} {spec.version}\nOperations:\n{operations}\n\n"
+        f"Review this {artifact.kind} produced by '{artifact.author}':\n"
+        f"<artifact>\n{rendering}\n</artifact>\n\n"
+        'Each finding\'s "item" must be one of the ids shown above (the text before the '
+        'colon on a line, or an operation id), or "general" for a suggestion.'
+    )
+
+
+def _operation_line(op: Operation) -> str:
+    """One line per operation, with the inputs a reviewer needs to judge cases."""
+    params = ", ".join(
+        f"{p.name} ({p.location}{', required' if p.required else ''}"
+        + (f", {_short(_constraints(p.schema_))}" if _constraints(p.schema_) else "")
+        + ")"
+        for p in op.parameters
+    )
+    body = ""
+    if op.request_body is not None:
+        body = f" body: {op.request_body.content_type}"
+        body += " (required)" if op.request_body.required else " (optional)"
+    return (
+        f"- {op.id}: {op.method.upper()} {op.path}"
+        + (f" ({op.summary})" if op.summary else "")
+        + (f" params: {params};" if params else " no params;")
+        + (body or " no request body;")
+        + f" statuses {[r.status for r in op.responses]}"
+    )
+
+
+def _constraints(schema: dict[str, Any] | None) -> dict[str, Any]:
+    keys = ("type", "enum", "minimum", "maximum", "minLength", "maxLength", "format")
+    return {k: v for k, v in (schema or {}).items() if k in keys}
+
+
+def _downgrade(finding: Finding, blockable: set[str]) -> Finding:
+    """A blocker on something the author cannot change becomes a suggestion."""
+    if finding.severity == "blocker" and finding.item not in blockable:
+        return finding.model_copy(update={"severity": "suggestion"})
+    return finding
+
+
+def _problems(output: ReviewFindings, items: set[str]) -> list[str]:
+    return [
+        f"finding about {finding.item!r}: {problem}"
+        for finding in output.findings
+        for problem in _finding_problems(finding, items)
+    ]
+
+
+def _finding_problems(finding: Finding, items: set[str]) -> list[str]:
+    if finding.item == GENERAL:
+        if finding.severity == "blocker":
+            return ["a blocker must name a specific item, not 'general'."]
+        return []
+    if finding.item not in items:
+        return ["item is not in the artifact; cite an id exactly as shown."]
+    return []
+
+
+def _short(value: Any) -> str:
+    text = json.dumps(value, default=str, ensure_ascii=False)
+    return text if len(text) <= MAX_VALUE_CHARS else text[:MAX_VALUE_CHARS] + "...(truncated)"
+
+
+def _fit(lines: list[str]) -> str:
+    kept: list[str] = []
+    size = 0
+    for line in lines:
+        if size + len(line) > MAX_CHARS:
+            kept.append(f"... ({len(lines) - len(kept)} more lines omitted)")
+            break
+        kept.append(line)
+        size += len(line) + 1
+    return "\n".join(kept)

api_shakedown/agents/stubs.py

@@ -0,0 +1,163 @@
+"""Agent that generates WireMock stub mappings from the spec.
+
+Request matching comes from code (`api_shakedown.wiremock`). The LLM writes realistic
+response bodies, which are validated against the spec's response schemas; problems are
+fed back for another attempt, and anything still invalid falls back to a schema sample.
+Without a model, the agent is fully deterministic.
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+from agentpanel import AgentContext
+from agentpanel.llm_agents import render_feedback
+from langchain_core.language_models import BaseChatModel
+from pydantic import BaseModel, Field
+
+from api_shakedown.agents._common import generate_checked
+from api_shakedown.spec import ApiSpec, Operation, chunk_operations
+from api_shakedown.spec.chunking import DEFAULT_MAX_CHARS
+from api_shakedown.spec.models import Response
+from api_shakedown.spec.schema_check import schema_errors
+from api_shakedown.testdata import Dataset
+from api_shakedown.wiremock import (
+    ResponseContent,
+    ResponseKey,
+    StubSet,
+    build_stub_set,
+    dataset_content,
+    expected_responses,
+)
+
+INSTRUCTIONS = """\
+You generate realistic mock HTTP response bodies for WireMock stubs, from an OpenAPI \
+description of each operation.
+
+Rules:
+- Every body must strictly conform to its response schema: required fields, types, \
+enums, formats, and min/max constraints.
+- Use realistic, internally consistent data: reuse the same ids and names across \
+related operations, and avoid placeholders like "string" or "foo".
+- Error bodies should read like real errors for their status code.
+- Give plausible values for documented response headers.
+- Use null as the body when a response has no content."""
+
+
+class GeneratedResponse(BaseModel):
+    operation_id: str
+    status: str
+    body: Any = Field(default=None, description="JSON value matching the response schema.")
+    headers: dict[str, str] = Field(default_factory=dict)
+
+
+class GeneratedResponses(BaseModel):
+    responses: list[GeneratedResponse]
+
+
+@dataclass
+class StubAgent:
+    model: BaseChatModel | None = None
+    name: str = "stubs"
+    kind: str = "WireMock stub mappings"
+    max_chars: int = DEFAULT_MAX_CHARS
+    max_attempts: int = 3
+
+    async def produce(self, ctx: AgentContext) -> StubSet:
+        spec: ApiSpec = ctx.inputs["spec"]
+        dataset: Dataset | None = ctx.inputs.get("dataset")
+        contents: dict[ResponseKey, ResponseContent] = {}
+        notes: list[str] = []
+        if self.model is not None:
+            feedback = ctx.feedback_for(self.name)
+            for chunk in chunk_operations(spec.operations, self.max_chars):
+                expected = _needs_llm(chunk, dataset)
+                if expected:
+                    contents |= await self._generate(
+                        self.model, spec, chunk, expected, render_feedback(feedback), notes
+                    )
+            notes += [
+                f"{op.id} {response.status}: used a schema-generated sample body."
+                for op, response in expected_responses(spec.operations)
+                if (op.id, response.status) not in contents
+                and dataset_content(response, dataset) is None
+            ]
+        return build_stub_set(spec, contents, notes, dataset)
+
+    async def _generate(
+        self,
+        model: BaseChatModel,
+        spec: ApiSpec,
+        chunk: list[Operation],
+        expected: dict[ResponseKey, Response],
+        feedback: str,
+        notes: list[str],
+    ) -> dict[ResponseKey, ResponseContent]:
+        output = await generate_checked(
+            model,
+            INSTRUCTIONS,
+            _prompt(spec, chunk, expected, feedback),
+            GeneratedResponses,
+            check=lambda out: _problems(spec, expected, out),
+            max_attempts=self.max_attempts,
+            notes=notes,
+        )
+        usable: dict[ResponseKey, ResponseContent] = {}
+        for item in output.responses if output else []:
+            key = (item.operation_id, item.status)
+            response = expected.get(key)
+            if response is None or (response.body is not None and item.body is None):
+                continue  # unknown entry, or a null body where the sample is better
+            if not _item_problems(spec, response, item):
+                usable[key] = ResponseContent(body=item.body, headers=item.headers, source="llm")
+        return usable
+
+
+def _needs_llm(chunk: list[Operation], dataset: Dataset | None) -> dict[ResponseKey, Response]:
+    """Responses the LLM should write: everything the dataset does not already cover."""
+    return {
+        (op.id, r.status): r
+        for op, r in expected_responses(chunk)
+        if dataset_content(r, dataset) is None
+    }
+
+
+def _prompt(
+    spec: ApiSpec, chunk: list[Operation], expected: dict[ResponseKey, Response], feedback: str
+) -> str:
+    entries = []
+    for (op_id, status), response in expected.items():
+        body = response.body
+        what = f"{body.content_type}" if body else "no body: use null"
+        entries.append(f"- operation_id={op_id} status={status} ({what})")
+    prompt = (
+        f"API: {spec.title} {spec.version}\n\n"
+        "<operations>\n" + "\n".join(op.to_prompt() for op in chunk) + "\n</operations>\n\n"
+        "Produce exactly one entry for each of these responses:\n" + "\n".join(entries)
+    )
+    if feedback:
+        prompt += f"\n\nReviewers rejected the previous stubs. Address this feedback:\n{feedback}"
+    return prompt
+
+
+def _problems(
+    spec: ApiSpec, expected: dict[ResponseKey, Response], output: GeneratedResponses
+) -> list[str]:
+    produced = {(item.operation_id, item.status): item for item in output.responses}
+    problems = [
+        f"Missing entry for operation_id={op_id} status={status}."
+        for op_id, status in expected
+        if (op_id, status) not in produced
+    ]
+    for key, item in produced.items():
+        response = expected.get(key)
+        if response is not None:
+            problems += [f"{key[0]} {key[1]}: {p}" for p in _item_problems(spec, response, item)]
+    return problems
+
+
+def _item_problems(spec: ApiSpec, response: Response, item: GeneratedResponse) -> list[str]:
+    if response.body is None or response.body.schema_ is None:
+        return []
+    if item.body is None:
+        return ["body is null but the response has a schema."]
+    return schema_errors(response.body.schema_, item.body, spec.openapi_version)

api_shakedown/agents/testdata.py

@@ -0,0 +1,146 @@
+"""Agent that builds the shared test dataset.
+
+The LLM writes realistic, varied records for each entity the API serves; code validates
+every record against its schema, enforces unique ids, and falls back to deterministic
+records where needed. Edge-case values come from code alone. The dataset is meant to run
+first and be passed to the other agents as `inputs["dataset"]`.
+"""
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from agentpanel import AgentContext
+from agentpanel.llm_agents import render_feedback
+from langchain_core.language_models import BaseChatModel
+from pydantic import BaseModel, Field
+
+from api_shakedown.agents._common import generate_checked
+from api_shakedown.spec import ApiSpec
+from api_shakedown.spec.chunking import DEFAULT_MAX_CHARS
+from api_shakedown.testdata import (
+    DEFAULT_RECORDS,
+    Dataset,
+    Entity,
+    build_dataset,
+    find_entities,
+    record_problems,
+)
+
+MAX_RECORDS = 10
+
+INSTRUCTIONS = """\
+You create realistic test data for an API: records for each entity, matching its JSON \
+schema exactly.
+
+Rules:
+- Every record must satisfy the schema: required fields, types, enums, formats, and \
+min/max constraints.
+- Records must have unique ids.
+- Make records realistic and varied: different enum values, plausible names, dates, \
+and amounts. Avoid placeholders like "string", "foo", or "test".
+- Keep data consistent across entities that refer to each other."""
+
+
+class EntityRecords(BaseModel):
+    entity: str
+    records: list[dict[str, Any]]
+
+
+class GeneratedDataset(BaseModel):
+    entities: list[EntityRecords] = Field(description="One entry per requested entity.")
+
+
+@dataclass
+class DatasetAgent:
+    model: BaseChatModel | None = None
+    name: str = "testdata"
+    kind: str = "test dataset"
+    records_per_entity: int = DEFAULT_RECORDS
+    max_chars: int = DEFAULT_MAX_CHARS
+    max_attempts: int = 3
+
+    async def produce(self, ctx: AgentContext) -> Dataset:
+        spec: ApiSpec = ctx.inputs["spec"]
+        entities = find_entities(spec)
+        notes: list[str] = []
+        if self.model is not None and entities:
+            feedback = render_feedback(ctx.feedback_for(self.name))
+            for group in _groups(list(entities.values()), self.max_chars):
+                await self._generate(self.model, spec, group, feedback, notes)
+            notes += [
+                f"{e.name}: used deterministic sample records."
+                for e in entities.values()
+                if not e.records
+            ]
+        return build_dataset(spec, entities, notes)
+
+    async def _generate(
+        self,
+        model: BaseChatModel,
+        spec: ApiSpec,
+        group: list[Entity],
+        feedback: str,
+        notes: list[str],
+    ) -> None:
+        count = max(1, min(self.records_per_entity, MAX_RECORDS))
+        by_name = {e.name: e for e in group}
+        output = await generate_checked(
+            model,
+            INSTRUCTIONS,
+            _prompt(spec, group, count, feedback),
+            GeneratedDataset,
+            check=lambda out: _problems(spec, by_name, out),
+            max_attempts=self.max_attempts,
+            notes=notes,
+        )
+        for item in output.entities if output else []:
+            entity = by_name.get(item.entity)
+            if entity is None:
+                continue
+            valid: list[dict[str, Any]] = []
+            for record in item.records[:MAX_RECORDS]:
+                if not record_problems(entity, [*valid, record], spec.openapi_version):
+                    valid.append(record)
+            entity.records = valid
+
+
+def _groups(entities: list[Entity], max_chars: int) -> list[list[Entity]]:
+    groups: list[list[Entity]] = []
+    size = 0
+    for entity in entities:
+        entity_size = len(json.dumps(entity.schema_))
+        if not groups or size + entity_size > max_chars:
+            groups.append([])
+            size = 0
+        groups[-1].append(entity)
+        size += entity_size
+    return groups
+
+
+def _prompt(spec: ApiSpec, group: list[Entity], count: int, feedback: str) -> str:
+    schemas = "\n".join(
+        f'<entity name="{e.name}" id_field="{e.id_field}">\n{json.dumps(e.schema_)}\n</entity>'
+        for e in group
+    )
+    prompt = (
+        f"API: {spec.title} {spec.version}\n\n{schemas}\n\n"
+        f"Create {count} records for each of: {', '.join(e.name for e in group)}."
+    )
+    if feedback:
+        prompt += f"\n\nReviewers rejected the previous dataset. Address this feedback:\n{feedback}"
+    return prompt
+
+
+def _problems(spec: ApiSpec, entities: dict[str, Entity], output: GeneratedDataset) -> list[str]:
+    produced = {item.entity: item for item in output.entities}
+    problems = [f"Missing records for entity {name}." for name in entities if name not in produced]
+    for name, item in produced.items():
+        entity = entities.get(name)
+        if entity is None:
+            problems.append(f"Unknown entity {name!r}; use one of {sorted(entities)}.")
+        elif not item.records:
+            problems.append(f"No records for {name}.")
+        else:
+            problems += record_problems(entity, item.records, spec.openapi_version)
+    return problems